shroud-privacy 2.2.11 → 2.2.13

package/README.md

@@ -17,7 +17,7 @@
   <a href="CHANGELOG.md">Changelog</a>
 </p>
 
-> Apache 2.0 &middot; Zero runtime dependencies &middot; Works with [OpenClaw](https://openclaw.ai) or any agent via [APP](#agent-privacy-protocol-app)
+> Apache 2.0 &middot; Zero runtime dependencies &middot; Anthropic + OpenAI + Google supported &middot; Prompt-caching friendly &middot; Works with [OpenClaw](https://openclaw.ai) or any agent via [APP](#agent-privacy-protocol-app)
 
 ---
 
@@ -61,6 +61,7 @@ Shroud does not guarantee compliance — regex-based detection has limitations (
 3. **Passes through public URLs** — external URLs (arxiv.org, docs.stripe.com, etc.) are not obfuscated. Shroud resolves FQDNs via DNS: public IPs pass through, RFC 1918 / NXDOMAIN / internal IPs are obfuscated. Well-known platforms (GitHub, YouTube, Wikipedia, etc.) are always passed through.
 4. **Deobfuscates** LLM responses and tool parameters so the user sees real values and tools receive real arguments.
 5. **Audit logs** every event with counts, categories, char deltas, and optional proof hashes — never logging raw sensitive values.
+6. **Preserves prompt caching.** Obfuscation is deterministic — same input + same key = same output every turn. The system prompt prefix stays identical across turns, so provider-side prompt caching (Anthropic, OpenAI, Bedrock) works normally. No cache-busting, no extra token costs.
 
 ### Hook lifecycle
 
@@ -355,10 +356,12 @@ APP is an open protocol for adding privacy and infrastructure protection to any
 |  (any language)   |                      |  (app-server.mjs)|
 +-------------------+                      +------------------+
         |                                        |
-        | 1. obfuscate(user_input)               | detects entities,
-        | 2. send to LLM                         | returns fakes
-        | 3. deobfuscate(llm_response)           | restores reals
-        | 4. show to user                        |
+        | 1. identify(agent, version)            | registers agent
+        | 2. obfuscate(user_input)               | detects entities,
+        | 3. send to LLM                         | returns fakes
+        | 4. tool_call(tool, args)               | scans tool call
+        | 5. deobfuscate(llm_response)           | restores reals
+        | 6. show to user                        |
 ```
 
 ### Protocol specification
@@ -371,15 +374,21 @@ APP is an open protocol for adding privacy and infrastructure protection to any
 
 | Method | Params | Returns | Description |
 |--------|--------|---------|-------------|
-| `obfuscate` | `{text}` | `{text, entityCount, categories, modified, audit}` | Replace real values with fakes |
-| `deobfuscate` | `{text}` | `{text, replacementCount, modified, audit}` | Restore fakes to real values |
+| `identify` | `{agent, version, channel?}` | `{ok, agent, buildId}` | Identify the agent (required before obfuscate/deobfuscate) |
+| `obfuscate` | `{text, partition?}` | `{text, entityCount, categories, modified, audit}` | Replace real values with fakes |
+| `deobfuscate` | `{text, partition?}` | `{text, replacementCount, modified, audit}` | Restore fakes to real values |
+| `tool_call` | `{tool, args?}` | `{allowed, blocked, tool, events?}` | Report a tool call for security scanning |
+| `tool_result` | `{tool, result}` | `{text, replacementCount}` | Obfuscate tool result before storing |
 | `reset` | `{}` | `{ok, summary}` | Clear all mappings |
 | `stats` | `{}` | `{storeMappings, ruleHits, ...}` | Engine statistics |
 | `health` | `{}` | `{uptime, requests, avgLatencyMs}` | Liveness check |
 | `configure` | `{config}` | `{ok}` | Hot-reload configuration |
 | `batch` | `{operations: [{direction, text}]}` | `{results: [...]}` | Batch obfuscate/deobfuscate |
+| `setPartition` | `{partition}` | `{ok}` | Switch mapping namespace (multi-tenant) |
 | `shutdown` | `{}` | `{ok}` | Graceful shutdown (flushes stats) |
 
+Agents should call `identify` first to register themselves. `tool_call` and `tool_result` are optional — they enable per-tool privacy scanning and mapping isolation for tool arguments.
+
 ### Python client
 
 ```python
@@ -408,16 +417,16 @@ client.stop()
 npm install
 npm run build               # compile TypeScript
 npm run lint                # type-check without emitting
-npm test                    # unit + harness (1,229 tests, no Docker)
+npm test                    # unit + harness (1,238 tests, no Docker)
 npm run test:docker         # Docker E2E — real OpenClaw, all channels (192 tests)
-npm run test:all            # everything (1,421 tests)
+npm run test:all            # everything (1,430 tests)
 ```
 
 ### Test layers
 
 | Layer | Command | Tests | What it covers |
 |-------|---------|-------|---------------|
-| Unit | `npm run test:unit` | 870 | Obfuscator, detectors, generators, store, config |
+| Unit | `npm run test:unit` | 879 | Obfuscator, detectors, generators, store, config |
 | APP Harness | `npm run test:integration` | 359 | 48 scenario files via mock LLM, no OpenClaw |
 | Docker E2E | `npm run test:docker` | 192 | Real OpenClaw gateway, Slack/WhatsApp/Cron/TUI channels, 153 regression scenarios |
 | Sandbox E2E | `run-compat.sh <ver> --sandbox` | +8 | Docker-in-Docker, sandboxed agent exec, tool call deobfuscation |

package/dist/hooks.js

@@ -299,25 +299,56 @@ export function registerHooks(api, obfuscator) {
                 totalEntities += result.entities.length;
             }
         }
-        // Pre-create mappings for PII in user messages WITHOUT mutating them.
-        // This seeds the mapping store so the fetch intercept's response
-        // deobfuscation can replace fakes with the correct real values.
+        // Obfuscate ALL messages in-place — seeds the mapping store AND mutates
+        // the message array so PII is replaced before OpenClaw builds the request.
+        // This is critical when the LLM SDK (e.g. OpenAI v6) captures fetch at
+        // construction time, bypassing Shroud's globalThis.fetch intercept.
+        // The fetch intercept is still the primary path for SDKs that use
+        // globalThis.fetch (Anthropic) — double-obfuscation is safe because
+        // already-obfuscated text has no detectable PII entities.
         if (Array.isArray(event?.messages)) {
             for (const msg of event.messages) {
-                const texts = [];
-                if (typeof msg.content === "string")
-                    texts.push(msg.content);
+                // String content (Anthropic/OpenAI)
+                if (typeof msg.content === "string") {
+                    const cleaned = stripSlackLinksForHook(msg.content);
+                    const result = ob().obfuscate(cleaned);
+                    totalEntities += result.entities.length;
+                    if (result.entities.length > 0 || cleaned !== msg.content) {
+                        msg.content = result.entities.length > 0 ? result.obfuscated : cleaned;
+                    }
+                }
+                // Array content blocks
                 else if (Array.isArray(msg.content)) {
                     for (const b of msg.content) {
-                        if (b?.type === "text" && typeof b.text === "string")
-                            texts.push(b.text);
+                        if (b?.type === "text" && typeof b.text === "string") {
+                            const cleaned = stripSlackLinksForHook(b.text);
+                            const result = ob().obfuscate(cleaned);
+                            totalEntities += result.entities.length;
+                            if (result.entities.length > 0 || cleaned !== b.text) {
+                                b.text = result.entities.length > 0 ? result.obfuscated : cleaned;
+                            }
+                        }
+                        // tool_result blocks with string content
+                        if (typeof b?.content === "string") {
+                            const cleaned = stripSlackLinksForHook(b.content);
+                            const result = ob().obfuscate(cleaned);
+                            totalEntities += result.entities.length;
+                            if (result.entities.length > 0 || cleaned !== b.content) {
+                                b.content = result.entities.length > 0 ? result.obfuscated : cleaned;
+                            }
+                        }
                     }
                 }
-                for (const text of texts) {
-                    const cleaned = stripSlackLinksForHook(text);
-                    const result = ob().obfuscate(cleaned);
-                    totalEntities += result.entities.length;
-                    // Do NOT mutate — just creating mappings in the store
+                // OpenAI tool_calls in assistant messages
+                if (Array.isArray(msg.tool_calls)) {
+                    for (const tc of msg.tool_calls) {
+                        if (typeof tc.function?.arguments === "string") {
+                            const result = ob().obfuscate(tc.function.arguments);
+                            totalEntities += result.entities.length;
+                            if (result.entities.length > 0)
+                                tc.function.arguments = result.obfuscated;
+                        }
+                    }
                 }
             }
         }
@@ -740,6 +771,8 @@ export function registerHooks(api, obfuscator) {
     const originalFetch = globalThis.fetch;
     if (originalFetch && !(globalThis.__shroudFetchPatched)) {
         globalThis.__shroudFetchPatched = true;
+        // Store original for comparison — SDK clients may have captured it
+        const _prePatchFetch = globalThis.fetch;
         globalThis.fetch = async function shroudFetchInterceptor(input, init) {
             // Only intercept POST requests with a body
             if (init?.method?.toUpperCase() !== "POST" || !init?.body) {
@@ -922,6 +955,25 @@ export function registerHooks(api, obfuscator) {
                             modified = true;
                         }
                     }
+                    // OpenAI: tool_calls in assistant messages (multi-turn re-obfuscation)
+                    if (Array.isArray(msg.tool_calls)) {
+                        for (const tc of msg.tool_calls) {
+                            if (typeof tc.function?.arguments === "string") {
+                                const r = obfuscateText(tc.function.arguments);
+                                if (r.modified) {
+                                    tc.function.arguments = r.text;
+                                    modified = true;
+                                }
+                            }
+                            if (typeof tc.function?.name === "string") {
+                                const r = obfuscateText(tc.function.name);
+                                if (r.modified) {
+                                    tc.function.name = r.text;
+                                    modified = true;
+                                }
+                            }
+                        }
+                    }
                 }
                 if (modified) {
                     const newBody = JSON.stringify(body);
@@ -968,6 +1020,9 @@ export function registerHooks(api, obfuscator) {
                 // OpenAI per-choice state
                 const choiceAccum = new Map();
                 const choiceBuffer = new Map();
+                // OpenAI tool_calls per-choice state: Map<choiceIdx, Map<toolCallIdx, argString>>
+                const toolCallAccum = new Map();
+                const toolCallBuffer = new Map();
                 let sseRemainder = "";
                 const transform = new TransformStream({
                     transform(chunk, controller) {
@@ -1038,7 +1093,7 @@ export function registerHooks(api, obfuscator) {
                                 controller.enqueue(new TextEncoder().encode(part + "\n\n"));
                                 continue;
                             }
-                            // OpenAI delta.content: buffer until finish_reason
+                            // OpenAI delta.content / delta.tool_calls: buffer until finish_reason
                             if (Array.isArray(json.choices)) {
                                 let buffered = false;
                                 for (const choice of json.choices) {
@@ -1050,8 +1105,25 @@ export function registerHooks(api, obfuscator) {
                                         choiceBuffer.get(idx).push(part);
                                         buffered = true;
                                     }
+                                    // Buffer tool_calls argument fragments
+                                    if (Array.isArray(choice.delta?.tool_calls)) {
+                                        for (const tc of choice.delta.tool_calls) {
+                                            const tcIdx = tc.index ?? 0;
+                                            if (!toolCallAccum.has(idx))
+                                                toolCallAccum.set(idx, new Map());
+                                            const tcMap = toolCallAccum.get(idx);
+                                            if (typeof tc.function?.arguments === "string") {
+                                                tcMap.set(tcIdx, (tcMap.get(tcIdx) || "") + tc.function.arguments);
+                                            }
+                                        }
+                                        if (!toolCallBuffer.has(idx))
+                                            toolCallBuffer.set(idx, []);
+                                        toolCallBuffer.get(idx).push(part);
+                                        buffered = true;
+                                    }
                                     // finish_reason signals block complete — flush
                                     if (choice.finish_reason) {
+                                        // Flush text content
                                         const accumulated = choiceAccum.get(idx);
                                         const buf = choiceBuffer.get(idx);
                                         if (accumulated && buf && buf.length > 0) {
@@ -1082,6 +1154,57 @@ export function registerHooks(api, obfuscator) {
                                             choiceAccum.delete(idx);
                                             choiceBuffer.delete(idx);
                                         }
+                                        // Flush tool_calls — deobfuscate accumulated arguments
+                                        const tcMap = toolCallAccum.get(idx);
+                                        const tcBuf = toolCallBuffer.get(idx);
+                                        if (tcMap && tcMap.size > 0 && tcBuf && tcBuf.length > 0) {
+                                            // Deobfuscate each tool call's accumulated arguments
+                                            const deobArgs = new Map();
+                                            for (const [tcIdx, args] of tcMap) {
+                                                const { text: deobArg, replacementCount: tcRc } = ob().deobfuscateWithStats(args);
+                                                deobArgs.set(tcIdx, deobArg);
+                                                // tcRc tracked via deobfuscateWithStats
+                                            }
+                                            // Track per-tool-call whether we've emitted the deobbed args
+                                            const tcEmitted = new Set();
+                                            for (const eventStr of tcBuf) {
+                                                const dLine = eventStr.split("\n").find((l) => l.startsWith("data: "));
+                                                if (dLine) {
+                                                    try {
+                                                        const dJson = JSON.parse(dLine.slice(6));
+                                                        if (Array.isArray(dJson.choices)) {
+                                                            for (const c of dJson.choices) {
+                                                                if (Array.isArray(c.delta?.tool_calls)) {
+                                                                    for (const tc of c.delta.tool_calls) {
+                                                                        const tcIdx = tc.index ?? 0;
+                                                                        if (typeof tc.function?.arguments === "string") {
+                                                                            const deob = deobArgs.get(tcIdx);
+                                                                            if (deob !== undefined) {
+                                                                                if (!tcEmitted.has(tcIdx)) {
+                                                                                    tc.function.arguments = deob;
+                                                                                    tcEmitted.add(tcIdx);
+                                                                                }
+                                                                                else {
+                                                                                    tc.function.arguments = "";
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                }
+                                                            }
+                                                            const nonDataLines = eventStr.split("\n").filter((l) => !l.startsWith("data: ")).join("\n");
+                                                            const rebuilt = (nonDataLines ? nonDataLines + "\n" : "") + "data: " + JSON.stringify(dJson);
+                                                            controller.enqueue(new TextEncoder().encode(rebuilt + "\n\n"));
+                                                            continue;
+                                                        }
+                                                    }
+                                                    catch { }
+                                                }
+                                                controller.enqueue(new TextEncoder().encode(eventStr + "\n\n"));
+                                            }
+                                            toolCallAccum.delete(idx);
+                                            toolCallBuffer.delete(idx);
+                                        }
                                         buffered = false;
                                     }
                                 }
@@ -1156,6 +1279,54 @@ export function registerHooks(api, obfuscator) {
                                 controller.enqueue(new TextEncoder().encode(eventStr + "\n\n"));
                             }
                         }
+                        // Flush any remaining tool_calls buffers
+                        for (const [idx, tcBuf] of toolCallBuffer) {
+                            const tcMap = toolCallAccum.get(idx);
+                            if (tcMap && tcMap.size > 0 && tcBuf.length > 0) {
+                                const deobArgs = new Map();
+                                for (const [tcIdx, args] of tcMap) {
+                                    const { text: deobArg, replacementCount: tcFlushRc } = ob().deobfuscateWithStats(args);
+                                    deobArgs.set(tcIdx, deobArg);
+                                    // tcFlushRc tracked via deobfuscateWithStats
+                                }
+                                const tcEmitted = new Set();
+                                for (const eventStr of tcBuf) {
+                                    const dLine = eventStr.split("\n").find((l) => l.startsWith("data: "));
+                                    if (dLine) {
+                                        try {
+                                            const dJson = JSON.parse(dLine.slice(6));
+                                            if (Array.isArray(dJson.choices)) {
+                                                for (const c of dJson.choices) {
+                                                    if (Array.isArray(c.delta?.tool_calls)) {
+                                                        for (const tc of c.delta.tool_calls) {
+                                                            const tcIdx = tc.index ?? 0;
+                                                            if (typeof tc.function?.arguments === "string") {
+                                                                const deob = deobArgs.get(tcIdx);
+                                                                if (deob !== undefined) {
+                                                                    if (!tcEmitted.has(tcIdx)) {
+                                                                        tc.function.arguments = deob;
+                                                                        tcEmitted.add(tcIdx);
+                                                                    }
+                                                                    else {
+                                                                        tc.function.arguments = "";
+                                                                    }
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                                const nonDataLines = eventStr.split("\n").filter((l) => !l.startsWith("data: ")).join("\n");
+                                                const rebuilt = (nonDataLines ? nonDataLines + "\n" : "") + "data: " + JSON.stringify(dJson);
+                                                controller.enqueue(new TextEncoder().encode(rebuilt + "\n\n"));
+                                                continue;
+                                            }
+                                        }
+                                        catch { }
+                                    }
+                                    controller.enqueue(new TextEncoder().encode(eventStr + "\n\n"));
+                                }
+                            }
+                        }
                         if (sseRemainder.trim()) {
                             controller.enqueue(new TextEncoder().encode(sseRemainder));
                         }
@@ -1185,6 +1356,16 @@ export function registerHooks(api, obfuscator) {
                             if (typeof choice.message?.content === "string") {
                                 choice.message.content = ob().deobfuscate(choice.message.content);
                             }
+                            // OpenAI: deobfuscate tool_calls arguments
+                            if (Array.isArray(choice.message?.tool_calls)) {
+                                for (const tc of choice.message.tool_calls) {
+                                    if (typeof tc.function?.arguments === "string") {
+                                        const { text: _jdt3, replacementCount: _jdrc3 } = ob().deobfuscateWithStats(tc.function.arguments);
+                                        tc.function.arguments = _jdt3;
+                                        // _jdrc3 tracked via deobfuscateWithStats
+                                    }
+                                }
+                            }
                         }
                     }
                     return new Response(JSON.stringify(json), {
@@ -1203,5 +1384,56 @@ export function registerHooks(api, obfuscator) {
             }
             return response;
         }
+        // ── Rebind SDK clients that captured the pre-patch fetch reference ──
+        // The OpenAI SDK (and others) capture `globalThis.fetch` at construction
+        // time via `this.fetch = options.fetch ?? getDefaultFetch()`. If the SDK
+        // was constructed before Shroud patched fetch, it holds the original
+        // unpatched reference and all LLM requests bypass the intercept.
+        // Walk all reachable objects looking for SDK client instances that still
+        // hold the old reference and rebind them to the patched interceptor.
+        try {
+            const patchedFetch = globalThis.fetch;
+            const visited = new WeakSet();
+            function rebindSdkClients(obj, depth) {
+                if (!obj || typeof obj !== "object" || depth > 4 || visited.has(obj))
+                    return;
+                visited.add(obj);
+                // OpenAI SDK: client.fetch === old unpatched fetch
+                if (obj.fetch === _prePatchFetch && obj.fetch !== patchedFetch) {
+                    obj.fetch = patchedFetch;
+                    api.logger?.info("[shroud] rebound SDK client fetch to patched interceptor");
+                }
+                // Check known extension paths
+                try {
+                    for (const key of Object.keys(obj)) {
+                        if (key.startsWith("_") || key === "constructor")
+                            continue;
+                        try {
+                            rebindSdkClients(obj[key], depth + 1);
+                        }
+                        catch { }
+                    }
+                }
+                catch { }
+            }
+            // Scan globalThis for SDK client instances
+            rebindSdkClients(globalThis.__ocClients, 0);
+            rebindSdkClients(globalThis.__openaiClient, 0);
+            // Scan all loaded modules for exported clients
+            if (typeof require !== "undefined" && require.cache) {
+                for (const modId of Object.keys(require.cache)) {
+                    if (modId.includes("openai") || modId.includes("lossless")) {
+                        try {
+                            const mod = require.cache[modId]?.exports;
+                            rebindSdkClients(mod, 0);
+                        }
+                        catch { }
+                    }
+                }
+            }
+        }
+        catch {
+            // Non-fatal — fetch intercept still works for globalThis.fetch callers
+        }
     }
 }

package/openclaw.plugin.json

@@ -1,7 +1,7 @@
 {
   "id": "shroud-privacy",
   "name": "Shroud",
-  "version": "2.2.11",
+  "version": "2.2.12",
   "description": "Privacy obfuscation with deterministic fake values and deobfuscation — PII never reaches the LLM, tool calls still work",
   "configSchema": {
     "type": "object",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "shroud-privacy",
-  "version": "2.2.11",
+  "version": "2.2.13",
   "description": "Privacy and infrastructure protection for AI agents — detects sensitive data (PII, network topology, credentials, OT/SCADA) and replaces with deterministic fakes before anything reaches the LLM.",
   "type": "module",
   "main": "dist/index.js",
@@ -22,7 +22,8 @@
     "test": "npm run test:unit && npm run test:integration",
     "test:unit": "vitest run",
     "test:integration": "node tests/harness/run.mjs",
-    "test:docker": "bash compat/run-compat.sh latest",
+    "test:docker": "bash compat/run-e2e.sh latest",
+    "test:docker:legacy": "bash compat/run-compat.sh latest",
     "test:all": "npm run test:unit && npm run test:integration && npm run test:docker",
     "test:watch": "vitest",
     "lint": "tsc --noEmit",