pi-rehydra 1.0.2 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/README.en.md +82 -59
  2. package/README.md +82 -59
  3. package/dist/core/anonymizer.d.ts +9 -1
  4. package/dist/core/anonymizer.d.ts.map +1 -1
  5. package/dist/core/anonymizer.js +169 -15
  6. package/dist/core/anonymizer.js.map +1 -1
  7. package/dist/core/observer.d.ts +33 -0
  8. package/dist/core/observer.d.ts.map +1 -0
  9. package/dist/core/observer.js +2 -0
  10. package/dist/core/observer.js.map +1 -0
  11. package/dist/host-agnostic/normalize-options.d.ts +28 -2
  12. package/dist/host-agnostic/normalize-options.d.ts.map +1 -1
  13. package/dist/host-agnostic/normalize-options.js +60 -4
  14. package/dist/host-agnostic/normalize-options.js.map +1 -1
  15. package/dist/host-agnostic/policy-input.d.ts +41 -0
  16. package/dist/host-agnostic/policy-input.d.ts.map +1 -0
  17. package/dist/host-agnostic/policy-input.js +172 -0
  18. package/dist/host-agnostic/policy-input.js.map +1 -0
  19. package/dist/host-agnostic/scrub.d.ts +42 -2
  20. package/dist/host-agnostic/scrub.d.ts.map +1 -1
  21. package/dist/host-agnostic/scrub.js +135 -17
  22. package/dist/host-agnostic/scrub.js.map +1 -1
  23. package/dist/ner/inference-server-client.d.ts.map +1 -1
  24. package/dist/ner/inference-server-client.js +18 -2
  25. package/dist/ner/inference-server-client.js.map +1 -1
  26. package/dist/ner/inference-server-ner-model.d.ts +8 -0
  27. package/dist/ner/inference-server-ner-model.d.ts.map +1 -1
  28. package/dist/ner/inference-server-ner-model.js +36 -3
  29. package/dist/ner/inference-server-ner-model.js.map +1 -1
  30. package/dist/ner/ner-model.d.ts.map +1 -1
  31. package/dist/ner/ner-model.js +5 -1
  32. package/dist/ner/ner-model.js.map +1 -1
  33. package/dist/pi-extension/extension.d.ts.map +1 -1
  34. package/dist/pi-extension/extension.js +280 -11
  35. package/dist/pi-extension/extension.js.map +1 -1
  36. package/dist/pi-extension/host-types.d.ts +5 -1
  37. package/dist/pi-extension/host-types.d.ts.map +1 -1
  38. package/dist/pi-extension/options.d.ts +14 -0
  39. package/dist/pi-extension/options.d.ts.map +1 -1
  40. package/dist/pipeline/prenormalize.d.ts.map +1 -1
  41. package/dist/pipeline/prenormalize.js +4 -5
  42. package/dist/pipeline/prenormalize.js.map +1 -1
  43. package/dist/pipeline/tagger.d.ts.map +1 -1
  44. package/dist/pipeline/tagger.js +1 -0
  45. package/dist/pipeline/tagger.js.map +1 -1
  46. package/dist/pipeline/title-extractor.d.ts.map +1 -1
  47. package/dist/pipeline/title-extractor.js +3 -0
  48. package/dist/pipeline/title-extractor.js.map +1 -1
  49. package/dist/recognizers/address.d.ts.map +1 -1
  50. package/dist/recognizers/address.js +1 -0
  51. package/dist/recognizers/address.js.map +1 -1
  52. package/dist/recognizers/base.d.ts +6 -0
  53. package/dist/recognizers/base.d.ts.map +1 -1
  54. package/dist/recognizers/base.js +1 -0
  55. package/dist/recognizers/base.js.map +1 -1
  56. package/dist/recognizers/bic-swift.d.ts.map +1 -1
  57. package/dist/recognizers/bic-swift.js +29 -0
  58. package/dist/recognizers/bic-swift.js.map +1 -1
  59. package/dist/recognizers/credit-card.d.ts.map +1 -1
  60. package/dist/recognizers/credit-card.js +1 -0
  61. package/dist/recognizers/credit-card.js.map +1 -1
  62. package/dist/recognizers/custom-id.d.ts.map +1 -1
  63. package/dist/recognizers/custom-id.js +7 -2
  64. package/dist/recognizers/custom-id.js.map +1 -1
  65. package/dist/recognizers/date.d.ts.map +1 -1
  66. package/dist/recognizers/date.js +1 -0
  67. package/dist/recognizers/date.js.map +1 -1
  68. package/dist/recognizers/email.d.ts.map +1 -1
  69. package/dist/recognizers/email.js +4 -0
  70. package/dist/recognizers/email.js.map +1 -1
  71. package/dist/recognizers/iban.d.ts.map +1 -1
  72. package/dist/recognizers/iban.js +1 -0
  73. package/dist/recognizers/iban.js.map +1 -1
  74. package/dist/recognizers/ip-address.d.ts.map +1 -1
  75. package/dist/recognizers/ip-address.js +1 -0
  76. package/dist/recognizers/ip-address.js.map +1 -1
  77. package/dist/recognizers/phone.d.ts.map +1 -1
  78. package/dist/recognizers/phone.js +1 -0
  79. package/dist/recognizers/phone.js.map +1 -1
  80. package/dist/recognizers/postal-code.d.ts.map +1 -1
  81. package/dist/recognizers/postal-code.js +1 -0
  82. package/dist/recognizers/postal-code.js.map +1 -1
  83. package/dist/recognizers/registry.d.ts.map +1 -1
  84. package/dist/recognizers/registry.js +3 -1
  85. package/dist/recognizers/registry.js.map +1 -1
  86. package/dist/recognizers/secrets/api-key.d.ts.map +1 -1
  87. package/dist/recognizers/secrets/api-key.js +1 -0
  88. package/dist/recognizers/secrets/api-key.js.map +1 -1
  89. package/dist/recognizers/secrets/aws-credentials.d.ts.map +1 -1
  90. package/dist/recognizers/secrets/aws-credentials.js +1 -0
  91. package/dist/recognizers/secrets/aws-credentials.js.map +1 -1
  92. package/dist/recognizers/secrets/config-secret.d.ts.map +1 -1
  93. package/dist/recognizers/secrets/config-secret.js +12 -5
  94. package/dist/recognizers/secrets/config-secret.js.map +1 -1
  95. package/dist/recognizers/secrets/connection-string.d.ts.map +1 -1
  96. package/dist/recognizers/secrets/connection-string.js +1 -0
  97. package/dist/recognizers/secrets/connection-string.js.map +1 -1
  98. package/dist/recognizers/secrets/env-var.d.ts.map +1 -1
  99. package/dist/recognizers/secrets/env-var.js +17 -3
  100. package/dist/recognizers/secrets/env-var.js.map +1 -1
  101. package/dist/recognizers/secrets/index.d.ts +1 -1
  102. package/dist/recognizers/secrets/index.d.ts.map +1 -1
  103. package/dist/recognizers/secrets/index.js +1 -1
  104. package/dist/recognizers/secrets/index.js.map +1 -1
  105. package/dist/recognizers/secrets/jwt.d.ts.map +1 -1
  106. package/dist/recognizers/secrets/jwt.js +1 -0
  107. package/dist/recognizers/secrets/jwt.js.map +1 -1
  108. package/dist/recognizers/secrets/key-patterns.d.ts +17 -0
  109. package/dist/recognizers/secrets/key-patterns.d.ts.map +1 -1
  110. package/dist/recognizers/secrets/key-patterns.js +151 -0
  111. package/dist/recognizers/secrets/key-patterns.js.map +1 -1
  112. package/dist/recognizers/secrets/literal-value.d.ts.map +1 -1
  113. package/dist/recognizers/secrets/literal-value.js +1 -0
  114. package/dist/recognizers/secrets/literal-value.js.map +1 -1
  115. package/dist/recognizers/secrets/private-key.d.ts.map +1 -1
  116. package/dist/recognizers/secrets/private-key.js +1 -0
  117. package/dist/recognizers/secrets/private-key.js.map +1 -1
  118. package/dist/recognizers/url.d.ts.map +1 -1
  119. package/dist/recognizers/url.js +1 -0
  120. package/dist/recognizers/url.js.map +1 -1
  121. package/dist/records/recorder.d.ts +87 -0
  122. package/dist/records/recorder.d.ts.map +1 -0
  123. package/dist/records/recorder.js +317 -0
  124. package/dist/records/recorder.js.map +1 -0
  125. package/dist/records/types.d.ts +80 -0
  126. package/dist/records/types.d.ts.map +1 -0
  127. package/dist/records/types.js +33 -0
  128. package/dist/records/types.js.map +1 -0
  129. package/dist/storage/session-base.d.ts +3 -2
  130. package/dist/storage/session-base.d.ts.map +1 -1
  131. package/dist/storage/session-base.js +2 -2
  132. package/dist/storage/session-base.js.map +1 -1
  133. package/dist/storage/sqlite.js +18 -18
  134. package/dist/storage/types.d.ts +2 -1
  135. package/dist/storage/types.d.ts.map +1 -1
  136. package/dist/types/index.d.ts +21 -0
  137. package/dist/types/index.d.ts.map +1 -1
  138. package/dist/types/index.js.map +1 -1
  139. package/index.d.ts +10 -10
  140. package/index.js +6 -6
  141. package/package.json +56 -58
package/README.en.md CHANGED
@@ -1,60 +1,83 @@
1
- # pi-rehydra
2
-
3
- Scrub detected secrets from Pi conversations before the main LLM request.
4
-
5
- [中文](./README.md)
6
-
7
- This extension intercepts the conversation between [Pi](https://pi.dev) and the LLM. Secrets from your `.env` files are replaced with placeholders before they leave your machine, and transparently restored before any tool (shell commands, file writes, etc.) executes locally; a placeholder that cannot be restored blocks that tool call.
8
-
9
- It uses Pi's `ExtensionAPI` events: `context_with_system`, `tool_call`, `tool_result`, `message_end`, and `session_start` / `session_shutdown`.
10
-
11
- Detection and rehydration are local. The model only sees placeholders; your tools always see real values.
12
-
13
- ## Install
14
-
15
- ```bash
16
- pi install npm:pi-rehydra
17
- ```
18
-
19
- Pi writes the package entry to `~/.pi/agent/settings.json`. Edit that file to add extension options:
20
-
21
- ```json
22
- {
23
- "packages": [
24
- {
25
- "source": "npm:pi-rehydra"
26
- }
27
- ],
28
- "rehydra": {
29
- "envFiles": [".env", ".env.local"],
30
- "redactValues": ["sk-live-abc123..."],
31
- "allowLiteralTagsIn": ["write", "edit"]
32
- }
33
- }
34
- ```
35
-
36
- Every key under `rehydra` is an extension option: `envFiles`, `redactValues`, `minValueLength`, `disableTypes`, `locale`, `vcsIdentities`, `policy`, `anonymizer`, `tagFormat`. `envFiles` defaults to `["**/.env*"]`, skipping `node_modules`, `.git`, and symlinks.
37
-
38
- `allowLiteralTagsIn` defaults to an empty array (always block). Tools listed here — typically `write` and `edit` — keep an **unresolvable** literal tag in their arguments instead of aborting, so you can write documentation or tests that intentionally show tag examples. Every allowed occurrence is logged (`allowed literal tag in tool args`, tool name and argument path only). Resolvable tags are still restored in every tool.
39
-
40
- ## How it works
41
-
42
- | Event | Effect |
43
- |---|---|
44
- | `session_start` | Instantiate the anonymizer, build the session map, parse the `rehydra` config |
45
- | `context_with_system` | Anonymize all request messages and inject the rehydra instruction when needed |
46
- | `tool_call` | Strictly restore PII tags in tool arguments before local execution; an unresolvable tag blocks the call |
47
- | `tool_result` | Anonymize the completed tool result before it re-enters model context |
48
- | `message_end` | Rewrite the final assistant message so PII tags are replaced with real values |
49
- | `session_shutdown` | Dispose of the anonymizer and session registry |
50
-
51
- ## Limitations
52
- - Tool-argument rehydration is strict: when a tag is absent from this session's map (or the map is missing, or decryption fails), the tool call is blocked rather than run with `<PII .../>` as a real value. A placeholder can only be restored from the same session's map; the default map is in memory, so tags from before a process restart cannot be restored.
53
-
54
- - Model calls that bypass these hooks (rare, e.g. user-initiated notes) are not intercepted.
55
- - `URL` and `IP_ADDRESS` are disabled by default; re-enable with `disableTypes: []`.
56
- - This extension cannot guarantee that title or compaction summaries stay sanitized — disable title agents if you need titles to be human-readable.
57
-
58
- ## License
59
-
1
+ # pi-rehydra
2
+
3
+ Scrub detected secrets from Pi conversations before the main LLM request.
4
+
5
+ [中文](./README.md)
6
+
7
+ This extension intercepts the conversation between [Pi](https://pi.dev) and the LLM. Secrets from your `.env` files are replaced with placeholders before they leave your machine, and transparently restored before any tool (shell commands, file writes, etc.) executes locally; a placeholder that cannot be restored blocks that tool call.
8
+
9
+ It uses Pi's `ExtensionAPI` events: `context_with_system`, `tool_call`, `tool_result`, `message_end`, and `session_start` / `session_shutdown`.
10
+
11
+ Detection and rehydration are local. The model only sees placeholders; your tools always see real values.
12
+
13
+ ## Install
14
+
15
+ ```bash
16
+ pi install npm:pi-rehydra
17
+ ```
18
+
19
+ Pi writes the package entry to `~/.pi/agent/settings.json`. Edit that file to add extension options:
20
+
21
+ ```json
22
+ {
23
+ "packages": [
24
+ {
25
+ "source": "npm:pi-rehydra"
26
+ }
27
+ ],
28
+ "rehydra": {
29
+ "envFiles": [".env", ".env.local"],
30
+ "redactValues": ["sk-live-abc123..."],
31
+ "allowLiteralTagsIn": ["write", "edit"]
32
+ }
33
+ }
34
+ ```
35
+
36
+ Every key under `rehydra` is an extension option: `envFiles`, `redactValues`, `minValueLength`, `disableTypes`, `locale`, `vcsIdentities`, `policy`, `anonymizer`, `tagFormat`. `envFiles` defaults to `["**/.env*"]`, skipping `node_modules`, `.git`, and symlinks.
37
+
38
+ `allowLiteralTagsIn` defaults to an empty array (always block). Tools listed here — typically `write` and `edit` — keep an **unresolvable** literal tag in their arguments instead of aborting, so you can write documentation or tests that intentionally show tag examples. Every allowed occurrence is logged (`allowed literal tag in tool args`, tool name and argument path only). Resolvable tags are still restored in every tool.
39
+
40
+ ## How it works
41
+
42
+ | Event | Effect |
43
+ |---|---|
44
+ | `session_start` | Instantiate the anonymizer, build the session map, parse the `rehydra` config |
45
+ | `context_with_system` | Anonymize all request messages and inject the rehydra instruction when needed |
46
+ | `tool_call` | Strictly restore PII tags in tool arguments before local execution; an unresolvable tag blocks the call |
47
+ | `tool_result` | Anonymize the completed tool result before it re-enters model context |
48
+ | `message_end` | Rewrite the final assistant message so PII tags are replaced with real values |
49
+ | `session_shutdown` | Dispose of the anonymizer and session registry |
50
+
51
+ ## Masking records and comparison
52
+
53
+ Once enabled, you can check request by request what was intercepted and what the model actually received:
54
+
55
+ ```json
56
+ {
57
+ "rehydra": {
58
+ "record": {
59
+ "enabled": true,
60
+ "includeOriginal": true
61
+ }
62
+ }
63
+ }
64
+ ```
65
+
66
+ `/rehydra log` lists the last 20 records for the current session, `/rehydra log <id>` shows one record (hit lines plus field-level before/after), `/rehydra log --type EMAIL` filters by type; `/rehydra clear` clears the current session and `/rehydra clear --all` clears everything (both ask for confirmation).
67
+
68
+ Recording is off by default. When enabled it writes to `~/.pi/agent/rehydra/records/` (directory 0700, files 0600), trimmed by both a 7-day and a 32 MiB cap, with 256 KiB per record for comparisons. **With `includeOriginal` set to true, real values are written to local disk**; set it to false to keep only the type, detector, placeholder, length and offsets. Terminal control characters are escaped when displayed, and record text is never fed back into model context.
69
+
70
+ Any recording failure (unwritable directory, symlink, half-written line) only produces a warning and never affects masking or the request.
71
+
72
+ ## Limitations
73
+ - Tool-argument rehydration is strict: when a tag is absent from this session's map (or the map is missing, or decryption fails), the tool call is blocked rather than run with `<PII .../>` as a real value. A placeholder can only be restored from the same session's map; the default map is in memory, so tags from before a process restart cannot be restored.
74
+
75
+ - Model calls that bypass these hooks (rare, e.g. user-initiated notes) are not intercepted.
76
+ - `URL`, `IP_ADDRESS`, `BIC_SWIFT`, `POSTAL_CODE` and `DATE` are disabled by default: recognizers that fire on shape alone misfire badly in code and documents. Restore them with `disableTypes: ["URL", "IP_ADDRESS"]`; a bank identifier still needs a full banking context word nearby.
77
+ - Only the values of sensitive fields in an `.env` file enter the global known-value set; project names and ports are not replaced later in the text. Explicitly configured `redactValues` are not affected by that filter.
78
+ - Collection fields in `policy` may be arrays and thresholds plain objects; an invalid field type fails startup with the field name instead of silently dropping the protection rule.
79
+ - This extension cannot guarantee that title or compaction summaries stay sanitized — disable title agents if you need titles to be human-readable.
80
+
81
+ ## License
82
+
60
83
  MIT
package/README.md CHANGED
@@ -1,60 +1,83 @@
1
- # pi-rehydra
2
-
3
- 在主 LLM 请求发出之前,脱敏 Pi 对话中检测到的密钥。
4
-
5
- [English](./README.en.md)
6
-
7
- 这个扩展拦截 [Pi](https://pi.dev) 与 LLM 之间的对话。你的 `.env` 文件中的密钥在离开本机前被替换为占位符,并在工具(shell 命令、文件写入等)在本地执行前透明还原;无法还原的占位符会阻止该次工具调用。
8
-
9
- 它依赖 Pi 的 `ExtensionAPI`:`context_with_system`、`tool_call`、`tool_result`、`message_end`、`session_start` / `session_shutdown`。
10
-
11
- 脱敏和还原都在本地进行;模型只用占位符推理,工具始终拿到真实值。
12
-
13
- ## 安装
14
-
15
- ```bash
16
- pi install npm:pi-rehydra
17
- ```
18
-
19
- Pi 会在 `~/.pi/agent/settings.json` 写入包声明。然后编辑该文件添加扩展配置:
20
-
21
- ```json
22
- {
23
- "packages": [
24
- {
25
- "source": "npm:pi-rehydra"
26
- }
27
- ],
28
- "rehydra": {
29
- "envFiles": [".env", ".env.local"],
30
- "redactValues": ["sk-live-abc123..."],
31
- "allowLiteralTagsIn": ["write", "edit"]
32
- }
33
- }
34
- ```
35
-
36
- `rehydra` 键的所有字段都是扩展配置项:`envFiles`、`redactValues`、`minValueLength`、`disableTypes`、`locale`、`vcsIdentities`、`policy`、`anonymizer`、`tagFormat`。`envFiles` 默认是 `["**/.env*"]`,会跳过 `node_modules`、`.git` 和符号链接。
37
-
38
- `allowLiteralTagsIn` 默认是空数组(一律阻断)。列进这个数组的工具(通常是 `write`、`edit`)在参数里出现**无法还原**的字面标签时不再阻断,而是原样保留——用于写文档或测试时故意写标签示例。放行会在日志里留痕(`allowed literal tag in tool args`,只带工具名与参数路径);能还原的标签在任何工具里都照常还原。
39
-
40
- ## 工作原理
41
-
42
- | 钩子 | 作用 |
43
- |---|---|
44
- | `session_start` | 实例化匿名器、装配会话映射、解析 `rehydra` 配置 |
45
- | `context_with_system` | 匿名化模型请求前的所有消息,并按需追加 rehydra 指令 |
46
- | `tool_call` | 在工具执行前严格还原工具入参中的 PII 标签;任一标签无法还原即阻断该次调用 |
47
- | `tool_result` | 在结果返回给模型前脱敏真实 PII |
48
- | `message_end` | 改写最终回答的文本块,把 PII 标签还原为真实值 |
49
- | `session_shutdown` | 释放匿名器与会话注册表 |
50
-
51
- ## 限制
52
- - 工具入参还原是严格的:标签不在本次会话映射中(或映射缺失、解密失败)时,该次工具调用会被阻止,不会把 `<PII .../>` 当作真实值执行。占位符只能由同一条会话的映射还原;默认使用内存映射,进程重启后旧标签无法还原。
53
-
54
- - 不会拦截用户在 UI 里粘贴的、且未走这些 hook 的模型调用。
55
- - 默认禁用 `URL` 和 `IP_ADDRESS`;通过 `disableTypes: []` 重新启用。
56
- - Pi 扩展无法保证消息标题(title)与元数据走相同的 hook,因此这些字段仍可能包含占位符。建议在脱敏时启用完整的 PII 脱敏策略。
57
-
58
- ## License
59
-
1
+ # pi-rehydra
2
+
3
+ 在主 LLM 请求发出之前,脱敏 Pi 对话中检测到的密钥。
4
+
5
+ [English](./README.en.md)
6
+
7
+ 这个扩展拦截 [Pi](https://pi.dev) 与 LLM 之间的对话。你的 `.env` 文件中的密钥在离开本机前被替换为占位符,并在工具(shell 命令、文件写入等)在本地执行前透明还原;无法还原的占位符会阻止该次工具调用。
8
+
9
+ 它依赖 Pi 的 `ExtensionAPI`:`context_with_system`、`tool_call`、`tool_result`、`message_end`、`session_start` / `session_shutdown`。
10
+
11
+ 脱敏和还原都在本地进行;模型只用占位符推理,工具始终拿到真实值。
12
+
13
+ ## 安装
14
+
15
+ ```bash
16
+ pi install npm:pi-rehydra
17
+ ```
18
+
19
+ Pi 会在 `~/.pi/agent/settings.json` 写入包声明。然后编辑该文件添加扩展配置:
20
+
21
+ ```json
22
+ {
23
+ "packages": [
24
+ {
25
+ "source": "npm:pi-rehydra"
26
+ }
27
+ ],
28
+ "rehydra": {
29
+ "envFiles": [".env", ".env.local"],
30
+ "redactValues": ["sk-live-abc123..."],
31
+ "allowLiteralTagsIn": ["write", "edit"]
32
+ }
33
+ }
34
+ ```
35
+
36
+ `rehydra` 键的所有字段都是扩展配置项:`envFiles`、`redactValues`、`minValueLength`、`disableTypes`、`locale`、`vcsIdentities`、`policy`、`anonymizer`、`tagFormat`。`envFiles` 默认是 `["**/.env*"]`,会跳过 `node_modules`、`.git` 和符号链接。
37
+
38
+ `allowLiteralTagsIn` 默认是空数组(一律阻断)。列进这个数组的工具(通常是 `write`、`edit`)在参数里出现**无法还原**的字面标签时不再阻断,而是原样保留——用于写文档或测试时故意写标签示例。放行会在日志里留痕(`allowed literal tag in tool args`,只带工具名与参数路径);能还原的标签在任何工具里都照常还原。
39
+
40
+ ## 工作原理
41
+
42
+ | 钩子 | 作用 |
43
+ |---|---|
44
+ | `session_start` | 实例化匿名器、装配会话映射、解析 `rehydra` 配置 |
45
+ | `context_with_system` | 匿名化模型请求前的所有消息,并按需追加 rehydra 指令 |
46
+ | `tool_call` | 在工具执行前严格还原工具入参中的 PII 标签;任一标签无法还原即阻断该次调用 |
47
+ | `tool_result` | 在结果返回给模型前脱敏真实 PII |
48
+ | `message_end` | 改写最终回答的文本块,把 PII 标签还原为真实值 |
49
+ | `session_shutdown` | 释放匿名器与会话注册表 |
50
+
51
+ ## 脱敏记录与对照
52
+
53
+ 开启后可以逐请求核对"哪些内容被拦截、模型实际收到了什么":
54
+
55
+ ```json
56
+ {
57
+ "rehydra": {
58
+ "record": {
59
+ "enabled": true,
60
+ "includeOriginal": true
61
+ }
62
+ }
63
+ }
64
+ ```
65
+
66
+ `/rehydra log` 列出当前会话最近 20 条记录,`/rehydra log <id>` 看单条详情(命中行 + 字段级 before/after),`/rehydra log --type EMAIL` 按类型筛选;`/rehydra clear` 清当前会话,`/rehydra clear --all` 清全部(都要确认)。
67
+
68
+ 记录默认关闭。开启后写入 `~/.pi/agent/rehydra/records/`(目录 0700、文件 0600),按 7 天与 32 MiB 双上限裁剪,单条记录对照上限 256 KiB。**`includeOriginal` 为 true 时真实值会写到本地磁盘**;不需要时设成 `false`,记录只保留类型、检测器、占位符、长度与偏移。记录里出现的终端控制字符在展示时会转义,原文不会回填到模型上下文。
69
+
70
+ 记录层任何失败(目录不可写、符号链接、半行文件)都只告警,不影响脱敏与请求。
71
+
72
+ ## 限制
73
+ - 工具入参还原是严格的:标签不在本次会话映射中(或映射缺失、解密失败)时,该次工具调用会被阻止,不会把 `<PII .../>` 当作真实值执行。占位符只能由同一条会话的映射还原;默认使用内存映射,进程重启后旧标签无法还原。
74
+
75
+ - 不会拦截用户在 UI 里粘贴的、且未走这些 hook 的模型调用。
76
+ - 默认禁用 `URL`、`IP_ADDRESS`、`BIC_SWIFT`、`POSTAL_CODE`、`DATE`;只靠形状就能命中的识别器在代码与文档里误报率高。需要它们时把 `disableTypes` 写成 `["URL", "IP_ADDRESS"]` 即可恢复。即便恢复,银行标识码仍要求附近出现完整的银行语境词。
77
+ - `.env` 里只有敏感字段的值会进入全局已知值集合:项目名、端口这类普通配置值不会被替换。显式配置的 `redactValues` 不受这条筛选影响。
78
+ - `policy` 的集合类字段可写成数组、阈值可写成对象;字段类型不合法时扩展启动报错并指出字段名,不会静默丢掉保护规则。
79
+ - Pi 扩展无法保证消息标题(title)与元数据走相同的 hook,因此这些字段仍可能包含占位符。建议在脱敏时启用完整的 PII 脱敏策略。
80
+
81
+ ## License
82
+
60
83
  MIT
@@ -6,6 +6,8 @@ import { AnonymizationResult, AnonymizationPolicy, AnonymizationMode, SemanticCo
6
6
  import { RecognizerRegistry } from "../recognizers/index.js";
7
7
  import { type INERModel, type OrtSessionOptions } from "../ner/index.js";
8
8
  import { type NERModelMode, type DownloadProgressCallback } from "../ner/model-manager.js";
9
+ import type { AnonymizeRun, PiiObserver } from "./observer.js";
10
+ export type { AnonymizeRun, ObservedEntity, PiiObserver } from "./observer.js";
9
11
  import { type RawPIIMap } from "../pipeline/tagger.js";
10
12
  import { type KeyProvider } from "../crypto/index.js";
11
13
  export type { RawPIIMap } from "../pipeline/tagger.js";
@@ -144,6 +146,11 @@ export interface AnonymizerConfig {
144
146
  * @default { enabled: false }
145
147
  */
146
148
  secrets?: SecretsConfig;
149
+ /**
150
+ * 模型检测失败时的处理。`rules` 降级为规则并标记检测不完整;
151
+ * `block` 直接把错误抛给调用方(fail-closed)。未配置时保持原有抛错行为。
152
+ */
153
+ nerFailure?: "rules" | "block";
147
154
  /** Key provider for encryption (generates random key if not provided) */
148
155
  keyProvider?: KeyProvider;
149
156
  /**
@@ -187,6 +194,7 @@ export declare class Anonymizer {
187
194
  private registry;
188
195
  private nerModel;
189
196
  private nerConfig;
197
+ private nerFailure;
190
198
  private semanticConfig;
191
199
  private secretsConfig;
192
200
  private mode;
@@ -218,7 +226,7 @@ export declare class Anonymizer {
218
226
  * @param existingPiiMap - Optional existing PII map for session-level ID reuse
219
227
  * @returns Anonymization result with anonymized text and encrypted PII map
220
228
  */
221
- anonymize(text: string, locale?: string, policy?: Partial<AnonymizationPolicy>, existingPiiMap?: RawPIIMap): Promise<AnonymizationResult>;
229
+ anonymize(text: string, locale?: string, policy?: Partial<AnonymizationPolicy>, existingPiiMap?: RawPIIMap, observer?: PiiObserver, run?: AnonymizeRun): Promise<AnonymizationResult>;
222
230
  /**
223
231
  * Disposes of resources
224
232
  */
@@ -1 +1 @@
1
- {"version":3,"file":"anonymizer.d.ts","sourceRoot":"","sources":["../../src/core/anonymizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EAEnB,iBAAiB,EAEjB,cAAc,EACd,aAAa,EAEb,OAAO,EACP,SAAS,EAIV,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAEL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EACL,KAAK,SAAS,EAId,KAAK,iBAAiB,EACvB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,KAAK,YAAY,EAGjB,KAAK,wBAAwB,EAC9B,MAAM,yBAAyB,CAAC;AAIjC,OAAO,EAGL,KAAK,SAAS,EACf,MAAM,uBAAuB,CAAC;AAY/B,OAAO,EAGL,KAAK,WAAW,EACjB,MAAM,oBAAoB,CAAC;AAI5B,YAAY,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAGvD,OAAO,KAAK,EACV,kBAAkB,EAClB,iBAAiB,EAClB,MAAM,qBAAqB,CAAC;AAG7B,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;AAsCtD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;;;;OAMG;IACH,IAAI,EAAE,YAAY,CAAC;IAEnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;OAEG;IACH,kBAAkB,CAAC,EAAE,wBAAwB,CAAC;IAE9C;;OAEG;IACH,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IAEpC;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IAE9C;;;;OAIG;IACH,cAAc,CAAC,EAAE,iBAAiB,CAAC;IAEnC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,OAAO,GAAG,kBAAkB,CAAC;IAEvC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;OAGG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;;OAKG;IACH,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CAC/E;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yDAAyD;IACzD,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAE9B;;;;;OAKG;IACH,IAAI,CAAC,EAAE,iBAAiB,CAAC;IAEzB;;;OAGG;IACH,GAAG,CAAC,EAAE,SAAS,CAAC;IAEhB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAE1B;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IAExB,yEAAyE;IACzE,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,+EAA+E;IAC/E,aAAa,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,4BAA4B;IAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,KAAK,IAAI,CAAC;IAEnF;;;;;OAKG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,kBAAkB,EAC3B,WAAW,EAAE,WAAW,KACrB,iBAAiB,CAAC;AAEvB;;;GAGG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,QAAQ,CAA0B;IAC1C,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,IAAI,CAAoB;IAChC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,kBAAkB,CAA4B;IACtD,OAAO,CAAC,aAAa,CAAsB;IAC3C,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAwB;IAC9C,OAAO,CAAC,mBAAmB,CAAwE;IACnG,OAAO,CAAC,SAAS,CAAY;gBAEjB,MAAM,GAAE,gBAAqB,EAAE,cAAc,CAAC,EAAE,cAAc;IA4E1E;;OAEG;IACH,IAAI,iBAAiB,IAAI,SAAS,CAEjC;IAED;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA6JjC;;;;;;;OAOG;IACG,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,EACrC,cAAc,CAAC,EAAE,SAAS,GACzB,OAAO,CAAC,mBAAmB,CAAC;IA4I/B;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;OAEG;IACH,WAAW,IAAI,kBAAkB;IAIjC;;OAEG;IACH,WAAW,IAAI,SAAS,GAAG,IAAI;IAI/B;;OAEG;IACH,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB;CAmD9C;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,CAAC,EAAE,gBAAgB,EACzB,cAAc,CAAC,EAAE,cAAc,GAC9B,UAAU,CAEZ;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAS9B;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAQ9B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,GAAG;IAAE,IAAI,CAAC,EAAE,UAAU,GAAG,WAAW,CAAA;CAAE,EACxE,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAe9B"}
1
+ {"version":3,"file":"anonymizer.d.ts","sourceRoot":"","sources":["../../src/core/anonymizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EAEnB,iBAAiB,EAEjB,cAAc,EACd,aAAa,EAEb,OAAO,EACP,SAAS,EAIV,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAEL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AAIjC,OAAO,EACL,KAAK,SAAS,EAKd,KAAK,iBAAiB,EACvB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,KAAK,YAAY,EAGjB,KAAK,wBAAwB,EAC9B,MAAM,yBAAyB,CAAC;AAGjC,OAAO,KAAK,EAAE,YAAY,EAAkB,WAAW,EAAE,MAAM,eAAe,CAAC;AAC/E,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE/E,OAAO,EAIL,KAAK,SAAS,EACf,MAAM,uBAAuB,CAAC;AAY/B,OAAO,EAGL,KAAK,WAAW,EACjB,MAAM,oBAAoB,CAAC;AAI5B,YAAY,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAGvD,OAAO,KAAK,EACV,kBAAkB,EAClB,iBAAiB,EAClB,MAAM,qBAAqB,CAAC;AAG7B,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;AAyItD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;;;;OAMG;IACH,IAAI,EAAE,YAAY,CAAC;IAEnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;OAEG;IACH,kBAAkB,CAAC,EAAE,wBAAwB,CAAC;IAE9C;;OAEG;IACH,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IAEpC;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IAE9C;;;;OAIG;IACH,cAAc,CAAC,EAAE,iBAAiB,CAAC;IAEnC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,OAAO,GAAG,kBAAkB,CAAC;IAEvC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;OAGG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;;OAKG;IACH,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CAC/E;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yDAAyD;IACzD,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAE9B;;;;;OAKG;IACH,IAAI,CAAC,EAAE,iBAAiB,CAAC;IAEzB;;;OAGG;IACH,GAAG,CAAC,EAAE,SAAS,CAAC;IAEhB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAE1B;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IAExB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC;IAE/B,yEAAyE;IACzE,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,+EAA+E;IAC/E,aAAa,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,4BAA4B;IAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,KAAK,IAAI,CAAC;IAEnF;;;;;OAKG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,kBAAkB,EAC3B,WAAW,EAAE,WAAW,KACrB,iBAAiB,CAAC;AAEvB;;;GAGG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,QAAQ,CAA0B;IAC1C,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,IAAI,CAAoB;IAChC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,kBAAkB,CAA4B;IACtD,OAAO,CAAC,aAAa,CAAsB;IAC3C,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAwB;IAC9C,OAAO,CAAC,mBAAmB,CAAwE;IACnG,OAAO,CAAC,SAAS,CAAY;gBAEjB,MAAM,GAAE,gBAAqB,EAAE,cAAc,CAAC,EAAE,cAAc;IA6E1E;;OAEG;IACH,IAAI,iBAAiB,IAAI,SAAS,CAEjC;IAED;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAsKjC;;;;;;;OAOG;IACG,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,EACrC,cAAc,CAAC,EAAE,SAAS,EAC1B,QAAQ,CAAC,EAAE,WAAW,EACtB,GAAG,CAAC,EAAE,YAAY,GACjB,OAAO,CAAC,mBAAmB,CAAC;IAgM/B;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;OAEG;IACH,WAAW,IAAI,kBAAkB;IAIjC;;OAEG;IACH,WAAW,IAAI,SAAS,GAAG,IAAI;IAI/B;;OAEG;IACH,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB;CAmD9C;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,CAAC,EAAE,gBAAgB,EACzB,cAAc,CAAC,EAAE,cAAc,GAC9B,UAAU,CAEZ;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAS9B;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAQ9B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,GAAG;IAAE,IAAI,CAAC,EAAE,UAAU,GAAG,WAAW,CAAA;CAAE,EACxE,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAe9B"}
@@ -5,11 +5,12 @@
5
5
  import { PIIType, DEFAULT_TAG_FORMAT, createDefaultPolicy, SECRET_PII_TYPES, } from "../types/index.js";
6
6
  import { createDefaultRegistry, createSecretRecognizers, } from "../recognizers/index.js";
7
7
  import { createLiteralValueRecognizer } from "../recognizers/secrets/literal-value.js";
8
+ import { isCredentialField } from "../recognizers/secrets/key-patterns.js";
8
9
  import { NERModelStub, createNERModel, createInferenceServerNERModel, } from "../ner/index.js";
9
10
  import { ensureModel, loadLabelMap, } from "../ner/model-manager.js";
10
- import { prenormalize } from "../pipeline/prenormalize.js";
11
+ import { createLineEndingMapping, prenormalize } from "../pipeline/prenormalize.js";
11
12
  import { resolveEntities } from "../pipeline/resolver.js";
12
- import { tagEntities, countEntitiesByType, } from "../pipeline/tagger.js";
13
+ import { generateTag, tagEntities, countEntitiesByType, } from "../pipeline/tagger.js";
13
14
  import { validateOutput } from "../pipeline/validator.js";
14
15
  import { enrichSemantics } from "../pipeline/semantic-enricher.js";
15
16
  import { ensureSemanticData, isSemanticDataAvailable, loadSemanticData, } from "../pipeline/semantic-data-loader.js";
@@ -44,6 +45,93 @@ function mergePolicyWithBase(base, partial) {
44
45
  excludeLocationScopes: partial.excludeLocationScopes ?? base.excludeLocationScopes,
45
46
  };
46
47
  }
48
+ /** 判断信号是否已取消(单独成函数,避免被 TS 的收窄影响)。 */
49
+ function isAborted(signal) {
50
+ return signal !== undefined && signal.aborted;
51
+ }
52
+ /** 把取消信号转成标准 AbortError。 */
53
+ function toAbortError(signal) {
54
+ const reason = signal.reason;
55
+ if (reason instanceof Error)
56
+ return reason;
57
+ const error = new Error("Anonymization aborted");
58
+ error.name = "AbortError";
59
+ return error;
60
+ }
61
+ /** 等待期间保持对取消信号的响应:取消时立即以 AbortError 结束。 */
62
+ function raceWithSignal(promise, signal) {
63
+ if (signal === undefined)
64
+ return promise;
65
+ if (signal.aborted)
66
+ return Promise.reject(toAbortError(signal));
67
+ return new Promise((resolve, reject) => {
68
+ const onAbort = () => {
69
+ reject(toAbortError(signal));
70
+ };
71
+ signal.addEventListener("abort", onAbort, { once: true });
72
+ promise.then((value) => {
73
+ signal.removeEventListener("abort", onAbort);
74
+ resolve(value);
75
+ }, (error) => {
76
+ signal.removeEventListener("abort", onAbort);
77
+ reject(error);
78
+ });
79
+ });
80
+ }
81
+ /** 给等待加一个上限;超时抛出可归类为 model-deadline 的错误。 */
82
+ function withTimeout(promise, ms) {
83
+ return new Promise((resolve, reject) => {
84
+ const timer = setTimeout(() => {
85
+ reject(new Error("inference deadline exceeded while waiting"));
86
+ }, ms);
87
+ promise.then((value) => {
88
+ clearTimeout(timer);
89
+ resolve(value);
90
+ }, (error) => {
91
+ clearTimeout(timer);
92
+ reject(error);
93
+ });
94
+ });
95
+ }
96
+ /** 把模型失败归类成不含正文的错误码。 */
97
+ function classifyNerError(error) {
98
+ const message = error instanceof Error ? error.message.toLowerCase() : "";
99
+ if (message.includes("deadline exceeded"))
100
+ return "model-deadline";
101
+ if (message.includes("timed out") || message.includes("timeout"))
102
+ return "model-timeout";
103
+ if (message.includes("failed to connect") || message.includes("econnrefused")) {
104
+ return "model-unreachable";
105
+ }
106
+ if (message.includes("protocol error"))
107
+ return "protocol-error";
108
+ if (message.includes("not loaded"))
109
+ return "model-unavailable";
110
+ return "model-failed";
111
+ }
112
+ /** 把命中回调给观察者;偏移从规范化文本折回原文空间。 */
113
+ function notifyObserver(observer, input) {
114
+ const mapping = createLineEndingMapping(input.text);
115
+ for (const entity of input.entities) {
116
+ const hit = {
117
+ type: entity.type,
118
+ recognizer: entity.recognizer,
119
+ start: mapping.toOriginal(entity.start),
120
+ end: mapping.toOriginal(entity.end),
121
+ replaced: generateTag(entity.type, entity.id, entity.semantic, input.tagFormat),
122
+ };
123
+ if (observer.includeOriginal === true) {
124
+ // 从原文切片,保留内部 CRLF 与规范化的差异
125
+ hit.original = input.text.slice(hit.start, hit.end);
126
+ }
127
+ try {
128
+ observer.onEntity(hit);
129
+ }
130
+ catch {
131
+ // 观察者属于诊断通道,异常不能影响脱敏本身
132
+ }
133
+ }
134
+ }
47
135
  /**
48
136
  * Anonymizer instance
49
137
  * Main class for performing PII anonymization
@@ -52,6 +140,7 @@ export class Anonymizer {
52
140
  registry;
53
141
  nerModel = null;
54
142
  nerConfig;
143
+ nerFailure;
55
144
  semanticConfig;
56
145
  secretsConfig;
57
146
  mode;
@@ -83,6 +172,7 @@ export class Anonymizer {
83
172
  }
84
173
  // Handle NER configuration
85
174
  this.nerConfig = config.ner ?? { mode: "disabled" };
175
+ this.nerFailure = config.nerFailure ?? "propagate";
86
176
  this.modelVersion = config.modelVersion ?? "1.0.0";
87
177
  // Merge NER thresholds into default policy if provided
88
178
  if (this.nerConfig.thresholds !== undefined) {
@@ -246,11 +336,16 @@ export class Anonymizer {
246
336
  }
247
337
  // Handle secrets .env file loading and literal value matching
248
338
  if (this.secretsConfig.enabled) {
249
- const literalValues = [
250
- ...(this.secretsConfig.redactValues ?? []),
251
- ];
339
+ // 显式指定的值按用户意图保留,不做自动筛选
340
+ const explicitValues = [];
341
+ for (const value of this.secretsConfig.redactValues ?? []) {
342
+ if (value.trim().length === 0)
343
+ continue;
344
+ explicitValues.push(value);
345
+ }
252
346
  const minLength = this.secretsConfig.minValueLength ?? 4;
253
347
  // Parse .env files if provided
348
+ const envValues = [];
254
349
  if (this.secretsConfig.envFiles !== undefined && this.secretsConfig.envFiles.length > 0) {
255
350
  const storage = await getStorageProvider();
256
351
  const envFiles = storage.resolveFiles !== undefined
@@ -260,13 +355,19 @@ export class Anonymizer {
260
355
  try {
261
356
  const content = await storage.readTextFile(envFile);
262
357
  const parsed = parseEnvFile(content);
263
- literalValues.push(...parsed);
358
+ // 只把敏感字段的值收进已知值集合:项目名、端口这类普通配置
359
+ // 值一旦进入集合,后续文本里出现就会被替换。
360
+ for (const entry of parsed) {
361
+ if (isCredentialField(entry.key))
362
+ envValues.push(entry.value);
363
+ }
264
364
  }
265
365
  catch {
266
366
  // Skip files that can't be read
267
367
  }
268
368
  }
269
369
  }
370
+ const literalValues = [...explicitValues, ...envValues];
270
371
  // Register literal value recognizer if we have values
271
372
  if (literalValues.length > 0) {
272
373
  this.registry.register(createLiteralValueRecognizer(literalValues, minLength));
@@ -283,10 +384,14 @@ export class Anonymizer {
283
384
  * @param existingPiiMap - Optional existing PII map for session-level ID reuse
284
385
  * @returns Anonymization result with anonymized text and encrypted PII map
285
386
  */
286
- async anonymize(text, locale, policy, existingPiiMap) {
387
+ async anonymize(text, locale, policy, existingPiiMap, observer, run) {
287
388
  if (!this.initialized) {
288
389
  await this.initialize();
289
390
  }
391
+ // 宿主取消:直接结束,不进入降级路径
392
+ if (run?.signal?.aborted === true) {
393
+ throw toAbortError(run.signal);
394
+ }
290
395
  const startTime = performance.now();
291
396
  // Merge policy with instance defaults (not global defaults)
292
397
  // This ensures semantic config from constructor is preserved
@@ -298,8 +403,42 @@ export class Anonymizer {
298
403
  const normalizedText = prenormalize(text);
299
404
  // Step 2: Run regex recognizers
300
405
  const regexMatches = this.registry.findAll(normalizedText, effectivePolicy);
301
- // Step 3: Run NER model
302
- const nerResult = await this.nerModel.predict(normalizedText, effectivePolicy);
406
+ // Step 3: Run NER model;失败按 nerFailure 策略处理
407
+ const detectionErrors = [];
408
+ let nerResult;
409
+ const remaining = run?.deadline === undefined ? undefined : run.deadline - Date.now();
410
+ try {
411
+ if (remaining !== undefined && remaining <= 0) {
412
+ // 预算已耗尽:不再发请求,直接走规则
413
+ throw new Error("inference deadline exceeded before request");
414
+ }
415
+ const prediction = this.nerModel.predict(normalizedText, effectivePolicy);
416
+ // shortcut: 预算只截断等待,底层请求仍会在自身超时后结束
417
+ nerResult =
418
+ remaining === undefined
419
+ ? await raceWithSignal(prediction, run?.signal)
420
+ : await raceWithSignal(withTimeout(prediction, remaining), run?.signal);
421
+ }
422
+ catch (error) {
423
+ // 取消不属于检测失败,始终向上传递
424
+ const signal = run?.signal;
425
+ if (signal !== undefined && isAborted(signal))
426
+ throw toAbortError(signal);
427
+ if (error instanceof Error && error.name === "AbortError")
428
+ throw error;
429
+ if (this.nerFailure === "block" || this.nerFailure === "propagate") {
430
+ throw error;
431
+ }
432
+ // 降级:本次只用规则结果,并标记检测不完整
433
+ detectionErrors.push({ finder: "model", code: classifyNerError(error) });
434
+ nerResult = {
435
+ spans: [],
436
+ processingTimeMs: 0,
437
+ modelVersion: this.modelVersion,
438
+ truncated: false,
439
+ coverageChars: 0,
440
+ };
441
+ }
303
442
  const nerMatches = nerResult.spans;
304
443
  // Step 4: Resolve and merge entities
305
444
  const resolvedMatches = resolveEntities(regexMatches, nerMatches, effectivePolicy, normalizedText);
@@ -337,6 +476,16 @@ export class Anonymizer {
337
476
  }));
338
477
  this.onValidationWarning(safeErrors);
339
478
  }
479
+ // Step 6.5: 把每条替换回调给可选的观察者(诊断/记录用)。
480
+ // 只在调用方显式注册时执行;命中偏移折回原始文本空间。
481
+ if (observer !== undefined && entities.length > 0) {
482
+ notifyObserver(observer, {
483
+ text,
484
+ normalizedText,
485
+ entities,
486
+ tagFormat: this.tagFormat,
487
+ });
488
+ }
340
489
  // Step 7: Build stats
341
490
  const endTime = performance.now();
342
491
  const stats = {
@@ -349,6 +498,7 @@ export class Anonymizer {
349
498
  nerTruncated: nerResult.truncated,
350
499
  nerCoverageChars: nerResult.coverageChars,
351
500
  };
501
+ const integrity = detectionErrors.length === 0 ? "complete" : "rules-only";
352
502
  // Step 8: Build result (without original text in entities)
353
503
  const safeEntities = entities.map(({ original: _original, ...rest }) => rest);
354
504
  // Step 9: Encrypt PII map (only in pseudonymize mode)
@@ -358,6 +508,9 @@ export class Anonymizer {
358
508
  anonymizedText,
359
509
  entities: safeEntities,
360
510
  stats,
511
+ ...(detectionErrors.length === 0
512
+ ? {}
513
+ : { detectionErrors, integrity }),
361
514
  };
362
515
  }
363
516
  // Pseudonymize mode - encrypt and return PII map
@@ -370,6 +523,7 @@ export class Anonymizer {
370
523
  entities: safeEntities,
371
524
  piiMap: encryptedPiiMap,
372
525
  stats,
526
+ ...(detectionErrors.length === 0 ? {} : { detectionErrors, integrity }),
373
527
  };
374
528
  }
375
529
  /**
@@ -534,10 +688,10 @@ export async function anonymizeWithNER(text, nerConfig, policy) {
534
688
  }
535
689
  }
536
690
  /**
537
- * Parses a .env file content and returns values
691
+ * Parses a .env file content and returns key/value pairs.
538
692
  */
539
693
  function parseEnvFile(content) {
540
- const values = [];
694
+ const entries = [];
541
695
  const lines = content.split("\n");
542
696
  for (const line of lines) {
543
697
  const trimmed = line.trim();
@@ -545,11 +699,11 @@ function parseEnvFile(content) {
545
699
  if (trimmed === "" || trimmed.startsWith("#"))
546
700
  continue;
547
701
  // Match KEY=VALUE (with optional export prefix and quotes)
548
- const match = trimmed.match(/^(?:export\s+)?[A-Za-z_][A-Za-z0-9_]*\s*=\s*["']?(.*?)["']?$/);
549
- if (match !== null && match[1] !== undefined && match[1] !== "") {
550
- values.push(match[1]);
702
+ const match = trimmed.match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*["']?(.*?)["']?$/);
703
+ if (match !== null && match[1] !== undefined && match[2] !== undefined && match[2] !== "") {
704
+ entries.push({ key: match[1], value: match[2] });
551
705
  }
552
706
  }
553
- return values;
707
+ return entries;
554
708
  }
555
709
  //# sourceMappingURL=anonymizer.js.map