helan 0.1.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- helan-0.1.0rc1/LICENSE +21 -0
- helan-0.1.0rc1/PKG-INFO +159 -0
- helan-0.1.0rc1/README.md +133 -0
- helan-0.1.0rc1/pyproject.toml +52 -0
- helan-0.1.0rc1/setup.cfg +4 -0
- helan-0.1.0rc1/src/helan/__init__.py +59 -0
- helan-0.1.0rc1/src/helan/__main__.py +3 -0
- helan-0.1.0rc1/src/helan/bench.py +126 -0
- helan-0.1.0rc1/src/helan/calibrate.py +79 -0
- helan-0.1.0rc1/src/helan/checksum.py +210 -0
- helan-0.1.0rc1/src/helan/cli.py +132 -0
- helan-0.1.0rc1/src/helan/compat_presidio.py +76 -0
- helan-0.1.0rc1/src/helan/data.py +120 -0
- helan-0.1.0rc1/src/helan/eval/__init__.py +6 -0
- helan-0.1.0rc1/src/helan/eval/corpus.py +279 -0
- helan-0.1.0rc1/src/helan/eval/metrics.py +91 -0
- helan-0.1.0rc1/src/helan/eval/report.py +97 -0
- helan-0.1.0rc1/src/helan/io_utils.py +96 -0
- helan-0.1.0rc1/src/helan/operators.py +131 -0
- helan-0.1.0rc1/src/helan/pipeline.py +163 -0
- helan-0.1.0rc1/src/helan/py.typed +0 -0
- helan-0.1.0rc1/src/helan/recognizers/__init__.py +35 -0
- helan-0.1.0rc1/src/helan/recognizers/base.py +31 -0
- helan-0.1.0rc1/src/helan/recognizers/chinese.py +233 -0
- helan-0.1.0rc1/src/helan/recognizers/llm.py +171 -0
- helan-0.1.0rc1/src/helan/recognizers/ner_jieba.py +65 -0
- helan-0.1.0rc1/src/helan/recognizers/official.py +120 -0
- helan-0.1.0rc1/src/helan/recognizers/rules.py +101 -0
- helan-0.1.0rc1/src/helan/resolver.py +85 -0
- helan-0.1.0rc1/src/helan/streaming.py +99 -0
- helan-0.1.0rc1/src/helan/types.py +90 -0
- helan-0.1.0rc1/src/helan/vault.py +67 -0
- helan-0.1.0rc1/src/helan.egg-info/PKG-INFO +159 -0
- helan-0.1.0rc1/src/helan.egg-info/SOURCES.txt +46 -0
- helan-0.1.0rc1/src/helan.egg-info/dependency_links.txt +1 -0
- helan-0.1.0rc1/src/helan.egg-info/entry_points.txt +2 -0
- helan-0.1.0rc1/src/helan.egg-info/requires.txt +7 -0
- helan-0.1.0rc1/src/helan.egg-info/top_level.txt +1 -0
- helan-0.1.0rc1/tests/test_checksum.py +130 -0
- helan-0.1.0rc1/tests/test_cli_eval.py +131 -0
- helan-0.1.0rc1/tests/test_compat_presidio.py +35 -0
- helan-0.1.0rc1/tests/test_fuzz.py +76 -0
- helan-0.1.0rc1/tests/test_iter3.py +158 -0
- helan-0.1.0rc1/tests/test_llm_mock.py +209 -0
- helan-0.1.0rc1/tests/test_pipeline.py +192 -0
- helan-0.1.0rc1/tests/test_recognizers.py +134 -0
- helan-0.1.0rc1/tests/test_rules.py +95 -0
- helan-0.1.0rc1/tests/test_streaming.py +98 -0
helan-0.1.0rc1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 helan contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
helan-0.1.0rc1/PKG-INFO
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: helan
|
|
3
|
+
Version: 0.1.0rc1
|
|
4
|
+
Summary: The Veil of Hidden Names (Helan) | Chinese-first personal data detection and masking
|
|
5
|
+
Author: helan contributors
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/cloudydreamland/TheVeilOfHiddenNames
|
|
8
|
+
Project-URL: Repository, https://github.com/cloudydreamland/TheVeilOfHiddenNames
|
|
9
|
+
Project-URL: Issues, https://github.com/cloudydreamland/TheVeilOfHiddenNames/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/cloudydreamland/TheVeilOfHiddenNames/blob/main/CHANGELOG.md
|
|
11
|
+
Project-URL: Security, https://github.com/cloudydreamland/TheVeilOfHiddenNames/security/policy
|
|
12
|
+
Keywords: pii,脱敏,privacy,chinese,nlp,llm,compliance
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Topic :: Security
|
|
16
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Provides-Extra: jieba
|
|
21
|
+
Requires-Dist: jieba>=0.42; extra == "jieba"
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
24
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# The Veil of Hidden Names — Helan
|
|
28
|
+
|
|
29
|
+
简体中文 · [English](README.en.md)
|
|
30
|
+
|
|
31
|
+
> 展示名 **The Veil of Hidden Names** 意为“隐名之幕”;Helan 是该项目的短名。
|
|
32
|
+
|
|
33
|
+
**中文优先的 PII 检测与脱敏库。校验和级识别,可逆 Vault 还原,格式保留假名——帮助你在把数据交给大模型或其他服务前发现并处理敏感信息。**
|
|
34
|
+
|
|
35
|
+
[](.github/workflows/ci.yml)
|
|
36
|
+
[](pyproject.toml)
|
|
37
|
+
[](LICENSE)
|
|
38
|
+
|
|
39
|
+
## 为什么需要它 / Why
|
|
40
|
+
|
|
41
|
+
在将中文业务文本送入 LLM、RAG 或其他外部处理服务前,开发者常需要识别并处理个人信息。通用识别框架可扩展,但中文证件校验、误报控制和可恢复脱敏通常需要额外规则与评测。
|
|
42
|
+
|
|
43
|
+
`helan`(Helan)把这件事做成零依赖标准件:
|
|
44
|
+
|
|
45
|
+
- **格式校验**:对身份证、银行卡和统一社会信用代码应用相应校验规则,减少仅凭位数与字符模式产生的误报;其他实体依赖上下文规则,仍可能漏检或误报
|
|
46
|
+
- **偏移不变量**:每个实体保证 `entity.text == source[start:end]`(fuzz 测试永久守护),脱敏结果可精确引用回原文
|
|
47
|
+
- **可逆 Vault**:用占位符脱敏并通过受保护的 Vault 还原;Vault 含有恢复敏感信息,必须与原文同等保护
|
|
48
|
+
- **格式保留假名**:`fake` 算子生成的假身份证能通过身份证校验、假银行卡能通过 Luhn——替换后的数据仍是"合法格式",适合造测试集与演示数据
|
|
49
|
+
- **零必装依赖**:核心纯 Python;jieba(人名 NER 补召回)与 LLM(语义级兜底)全部可选
|
|
50
|
+
- **自带评测**:内置 12 篇中文语料 + P/R/F1 报告,数字如实
|
|
51
|
+
|
|
52
|
+
## 安装 / Install
|
|
53
|
+
|
|
54
|
+
> 当前尚未发布到 PyPI;下方给出从 GitHub 获取并本地安装的命令。
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
git clone https://github.com/cloudydreamland/TheVeilOfHiddenNames.git
|
|
58
|
+
cd TheVeilOfHiddenNames
|
|
59
|
+
python -m pip install .
|
|
60
|
+
# PyPI 首发后:python -m pip install helan
|
|
61
|
+
python -m pip install ".[jieba]"
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## 快速开始 / Quickstart
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
from helan import mask, restore, recognize
|
|
68
|
+
|
|
69
|
+
text = "出租方张伟明(身份证 11010519491231002X,电话 13812345678)同意将房屋出租。"
|
|
70
|
+
|
|
71
|
+
# 只识别
|
|
72
|
+
|
|
73
|
+
for e in recognize(text):
|
|
74
|
+
print(e.type, e.start, e.end, e.text)
|
|
75
|
+
|
|
76
|
+
# 可逆脱敏:身份证换成占位符,可精确还原
|
|
77
|
+
|
|
78
|
+
masked, vault = mask(text, ops={"ID_CARD": "vault"})
|
|
79
|
+
original = restore(masked, vault)
|
|
80
|
+
assert original == text
|
|
81
|
+
|
|
82
|
+
# 不可逆脱敏:手机号换成"合法格式"的假号码
|
|
83
|
+
|
|
84
|
+
masked, _ = mask(text, ops={"PHONE": "fake"})
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
命令行:
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
helan scan 合同.txt --json # 只识别
|
|
91
|
+
helan mask 合同.txt -o 脱敏.txt --ops ID_CARD:vault,PHONE:fake --vault-out vault.json
|
|
92
|
+
helan restore 脱敏.txt --vault vault.json -o 还原.txt
|
|
93
|
+
helan eval # 内置基准报告
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## 实体类型与算子
|
|
97
|
+
|
|
98
|
+
| 类型 | 识别方式 | 默认算子 |
|
|
99
|
+
|---|---|---|
|
|
100
|
+
| ID_CARD 身份证 | 区划+出生日期+MOD 11-2 校验码 | partial(前3后4) |
|
|
101
|
+
| BANK_CARD 银行卡 | Luhn 校验(支持空格/连字符分组) | partial(留后4) |
|
|
102
|
+
| USCC 统一社会信用代码 | GB 32100 MOD 31-3 校验 | partial |
|
|
103
|
+
| PHONE 手机号 | 号段表严格校验 | partial(138\*\*\*\*5678) |
|
|
104
|
+
| PERSON_NAME 人名 | 称谓/引导词/顿号枚举上下文;jieba nr 可选 | partial(张\*\*) |
|
|
105
|
+
| ADDRESS 地址 | 引导词上下文 | redact |
|
|
106
|
+
| LANDLINE / EMAIL / IP / URL / PASSPORT / LICENSE_PLATE | 规则+守卫 | partial/redact |
|
|
107
|
+
| TW_ID_CARD 台湾身份证 | 地区字母码+性别位+加权 MOD 10 校验 | partial |
|
|
108
|
+
| POSTAL_CODE / QQ / 微信号 / OFFICER_ID | 关键词上下文(军官证为 format-only,如实标注) | redact/partial |
|
|
109
|
+
| ID_CARD 全角/符号分隔写法 | 1101 0519 4912 3100 2X 等分隔归一化后过校验和 | partial |
|
|
110
|
+
|
|
111
|
+
算子:`redact`(标签替换)/ `partial`(部分保留)/ `hash`(加盐稳定假名)/ `fake`(合法格式假数据)/ `vault`(可逆占位)/ 任意自定义 callable。
|
|
112
|
+
|
|
113
|
+
## 与现有方案的关系 / Landscape
|
|
114
|
+
|
|
115
|
+
我们曾用固定版本的 `presidio-analyzer` 与本项目内置语料做对照。测试范围、配置、语料及局限见[方法和完整结果](docs/presidio_zh.md);这些结果只适用于该次设置,不代表所有 Presidio 中文部署:
|
|
116
|
+
|
|
117
|
+
| 方案 | 实测/事实 |
|
|
118
|
+
|---|---|
|
|
119
|
+
| Presidio 等通用框架 | 提供可配置的识别与匿名化管线;中文效果取决于所选 recognizer、规则和评测语料 |
|
|
120
|
+
| 自定义正则 | 易于嵌入,但需要自行实现格式校验、上下文规则、偏移处理和评测 |
|
|
121
|
+
| Helan | 聚焦中文规则、原文偏移以及多种脱敏算子;适用范围和评测边界见下文 |
|
|
122
|
+
|
|
123
|
+
选题取证(为什么这个缺口是真的)见 [GAP_PROOF.md](GAP_PROOF.md)。
|
|
124
|
+
|
|
125
|
+
## 评测 / Evaluation
|
|
126
|
+
|
|
127
|
+
内置基准(14 篇中文合成文档(含散文体)/ 48 个 gold 实体)真实结果:[benchmarks/results.md](benchmarks/results.md)
|
|
128
|
+
|
|
129
|
+
- 当前快照:default 配置 **P 1.000 / R 1.000 / F1 1.000**;无上下文配置 R 0.644(人名/地址全靠上下文层)
|
|
130
|
+
- **诚实声明**:语料为合成文档(由本库假数据生成器构造,标注零噪声),分布窄于真实业务文档,数字代表格式级能力上限,不外推为生产效果
|
|
131
|
+
|
|
132
|
+
## 性能 / Performance
|
|
133
|
+
|
|
134
|
+
2MB 混合文本、单核([benchmarks/perf.md](benchmarks/perf.md)):完整管线 **2.35 MB/s**(3.4 万实体),比"同正则、零校验"的手搓基线慢 2.8 倍——这个代价买的是误报治理。
|
|
135
|
+
|
|
136
|
+
## 从 presidio 迁移 / Migration
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
from helan.compat_presidio import MianjuAnalyzer
|
|
140
|
+
|
|
141
|
+
results = MianjuAnalyzer().analyze(text="证件号 23144319731204692X", entities=["ID_CARD"])
|
|
142
|
+
for r in results:
|
|
143
|
+
print(r.entity_type, r.start, r.end, r.score) # 属性面与 presidio 一致
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
## 大文本 / Streaming
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
from helan import recognize_iter, read_file_chunks
|
|
150
|
+
|
|
151
|
+
for e in recognize_iter(read_file_chunks("huge.txt"), chunk_size=65536, overlap=512):
|
|
152
|
+
print(e.type, e.start, e.end, e.text) # 返回绝对偏移;该测试样本与整读结果一致,详见性能报告
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
约束:单实体长度须小于 overlap(URL 已加 512 上限与之匹配)。
|
|
156
|
+
|
|
157
|
+
## 路线图 / Roadmap
|
|
158
|
+
|
|
159
|
+
见 [ROADMAP.md](ROADMAP.md)。当前 v0.1.0:17 类型识别(含台湾身份证校验和、全角分隔身份证写法)+ 4 类算子 + Vault 还原 + 内置评测 + 流式 API + 黑名单校准,194 项测试全绿。
|
helan-0.1.0rc1/README.md
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# The Veil of Hidden Names — Helan
|
|
2
|
+
|
|
3
|
+
简体中文 · [English](README.en.md)
|
|
4
|
+
|
|
5
|
+
> 展示名 **The Veil of Hidden Names** 意为“隐名之幕”;Helan 是该项目的短名。
|
|
6
|
+
|
|
7
|
+
**中文优先的 PII 检测与脱敏库。校验和级识别,可逆 Vault 还原,格式保留假名——帮助你在把数据交给大模型或其他服务前发现并处理敏感信息。**
|
|
8
|
+
|
|
9
|
+
[](.github/workflows/ci.yml)
|
|
10
|
+
[](pyproject.toml)
|
|
11
|
+
[](LICENSE)
|
|
12
|
+
|
|
13
|
+
## 为什么需要它 / Why
|
|
14
|
+
|
|
15
|
+
在将中文业务文本送入 LLM、RAG 或其他外部处理服务前,开发者常需要识别并处理个人信息。通用识别框架可扩展,但中文证件校验、误报控制和可恢复脱敏通常需要额外规则与评测。
|
|
16
|
+
|
|
17
|
+
`helan`(Helan)把这件事做成零依赖标准件:
|
|
18
|
+
|
|
19
|
+
- **格式校验**:对身份证、银行卡和统一社会信用代码应用相应校验规则,减少仅凭位数与字符模式产生的误报;其他实体依赖上下文规则,仍可能漏检或误报
|
|
20
|
+
- **偏移不变量**:每个实体保证 `entity.text == source[start:end]`(fuzz 测试永久守护),脱敏结果可精确引用回原文
|
|
21
|
+
- **可逆 Vault**:用占位符脱敏并通过受保护的 Vault 还原;Vault 含有恢复敏感信息,必须与原文同等保护
|
|
22
|
+
- **格式保留假名**:`fake` 算子生成的假身份证能通过身份证校验、假银行卡能通过 Luhn——替换后的数据仍是"合法格式",适合造测试集与演示数据
|
|
23
|
+
- **零必装依赖**:核心纯 Python;jieba(人名 NER 补召回)与 LLM(语义级兜底)全部可选
|
|
24
|
+
- **自带评测**:内置 12 篇中文语料 + P/R/F1 报告,数字如实
|
|
25
|
+
|
|
26
|
+
## 安装 / Install
|
|
27
|
+
|
|
28
|
+
> 当前尚未发布到 PyPI;下方给出从 GitHub 获取并本地安装的命令。
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
git clone https://github.com/cloudydreamland/TheVeilOfHiddenNames.git
|
|
32
|
+
cd TheVeilOfHiddenNames
|
|
33
|
+
python -m pip install .
|
|
34
|
+
# PyPI 首发后:python -m pip install helan
|
|
35
|
+
python -m pip install ".[jieba]"
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## 快速开始 / Quickstart
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from helan import mask, restore, recognize
|
|
42
|
+
|
|
43
|
+
text = "出租方张伟明(身份证 11010519491231002X,电话 13812345678)同意将房屋出租。"
|
|
44
|
+
|
|
45
|
+
# 只识别
|
|
46
|
+
|
|
47
|
+
for e in recognize(text):
|
|
48
|
+
print(e.type, e.start, e.end, e.text)
|
|
49
|
+
|
|
50
|
+
# 可逆脱敏:身份证换成占位符,可精确还原
|
|
51
|
+
|
|
52
|
+
masked, vault = mask(text, ops={"ID_CARD": "vault"})
|
|
53
|
+
original = restore(masked, vault)
|
|
54
|
+
assert original == text
|
|
55
|
+
|
|
56
|
+
# 不可逆脱敏:手机号换成"合法格式"的假号码
|
|
57
|
+
|
|
58
|
+
masked, _ = mask(text, ops={"PHONE": "fake"})
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
命令行:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
helan scan 合同.txt --json # 只识别
|
|
65
|
+
helan mask 合同.txt -o 脱敏.txt --ops ID_CARD:vault,PHONE:fake --vault-out vault.json
|
|
66
|
+
helan restore 脱敏.txt --vault vault.json -o 还原.txt
|
|
67
|
+
helan eval # 内置基准报告
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## 实体类型与算子
|
|
71
|
+
|
|
72
|
+
| 类型 | 识别方式 | 默认算子 |
|
|
73
|
+
|---|---|---|
|
|
74
|
+
| ID_CARD 身份证 | 区划+出生日期+MOD 11-2 校验码 | partial(前3后4) |
|
|
75
|
+
| BANK_CARD 银行卡 | Luhn 校验(支持空格/连字符分组) | partial(留后4) |
|
|
76
|
+
| USCC 统一社会信用代码 | GB 32100 MOD 31-3 校验 | partial |
|
|
77
|
+
| PHONE 手机号 | 号段表严格校验 | partial(138\*\*\*\*5678) |
|
|
78
|
+
| PERSON_NAME 人名 | 称谓/引导词/顿号枚举上下文;jieba nr 可选 | partial(张\*\*) |
|
|
79
|
+
| ADDRESS 地址 | 引导词上下文 | redact |
|
|
80
|
+
| LANDLINE / EMAIL / IP / URL / PASSPORT / LICENSE_PLATE | 规则+守卫 | partial/redact |
|
|
81
|
+
| TW_ID_CARD 台湾身份证 | 地区字母码+性别位+加权 MOD 10 校验 | partial |
|
|
82
|
+
| POSTAL_CODE / QQ / 微信号 / OFFICER_ID | 关键词上下文(军官证为 format-only,如实标注) | redact/partial |
|
|
83
|
+
| ID_CARD 全角/符号分隔写法 | 1101 0519 4912 3100 2X 等分隔归一化后过校验和 | partial |
|
|
84
|
+
|
|
85
|
+
算子:`redact`(标签替换)/ `partial`(部分保留)/ `hash`(加盐稳定假名)/ `fake`(合法格式假数据)/ `vault`(可逆占位)/ 任意自定义 callable。
|
|
86
|
+
|
|
87
|
+
## 与现有方案的关系 / Landscape
|
|
88
|
+
|
|
89
|
+
我们曾用固定版本的 `presidio-analyzer` 与本项目内置语料做对照。测试范围、配置、语料及局限见[方法和完整结果](docs/presidio_zh.md);这些结果只适用于该次设置,不代表所有 Presidio 中文部署:
|
|
90
|
+
|
|
91
|
+
| 方案 | 实测/事实 |
|
|
92
|
+
|---|---|
|
|
93
|
+
| Presidio 等通用框架 | 提供可配置的识别与匿名化管线;中文效果取决于所选 recognizer、规则和评测语料 |
|
|
94
|
+
| 自定义正则 | 易于嵌入,但需要自行实现格式校验、上下文规则、偏移处理和评测 |
|
|
95
|
+
| Helan | 聚焦中文规则、原文偏移以及多种脱敏算子;适用范围和评测边界见下文 |
|
|
96
|
+
|
|
97
|
+
选题取证(为什么这个缺口是真的)见 [GAP_PROOF.md](GAP_PROOF.md)。
|
|
98
|
+
|
|
99
|
+
## 评测 / Evaluation
|
|
100
|
+
|
|
101
|
+
内置基准(14 篇中文合成文档(含散文体)/ 48 个 gold 实体)真实结果:[benchmarks/results.md](benchmarks/results.md)
|
|
102
|
+
|
|
103
|
+
- 当前快照:default 配置 **P 1.000 / R 1.000 / F1 1.000**;无上下文配置 R 0.644(人名/地址全靠上下文层)
|
|
104
|
+
- **诚实声明**:语料为合成文档(由本库假数据生成器构造,标注零噪声),分布窄于真实业务文档,数字代表格式级能力上限,不外推为生产效果
|
|
105
|
+
|
|
106
|
+
## 性能 / Performance
|
|
107
|
+
|
|
108
|
+
2MB 混合文本、单核([benchmarks/perf.md](benchmarks/perf.md)):完整管线 **2.35 MB/s**(3.4 万实体),比"同正则、零校验"的手搓基线慢 2.8 倍——这个代价买的是误报治理。
|
|
109
|
+
|
|
110
|
+
## 从 presidio 迁移 / Migration
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
from helan.compat_presidio import MianjuAnalyzer
|
|
114
|
+
|
|
115
|
+
results = MianjuAnalyzer().analyze(text="证件号 23144319731204692X", entities=["ID_CARD"])
|
|
116
|
+
for r in results:
|
|
117
|
+
print(r.entity_type, r.start, r.end, r.score) # 属性面与 presidio 一致
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
## 大文本 / Streaming
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
from helan import recognize_iter, read_file_chunks
|
|
124
|
+
|
|
125
|
+
for e in recognize_iter(read_file_chunks("huge.txt"), chunk_size=65536, overlap=512):
|
|
126
|
+
print(e.type, e.start, e.end, e.text) # 返回绝对偏移;该测试样本与整读结果一致,详见性能报告
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
约束:单实体长度须小于 overlap(URL 已加 512 上限与之匹配)。
|
|
130
|
+
|
|
131
|
+
## 路线图 / Roadmap
|
|
132
|
+
|
|
133
|
+
见 [ROADMAP.md](ROADMAP.md)。当前 v0.1.0:17 类型识别(含台湾身份证校验和、全角分隔身份证写法)+ 4 类算子 + Vault 还原 + 内置评测 + 流式 API + 黑名单校准,194 项测试全绿。
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "helan"
|
|
7
|
+
version = "0.1.0rc1"
|
|
8
|
+
description = "The Veil of Hidden Names (Helan) | Chinese-first personal data detection and masking"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "helan contributors" }]
|
|
13
|
+
keywords = ["pii", "脱敏", "privacy", "chinese", "nlp", "llm", "compliance"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Topic :: Security",
|
|
18
|
+
"Topic :: Text Processing :: Linguistic",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.optional-dependencies]
|
|
22
|
+
jieba = ["jieba>=0.42"]
|
|
23
|
+
dev = ["pytest>=8.0", "ruff>=0.6"]
|
|
24
|
+
|
|
25
|
+
[project.scripts]
|
|
26
|
+
helan = "helan.cli:main"
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://github.com/cloudydreamland/TheVeilOfHiddenNames"
|
|
30
|
+
Repository = "https://github.com/cloudydreamland/TheVeilOfHiddenNames"
|
|
31
|
+
Issues = "https://github.com/cloudydreamland/TheVeilOfHiddenNames/issues"
|
|
32
|
+
Changelog = "https://github.com/cloudydreamland/TheVeilOfHiddenNames/blob/main/CHANGELOG.md"
|
|
33
|
+
Security = "https://github.com/cloudydreamland/TheVeilOfHiddenNames/security/policy"
|
|
34
|
+
|
|
35
|
+
[tool.setuptools.packages.find]
|
|
36
|
+
where = ["src"]
|
|
37
|
+
|
|
38
|
+
[tool.setuptools.package-data]
|
|
39
|
+
helan = ["py.typed"]
|
|
40
|
+
|
|
41
|
+
[tool.ruff]
|
|
42
|
+
line-length = 100
|
|
43
|
+
target-version = "py310"
|
|
44
|
+
|
|
45
|
+
[tool.ruff.lint]
|
|
46
|
+
ignore = [
|
|
47
|
+
"DTZ011", # 出生日期校验用 date.today() 足够,与时区无关
|
|
48
|
+
"RUF007", # 测试中 zip(seq, seq[1:]) 比 pairwise 更直观
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
[tool.pytest.ini_options]
|
|
52
|
+
testpaths = ["tests"]
|
helan-0.1.0rc1/setup.cfg
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Helan — 中文优先的 PII 检测与脱敏库。
|
|
2
|
+
|
|
3
|
+
快速上手::
|
|
4
|
+
|
|
5
|
+
from helan import mask, restore, Vault
|
|
6
|
+
|
|
7
|
+
masked, vault = mask(text, ops={"ID_CARD": "vault", "PHONE": "fake"})
|
|
8
|
+
original = restore(masked, vault)
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from .pipeline import DEFAULT_OPS, build_recognizers, mask, recognize, restore
|
|
12
|
+
from .streaming import read_file_chunks, recognize_iter
|
|
13
|
+
from .types import (
|
|
14
|
+
ADDRESS,
|
|
15
|
+
ALL_TYPES,
|
|
16
|
+
BANK_CARD,
|
|
17
|
+
EMAIL,
|
|
18
|
+
ENTITY_TYPES,
|
|
19
|
+
ID_CARD,
|
|
20
|
+
IP_ADDRESS,
|
|
21
|
+
LANDLINE,
|
|
22
|
+
LICENSE_PLATE,
|
|
23
|
+
PASSPORT,
|
|
24
|
+
PERSON_NAME,
|
|
25
|
+
PHONE,
|
|
26
|
+
URL,
|
|
27
|
+
USCC,
|
|
28
|
+
Entity,
|
|
29
|
+
)
|
|
30
|
+
from .vault import Vault
|
|
31
|
+
|
|
32
|
+
__version__ = "0.1.0"
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"ADDRESS",
|
|
36
|
+
"ALL_TYPES",
|
|
37
|
+
"BANK_CARD",
|
|
38
|
+
"DEFAULT_OPS",
|
|
39
|
+
"EMAIL",
|
|
40
|
+
"ENTITY_TYPES",
|
|
41
|
+
"ID_CARD",
|
|
42
|
+
"IP_ADDRESS",
|
|
43
|
+
"LANDLINE",
|
|
44
|
+
"LICENSE_PLATE",
|
|
45
|
+
"PASSPORT",
|
|
46
|
+
"PERSON_NAME",
|
|
47
|
+
"PHONE",
|
|
48
|
+
"URL",
|
|
49
|
+
"USCC",
|
|
50
|
+
"Entity",
|
|
51
|
+
"Vault",
|
|
52
|
+
"__version__",
|
|
53
|
+
"build_recognizers",
|
|
54
|
+
"mask",
|
|
55
|
+
"read_file_chunks",
|
|
56
|
+
"recognize",
|
|
57
|
+
"recognize_iter",
|
|
58
|
+
"restore",
|
|
59
|
+
]
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""吞吐基准:helan 完整管线 vs 纯正则基线。
|
|
2
|
+
|
|
3
|
+
诚实口径:
|
|
4
|
+
- "纯正则基线"= 用与 helan 完全相同的正则抓候选,但**不做任何校验和验证、
|
|
5
|
+
不做冲突消解**——它代表"手搓正则脱敏"的成本上限(它更快,但会产出大量误报)。
|
|
6
|
+
- 差值就是校验和验证 + 冲突消解的安全代价。
|
|
7
|
+
- 单核、CPython、Windows;数字只代表本机,不外推。
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import random
|
|
14
|
+
import time
|
|
15
|
+
|
|
16
|
+
from .eval.corpus import build_corpus
|
|
17
|
+
from .pipeline import recognize
|
|
18
|
+
|
|
19
|
+
MACHINE = "Windows / CPython 3.13 / 单核"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def build_mixed_text(target_mb: float, seed: int = 2026) -> str:
|
|
23
|
+
"""语料文档 + 随机中文数字噪声,拼到目标大小。噪声保证正则层有真实扫描压力。"""
|
|
24
|
+
rng = random.Random(seed)
|
|
25
|
+
base_docs = [d.text for d in build_corpus()]
|
|
26
|
+
filler_pool = (
|
|
27
|
+
"本段为普通业务文本,用于模拟真实文档中的非敏感内容,包含数字 12345 与标点。",
|
|
28
|
+
"会议决定、预算安排、项目进度、交付节点、验收标准等事项。",
|
|
29
|
+
"The quick brown fox jumps over the lazy dog 9988776655.",
|
|
30
|
+
"账户余额、汇率、结算周期、手续费率等财务信息。",
|
|
31
|
+
)
|
|
32
|
+
parts: list[str] = []
|
|
33
|
+
total = 0
|
|
34
|
+
target = int(target_mb * 1024 * 1024)
|
|
35
|
+
while total < target:
|
|
36
|
+
if rng.random() < 0.25:
|
|
37
|
+
parts.append(rng.choice(base_docs))
|
|
38
|
+
else:
|
|
39
|
+
parts.append(rng.choice(filler_pool))
|
|
40
|
+
total += len(parts[-1])
|
|
41
|
+
return "\n".join(parts)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def regex_baseline_scan(text: str) -> int:
|
|
45
|
+
"""纯正则基线:与 helan 相同的正则抓候选,不验证、不消解。返回候选数。"""
|
|
46
|
+
import re
|
|
47
|
+
|
|
48
|
+
from helan.data import HONORIFICS, LEAD_WORDS, SURNAMES
|
|
49
|
+
from helan.recognizers import chinese
|
|
50
|
+
from helan.recognizers.base import guarded_pattern
|
|
51
|
+
|
|
52
|
+
patterns = [
|
|
53
|
+
re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+"),
|
|
54
|
+
guarded_pattern(r"(?:\d{1,3}\.){3}\d{1,3}", left="0-9.", right="0-9"),
|
|
55
|
+
re.compile(r"https?://[^\s<>\"',。;!?、]{1,512}"),
|
|
56
|
+
guarded_pattern(r"(?:0\d{2,3})-?[2-9]\d{6,7}"),
|
|
57
|
+
guarded_pattern(r"\d{17}[0-9Xx]"),
|
|
58
|
+
guarded_pattern(r"\d{15}"),
|
|
59
|
+
guarded_pattern(r"\d{13,19}", right="0-9Xx"),
|
|
60
|
+
guarded_pattern(r"1[3-9]\d{9}"),
|
|
61
|
+
chinese._ADDR_RE,
|
|
62
|
+
]
|
|
63
|
+
count = 0
|
|
64
|
+
for pat in patterns:
|
|
65
|
+
for _m in pat.finditer(text):
|
|
66
|
+
count += 1
|
|
67
|
+
_ = HONORIFICS, LEAD_WORDS, SURNAMES
|
|
68
|
+
return count
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def run(target_mb: float) -> dict:
|
|
72
|
+
text = build_mixed_text(target_mb)
|
|
73
|
+
size_mb = len(text) / 1024 / 1024
|
|
74
|
+
|
|
75
|
+
t0 = time.perf_counter()
|
|
76
|
+
entities = recognize(text)
|
|
77
|
+
t1 = time.perf_counter()
|
|
78
|
+
full_secs = t1 - t0
|
|
79
|
+
|
|
80
|
+
t0 = time.perf_counter()
|
|
81
|
+
candidates = regex_baseline_scan(text)
|
|
82
|
+
baseline_secs = time.perf_counter() - t0
|
|
83
|
+
|
|
84
|
+
return {
|
|
85
|
+
"size_mb": size_mb,
|
|
86
|
+
"entities": len(entities),
|
|
87
|
+
"full_secs": full_secs,
|
|
88
|
+
"full_mbps": size_mb / full_secs,
|
|
89
|
+
"baseline_secs": baseline_secs,
|
|
90
|
+
"baseline_mbps": size_mb / baseline_secs,
|
|
91
|
+
"baseline_candidates": candidates,
|
|
92
|
+
"machine": MACHINE,
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def render(result: dict) -> str:
|
|
97
|
+
return (
|
|
98
|
+
"# helan 性能基准\n\n"
|
|
99
|
+
f"- 机器:{result['machine']}(数字只代表本机,不外推)\n"
|
|
100
|
+
f"- 文本:{result['size_mb']:.2f} MB 合成混合文本(内置语料 25% + 普通业务噪声 75%)\n"
|
|
101
|
+
f"- helan 完整管线(校验和+规则+上下文+消解):**{result['full_mbps']:.2f} MB/s**"
|
|
102
|
+
f"({result['full_secs']:.2f}s,{result['entities']} 个实体)\n"
|
|
103
|
+
f"- 纯正则基线(同正则、零校验、零消解,手搓方案的速度上限):{result['baseline_mbps']:.2f} MB/s"
|
|
104
|
+
f"({result['baseline_secs']:.2f}s,{result['baseline_candidates']} 个未验证候选)\n"
|
|
105
|
+
f"- **安全代价:完整管线比纯正则慢 {result['full_secs'] / result['baseline_secs']:.1f} 倍**——"
|
|
106
|
+
"这个代价买的是误报治理(发票号不再被当成手机号)。\n"
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def main(argv: list[str] | None = None) -> int:
|
|
111
|
+
parser = argparse.ArgumentParser(prog="helan.bench")
|
|
112
|
+
parser.add_argument("--mb", type=float, default=2.0)
|
|
113
|
+
parser.add_argument("--write", help="markdown 结果写入路径")
|
|
114
|
+
args = parser.parse_args(argv)
|
|
115
|
+
result = run(args.mb)
|
|
116
|
+
report = render(result)
|
|
117
|
+
print(report)
|
|
118
|
+
if args.write:
|
|
119
|
+
from .io_utils import write_text
|
|
120
|
+
|
|
121
|
+
write_text(args.write, report)
|
|
122
|
+
return 0
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
if __name__ == "__main__":
|
|
126
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""停用词黑名单校准:在内置语料上找出 PERSON_NAME 的真实 FP,自动建议扩充。
|
|
2
|
+
|
|
3
|
+
原理:合成语料有零噪声 gold,所以这里统计的是**真实 FP**(不是猜测):
|
|
4
|
+
- 逐条列出 FP 的触发 cue 与上下文;
|
|
5
|
+
- 按 `text[:2]` 聚合,出现 ≥2 次的前缀自动建议加入 `PERSON_NAME_BLOCKLIST`;
|
|
6
|
+
- 单次出现的 FP 交人工判断(宁可漏建议,不可误杀真人名)。
|
|
7
|
+
|
|
8
|
+
用法:`helan calibrate`(或 `python -m helan.calibrate`)。
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from collections import Counter
|
|
14
|
+
|
|
15
|
+
from .data import PERSON_NAME_BLOCKLIST
|
|
16
|
+
from .eval.corpus import build_corpus
|
|
17
|
+
from .pipeline import recognize
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def collect_person_fps() -> list[dict]:
|
|
21
|
+
"""跑内置语料,收集 PERSON_NAME 的 FP(与 gold 不匹配的命中)。"""
|
|
22
|
+
fps: list[dict] = []
|
|
23
|
+
for doc in build_corpus():
|
|
24
|
+
gold = {
|
|
25
|
+
(g.start, g.end) for g in doc.gold if g.type == "PERSON_NAME"
|
|
26
|
+
}
|
|
27
|
+
for e in recognize(doc.text, jieba=False):
|
|
28
|
+
if e.type == "PERSON_NAME" and (e.start, e.end) not in gold:
|
|
29
|
+
fps.append(
|
|
30
|
+
{
|
|
31
|
+
"text": e.text,
|
|
32
|
+
"cue": e.meta.get("cue", "?"),
|
|
33
|
+
"doc": doc.doc_id,
|
|
34
|
+
"context": doc.text[max(0, e.start - 10) : e.end + 10].replace("\n", "|"),
|
|
35
|
+
}
|
|
36
|
+
)
|
|
37
|
+
return fps
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def suggest_blocklist(fps: list[dict], min_count: int = 2) -> list[str]:
|
|
41
|
+
"""出现 ≥ min_count 次的两字前缀,自动建议加入黑名单。"""
|
|
42
|
+
counter = Counter(fp["text"][:2] for fp in fps)
|
|
43
|
+
return [prefix for prefix, n in counter.most_common() if n >= min_count]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def render() -> str:
|
|
47
|
+
fps = collect_person_fps()
|
|
48
|
+
lines = [
|
|
49
|
+
"# PERSON_NAME 停用词黑名单校准报告",
|
|
50
|
+
"",
|
|
51
|
+
f"- 数据:内置语料(gold 零噪声),本次 FP 总数 **{len(fps)}**",
|
|
52
|
+
f"- 现行黑名单规模:{len(PERSON_NAME_BLOCKLIST)} 条",
|
|
53
|
+
"",
|
|
54
|
+
]
|
|
55
|
+
if not fps:
|
|
56
|
+
lines.append("当前语料上 PERSON_NAME 无 FP,黑名单无需扩充。")
|
|
57
|
+
return "\n".join(lines)
|
|
58
|
+
|
|
59
|
+
suggestions = suggest_blocklist(fps)
|
|
60
|
+
lines.append("## 自动建议(出现 ≥2 次的前缀)")
|
|
61
|
+
lines.append("")
|
|
62
|
+
lines.append(f"`{suggestions}`" if suggestions else "无")
|
|
63
|
+
lines.append("")
|
|
64
|
+
lines.append("## 全部 FP 明细(单次出现者请人工判断)")
|
|
65
|
+
lines.append("")
|
|
66
|
+
lines.append("| 触发 cue | 文本 | 上下文 | 文档 |")
|
|
67
|
+
lines.append("|---|---|---|---|")
|
|
68
|
+
for fp in fps:
|
|
69
|
+
lines.append(f"| {fp['cue']} | {fp['text']} | `{fp['context']}` | {fp['doc']} |")
|
|
70
|
+
return "\n".join(lines)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def main() -> int:
|
|
74
|
+
print(render())
|
|
75
|
+
return 0
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
if __name__ == "__main__":
|
|
79
|
+
main()
|