agenthandler 0.92.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of agenthandler might be problematic. Click here for more details.
- agenthandler/__init__.py +163 -0
- agenthandler/approval.py +135 -0
- agenthandler/audit.py +144 -0
- agenthandler/auth.py +188 -0
- agenthandler/budget.py +94 -0
- agenthandler/builder.py +178 -0
- agenthandler/circuit_breaker.py +88 -0
- agenthandler/connectors.py +425 -0
- agenthandler/context.py +410 -0
- agenthandler/errors.py +130 -0
- agenthandler/guardrails/__init__.py +79 -0
- agenthandler/guardrails/deterministic.py +472 -0
- agenthandler/guardrails/llm_based.py +549 -0
- agenthandler/guardrails/types.py +83 -0
- agenthandler/observe.py +525 -0
- agenthandler/openclaw.py +370 -0
- agenthandler/pipeline.py +532 -0
- agenthandler/pipeline_poml.py +593 -0
- agenthandler/policy.py +251 -0
- agenthandler/py.typed +0 -0
- agenthandler/redact.py +260 -0
- agenthandler/reflection.py +371 -0
- agenthandler/replay.py +225 -0
- agenthandler/server.py +752 -0
- agenthandler/session.py +391 -0
- agenthandler/skill.py +265 -0
- agenthandler/store.py +341 -0
- agenthandler/streams.py +339 -0
- agenthandler/supervisor.py +882 -0
- agenthandler/triggers.py +471 -0
- agenthandler-0.92.0.dist-info/METADATA +1355 -0
- agenthandler-0.92.0.dist-info/RECORD +34 -0
- agenthandler-0.92.0.dist-info/WHEEL +4 -0
- agenthandler-0.92.0.dist-info/licenses/LICENSE +201 -0
agenthandler/policy.py
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"""Supervision policy — what limits to enforce and what to do when they're hit."""
|
|
2
|
+
|
|
3
|
+
import fnmatch
|
|
4
|
+
import re
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from enum import Enum
|
|
7
|
+
from typing import Any, Dict, List, Optional
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Action(Enum):
|
|
11
|
+
"""What to do when a limit is hit."""
|
|
12
|
+
|
|
13
|
+
RETURN_ERROR = "return_error"
|
|
14
|
+
RESPOND_WITH_BEST_EFFORT = "respond_with_best_effort"
|
|
15
|
+
KILL = "kill"
|
|
16
|
+
|
|
17
|
+
@classmethod
|
|
18
|
+
def from_str(cls, s: str) -> "Action":
|
|
19
|
+
mapping = {
|
|
20
|
+
"return_error": cls.RETURN_ERROR,
|
|
21
|
+
"respond_with_best_effort": cls.RESPOND_WITH_BEST_EFFORT,
|
|
22
|
+
"kill": cls.KILL,
|
|
23
|
+
}
|
|
24
|
+
return mapping.get(s, cls.RETURN_ERROR)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class OnFailure:
|
|
29
|
+
"""What to do on failure + how many retries."""
|
|
30
|
+
|
|
31
|
+
action: Action = Action.RETURN_ERROR
|
|
32
|
+
retries: int = 0
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class ScopeRule:
|
|
37
|
+
"""A single scope constraint on a tool parameter."""
|
|
38
|
+
|
|
39
|
+
param: str # parameter name to constrain
|
|
40
|
+
constraint: str # "allow", "block", or "match"
|
|
41
|
+
values: List[str] # patterns (glob-style for allow/block, regex for match)
|
|
42
|
+
|
|
43
|
+
def check(self, param_value: Any) -> bool:
|
|
44
|
+
"""Returns True if the value is allowed by this rule."""
|
|
45
|
+
str_value = str(param_value)
|
|
46
|
+
if self.constraint == "allow":
|
|
47
|
+
return any(fnmatch.fnmatch(str_value, p) for p in self.values)
|
|
48
|
+
if self.constraint == "block":
|
|
49
|
+
return not any(fnmatch.fnmatch(str_value, p) for p in self.values)
|
|
50
|
+
if self.constraint == "match":
|
|
51
|
+
return any(re.search(p, str_value) for p in self.values)
|
|
52
|
+
return True
|
|
53
|
+
|
|
54
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
55
|
+
return {
|
|
56
|
+
"param": self.param,
|
|
57
|
+
"constraint": self.constraint,
|
|
58
|
+
"values": list(self.values),
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class ToolScope:
|
|
64
|
+
"""Scoped permissions for a specific tool."""
|
|
65
|
+
|
|
66
|
+
tool_name: str
|
|
67
|
+
rules: List[ScopeRule]
|
|
68
|
+
max_calls_per_minute: Optional[int] = None # rate limit
|
|
69
|
+
|
|
70
|
+
def check_args(self, kwargs: Dict[str, Any]) -> Optional[str]:
|
|
71
|
+
"""Check if kwargs satisfy all rules.
|
|
72
|
+
|
|
73
|
+
Returns None if OK, error message if violated.
|
|
74
|
+
"""
|
|
75
|
+
for rule in self.rules:
|
|
76
|
+
if rule.param in kwargs:
|
|
77
|
+
if not rule.check(kwargs[rule.param]):
|
|
78
|
+
return (
|
|
79
|
+
f"Parameter '{rule.param}' value "
|
|
80
|
+
f"'{kwargs[rule.param]}' blocked by scope rule"
|
|
81
|
+
)
|
|
82
|
+
return None
|
|
83
|
+
|
|
84
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
85
|
+
d: Dict[str, Any] = {
|
|
86
|
+
"tool_name": self.tool_name,
|
|
87
|
+
"rules": [r.to_dict() for r in self.rules],
|
|
88
|
+
}
|
|
89
|
+
if self.max_calls_per_minute is not None:
|
|
90
|
+
d["max_calls_per_minute"] = self.max_calls_per_minute
|
|
91
|
+
return d
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@dataclass
|
|
95
|
+
class Policy:
|
|
96
|
+
"""Supervision policy for a skill or agent.
|
|
97
|
+
|
|
98
|
+
Configure limits, failure handlers, and confirmation gates.
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
# Tool execution limits
|
|
102
|
+
tool_timeout: float = 30.0 # seconds per tool call
|
|
103
|
+
max_iterations: int = 5 # max tool call rounds
|
|
104
|
+
token_budget: Optional[int] = None # max tokens for the whole skill
|
|
105
|
+
|
|
106
|
+
# Request-level limits
|
|
107
|
+
request_timeout: float = 300.0 # seconds for the entire request
|
|
108
|
+
silence_timeout: Optional[float] = None # dead man's switch (seconds)
|
|
109
|
+
|
|
110
|
+
# Circuit breaker
|
|
111
|
+
circuit_breaker_threshold: int = 5 # failures before opening
|
|
112
|
+
circuit_breaker_reset: float = 60.0 # seconds before half-open
|
|
113
|
+
|
|
114
|
+
# Failure handlers
|
|
115
|
+
on_timeout: OnFailure = field(default_factory=lambda: OnFailure(Action.RETURN_ERROR))
|
|
116
|
+
on_budget_exceeded: OnFailure = field(
|
|
117
|
+
default_factory=lambda: OnFailure(Action.RESPOND_WITH_BEST_EFFORT)
|
|
118
|
+
)
|
|
119
|
+
on_max_iterations: OnFailure = field(default_factory=lambda: OnFailure(Action.RETURN_ERROR))
|
|
120
|
+
on_circuit_open: OnFailure = field(default_factory=lambda: OnFailure(Action.RETURN_ERROR))
|
|
121
|
+
on_silence: OnFailure = field(default_factory=lambda: OnFailure(Action.RETURN_ERROR))
|
|
122
|
+
|
|
123
|
+
# Crash loop protection
|
|
124
|
+
max_resumes: int = 3 # max times a session can be resumed before marked failed
|
|
125
|
+
|
|
126
|
+
# Audit
|
|
127
|
+
audit_enabled: bool = True
|
|
128
|
+
|
|
129
|
+
# Confirmation gates (tool names that require user confirm)
|
|
130
|
+
require_confirm: List[str] = field(default_factory=list)
|
|
131
|
+
|
|
132
|
+
# Tool scopes (parameter-level permission rules)
|
|
133
|
+
tool_scopes: List[ToolScope] = field(default_factory=list)
|
|
134
|
+
|
|
135
|
+
# Privacy shielding — PII redaction categories
|
|
136
|
+
# Options: "email", "phone", "ssn", "credit_card", "api_key", "ip_address"
|
|
137
|
+
# Empty list = no redaction. None = redact all categories.
|
|
138
|
+
redact: Optional[List[str]] = field(default_factory=list)
|
|
139
|
+
redact_direction: str = "both" # "inbound", "outbound", or "both"
|
|
140
|
+
|
|
141
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
142
|
+
"""Serialize to a plain dictionary (round-trips with from_dict)."""
|
|
143
|
+
d: Dict[str, Any] = {
|
|
144
|
+
"tool_timeout": self.tool_timeout,
|
|
145
|
+
"max_iterations": self.max_iterations,
|
|
146
|
+
"request_timeout": self.request_timeout,
|
|
147
|
+
"circuit_breaker_threshold": self.circuit_breaker_threshold,
|
|
148
|
+
"circuit_breaker_reset": self.circuit_breaker_reset,
|
|
149
|
+
"audit_enabled": self.audit_enabled,
|
|
150
|
+
}
|
|
151
|
+
if self.token_budget is not None:
|
|
152
|
+
d["token_budget"] = self.token_budget
|
|
153
|
+
if self.silence_timeout is not None:
|
|
154
|
+
d["silence_timeout"] = self.silence_timeout
|
|
155
|
+
for key in (
|
|
156
|
+
"on_timeout",
|
|
157
|
+
"on_budget_exceeded",
|
|
158
|
+
"on_max_iterations",
|
|
159
|
+
"on_circuit_open",
|
|
160
|
+
"on_silence",
|
|
161
|
+
):
|
|
162
|
+
val: OnFailure = getattr(self, key)
|
|
163
|
+
d[key] = {"action": val.action.value, "retries": val.retries}
|
|
164
|
+
d["max_resumes"] = self.max_resumes
|
|
165
|
+
if self.require_confirm:
|
|
166
|
+
d["require_confirm"] = list(self.require_confirm)
|
|
167
|
+
if self.tool_scopes:
|
|
168
|
+
d["tool_scopes"] = [s.to_dict() for s in self.tool_scopes]
|
|
169
|
+
if self.redact is not None and self.redact:
|
|
170
|
+
d["redact"] = list(self.redact)
|
|
171
|
+
elif self.redact is None:
|
|
172
|
+
d["redact"] = None
|
|
173
|
+
d["redact_direction"] = self.redact_direction
|
|
174
|
+
return d
|
|
175
|
+
|
|
176
|
+
@classmethod
|
|
177
|
+
def from_dict(cls, d: Dict[str, Any]) -> "Policy":
|
|
178
|
+
"""Create a Policy from a plain dictionary.
|
|
179
|
+
|
|
180
|
+
Recognised keys (all optional — missing keys use defaults):
|
|
181
|
+
tool_timeout, max_iterations, token_budget,
|
|
182
|
+
request_timeout, silence_timeout,
|
|
183
|
+
circuit_breaker_threshold, circuit_breaker_reset,
|
|
184
|
+
max_resumes,
|
|
185
|
+
on_timeout, on_budget_exceeded, on_max_iterations,
|
|
186
|
+
on_circuit_open, on_silence, audit_enabled, require_confirm
|
|
187
|
+
"""
|
|
188
|
+
kwargs: Dict[str, Any] = {}
|
|
189
|
+
# Simple scalar fields
|
|
190
|
+
for key in (
|
|
191
|
+
"tool_timeout",
|
|
192
|
+
"max_iterations",
|
|
193
|
+
"token_budget",
|
|
194
|
+
"request_timeout",
|
|
195
|
+
"silence_timeout",
|
|
196
|
+
"circuit_breaker_threshold",
|
|
197
|
+
"circuit_breaker_reset",
|
|
198
|
+
"max_resumes",
|
|
199
|
+
"audit_enabled",
|
|
200
|
+
):
|
|
201
|
+
if key in d:
|
|
202
|
+
kwargs[key] = d[key]
|
|
203
|
+
|
|
204
|
+
# OnFailure fields (accept string or dict with action + retries)
|
|
205
|
+
for key in (
|
|
206
|
+
"on_timeout",
|
|
207
|
+
"on_budget_exceeded",
|
|
208
|
+
"on_max_iterations",
|
|
209
|
+
"on_circuit_open",
|
|
210
|
+
"on_silence",
|
|
211
|
+
):
|
|
212
|
+
if key in d:
|
|
213
|
+
val = d[key]
|
|
214
|
+
if isinstance(val, str):
|
|
215
|
+
kwargs[key] = OnFailure(Action.from_str(val))
|
|
216
|
+
elif isinstance(val, dict):
|
|
217
|
+
kwargs[key] = OnFailure(
|
|
218
|
+
Action.from_str(val.get("action", "return_error")),
|
|
219
|
+
retries=val.get("retries", 0),
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
if "require_confirm" in d:
|
|
223
|
+
kwargs["require_confirm"] = list(d["require_confirm"])
|
|
224
|
+
|
|
225
|
+
if "tool_scopes" in d:
|
|
226
|
+
scopes = []
|
|
227
|
+
for scope_dict in d["tool_scopes"]:
|
|
228
|
+
rules = []
|
|
229
|
+
for r in scope_dict.get("rules", []):
|
|
230
|
+
rules.append(
|
|
231
|
+
ScopeRule(
|
|
232
|
+
param=r["param"],
|
|
233
|
+
constraint=r.get("constraint", "allow"),
|
|
234
|
+
values=r.get("values", []),
|
|
235
|
+
)
|
|
236
|
+
)
|
|
237
|
+
scopes.append(
|
|
238
|
+
ToolScope(
|
|
239
|
+
tool_name=scope_dict["tool_name"],
|
|
240
|
+
rules=rules,
|
|
241
|
+
max_calls_per_minute=scope_dict.get("max_calls_per_minute"),
|
|
242
|
+
)
|
|
243
|
+
)
|
|
244
|
+
kwargs["tool_scopes"] = scopes
|
|
245
|
+
|
|
246
|
+
if "redact" in d:
|
|
247
|
+
kwargs["redact"] = d["redact"] # None or list of strings
|
|
248
|
+
if "redact_direction" in d:
|
|
249
|
+
kwargs["redact_direction"] = d["redact_direction"]
|
|
250
|
+
|
|
251
|
+
return cls(**kwargs)
|
agenthandler/py.typed
ADDED
|
File without changes
|
agenthandler/redact.py
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
"""Privacy shielding — PII detection and redaction for tool call data.
|
|
2
|
+
|
|
3
|
+
Regex-based pattern matching that intercepts tool call inputs and outputs,
|
|
4
|
+
replacing sensitive data with placeholders. Fast, zero dependencies,
|
|
5
|
+
catches the common cases.
|
|
6
|
+
|
|
7
|
+
For HIPAA/GDPR-grade redaction, use a dedicated NER service — this
|
|
8
|
+
catches accidental leakage, not adversarial exfiltration.
|
|
9
|
+
|
|
10
|
+
Usage:
|
|
11
|
+
from agenthandler.redact import Redactor
|
|
12
|
+
|
|
13
|
+
redactor = Redactor(categories=["email", "phone", "ssn"])
|
|
14
|
+
clean = redactor.redact("Call me at 555-123-4567 or bob@example.com")
|
|
15
|
+
# "Call me at [PHONE] or [EMAIL]"
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from typing import Any, Dict, List, Optional, Set, Tuple
|
|
21
|
+
|
|
22
|
+
# ---------------------------------------------------------------------------
|
|
23
|
+
# Pattern registry
|
|
24
|
+
# ---------------------------------------------------------------------------
|
|
25
|
+
|
|
26
|
+
# Each pattern: (category_name, compiled_regex, placeholder)
|
|
27
|
+
_PATTERNS: List[Tuple[str, "re.Pattern[str]", str]] = [
|
|
28
|
+
(
|
|
29
|
+
"email",
|
|
30
|
+
re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b"),
|
|
31
|
+
"[EMAIL]",
|
|
32
|
+
),
|
|
33
|
+
(
|
|
34
|
+
"phone",
|
|
35
|
+
re.compile(
|
|
36
|
+
r"(?<!\d)" # not preceded by digit
|
|
37
|
+
r"(?:"
|
|
38
|
+
r"\+?1[-.\s]?" # optional country code
|
|
39
|
+
r")?"
|
|
40
|
+
r"(?:\(?\d{3}\)?[-.\s]?)" # area code
|
|
41
|
+
r"\d{3}[-.\s]?\d{4}" # number
|
|
42
|
+
r"(?!\d)" # not followed by digit
|
|
43
|
+
),
|
|
44
|
+
"[PHONE]",
|
|
45
|
+
),
|
|
46
|
+
(
|
|
47
|
+
"ssn",
|
|
48
|
+
re.compile(r"\b\d{3}-\d{2}-\d{4}\b"),
|
|
49
|
+
"[SSN]",
|
|
50
|
+
),
|
|
51
|
+
(
|
|
52
|
+
"credit_card",
|
|
53
|
+
re.compile(
|
|
54
|
+
r"\b(?:"
|
|
55
|
+
r"\d{4}[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{4}" # 16 digits
|
|
56
|
+
r"|"
|
|
57
|
+
r"\d{4}[-\s]?\d{6}[-\s]?\d{5}" # amex 15 digits
|
|
58
|
+
r")\b"
|
|
59
|
+
),
|
|
60
|
+
"[CREDIT_CARD]",
|
|
61
|
+
),
|
|
62
|
+
(
|
|
63
|
+
"api_key",
|
|
64
|
+
re.compile(
|
|
65
|
+
r"(?:"
|
|
66
|
+
r"sk-[A-Za-z0-9]{20,}" # OpenAI-style
|
|
67
|
+
r"|"
|
|
68
|
+
r"sk-ant-[A-Za-z0-9-]{20,}" # Anthropic-style
|
|
69
|
+
r"|"
|
|
70
|
+
r"AKIA[A-Z0-9]{16}" # AWS access key
|
|
71
|
+
r"|"
|
|
72
|
+
r"ghp_[A-Za-z0-9]{36}" # GitHub PAT
|
|
73
|
+
r"|"
|
|
74
|
+
r"xox[boaprs]-[A-Za-z0-9-]{10,}" # Slack token
|
|
75
|
+
r")"
|
|
76
|
+
),
|
|
77
|
+
"[API_KEY]",
|
|
78
|
+
),
|
|
79
|
+
(
|
|
80
|
+
"ip_address",
|
|
81
|
+
re.compile(
|
|
82
|
+
r"\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}"
|
|
83
|
+
r"(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b"
|
|
84
|
+
),
|
|
85
|
+
"[IP_ADDR]",
|
|
86
|
+
),
|
|
87
|
+
# -- International patterns --
|
|
88
|
+
(
|
|
89
|
+
"phone", # international phone numbers: +XX XXXX XXXXXX etc.
|
|
90
|
+
re.compile(
|
|
91
|
+
r"(?<!\d)"
|
|
92
|
+
r"\+(?:44|49|33|61|81|86|91|7|34|39|55|82|65|852)" # country codes
|
|
93
|
+
r"[-.\s]?\d{1,5}[-.\s]?\d{3,5}[-.\s]?\d{3,5}"
|
|
94
|
+
r"(?!\d)"
|
|
95
|
+
),
|
|
96
|
+
"[PHONE]",
|
|
97
|
+
),
|
|
98
|
+
(
|
|
99
|
+
"national_id", # UK National Insurance Number
|
|
100
|
+
re.compile(r"\b[A-CEGHJ-PR-TW-Z]{2}\s?\d{2}\s?\d{2}\s?\d{2}\s?[A-D]\b"),
|
|
101
|
+
"[NATIONAL_ID]",
|
|
102
|
+
),
|
|
103
|
+
(
|
|
104
|
+
"national_id", # German Tax ID (Steuerliche Identifikationsnummer)
|
|
105
|
+
re.compile(r"\b\d{2}\s?\d{3}\s?\d{3}\s?\d{3}\b"),
|
|
106
|
+
"[NATIONAL_ID]",
|
|
107
|
+
),
|
|
108
|
+
(
|
|
109
|
+
"iban", # International Bank Account Number
|
|
110
|
+
re.compile(
|
|
111
|
+
r"\b[A-Z]{2}\d{2}[\s]?[\dA-Z]{4}[\s]?[\dA-Z]{4}[\s]?[\dA-Z]{4}(?:[\s]?[\dA-Z]{1,4}){0,5}\b"
|
|
112
|
+
),
|
|
113
|
+
"[IBAN]",
|
|
114
|
+
),
|
|
115
|
+
(
|
|
116
|
+
"passport", # Common passport number patterns
|
|
117
|
+
re.compile(r"\b[A-Z]{1,2}\d{6,9}\b"),
|
|
118
|
+
"[PASSPORT]",
|
|
119
|
+
),
|
|
120
|
+
]
|
|
121
|
+
|
|
122
|
+
# All known category names
|
|
123
|
+
ALL_CATEGORIES: Set[str] = {name for name, _, _ in _PATTERNS}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# ---------------------------------------------------------------------------
|
|
127
|
+
# Redactor
|
|
128
|
+
# ---------------------------------------------------------------------------
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
@dataclass
|
|
132
|
+
class RedactionResult:
|
|
133
|
+
"""Result of a redaction pass."""
|
|
134
|
+
|
|
135
|
+
text: str
|
|
136
|
+
redacted_count: int = 0
|
|
137
|
+
categories_found: List[str] = field(default_factory=list)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class Redactor:
|
|
141
|
+
"""PII redactor with configurable categories and optional NLP augmentation.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
categories: Which PII types to detect. Defaults to all.
|
|
145
|
+
Options: "email", "phone", "ssn", "credit_card",
|
|
146
|
+
"api_key", "ip_address", "national_id", "iban", "passport"
|
|
147
|
+
use_nlp: If True, augment regex with spaCy NER for names, locations,
|
|
148
|
+
and other context-dependent PII. Requires spacy + a model.
|
|
149
|
+
Install with: pip install spacy && python -m spacy download en_core_web_sm
|
|
150
|
+
"""
|
|
151
|
+
|
|
152
|
+
def __init__(self, categories: Optional[List[str]] = None, use_nlp: bool = False):
|
|
153
|
+
if categories is None:
|
|
154
|
+
self._patterns = _PATTERNS
|
|
155
|
+
else:
|
|
156
|
+
cats = set(categories)
|
|
157
|
+
self._patterns = [(n, p, r) for n, p, r in _PATTERNS if n in cats]
|
|
158
|
+
|
|
159
|
+
self._nlp = None
|
|
160
|
+
if use_nlp:
|
|
161
|
+
self._init_nlp()
|
|
162
|
+
|
|
163
|
+
def _init_nlp(self) -> None:
|
|
164
|
+
"""Load spaCy model for NER. Fails silently if not installed."""
|
|
165
|
+
try:
|
|
166
|
+
import spacy # type: ignore[import-not-found]
|
|
167
|
+
|
|
168
|
+
self._nlp = spacy.load("en_core_web_sm")
|
|
169
|
+
except (ImportError, OSError):
|
|
170
|
+
pass # spaCy not installed or model not downloaded
|
|
171
|
+
|
|
172
|
+
@property
|
|
173
|
+
def categories(self) -> List[str]:
|
|
174
|
+
return [name for name, _, _ in self._patterns]
|
|
175
|
+
|
|
176
|
+
def redact(self, text: str) -> RedactionResult:
|
|
177
|
+
"""Redact PII from a string.
|
|
178
|
+
|
|
179
|
+
Returns a RedactionResult with the cleaned text and metadata
|
|
180
|
+
about what was found.
|
|
181
|
+
"""
|
|
182
|
+
if not isinstance(text, str):
|
|
183
|
+
return RedactionResult(text=str(text))
|
|
184
|
+
|
|
185
|
+
result_text = text
|
|
186
|
+
total_count = 0
|
|
187
|
+
found_categories: List[str] = []
|
|
188
|
+
|
|
189
|
+
for name, pattern, placeholder in self._patterns:
|
|
190
|
+
new_text, count = pattern.subn(placeholder, result_text)
|
|
191
|
+
if count > 0:
|
|
192
|
+
total_count += count
|
|
193
|
+
if name not in found_categories:
|
|
194
|
+
found_categories.append(name)
|
|
195
|
+
result_text = new_text
|
|
196
|
+
|
|
197
|
+
# NLP augmentation — catch names, locations, orgs that regex misses
|
|
198
|
+
if self._nlp is not None:
|
|
199
|
+
nlp_text, nlp_count, nlp_cats = self._nlp_redact(result_text)
|
|
200
|
+
if nlp_count > 0:
|
|
201
|
+
result_text = nlp_text
|
|
202
|
+
total_count += nlp_count
|
|
203
|
+
for cat in nlp_cats:
|
|
204
|
+
if cat not in found_categories:
|
|
205
|
+
found_categories.append(cat)
|
|
206
|
+
|
|
207
|
+
return RedactionResult(
|
|
208
|
+
text=result_text,
|
|
209
|
+
redacted_count=total_count,
|
|
210
|
+
categories_found=found_categories,
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
# NER entity label → placeholder mapping
|
|
214
|
+
_NER_LABELS = {
|
|
215
|
+
"PERSON": "[PERSON]",
|
|
216
|
+
"GPE": "[LOCATION]",
|
|
217
|
+
"LOC": "[LOCATION]",
|
|
218
|
+
"ORG": "[ORG]",
|
|
219
|
+
"NORP": "[GROUP]",
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
def _nlp_redact(self, text: str) -> Tuple[str, int, List[str]]:
|
|
223
|
+
"""Apply spaCy NER to catch context-dependent PII."""
|
|
224
|
+
if self._nlp is None:
|
|
225
|
+
return text, 0, []
|
|
226
|
+
doc = self._nlp(text)
|
|
227
|
+
replacements = []
|
|
228
|
+
categories: List[str] = []
|
|
229
|
+
for ent in doc.ents:
|
|
230
|
+
if ent.label_ in self._NER_LABELS:
|
|
231
|
+
replacements.append((ent.start_char, ent.end_char, self._NER_LABELS[ent.label_]))
|
|
232
|
+
cat = self._NER_LABELS[ent.label_].strip("[]").lower()
|
|
233
|
+
if cat not in categories:
|
|
234
|
+
categories.append(cat)
|
|
235
|
+
|
|
236
|
+
if not replacements:
|
|
237
|
+
return text, 0, []
|
|
238
|
+
|
|
239
|
+
# Apply replacements in reverse order to preserve offsets
|
|
240
|
+
result = text
|
|
241
|
+
for start, end, placeholder in sorted(replacements, reverse=True):
|
|
242
|
+
result = result[:start] + placeholder + result[end:]
|
|
243
|
+
|
|
244
|
+
return result, len(replacements), categories
|
|
245
|
+
|
|
246
|
+
def redact_dict(self, d: Dict[str, Any]) -> Tuple[Dict[str, Any], int]:
|
|
247
|
+
"""Redact PII from all string values in a dict (shallow).
|
|
248
|
+
|
|
249
|
+
Returns (cleaned_dict, total_redaction_count).
|
|
250
|
+
"""
|
|
251
|
+
cleaned = {}
|
|
252
|
+
total = 0
|
|
253
|
+
for k, v in d.items():
|
|
254
|
+
if isinstance(v, str):
|
|
255
|
+
result = self.redact(v)
|
|
256
|
+
cleaned[k] = result.text
|
|
257
|
+
total += result.redacted_count
|
|
258
|
+
else:
|
|
259
|
+
cleaned[k] = v
|
|
260
|
+
return cleaned, total
|