dpdp-scrub 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dpdp_scrub/__init__.py ADDED
@@ -0,0 +1,23 @@
1
+ """dpdp-scrub: detect, validate, and redact Indian PII.
2
+
3
+ Self-hosted, checksum-validated, DPDP-ready. The LLM never sees the real value;
4
+ your systems still do.
5
+ """
6
+
7
+ __version__ = "0.0.1"
8
+
9
+ from . import validators
10
+ from .detectors import detect
11
+ from .scrubber import Scrubber, ScrubResult
12
+ from .spans import Span
13
+ from .vault import MemoryVault
14
+
15
+ __all__ = [
16
+ "validators",
17
+ "detect",
18
+ "Scrubber",
19
+ "ScrubResult",
20
+ "MemoryVault",
21
+ "Span",
22
+ "__version__",
23
+ ]
dpdp_scrub/data.py ADDED
@@ -0,0 +1,41 @@
1
+ """Reference data for detectors.
2
+
3
+ These are public identifiers of payment infrastructure (PSP handles), not PII.
4
+ Community-maintainable: adding a new UPI handle is a one-line PR.
5
+ """
6
+
7
+ # Known UPI PSP handles (the part after '@' in a VPA). Not exhaustive —
8
+ # curated from NPCI's ecosystem. The handle list is what separates a UPI ID
9
+ # from an email-shaped string, so keep entries lowercase and dot-free.
10
+ UPI_PSP_HANDLES = frozenset(
11
+ {
12
+ # PhonePe
13
+ "ybl", "ibl", "axl",
14
+ # Google Pay
15
+ "okaxis", "okhdfcbank", "okicici", "oksbi",
16
+ # Paytm
17
+ "paytm", "ptyes", "ptaxis", "pthdfc", "ptsbi",
18
+ # Amazon Pay
19
+ "apl", "yapl", "rapl",
20
+ # WhatsApp Pay
21
+ "waaxis", "wahdfcbank", "waicici", "wasbi",
22
+ # Bank / app handles
23
+ "upi", "sbi", "hdfcbank", "icici", "axisbank", "barodampay",
24
+ "fbl", "idfcbank", "indus", "kotak", "dbs", "yesbank", "rbl",
25
+ "aubank", "bandhan", "federal", "uboi", "cnrb", "pnb", "boi",
26
+ # Apps
27
+ "freecharge", "mobikwik", "airtel", "jio", "slice", "jupiteraxis",
28
+ "naviaxis", "niyoicici", "tapicici", "timecosmos", "zoicici",
29
+ }
30
+ )
31
+
32
+ # RTO state/UT codes for vehicle registration plates, plus the BH (Bharat)
33
+ # series. Includes legacy codes still on the road (OR, UA, TS).
34
+ VEHICLE_STATE_CODES = frozenset(
35
+ {
36
+ "AN", "AP", "AR", "AS", "BR", "CG", "CH", "DD", "DL", "DN", "GA",
37
+ "GJ", "HP", "HR", "JH", "JK", "KA", "KL", "LA", "LD", "MH", "ML",
38
+ "MN", "MP", "MZ", "NL", "OD", "OR", "PB", "PY", "RJ", "SK", "TN",
39
+ "TR", "TS", "TG", "UK", "UA", "UP", "WB",
40
+ }
41
+ )
@@ -0,0 +1,311 @@
1
+ """Text-level detection: find candidate identifiers in running text, normalize
2
+ separators, validate with checksums, and emit typed Spans.
3
+
4
+ Detection is two-stage by design:
5
+ 1. A deliberately permissive regex finds candidates in text (spaces/hyphens
6
+ allowed, because humans write "2345 6789 0124").
7
+ 2. The normalized candidate must pass the entity's validator before a Span
8
+ is emitted. Regex finds; checksums decide.
9
+
10
+ Overlapping spans are resolved by confidence, then length (a Luhn-valid
11
+ 16-digit card beats the 12-digit Aadhaar candidate hiding inside it unless
12
+ the Aadhaar checksum actually passes and the card's doesn't).
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import re
18
+ from typing import Callable, NamedTuple
19
+
20
+ from .data import UPI_PSP_HANDLES, VEHICLE_STATE_CODES
21
+ from .spans import Span
22
+ from .validators import luhn, gstin, pan, verhoeff
23
+
24
+
25
+ def _upi_handle_known(vpa: str) -> bool:
26
+ return vpa.split("@", 1)[1].lower() in UPI_PSP_HANDLES
27
+
28
+
29
+ def _vehicle_state_known(reg: str) -> bool:
30
+ # BH-series plates start with a 2-digit year instead of a state code.
31
+ return reg[:2] in VEHICLE_STATE_CODES or (reg[:2].isdigit() and reg[2:4] == "BH")
32
+
33
+
34
+ def _always(_: str) -> bool:
35
+ return True
36
+
37
+
38
+ def _digits(s: str) -> str:
39
+ return re.sub(r"[ \-]", "", s)
40
+
41
+
42
+ def _upper(s: str) -> str:
43
+ return s.upper()
44
+
45
+
46
+ class _Detector(NamedTuple):
47
+ entity: str
48
+ pattern: re.Pattern
49
+ normalize: Callable[[str], str]
50
+ validate: Callable[[str], bool]
51
+ confidence: float
52
+
53
+
54
+ class _CtxRule(NamedTuple):
55
+ """Context evidence for entities without checksums. Negative dominates positive."""
56
+
57
+ positive: frozenset
58
+ negative: frozenset
59
+ boost: float = 0.30
60
+ penalty: float = 0.30
61
+ window: int = 40 # chars on each side of the span
62
+ # Checksummed entities set this: their own name nearby ("aadhaar") is
63
+ # definitive and beats distant negative words ("...pe invoice bana dena").
64
+ positive_first: bool = False
65
+
66
+
67
+ _CTX_RULES: dict[str, _CtxRule] = {
68
+ # Even Verhoeff can be passed by luck (~10% of random 12-digit strings),
69
+ # so strong non-ID context must be able to sink an Aadhaar candidate
70
+ # below the default threshold (0.99 - 0.55 = 0.44 < 0.5).
71
+ "AADHAAR": _CtxRule(
72
+ positive=frozenset("aadhaar aadhar adhar uid uidai kyc आधार".split()),
73
+ negative=frozenset(
74
+ "order txn transaction invoice ref reference awb tracking docket "
75
+ "batch lot qty amount bill".split()
76
+ ),
77
+ boost=0.0,
78
+ penalty=0.55,
79
+ positive_first=True,
80
+ ),
81
+ "PHONE_IN": _CtxRule(
82
+ positive=frozenset(
83
+ "call phone mobile mob whatsapp wa contact number num no sms msg "
84
+ "message baat sampark helpline care फोन मोबाइल नंबर".split()
85
+ ),
86
+ negative=frozenset(
87
+ "order txn transaction invoice ref reference qty quantity amount "
88
+ "bill id account acc a/c awb tracking docket".split()
89
+ ),
90
+ ),
91
+ "PASSPORT_IN": _CtxRule(
92
+ positive=frozenset("passport पासपोर्ट visa".split()),
93
+ negative=frozenset("order txn invoice ref batch lot model part".split()),
94
+ ),
95
+ "VOTER_ID": _CtxRule(
96
+ positive=frozenset("voter epic election मतदाता".split()),
97
+ negative=frozenset("order txn invoice ref batch lot model part".split()),
98
+ boost=0.25,
99
+ ),
100
+ # The three context-REQUIRED entities: base 0.30 sits below the default
101
+ # threshold, so they only surface when nearby words earn it.
102
+ "BANK_ACCOUNT": _CtxRule(
103
+ positive=frozenset(
104
+ "a/c ac acc account accno khata bank ifsc neft rtgs imps savings "
105
+ "current sb ca खाता बैंक".split()
106
+ ),
107
+ negative=frozenset("order txn invoice ref awb tracking docket qty".split()),
108
+ boost=0.45,
109
+ ),
110
+ "EPF_UAN": _CtxRule(
111
+ positive=frozenset("uan pf epf epfo provident यूएएन".split()),
112
+ negative=frozenset(),
113
+ boost=0.50,
114
+ ),
115
+ "ABHA": _CtxRule(
116
+ positive=frozenset("abha health ayushman आभा स्वास्थ्य".split()),
117
+ negative=frozenset(),
118
+ boost=0.50,
119
+ ),
120
+ }
121
+
122
+ _TOKEN_RE = re.compile(r"[a-z0-9/]+|[ऀ-ॿ]+")
123
+
124
+
125
+ def _context_adjust(text: str, start: int, end: int, entity: str, base: float) -> float:
126
+ rule = _CTX_RULES.get(entity)
127
+ if rule is None:
128
+ return base
129
+ lo, hi = max(0, start - rule.window), min(len(text), end + rule.window)
130
+ tokens = set(_TOKEN_RE.findall(text[lo:hi].lower()))
131
+ if rule.positive_first and tokens & rule.positive:
132
+ return min(0.99, base + rule.boost)
133
+ if tokens & rule.negative:
134
+ return max(0.0, base - rule.penalty)
135
+ if tokens & rule.positive:
136
+ return min(0.99, base + rule.boost)
137
+ return base
138
+
139
+
140
+ # NOTE on lookarounds: (?<![\d]) / (?![\d]) stop us matching inside longer
141
+ # digit runs (a 12-digit slice of a 16-digit number written without spaces).
142
+ # Spaced collisions (card "4111 1111 1111 1111" containing an Aadhaar-shaped
143
+ # prefix) are handled by validation + overlap resolution instead.
144
+
145
+ _DETECTORS: list[_Detector] = [
146
+ _Detector(
147
+ entity="AADHAAR",
148
+ pattern=re.compile(r"(?<!\d)(\d{4}[ \-]?\d{4}[ \-]?\d{4})(?!\d)"),
149
+ normalize=_digits,
150
+ validate=verhoeff.validate_aadhaar,
151
+ confidence=0.99, # Verhoeff-validated
152
+ ),
153
+ _Detector(
154
+ entity="AADHAAR_VID",
155
+ pattern=re.compile(r"(?<!\d)(\d{4}[ \-]?\d{4}[ \-]?\d{4}[ \-]?\d{4})(?!\d)"),
156
+ normalize=_digits,
157
+ validate=verhoeff.validate_vid,
158
+ confidence=0.99,
159
+ ),
160
+ _Detector(
161
+ entity="CREDIT_CARD",
162
+ pattern=re.compile(r"(?<!\d)(\d(?:[ \-]?\d){12,18})(?!\d)"),
163
+ normalize=_digits,
164
+ validate=luhn.validate_card,
165
+ confidence=0.95, # Luhn is weaker than Verhoeff (10% random pass, no length lock)
166
+ ),
167
+ _Detector(
168
+ entity="GSTIN",
169
+ pattern=re.compile(
170
+ r"(?<![A-Za-z0-9])(\d{2}[A-Za-z]{5}\d{4}[A-Za-z][1-9A-Za-z][Zz][0-9A-Za-z])(?![A-Za-z0-9])"
171
+ ),
172
+ normalize=_upper,
173
+ validate=gstin.validate,
174
+ confidence=0.99, # mod-36 check char + embedded PAN + state code
175
+ ),
176
+ _Detector(
177
+ entity="PAN",
178
+ pattern=re.compile(r"(?<![A-Za-z0-9])([A-Za-z]{5}\d{4}[A-Za-z])(?![A-Za-z0-9])"),
179
+ normalize=_upper,
180
+ validate=pan.validate,
181
+ confidence=0.85, # structural only — PAN's check letter isn't public
182
+ ),
183
+ _Detector(
184
+ entity="UPI_ID",
185
+ # username@handle, where the handle has no dot. The PSP handle list is
186
+ # what separates a real VPA from an email-shaped string.
187
+ pattern=re.compile(r"(?<![A-Za-z0-9.@_\-])([A-Za-z0-9._\-]{2,256}@[A-Za-z]{2,64})(?![A-Za-z0-9.@])"),
188
+ normalize=str.lower,
189
+ validate=_upi_handle_known,
190
+ confidence=0.92, # handle verified against known PSP list
191
+ ),
192
+ _Detector(
193
+ entity="EMAIL",
194
+ pattern=re.compile(
195
+ r"(?<![A-Za-z0-9.@_\-])([A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,})(?![A-Za-z0-9.@])"
196
+ ),
197
+ normalize=str.lower,
198
+ validate=_always,
199
+ confidence=0.90,
200
+ ),
201
+ _Detector(
202
+ entity="PHONE_IN",
203
+ # Indian mobile: 10 digits starting 6-9, optional +91/0 prefix.
204
+ # (?![\d@]) stops us flagging the phone-shaped username of a UPI VPA.
205
+ pattern=re.compile(r"(?<![\d])((?:\+91[ \-]?|0)?[6-9]\d{9})(?![\d@])"),
206
+ normalize=_digits,
207
+ validate=_always,
208
+ confidence=0.70, # no checksum exists; context scoring will adjust later
209
+ ),
210
+ _Detector(
211
+ entity="IFSC",
212
+ # 4-letter bank code + '0' + 6-char branch code.
213
+ pattern=re.compile(r"(?<![A-Za-z0-9])([A-Za-z]{4}0[A-Za-z0-9]{6})(?![A-Za-z0-9])"),
214
+ normalize=_upper,
215
+ validate=_always,
216
+ confidence=0.80, # structural; RBI bank-code list lands with the data files
217
+ ),
218
+ _Detector(
219
+ entity="VOTER_ID",
220
+ # EPIC: 3 letters + 7 digits.
221
+ pattern=re.compile(r"(?<![A-Za-z0-9])([A-Za-z]{3}\d{7})(?![A-Za-z0-9])"),
222
+ normalize=_upper,
223
+ validate=_always,
224
+ confidence=0.70, # structural only; context engine will refine
225
+ ),
226
+ _Detector(
227
+ entity="PASSPORT_IN",
228
+ # 1 letter + 7 digits — deliberately conservative confidence: the
229
+ # shape is generic, so this one leans on the context engine most.
230
+ pattern=re.compile(r"(?<![A-Za-z0-9])([A-Za-z]\d{7})(?![A-Za-z0-9])"),
231
+ normalize=_upper,
232
+ validate=_always,
233
+ confidence=0.60,
234
+ ),
235
+ _Detector(
236
+ entity="BANK_ACCOUNT",
237
+ # 9-18 solid digits, NO checksum exists — context-required (base 0.30).
238
+ pattern=re.compile(r"(?<!\d)(\d{9,18})(?!\d)"),
239
+ normalize=_digits,
240
+ validate=_always,
241
+ confidence=0.30,
242
+ ),
243
+ _Detector(
244
+ entity="EPF_UAN",
245
+ # 12 digits, same shape as Aadhaar but no public checksum — context-required.
246
+ pattern=re.compile(r"(?<!\d)(\d{12})(?!\d)"),
247
+ normalize=_digits,
248
+ validate=_always,
249
+ confidence=0.30,
250
+ ),
251
+ _Detector(
252
+ entity="ABHA",
253
+ # 14-digit health ID, often written 91-XXXX-XXXX-XXXX — context-required.
254
+ pattern=re.compile(r"(?<!\d)(\d{2}[ \-]?\d{4}[ \-]?\d{4}[ \-]?\d{4})(?!\d)"),
255
+ normalize=_digits,
256
+ validate=_always,
257
+ confidence=0.30,
258
+ ),
259
+ _Detector(
260
+ entity="VEHICLE_REG",
261
+ # Standard: SS RR L(LL) NNNN (MH 12 AB 1234). BH series: YY BH NNNN L(L).
262
+ pattern=re.compile(
263
+ r"(?<![A-Za-z0-9])"
264
+ r"([A-Za-z]{2}[ \-]?\d{1,2}[ \-]?[A-Za-z]{1,3}[ \-]?\d{4}"
265
+ r"|\d{2}[ \-]?[Bb][Hh][ \-]?\d{4}[ \-]?[A-Za-z]{1,2})"
266
+ r"(?![A-Za-z0-9])"
267
+ ),
268
+ normalize=lambda s: re.sub(r"[ \-]", "", s).upper(),
269
+ validate=_vehicle_state_known,
270
+ confidence=0.85, # state/RTO code verified against reference list
271
+ ),
272
+ ]
273
+
274
+
275
+ def _resolve_overlaps(spans: list[Span]) -> list[Span]:
276
+ """Greedy resolution: highest confidence wins, then longest, then leftmost."""
277
+ ordered = sorted(spans, key=lambda s: (-s.confidence, -len(s), s.start))
278
+ kept: list[Span] = []
279
+ for span in ordered:
280
+ if not any(span.overlaps(k) for k in kept):
281
+ kept.append(span)
282
+ return sorted(kept, key=lambda s: s.start)
283
+
284
+
285
+ def detect(text: str, min_confidence: float = 0.5) -> list[Span]:
286
+ """Detect validated PII spans in `text`, sorted by position.
287
+
288
+ `min_confidence` is the recall-vs-precision dial: context-required
289
+ entities (bank account, UAN, ABHA) start below the default threshold
290
+ and only surface when nearby words earn them a boost.
291
+ """
292
+ candidates: list[Span] = []
293
+ for det in _DETECTORS:
294
+ for m in det.pattern.finditer(text):
295
+ raw = m.group(1)
296
+ normalized = det.normalize(raw)
297
+ if not det.validate(normalized):
298
+ continue
299
+ conf = _context_adjust(text, m.start(1), m.end(1), det.entity, det.confidence)
300
+ if conf >= min_confidence:
301
+ candidates.append(
302
+ Span(
303
+ entity=det.entity,
304
+ start=m.start(1),
305
+ end=m.end(1),
306
+ raw=raw,
307
+ normalized=normalized,
308
+ confidence=conf,
309
+ )
310
+ )
311
+ return _resolve_overlaps(candidates)
@@ -0,0 +1,73 @@
1
+ """Framework adapters. v1: LiteLLM.
2
+
3
+ Usage (LiteLLM proxy config.yaml):
4
+ litellm_settings:
5
+ callbacks: dpdp_scrub.integrations.litellm_scrub_hook
6
+
7
+ Or in code:
8
+ import litellm
9
+ from dpdp_scrub.integrations import LiteLLMScrubHook
10
+ litellm.callbacks = [LiteLLMScrubHook()]
11
+
12
+ Every outbound message is scrubbed before the provider sees it; successful
13
+ responses are rehydrated on the way back (same session, keyed by call id or
14
+ metadata["dpdp_session"]).
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import uuid
20
+
21
+ from .scrubber import Scrubber
22
+
23
+ try: # litellm is optional; the hook class degrades to a plain object base.
24
+ from litellm.integrations.custom_logger import CustomLogger as _Base
25
+ except ImportError: # pragma: no cover
26
+ _Base = object
27
+
28
+
29
+ class LiteLLMScrubHook(_Base):
30
+ """LiteLLM CustomLogger that scrubs prompts and rehydrates responses."""
31
+
32
+ def __init__(self, scrubber: Scrubber | None = None, rehydrate_responses: bool = True):
33
+ self.scrubber = scrubber or Scrubber()
34
+ self.rehydrate_responses = rehydrate_responses
35
+
36
+ def _session(self, data: dict) -> str:
37
+ meta = data.setdefault("metadata", {}) if isinstance(data, dict) else {}
38
+ if "dpdp_session" not in meta:
39
+ meta["dpdp_session"] = data.get("litellm_call_id") or uuid.uuid4().hex
40
+ return meta["dpdp_session"]
41
+
42
+ def scrub_request(self, data: dict) -> dict:
43
+ """Scrub every string message content in a completion request dict."""
44
+ session = self._session(data)
45
+ for msg in data.get("messages", []) or []:
46
+ content = msg.get("content")
47
+ if isinstance(content, str):
48
+ msg["content"] = self.scrubber.scrub(content, session_id=session).text
49
+ return data
50
+
51
+ def rehydrate_response(self, data: dict, response) -> None:
52
+ """Rehydrate placeholder text in a ModelResponse, in place."""
53
+ session = self._session(data)
54
+ try:
55
+ for choice in getattr(response, "choices", []) or []:
56
+ message = getattr(choice, "message", None)
57
+ if message is not None and isinstance(getattr(message, "content", None), str):
58
+ message.content = self.scrubber.rehydrate(message.content, session)
59
+ except Exception: # never let privacy plumbing break the response path
60
+ pass
61
+
62
+ # --- LiteLLM proxy hook surface ---
63
+
64
+ async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): # noqa: D401
65
+ return self.scrub_request(data)
66
+
67
+ async def async_post_call_success_hook(self, data, user_api_key_dict, response):
68
+ if self.rehydrate_responses:
69
+ self.rehydrate_response(data, response)
70
+ return response
71
+
72
+
73
+ litellm_scrub_hook = LiteLLMScrubHook()
dpdp_scrub/policies.py ADDED
@@ -0,0 +1,27 @@
1
+ """Named policy profiles — compliance goals expressed as per-entity actions.
2
+
3
+ Users think "make this DPDP-safe", not "tokenize entity #7". A profile is a
4
+ dict of action overrides applied on top of DEFAULT_ACTIONS; "__all__" sets a
5
+ baseline for every entity first.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ PROFILES: dict[str, dict[str, str]] = {
11
+ # The default: reversible tokens, UIDAI/RBI-format masks, ABHA redacted.
12
+ "dpdp-strict": {},
13
+ # Detect and audit-log everything, change nothing. The zero-risk first
14
+ # week of a rollout: turn it on, read the report, then tighten.
15
+ "audit-only": {"__all__": "ignore"},
16
+ # Only the identifiers with an existing legal masking mandate; the rest
17
+ # pass through. For teams that need Aadhaar/card hygiene and nothing else.
18
+ "uidai-mask": {
19
+ "__all__": "ignore",
20
+ "AADHAAR": "mask",
21
+ "AADHAAR_VID": "mask",
22
+ "CREDIT_CARD": "mask",
23
+ },
24
+ # Nothing survives, nothing is reversible. For datasets leaving forever
25
+ # (fine-tuning exports, data shares).
26
+ "max-redact": {"__all__": "redact"},
27
+ }
dpdp_scrub/scrubber.py ADDED
@@ -0,0 +1,158 @@
1
+ """The scrub -> LLM -> rehydrate loop.
2
+
3
+ scrub(): replace detected PII with placeholders (tokenize), partial
4
+ masks (mask), or removal (redact), per entity policy.
5
+ rehydrate(): swap placeholders back to real values — only ever called on
6
+ authorized surfaces, inside the adopter's boundary.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import re
13
+ import time
14
+ import uuid
15
+ from dataclasses import dataclass, field
16
+ from pathlib import Path
17
+
18
+ from .detectors import detect
19
+ from .policies import PROFILES
20
+ from .spans import Span
21
+ from .vault import MemoryVault
22
+
23
+ # Per-entity default actions. "mask" keeps a legally-sanctioned partial view
24
+ # (UIDAI's own masked-Aadhaar format; RBI-style last-4 for cards); everything
25
+ # else tokenizes reversibly via the vault.
26
+ DEFAULT_ACTIONS: dict[str, str] = {
27
+ "AADHAAR": "mask",
28
+ "AADHAAR_VID": "mask",
29
+ "CREDIT_CARD": "mask",
30
+ "PAN": "tokenize",
31
+ "GSTIN": "tokenize",
32
+ "UPI_ID": "tokenize",
33
+ "EMAIL": "tokenize",
34
+ "PHONE_IN": "tokenize",
35
+ "IFSC": "tokenize",
36
+ "VOTER_ID": "tokenize",
37
+ "PASSPORT_IN": "tokenize",
38
+ "VEHICLE_REG": "tokenize",
39
+ "BANK_ACCOUNT": "tokenize",
40
+ "EPF_UAN": "tokenize",
41
+ "ABHA": "redact", # health IDs: highest sensitivity, no partial view
42
+ }
43
+
44
+ _PLACEHOLDER_RE = re.compile(r"<[A-Z_]+_\d+>")
45
+
46
+
47
+ def _mask(entity: str, normalized: str) -> str:
48
+ if entity in ("AADHAAR", "AADHAAR_VID"):
49
+ # UIDAI masked-Aadhaar format: only the last 4 digits visible.
50
+ return "XXXX XXXX " + normalized[-4:]
51
+ if entity == "CREDIT_CARD":
52
+ return "X" * (len(normalized) - 4) + normalized[-4:]
53
+ # Generic fallback: keep last 4.
54
+ return "X" * max(0, len(normalized) - 4) + normalized[-4:]
55
+
56
+
57
+ @dataclass
58
+ class ScrubResult:
59
+ text: str
60
+ spans: list[Span]
61
+ session_id: str
62
+ original: str = field(repr=False, default="")
63
+
64
+
65
+ class Scrubber:
66
+ """Detects and transforms Indian PII; holds the vault for rehydration."""
67
+
68
+ def __init__(
69
+ self,
70
+ policy: str = "dpdp-strict",
71
+ actions: dict[str, str] | None = None,
72
+ vault: MemoryVault | None = None,
73
+ min_confidence: float = 0.5,
74
+ audit_log: str | Path | None = None,
75
+ ) -> None:
76
+ if policy not in PROFILES:
77
+ raise ValueError(f"unknown policy {policy!r}; available: {sorted(PROFILES)}")
78
+ profile = dict(PROFILES[policy])
79
+ baseline = profile.pop("__all__", None)
80
+ merged = (
81
+ {e: baseline for e in DEFAULT_ACTIONS} if baseline else dict(DEFAULT_ACTIONS)
82
+ )
83
+ merged.update(profile)
84
+ merged.update(actions or {})
85
+ self.policy = policy
86
+ self.actions = merged
87
+ self.vault = vault or MemoryVault()
88
+ self.min_confidence = min_confidence
89
+ self.audit_log = Path(audit_log) if audit_log else None
90
+
91
+ def _audit(self, session_id: str, spans: list[Span]) -> None:
92
+ """Append one JSONL line per detection. Never logs raw values."""
93
+ if self.audit_log is None or not spans:
94
+ return
95
+ with self.audit_log.open("a", encoding="utf-8") as f:
96
+ for s in spans:
97
+ f.write(
98
+ json.dumps(
99
+ {
100
+ "ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
101
+ "session": session_id,
102
+ "entity": s.entity,
103
+ "action": self.actions.get(s.entity, "tokenize"),
104
+ "confidence": round(s.confidence, 2),
105
+ "start": s.start,
106
+ "end": s.end,
107
+ "policy": self.policy,
108
+ }
109
+ )
110
+ + "\n"
111
+ )
112
+
113
+ def scrub(self, text: str, session_id: str | None = None) -> ScrubResult:
114
+ session_id = session_id or uuid.uuid4().hex
115
+ spans = detect(text, min_confidence=self.min_confidence)
116
+ out = text
117
+ # Replace right-to-left so earlier offsets stay valid.
118
+ for s in sorted(spans, key=lambda x: -x.start):
119
+ action = self.actions.get(s.entity, "tokenize")
120
+ if action == "ignore":
121
+ continue
122
+ if action == "redact":
123
+ replacement = f"[{s.entity} REDACTED]"
124
+ elif action == "mask":
125
+ replacement = _mask(s.entity, s.normalized)
126
+ else: # tokenize
127
+ replacement = self.vault.placeholder_for(session_id, s.entity, s.normalized)
128
+ out = out[: s.start] + replacement + out[s.end :]
129
+ self._audit(session_id, spans)
130
+ return ScrubResult(text=out, spans=spans, session_id=session_id, original=text)
131
+
132
+ def scrub_json(self, payload, session_id: str | None = None):
133
+ """Recursively scrub every string value in a dict/list payload.
134
+
135
+ Keys are left untouched; only values are scrubbed. Returns
136
+ (scrubbed_payload, session_id) so multi-field payloads share one vault
137
+ session and placeholder numbering stays consistent across fields.
138
+ """
139
+ session_id = session_id or uuid.uuid4().hex
140
+
141
+ def _walk(node):
142
+ if isinstance(node, str):
143
+ return self.scrub(node, session_id=session_id).text
144
+ if isinstance(node, dict):
145
+ return {k: _walk(v) for k, v in node.items()}
146
+ if isinstance(node, list):
147
+ return [_walk(v) for v in node]
148
+ return node
149
+
150
+ return _walk(payload), session_id
151
+
152
+ def rehydrate(self, text: str, session_id: str) -> str:
153
+ """Swap placeholders back to real values (authorized surfaces only)."""
154
+
155
+ def _sub(m: re.Match) -> str:
156
+ return self.vault.resolve(session_id, m.group(0)) or m.group(0)
157
+
158
+ return _PLACEHOLDER_RE.sub(_sub, text)
dpdp_scrub/spans.py ADDED
@@ -0,0 +1,29 @@
1
+ """The typed span — the unit of output for every detector."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+
7
+
8
+ @dataclass(frozen=True)
9
+ class Span:
10
+ """A detected PII occurrence in the original text.
11
+
12
+ `start`/`end` index the ORIGINAL text (end exclusive), so callers can
13
+ slice, replace, or highlight without re-searching. `raw` is the exact
14
+ matched substring including separators; `normalized` is the cleaned
15
+ value that passed validation (digits/uppercase only).
16
+ """
17
+
18
+ entity: str
19
+ start: int
20
+ end: int
21
+ raw: str
22
+ normalized: str
23
+ confidence: float
24
+
25
+ def __len__(self) -> int:
26
+ return self.end - self.start
27
+
28
+ def overlaps(self, other: "Span") -> bool:
29
+ return self.start < other.end and other.start < self.end
@@ -0,0 +1,28 @@
1
+ """Checksum and structural validators for Indian identifiers.
2
+
3
+ These are pure functions with zero dependencies — the trust foundation of
4
+ dpdp-scrub. Every detector consults its validator before flagging a span.
5
+ """
6
+
7
+ from .verhoeff import validate_aadhaar, validate_vid
8
+ from .verhoeff import validate as validate_verhoeff
9
+ from .verhoeff import generate_check_digit as verhoeff_check_digit
10
+ from .luhn import validate_card
11
+ from .luhn import validate as validate_luhn
12
+ from .gstin import validate as validate_gstin
13
+ from .gstin import check_character as gstin_check_character
14
+ from .pan import validate as validate_pan
15
+ from .pan import holder_type as pan_holder_type
16
+
17
+ __all__ = [
18
+ "validate_aadhaar",
19
+ "validate_vid",
20
+ "validate_verhoeff",
21
+ "verhoeff_check_digit",
22
+ "validate_card",
23
+ "validate_luhn",
24
+ "validate_gstin",
25
+ "gstin_check_character",
26
+ "validate_pan",
27
+ "pan_holder_type",
28
+ ]
@@ -0,0 +1,48 @@
1
+ """GSTIN validation — 15-character Goods and Services Tax Identification Number.
2
+
3
+ Structure: SSPPPPPPPPPPENZ C
4
+ - chars 1-2 (SS): state code, 01-38 (or 97 for "other territory" / 99 for centre jurisdiction)
5
+ - chars 3-12 (P*): the holder's PAN
6
+ - char 13 (E) : entity/registration number within the state (1-9, A-Z)
7
+ - char 14 (N) : 'Z' by default (reserved)
8
+ - char 15 (C) : mod-36 check character (public algorithm)
9
+
10
+ The embedded PAN plus the check character makes GSTIN one of the most
11
+ validate-able identifiers in the catalog.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import re
17
+
18
+ from . import pan as _pan
19
+
20
+ _CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
21
+
22
+ # 01-38 are states/UTs; 97 = other territory, 99 = centre jurisdiction.
23
+ _VALID_STATE_CODES = {f"{i:02d}" for i in range(1, 39)} | {"97", "99"}
24
+
25
+ _GSTIN_RE = re.compile(r"^\d{2}[A-Z]{5}\d{4}[A-Z][1-9A-Z]Z[0-9A-Z]$")
26
+
27
+
28
+ def check_character(gstin14: str) -> str:
29
+ """Compute the mod-36 check character for the first 14 characters of a GSTIN."""
30
+ total = 0
31
+ for i, ch in enumerate(gstin14):
32
+ code = _CHARS.index(ch)
33
+ factor = 2 if i % 2 == 1 else 1
34
+ addend = factor * code
35
+ total += addend // 36 + addend % 36
36
+ return _CHARS[(36 - total % 36) % 36]
37
+
38
+
39
+ def validate(gstin: str) -> bool:
40
+ """Full GSTIN validation: format + state code + embedded PAN structure + checksum."""
41
+ gstin = gstin.upper()
42
+ if not _GSTIN_RE.match(gstin):
43
+ return False
44
+ if gstin[:2] not in _VALID_STATE_CODES:
45
+ return False
46
+ if not _pan.validate(gstin[2:12]):
47
+ return False
48
+ return check_character(gstin[:14]) == gstin[14]
@@ -0,0 +1,31 @@
1
+ """Luhn checksum — used by payment card numbers (13-19 digits).
2
+
3
+ RBI card-masking policy relies on knowing a digit run really is a card number;
4
+ Luhn validation is the first gate before any card span is flagged.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+
10
+ def validate(number: str) -> bool:
11
+ """Return True if `number` (digits only) passes the Luhn checksum."""
12
+ if not number.isdigit():
13
+ return False
14
+ total = 0
15
+ # Double every second digit from the right (excluding the check digit).
16
+ for i, digit in enumerate(reversed(number)):
17
+ d = int(digit)
18
+ if i % 2 == 1:
19
+ d *= 2
20
+ if d > 9:
21
+ d -= 9
22
+ total += d
23
+ return total % 10 == 0
24
+
25
+
26
+ def validate_card(number: str) -> bool:
27
+ """Validate a payment card number: 13-19 digits, Luhn-valid.
28
+
29
+ Accepts digits only — strip spaces/hyphens before calling.
30
+ """
31
+ return 13 <= len(number) <= 19 and validate(number)
@@ -0,0 +1,47 @@
1
+ """PAN (Permanent Account Number) structural validation.
2
+
3
+ Format: AAAHT1234C
4
+ - chars 1-3: alphabetic series (A-Z)
5
+ - char 4 : holder type — P (person), C (company), H (HUF), F (firm), A (AOP),
6
+ T (trust), B (BOI), L (local authority), J (artificial juridical
7
+ person), G (government)
8
+ - char 5 : first letter of holder's surname (individuals) or entity name
9
+ - chars 6-9: sequential digits
10
+ - char 10 : alphabetic check letter (algorithm not publicly documented, so we
11
+ validate structure only)
12
+
13
+ The 4th-character holder-type rule rejects ~62% of random 5-letter prefixes that
14
+ a bare [A-Z]{5}\\d{4}[A-Z] regex would accept.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import re
20
+
21
+ _PAN_RE = re.compile(r"^[A-Z]{3}[PCHFATBLJG][A-Z]\d{4}[A-Z]$")
22
+
23
+ _HOLDER_TYPES = {
24
+ "P": "person",
25
+ "C": "company",
26
+ "H": "hindu_undivided_family",
27
+ "F": "firm",
28
+ "A": "association_of_persons",
29
+ "T": "trust",
30
+ "B": "body_of_individuals",
31
+ "L": "local_authority",
32
+ "J": "artificial_juridical_person",
33
+ "G": "government",
34
+ }
35
+
36
+
37
+ def validate(pan: str) -> bool:
38
+ """Return True if `pan` is structurally a valid PAN (incl. holder-type rule)."""
39
+ return bool(_PAN_RE.match(pan.upper()))
40
+
41
+
42
+ def holder_type(pan: str) -> str | None:
43
+ """Return the holder type encoded in a valid PAN's 4th character, else None."""
44
+ pan = pan.upper()
45
+ if not validate(pan):
46
+ return None
47
+ return _HOLDER_TYPES[pan[3]]
@@ -0,0 +1,71 @@
1
+ """Verhoeff checksum — used by UIDAI for Aadhaar (12 digits) and Virtual ID (16 digits).
2
+
3
+ The Verhoeff scheme catches all single-digit errors and adjacent transpositions,
4
+ which is why a random 12-digit number has only a ~10% chance of passing. Combined
5
+ with the first-digit rule (Aadhaar never starts with 0 or 1), this kills the vast
6
+ majority of false positives from order IDs, invoice numbers, etc.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ # Dihedral group D5 multiplication table
12
+ _D = (
13
+ (0, 1, 2, 3, 4, 5, 6, 7, 8, 9),
14
+ (1, 2, 3, 4, 0, 6, 7, 8, 9, 5),
15
+ (2, 3, 4, 0, 1, 7, 8, 9, 5, 6),
16
+ (3, 4, 0, 1, 2, 8, 9, 5, 6, 7),
17
+ (4, 0, 1, 2, 3, 9, 5, 6, 7, 8),
18
+ (5, 9, 8, 7, 6, 0, 4, 3, 2, 1),
19
+ (6, 5, 9, 8, 7, 1, 0, 4, 3, 2),
20
+ (7, 6, 5, 9, 8, 2, 1, 0, 4, 3),
21
+ (8, 7, 6, 5, 9, 3, 2, 1, 0, 4),
22
+ (9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
23
+ )
24
+
25
+ # Permutation table
26
+ _P = (
27
+ (0, 1, 2, 3, 4, 5, 6, 7, 8, 9),
28
+ (1, 5, 7, 6, 2, 8, 3, 0, 9, 4),
29
+ (5, 8, 0, 3, 7, 9, 6, 1, 4, 2),
30
+ (8, 9, 1, 6, 0, 4, 3, 5, 2, 7),
31
+ (9, 4, 5, 3, 1, 2, 6, 8, 7, 0),
32
+ (4, 2, 8, 6, 5, 7, 3, 9, 0, 1),
33
+ (2, 7, 9, 3, 8, 0, 6, 4, 1, 5),
34
+ (7, 0, 4, 6, 9, 1, 3, 2, 5, 8),
35
+ )
36
+
37
+ # Multiplicative inverse table
38
+ _INV = (0, 4, 3, 2, 1, 5, 6, 7, 8, 9)
39
+
40
+
41
+ def validate(number: str) -> bool:
42
+ """Return True if `number` (digits only) passes the Verhoeff checksum."""
43
+ if not number.isdigit():
44
+ return False
45
+ c = 0
46
+ for i, digit in enumerate(reversed(number)):
47
+ c = _D[c][_P[i % 8][int(digit)]]
48
+ return c == 0
49
+
50
+
51
+ def generate_check_digit(number: str) -> str:
52
+ """Return the Verhoeff check digit for `number` (digits only, without check digit)."""
53
+ if not number.isdigit():
54
+ raise ValueError("Verhoeff check digit requires a numeric string")
55
+ c = 0
56
+ for i, digit in enumerate(reversed(number)):
57
+ c = _D[c][_P[(i + 1) % 8][int(digit)]]
58
+ return str(_INV[c])
59
+
60
+
61
+ def validate_aadhaar(number: str) -> bool:
62
+ """Validate an Aadhaar number: 12 digits, first digit 2-9, Verhoeff-valid.
63
+
64
+ Accepts digits only — strip spaces/hyphens before calling.
65
+ """
66
+ return len(number) == 12 and number[0] not in "01" and validate(number)
67
+
68
+
69
+ def validate_vid(number: str) -> bool:
70
+ """Validate an Aadhaar Virtual ID: 16 digits, first digit 2-9, Verhoeff-valid."""
71
+ return len(number) == 16 and number[0] not in "01" and validate(number)
dpdp_scrub/vault.py ADDED
@@ -0,0 +1,94 @@
1
+ """Session-scoped placeholder vault.
2
+
3
+ The vault is Ring 0 of the trust model: it maps <ENTITY_n> placeholders back
4
+ to real values, per session, inside the adopter's own infrastructure. This
5
+ module ships the in-memory backend; the interface is deliberately tiny so a
6
+ Redis/KMS backend is a drop-in (v1.5).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+
12
+ class MemoryVault:
13
+ """In-memory vault — for development and single-process apps.
14
+
15
+ Guarantees within a session:
16
+ - the same value always gets the same placeholder (multi-turn consistency)
17
+ - placeholders are numbered per entity type: <PHONE_IN_1>, <PHONE_IN_2>...
18
+ """
19
+
20
+ def __init__(self) -> None:
21
+ # session_id -> {placeholder -> value}
22
+ self._forward: dict[str, dict[str, str]] = {}
23
+ # session_id -> {(entity, value) -> placeholder}
24
+ self._reverse: dict[str, dict[tuple, str]] = {}
25
+
26
+ def placeholder_for(self, session_id: str, entity: str, value: str) -> str:
27
+ """Return the stable placeholder for `value`, minting one if new."""
28
+ fwd = self._forward.setdefault(session_id, {})
29
+ rev = self._reverse.setdefault(session_id, {})
30
+ key = (entity, value)
31
+ if key in rev:
32
+ return rev[key]
33
+ n = sum(1 for (e, _) in rev if e == entity) + 1
34
+ placeholder = f"<{entity}_{n}>"
35
+ rev[key] = placeholder
36
+ fwd[placeholder] = value
37
+ return placeholder
38
+
39
+ def resolve(self, session_id: str, placeholder: str) -> str | None:
40
+ """Resolve one placeholder back to its real value (None if unknown)."""
41
+ return self._forward.get(session_id, {}).get(placeholder)
42
+
43
+ def mappings(self, session_id: str) -> dict[str, str]:
44
+ """All placeholder->value mappings for a session (for authorized surfaces)."""
45
+ return dict(self._forward.get(session_id, {}))
46
+
47
+ def drop(self, session_id: str) -> None:
48
+ """Forget a session — after this, its placeholders are unresolvable forever."""
49
+ self._forward.pop(session_id, None)
50
+ self._reverse.pop(session_id, None)
51
+
52
+
53
+ class RedisVault:
54
+ """Redis-backed vault: shared across processes/VMs inside your boundary.
55
+
56
+ Same interface as MemoryVault. Keys are namespaced `dpdp:*:{session}` and
57
+ every write refreshes the session TTL — expired sessions become
58
+ permanently unresolvable (retroactive redaction).
59
+
60
+ Requires `redis` (pip install redis) unless a client is injected.
61
+ """
62
+
63
+ def __init__(self, url: str = "redis://localhost:6379", ttl: int = 86400, client=None):
64
+ if client is None:
65
+ import redis # lazy: optional dependency
66
+
67
+ client = redis.Redis.from_url(url, decode_responses=True)
68
+ self.r = client
69
+ self.ttl = ttl
70
+
71
+ def _keys(self, session_id: str) -> tuple[str, str, str]:
72
+ return (f"dpdp:fwd:{session_id}", f"dpdp:rev:{session_id}", f"dpdp:cnt:{session_id}")
73
+
74
+ def placeholder_for(self, session_id: str, entity: str, value: str) -> str:
75
+ fwd, rev, cnt = self._keys(session_id)
76
+ existing = self.r.hget(rev, f"{entity}|{value}")
77
+ if existing:
78
+ return existing
79
+ n = self.r.hincrby(cnt, entity, 1)
80
+ placeholder = f"<{entity}_{n}>"
81
+ self.r.hset(rev, f"{entity}|{value}", placeholder)
82
+ self.r.hset(fwd, placeholder, value)
83
+ for k in (fwd, rev, cnt):
84
+ self.r.expire(k, self.ttl)
85
+ return placeholder
86
+
87
+ def resolve(self, session_id: str, placeholder: str):
88
+ return self.r.hget(self._keys(session_id)[0], placeholder)
89
+
90
+ def mappings(self, session_id: str) -> dict:
91
+ return self.r.hgetall(self._keys(session_id)[0])
92
+
93
+ def drop(self, session_id: str) -> None:
94
+ self.r.delete(*self._keys(session_id))
@@ -0,0 +1,164 @@
1
+ Metadata-Version: 2.4
2
+ Name: dpdp-scrub
3
+ Version: 0.0.1
4
+ Summary: Detect, validate, and redact Indian PII (Aadhaar, PAN, GSTIN, UPI, and more) — self-hosted, checksum-validated, DPDP-ready.
5
+ Author: Palkush Dave
6
+ License: MIT
7
+ Keywords: pii,dpdp,aadhaar,pan,gstin,privacy,redaction,india
8
+ Classifier: Development Status :: 2 - Pre-Alpha
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Topic :: Security
13
+ Classifier: Topic :: Text Processing
14
+ Requires-Python: >=3.9
15
+ Description-Content-Type: text/markdown
16
+ License-File: LICENSE
17
+ Provides-Extra: dev
18
+ Requires-Dist: pytest>=8; extra == "dev"
19
+ Requires-Dist: ruff>=0.4; extra == "dev"
20
+ Requires-Dist: mypy>=1.10; extra == "dev"
21
+ Dynamic: license-file
22
+
23
+ # dpdp-scrub
24
+
25
+ **Detect, validate, and redact Indian PII before it reaches an LLM — self-hosted, checksum-validated, DPDP-ready.**
26
+
27
+ Your org keeps the keys; the LLM gets the placeholders.
28
+
29
+ ```
30
+ IN : Sir aadhaar 2345 6789 0124 hai, call karo 9876543210 pe, PAN ABCPE1234F
31
+ LLM : Sir aadhaar XXXX XXXX 0124 hai, call karo <PHONE_IN_1> pe, PAN <PAN_1>
32
+ BACK: Done! 9876543210 pe call schedule ho gaya. ← rehydrated, agent-side
33
+ ```
34
+
35
+ ## Why this exists
36
+
37
+ Every Indian company wiring LLMs into support, sales, and ops is streaming
38
+ Aadhaar numbers, PANs, GSTINs, UPI IDs, and phone numbers to third-party model
39
+ providers — and copying them into logs, vector DBs, and fine-tuning sets.
40
+ The DPDP Act's substantive obligations become enforceable in **May 2027**
41
+ (penalties up to ₹250 crore); UIDAI's Aadhaar-masking rules and RBI's card
42
+ rules already apply.
43
+
44
+ The redaction loop itself is well-established engineering (Presidio, Google
45
+ DLP). What didn't exist is **eyes that can see Indian PII** — validated,
46
+ context-aware, and open. Google Cloud DLP ships 3 Indian infoTypes, AWS
47
+ Comprehend 4, Presidio 6. dpdp-scrub detects **15**, most with mathematical
48
+ or reference-data validation:
49
+
50
+ | Entity | Validation |
51
+ |---|---|
52
+ | Aadhaar / Aadhaar VID | **Verhoeff checksum** + first-digit rule |
53
+ | GSTIN | **mod-36 check character** + embedded-PAN structure + state code |
54
+ | Credit/debit card | **Luhn checksum** |
55
+ | PAN | holder-type structure rule |
56
+ | UPI ID | handle verified against **known PSP list** (`@ybl`, `@oksbi`, …) |
57
+ | IFSC, phone, email, voter ID, passport | structural + context scoring |
58
+ | Vehicle registration | state/RTO code verified against reference list (incl. BH series) |
59
+ | Bank account, EPF UAN, ABHA | **context-required** (surface only with evidence like "A/c", "UAN") |
60
+
61
+ The context engine is what separates a phone number from an order ID:
62
+
63
+ ```
64
+ order 9876543210 ka status batao → nothing (suppressed by "order")
65
+ call karo 9876543210 pe → PHONE_IN 0.99 (boosted by "call")
66
+ mera a/c 34512345678901 hai → BANK_ACCOUNT (earned by "a/c")
67
+ code 34512345678901 likha hai → nothing
68
+ ```
69
+
70
+ ## Quick start
71
+
72
+ ```python
73
+ from dpdp_scrub import Scrubber
74
+
75
+ s = Scrubber()
76
+
77
+ result = s.scrub("mera PAN ABCPE1234F hai, call 9876543210", session_id="ticket-42")
78
+ result.text # "mera PAN <PAN_1> hai, call <PHONE_IN_1>"
79
+ result.spans # typed spans: entity, offsets, confidence
80
+
81
+ reply = call_your_llm(result.text) # provider never sees real values
82
+ s.rehydrate(reply, "ticket-42") # real values restored, your side only
83
+ ```
84
+
85
+ Detection only:
86
+
87
+ ```python
88
+ from dpdp_scrub import detect
89
+ detect("payment 9876543210@ybl pe karo")
90
+ # [Span(entity='UPI_ID', start=8, end=22, ..., confidence=0.92)]
91
+ ```
92
+
93
+ Zero runtime dependencies. Python 3.9+.
94
+
95
+ ## Trust model
96
+
97
+ ```
98
+ Ring 0: the vault (your Redis/memory) — real values, session-scoped, droppable
99
+ Ring 1: your org's services — placeholders + scoped right to resolve
100
+ Ring 2: LLM providers, logs, vector DBs — placeholders only, no resolution path
101
+ ```
102
+
103
+ - Same value → same placeholder within a session (multi-turn consistency).
104
+ - `vault.drop(session)` makes every old copy of its placeholders permanently
105
+ unresolvable — retroactive redaction for logs you already wrote.
106
+ - No cloud, no telemetry, no phoning home. This is a library, not a service:
107
+ **your data never leaves your infrastructure.**
108
+
109
+ Per-entity actions are policy: `tokenize` (reversible), `mask` (UIDAI-format
110
+ `XXXX XXXX 0124` for Aadhaar, last-4 for cards), `redact` (ABHA health IDs by
111
+ default), `ignore`.
112
+
113
+ ## Benchmark
114
+
115
+ IndicPII-Bench v0 (in `bench/`): 500 synthetic messages — Hinglish, English,
116
+ Devanagari, OCR-noise spacing, multi-entity — with 20% traps (order IDs,
117
+ invoice refs, phones-in-txn-context). Presidio runs at its best
118
+ configuration — its IN_* recognizers explicitly registered (they are shipped
119
+ but **not loaded by default**). F1, value-level matching:
120
+
121
+ | entity | dpdp-scrub | presidio |
122
+ |---|---|---|
123
+ | AADHAAR | **1.000** | 1.000 |
124
+ | GSTIN | **1.000** | 0.971 |
125
+ | PAN | **1.000** | 0.383 |
126
+ | PHONE_IN | **1.000** | 0.602 |
127
+ | UPI_ID | **1.000** | 0.000 |
128
+ | IFSC | **1.000** | 0.000 |
129
+ | EMAIL | **1.000** | 1.000 |
130
+ | **OVERALL** | **1.000** | 0.595 |
131
+ | trap false positives | **0/100** | 200/100 |
132
+
133
+ Honest caveats: this is our own v0 corpus (templated, synthetic, Hinglish-
134
+ heavy) — a perfect self-score mainly means the corpus is still too easy, and
135
+ it will get harder (Devanagari, OCR noise, multi-entity). Presidio was not
136
+ built for code-mixed text; the 107 trap FPs come from it having no context
137
+ suppression (e.g. any 10-digit number near "txn" flags as phone). Reproduce:
138
+ `python3 bench/compare.py 500`.
139
+
140
+ ## Status
141
+
142
+ Pre-alpha, moving fast. Built so far: 15 entities, checksum validators,
143
+ context-scoring engine, scrub/rehydrate loop with session vault — 86 tests,
144
+ all synthetic data. Roadmap:
145
+
146
+ - [ ] YAML policy profiles (`dpdp-strict`, `uidai-mask`, `audit-only`) + audit log
147
+ - [ ] `scrub_json()` for structured payloads
148
+ - [ ] Redis vault backend
149
+ - [ ] LiteLLM / LangChain / Presidio adapters
150
+ - [ ] **IndicPII-Bench** — a public benchmark for Indian PII detection
151
+ - [ ] Indic-script & code-mixed NER for names/addresses (AI4Bharat IndicNER)
152
+
153
+ ## Contributing
154
+
155
+ The reference-data files are designed for one-line PRs: new UPI PSP handles,
156
+ RTO codes, context words (English/Hindi/regional). Entity requests welcome —
157
+ include format, any checksum, and public documentation.
158
+
159
+ All test data must be synthetic (checksum-generated) or published samples.
160
+ Never commit real PII, including your own.
161
+
162
+ ## License
163
+
164
+ MIT
@@ -0,0 +1,18 @@
1
+ dpdp_scrub/__init__.py,sha256=0T-0xwvdAVIGBCpvo4jJsiZJ13HXiH-xo2tOsA--ULw,474
2
+ dpdp_scrub/data.py,sha256=1lgU0gZBosNpUpWlMQhlh4tzefJ9ggzG8ElXbfXBQ-Y,1623
3
+ dpdp_scrub/detectors.py,sha256=mpbOkzsaqjtCuDB7xIZhgl3jkPtOgr8AxBZCwQRJBmw,11477
4
+ dpdp_scrub/integrations.py,sha256=HcdfwL25FmPUoa4MoUss_1cqSXm1_YBMyKe-Em60IVg,2771
5
+ dpdp_scrub/policies.py,sha256=_0Ri-Aanpob0Tv1nBMSz5GhohFAYjodnCVocy07JRhA,1092
6
+ dpdp_scrub/scrubber.py,sha256=l1LI1K3yPiQtaJUNNSQ1fl29cKlgBtzY6URzbWredkk,5835
7
+ dpdp_scrub/spans.py,sha256=Fb-FpSjWQodee8ojqMS4IZdSOlrnIya2nYmp85WslGc,796
8
+ dpdp_scrub/vault.py,sha256=SAi_dMbMmTUiY94nLSXOu91LKVV-Vzn_N9pJVLJgByI,3741
9
+ dpdp_scrub/validators/__init__.py,sha256=6i-LgwVTCpY-aewSzbSKD_uu_3HBSISoa-97Xwqhizg,910
10
+ dpdp_scrub/validators/gstin.py,sha256=1g7JSjCSAdm7ZL8-_SLL896rT-J9xFLHqAKuALpLESo,1644
11
+ dpdp_scrub/validators/luhn.py,sha256=HmMm2vTT-0ZfFWH2A7Sn3zE2ebi18R3smliFnbyw-XI,946
12
+ dpdp_scrub/validators/pan.py,sha256=ew3FLmUlt6SdBOoVq_bcaftoul8IupzSVD3ctAunzJg,1473
13
+ dpdp_scrub/validators/verhoeff.py,sha256=GQUrQqNI7OOriPZhdHm5bZYiO9H-1B1a7pxx4OKqNck,2375
14
+ dpdp_scrub-0.0.1.dist-info/licenses/LICENSE,sha256=Fp2GhwIZEQFhUP1MpwzNmF_4I84jgX54mRFu5L7mQVI,1080
15
+ dpdp_scrub-0.0.1.dist-info/METADATA,sha256=RO_akrpTIxOxDuZ9OJfHxG7iF5tKpSt8G3Q2SctPAcI,6437
16
+ dpdp_scrub-0.0.1.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
17
+ dpdp_scrub-0.0.1.dist-info/top_level.txt,sha256=0ZIhxqrTZ1Q4xdhcWEF2nEWkcPSQAMAF9cxAnDJgyvQ,11
18
+ dpdp_scrub-0.0.1.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (83.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 dpdp-scrub contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ dpdp_scrub