chatsee-redact 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- chatsee_redact/__init__.py +35 -0
- chatsee_redact/cli.py +262 -0
- chatsee_redact/files.py +173 -0
- chatsee_redact/rules.py +84 -0
- chatsee_redact-0.2.0.dist-info/METADATA +130 -0
- chatsee_redact-0.2.0.dist-info/RECORD +9 -0
- chatsee_redact-0.2.0.dist-info/WHEEL +5 -0
- chatsee_redact-0.2.0.dist-info/entry_points.txt +2 -0
- chatsee_redact-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""chatsee-redact: local JSON/JSONL log redaction using ChatSee's rules.
|
|
2
|
+
|
|
3
|
+
Redaction runs entirely on your machine. The only network call is a read-only
|
|
4
|
+
fetch of the regex classifiers; your log content is never transmitted.
|
|
5
|
+
"""
|
|
6
|
+
from chatsee_redact.files import (
|
|
7
|
+
REDACT_ALL,
|
|
8
|
+
detect_format,
|
|
9
|
+
fields_to_redact,
|
|
10
|
+
redact_csv_file,
|
|
11
|
+
redact_file,
|
|
12
|
+
redact_json_file,
|
|
13
|
+
redact_jsonl_file,
|
|
14
|
+
)
|
|
15
|
+
from chatsee_redact.rules import (
|
|
16
|
+
build_classifiers_url,
|
|
17
|
+
fetch_raw_classifiers,
|
|
18
|
+
fetch_rules,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
__version__ = "0.2.0"
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"redact_file",
|
|
25
|
+
"redact_json_file",
|
|
26
|
+
"redact_jsonl_file",
|
|
27
|
+
"redact_csv_file",
|
|
28
|
+
"fields_to_redact",
|
|
29
|
+
"detect_format",
|
|
30
|
+
"REDACT_ALL",
|
|
31
|
+
"build_classifiers_url",
|
|
32
|
+
"fetch_rules",
|
|
33
|
+
"fetch_raw_classifiers",
|
|
34
|
+
"__version__",
|
|
35
|
+
]
|
chatsee_redact/cli.py
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"""Command-line entry point: ``chatsee-redact``.
|
|
2
|
+
|
|
3
|
+
Two modes:
|
|
4
|
+
chatsee-redact <files...> [opts] redact JSON/JSONL/CSV files (default)
|
|
5
|
+
chatsee-redact dump-rules [opts] fetch + print/export the raw rules
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import json
|
|
11
|
+
import logging
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
from typing import List, Optional
|
|
15
|
+
|
|
16
|
+
from chatsee_redact import __version__
|
|
17
|
+
from chatsee_redact.files import fields_to_redact, redact_file
|
|
18
|
+
from chatsee_redact.rules import (
|
|
19
|
+
build_classifiers_url,
|
|
20
|
+
fetch_raw_classifiers,
|
|
21
|
+
fetch_rules,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
logger = logging.getLogger("chatsee_redact")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _add_rule_source_args(p: argparse.ArgumentParser) -> None:
|
|
28
|
+
p.add_argument(
|
|
29
|
+
"--env",
|
|
30
|
+
default="qa",
|
|
31
|
+
help="ChatSee environment for the rule source: dev|qa|demo|poc "
|
|
32
|
+
"(default: qa). Ignored if --classifiers-url is given.",
|
|
33
|
+
)
|
|
34
|
+
p.add_argument(
|
|
35
|
+
"--classifiers-url",
|
|
36
|
+
help="Full URL of the classifiers endpoint (overrides --env).",
|
|
37
|
+
)
|
|
38
|
+
p.add_argument(
|
|
39
|
+
"--timeout",
|
|
40
|
+
type=float,
|
|
41
|
+
default=5.0,
|
|
42
|
+
help="Rule-fetch HTTP timeout in seconds (default: 5).",
|
|
43
|
+
)
|
|
44
|
+
p.add_argument(
|
|
45
|
+
"--no-verify-ssl",
|
|
46
|
+
action="store_true",
|
|
47
|
+
help="Disable TLS certificate verification for the rule fetch.",
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _resolve_output(
|
|
52
|
+
in_path: str, out_dir: Optional[str], out_file: Optional[str]
|
|
53
|
+
) -> str:
|
|
54
|
+
if out_file:
|
|
55
|
+
return out_file
|
|
56
|
+
if out_dir:
|
|
57
|
+
os.makedirs(out_dir, exist_ok=True)
|
|
58
|
+
return os.path.join(out_dir, os.path.basename(in_path))
|
|
59
|
+
stem, ext = os.path.splitext(in_path)
|
|
60
|
+
return f"{stem}.redacted{ext}"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def build_redact_parser() -> argparse.ArgumentParser:
|
|
64
|
+
p = argparse.ArgumentParser(
|
|
65
|
+
prog="chatsee-redact",
|
|
66
|
+
description=(
|
|
67
|
+
"Redact PII in your own JSON/JSONL/CSV files locally, using "
|
|
68
|
+
"ChatSee's redaction rules. Your file content never leaves this "
|
|
69
|
+
"machine; the only network call is a read-only fetch of the regex "
|
|
70
|
+
"rules. Use 'chatsee-redact dump-rules' to inspect those rules."
|
|
71
|
+
),
|
|
72
|
+
)
|
|
73
|
+
p.add_argument("inputs", nargs="+", help="Input JSON, JSONL or CSV file(s).")
|
|
74
|
+
p.add_argument(
|
|
75
|
+
"-o",
|
|
76
|
+
"--out",
|
|
77
|
+
metavar="DIR",
|
|
78
|
+
help="Output directory (created if missing). Default: write "
|
|
79
|
+
"'<name>.redacted.<ext>' next to each input.",
|
|
80
|
+
)
|
|
81
|
+
p.add_argument(
|
|
82
|
+
"--out-file",
|
|
83
|
+
metavar="PATH",
|
|
84
|
+
help="Explicit output path (only valid with a single input).",
|
|
85
|
+
)
|
|
86
|
+
_add_rule_source_args(p)
|
|
87
|
+
p.add_argument(
|
|
88
|
+
"--fields",
|
|
89
|
+
metavar="A,B,C",
|
|
90
|
+
help="Comma-separated keys/columns to redact. Default: redact every "
|
|
91
|
+
"string value, recursively.",
|
|
92
|
+
)
|
|
93
|
+
p.add_argument(
|
|
94
|
+
"--format",
|
|
95
|
+
choices=["auto", "json", "jsonl", "csv"],
|
|
96
|
+
default="auto",
|
|
97
|
+
help="Input format. 'auto' uses the extension (.jsonl/.ndjson -> "
|
|
98
|
+
"jsonl, .json -> json, .csv -> csv, else jsonl).",
|
|
99
|
+
)
|
|
100
|
+
p.add_argument(
|
|
101
|
+
"--indent",
|
|
102
|
+
type=int,
|
|
103
|
+
default=2,
|
|
104
|
+
help="Indent for JSON output (default: 2). Use -1 for compact output.",
|
|
105
|
+
)
|
|
106
|
+
p.add_argument(
|
|
107
|
+
"--cache-ttl",
|
|
108
|
+
type=int,
|
|
109
|
+
default=300,
|
|
110
|
+
help="In-process rule cache TTL in seconds (default: 300).",
|
|
111
|
+
)
|
|
112
|
+
p.add_argument(
|
|
113
|
+
"--strict",
|
|
114
|
+
action="store_true",
|
|
115
|
+
help="Abort (write nothing) if no redaction rules could be fetched, "
|
|
116
|
+
"instead of copying unredacted (the default fail-open behaviour).",
|
|
117
|
+
)
|
|
118
|
+
p.add_argument(
|
|
119
|
+
"-q", "--quiet", action="store_true", help="Only log warnings and errors."
|
|
120
|
+
)
|
|
121
|
+
p.add_argument(
|
|
122
|
+
"--version", action="version", version=f"chatsee-redact {__version__}"
|
|
123
|
+
)
|
|
124
|
+
return p
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def build_dump_parser() -> argparse.ArgumentParser:
|
|
128
|
+
p = argparse.ArgumentParser(
|
|
129
|
+
prog="chatsee-redact dump-rules",
|
|
130
|
+
description=(
|
|
131
|
+
"Fetch the raw redaction classifier documents (as the API serves "
|
|
132
|
+
"them) so you can audit exactly which regex patterns run. Prints "
|
|
133
|
+
"to stdout unless -o/--out-file is given."
|
|
134
|
+
),
|
|
135
|
+
)
|
|
136
|
+
_add_rule_source_args(p)
|
|
137
|
+
p.add_argument(
|
|
138
|
+
"-o",
|
|
139
|
+
"--out-file",
|
|
140
|
+
metavar="PATH",
|
|
141
|
+
help="Write the rules JSON here instead of stdout.",
|
|
142
|
+
)
|
|
143
|
+
p.add_argument(
|
|
144
|
+
"--indent",
|
|
145
|
+
type=int,
|
|
146
|
+
default=2,
|
|
147
|
+
help="Indent for the rules JSON (default: 2). Use -1 for compact.",
|
|
148
|
+
)
|
|
149
|
+
p.add_argument(
|
|
150
|
+
"-q", "--quiet", action="store_true", help="Only log warnings and errors."
|
|
151
|
+
)
|
|
152
|
+
return p
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _dump_rules_main(argv: Optional[List[str]]) -> int:
|
|
156
|
+
args = build_dump_parser().parse_args(argv)
|
|
157
|
+
logging.basicConfig(
|
|
158
|
+
level=logging.WARNING if args.quiet else logging.INFO,
|
|
159
|
+
format="%(levelname)s %(message)s",
|
|
160
|
+
)
|
|
161
|
+
try:
|
|
162
|
+
url, docs = fetch_raw_classifiers(
|
|
163
|
+
env=args.env,
|
|
164
|
+
classifiers_url=args.classifiers_url,
|
|
165
|
+
timeout_seconds=args.timeout,
|
|
166
|
+
verify_ssl=not args.no_verify_ssl,
|
|
167
|
+
)
|
|
168
|
+
except Exception as exc: # noqa: BLE001 — report any fetch/parse failure
|
|
169
|
+
logger.error("Failed to fetch classifiers: %s", exc)
|
|
170
|
+
return 1
|
|
171
|
+
|
|
172
|
+
indent = None if args.indent < 0 else args.indent
|
|
173
|
+
text = json.dumps(docs, ensure_ascii=False, indent=indent)
|
|
174
|
+
if args.out_file:
|
|
175
|
+
tmp = args.out_file + ".tmp"
|
|
176
|
+
with open(tmp, "w", encoding="utf-8") as fh:
|
|
177
|
+
fh.write(text)
|
|
178
|
+
os.replace(tmp, args.out_file)
|
|
179
|
+
logger.info(
|
|
180
|
+
"Wrote %d classifier(s) from %s -> %s", len(docs), url, args.out_file
|
|
181
|
+
)
|
|
182
|
+
else:
|
|
183
|
+
logger.info("Fetched %d classifier(s) from %s", len(docs), url)
|
|
184
|
+
print(text)
|
|
185
|
+
return 0
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
189
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
190
|
+
if argv and argv[0] == "dump-rules":
|
|
191
|
+
return _dump_rules_main(argv[1:])
|
|
192
|
+
return _redact_main(argv)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _redact_main(argv: Optional[List[str]]) -> int:
|
|
196
|
+
args = build_redact_parser().parse_args(argv)
|
|
197
|
+
logging.basicConfig(
|
|
198
|
+
level=logging.WARNING if args.quiet else logging.INFO,
|
|
199
|
+
format="%(levelname)s %(message)s",
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
if args.out_file and len(args.inputs) != 1:
|
|
203
|
+
logger.error("--out-file can only be used with exactly one input file.")
|
|
204
|
+
return 2
|
|
205
|
+
|
|
206
|
+
fields = fields_to_redact(args.fields.split(",") if args.fields else None)
|
|
207
|
+
indent = None if args.indent < 0 else args.indent
|
|
208
|
+
|
|
209
|
+
url = build_classifiers_url(args.env, args.classifiers_url)
|
|
210
|
+
logger.info("Fetching redaction rules from %s", url)
|
|
211
|
+
_, rules = fetch_rules(
|
|
212
|
+
env=args.env,
|
|
213
|
+
classifiers_url=args.classifiers_url,
|
|
214
|
+
cache_ttl_seconds=args.cache_ttl,
|
|
215
|
+
timeout_seconds=args.timeout,
|
|
216
|
+
verify_ssl=not args.no_verify_ssl,
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
if not rules:
|
|
220
|
+
if args.strict:
|
|
221
|
+
logger.error(
|
|
222
|
+
"No redaction rules available from %s and --strict is set. "
|
|
223
|
+
"Aborting without writing any output.",
|
|
224
|
+
url,
|
|
225
|
+
)
|
|
226
|
+
return 3
|
|
227
|
+
logger.warning(
|
|
228
|
+
"!! No redaction rules available from %s. Output will be an "
|
|
229
|
+
"UNREDACTED copy (fail-open). Re-run with --strict to abort instead.",
|
|
230
|
+
url,
|
|
231
|
+
)
|
|
232
|
+
else:
|
|
233
|
+
logger.info("Loaded %d redaction rule(s).", len(rules))
|
|
234
|
+
|
|
235
|
+
exit_code = 0
|
|
236
|
+
for in_path in args.inputs:
|
|
237
|
+
if not os.path.isfile(in_path):
|
|
238
|
+
logger.error("Input not found: %s", in_path)
|
|
239
|
+
exit_code = 2
|
|
240
|
+
continue
|
|
241
|
+
out_path = _resolve_output(in_path, args.out, args.out_file)
|
|
242
|
+
if os.path.abspath(out_path) == os.path.abspath(in_path):
|
|
243
|
+
logger.error("Refusing to overwrite the source file: %s", in_path)
|
|
244
|
+
exit_code = 2
|
|
245
|
+
continue
|
|
246
|
+
try:
|
|
247
|
+
fmt, n = redact_file(
|
|
248
|
+
in_path, out_path, rules, fields, fmt=args.format, indent=indent
|
|
249
|
+
)
|
|
250
|
+
except (OSError, ValueError) as exc:
|
|
251
|
+
logger.error("Failed to redact %s: %s", in_path, exc)
|
|
252
|
+
exit_code = 1
|
|
253
|
+
continue
|
|
254
|
+
logger.info(
|
|
255
|
+
"Redacted %s (%s, %d record(s)) -> %s", in_path, fmt, n, out_path
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
return exit_code
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
if __name__ == "__main__": # pragma: no cover
|
|
262
|
+
raise SystemExit(main())
|
chatsee_redact/files.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""
|
|
2
|
+
File-level redaction adapters.
|
|
3
|
+
|
|
4
|
+
These functions are pure and offline-testable: they take an already-compiled
|
|
5
|
+
list of ``RedactionRule`` objects and never touch the network. Fetching the
|
|
6
|
+
rules (the single outbound call this tool makes) lives in :mod:`rules`.
|
|
7
|
+
|
|
8
|
+
The actual masking is delegated to the ChatSee SDK's engine
|
|
9
|
+
(``chatsee.redaction.redact_payload_sync``) so redaction here is byte-for-byte
|
|
10
|
+
identical to what the ChatSee pipeline applies. We only add the JSON/JSONL
|
|
11
|
+
file plumbing on top.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import csv
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
from typing import List, Optional, Tuple, Union
|
|
19
|
+
|
|
20
|
+
from chatsee.redaction import RedactionRule, redact_payload_sync
|
|
21
|
+
|
|
22
|
+
# Sentinel understood by the engine: "redact every string value, recursively".
|
|
23
|
+
REDACT_ALL = "*"
|
|
24
|
+
|
|
25
|
+
_JSONL_EXTS = {".jsonl", ".ndjson"}
|
|
26
|
+
_JSON_EXTS = {".json"}
|
|
27
|
+
_CSV_EXTS = {".csv"}
|
|
28
|
+
|
|
29
|
+
FieldsArg = Union[str, List[str]]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def fields_to_redact(fields: Optional[List[str]]) -> FieldsArg:
|
|
33
|
+
"""Map a ``--fields`` list (or ``None``) to the engine's argument.
|
|
34
|
+
|
|
35
|
+
- ``None`` / empty -> ``"*"`` (redact every string value, recursively)
|
|
36
|
+
- ``["a", "b"]`` -> redact only those top-level keys
|
|
37
|
+
"""
|
|
38
|
+
cleaned = [f.strip() for f in (fields or []) if f and f.strip()]
|
|
39
|
+
return cleaned or REDACT_ALL
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def detect_format(path: str, override: Optional[str] = None) -> str:
|
|
43
|
+
"""Resolve the input format to ``"json"``, ``"jsonl"`` or ``"csv"``.
|
|
44
|
+
|
|
45
|
+
With ``override`` unset (or ``"auto"``), the file extension decides:
|
|
46
|
+
``.jsonl``/``.ndjson`` -> jsonl, ``.json`` -> json, ``.csv`` -> csv,
|
|
47
|
+
anything else -> jsonl (log files are usually line-delimited JSON).
|
|
48
|
+
"""
|
|
49
|
+
if override and override != "auto":
|
|
50
|
+
return override
|
|
51
|
+
ext = os.path.splitext(path)[1].lower()
|
|
52
|
+
if ext in _JSONL_EXTS:
|
|
53
|
+
return "jsonl"
|
|
54
|
+
if ext in _JSON_EXTS:
|
|
55
|
+
return "json"
|
|
56
|
+
if ext in _CSV_EXTS:
|
|
57
|
+
return "csv"
|
|
58
|
+
return "jsonl"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _atomic_write_text(out_path: str, text: str) -> None:
|
|
62
|
+
"""Write ``text`` to ``out_path`` via a temp file + atomic replace, so a
|
|
63
|
+
crash mid-write never leaves a half-written 'redacted' file behind."""
|
|
64
|
+
tmp = out_path + ".tmp"
|
|
65
|
+
with open(tmp, "w", encoding="utf-8") as fh:
|
|
66
|
+
fh.write(text)
|
|
67
|
+
os.replace(tmp, out_path)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def redact_json_file(
|
|
71
|
+
in_path: str,
|
|
72
|
+
out_path: str,
|
|
73
|
+
rules: List[RedactionRule],
|
|
74
|
+
fields: FieldsArg = REDACT_ALL,
|
|
75
|
+
indent: Optional[int] = 2,
|
|
76
|
+
) -> int:
|
|
77
|
+
"""Redact a whole-document JSON file (array, object, or scalar).
|
|
78
|
+
|
|
79
|
+
Returns the number of top-level records (``len`` for a list, else ``1``).
|
|
80
|
+
"""
|
|
81
|
+
# utf-8-sig tolerates (and strips) a leading BOM, common in files
|
|
82
|
+
# authored on Windows; plain utf-8 input is unaffected.
|
|
83
|
+
with open(in_path, "r", encoding="utf-8-sig") as fh:
|
|
84
|
+
data = json.load(fh)
|
|
85
|
+
|
|
86
|
+
redacted = redact_payload_sync(data, rules, fields_to_redact=fields)
|
|
87
|
+
|
|
88
|
+
_atomic_write_text(
|
|
89
|
+
out_path, json.dumps(redacted, ensure_ascii=False, indent=indent)
|
|
90
|
+
)
|
|
91
|
+
return len(redacted) if isinstance(redacted, list) else 1
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def redact_jsonl_file(
|
|
95
|
+
in_path: str,
|
|
96
|
+
out_path: str,
|
|
97
|
+
rules: List[RedactionRule],
|
|
98
|
+
fields: FieldsArg = REDACT_ALL,
|
|
99
|
+
) -> int:
|
|
100
|
+
"""Redact a JSON Lines file, streaming one record per line.
|
|
101
|
+
|
|
102
|
+
Never loads the whole file into memory. Blank lines are preserved.
|
|
103
|
+
Returns the number of JSON records redacted.
|
|
104
|
+
"""
|
|
105
|
+
count = 0
|
|
106
|
+
tmp = out_path + ".tmp"
|
|
107
|
+
with open(in_path, "r", encoding="utf-8-sig") as src, open(
|
|
108
|
+
tmp, "w", encoding="utf-8"
|
|
109
|
+
) as dst:
|
|
110
|
+
for line in src:
|
|
111
|
+
stripped = line.strip()
|
|
112
|
+
if not stripped:
|
|
113
|
+
dst.write("\n")
|
|
114
|
+
continue
|
|
115
|
+
record = json.loads(stripped)
|
|
116
|
+
redacted = redact_payload_sync(record, rules, fields_to_redact=fields)
|
|
117
|
+
dst.write(json.dumps(redacted, ensure_ascii=False) + "\n")
|
|
118
|
+
count += 1
|
|
119
|
+
os.replace(tmp, out_path)
|
|
120
|
+
return count
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def redact_csv_file(
|
|
124
|
+
in_path: str,
|
|
125
|
+
out_path: str,
|
|
126
|
+
rules: List[RedactionRule],
|
|
127
|
+
fields: FieldsArg = REDACT_ALL,
|
|
128
|
+
) -> int:
|
|
129
|
+
"""Redact a CSV file, streaming row by row. The header is preserved.
|
|
130
|
+
|
|
131
|
+
With ``fields == "*"`` every cell is scanned; with a list of column names
|
|
132
|
+
only those columns are redacted. Returns the number of data rows written.
|
|
133
|
+
"""
|
|
134
|
+
count = 0
|
|
135
|
+
tmp = out_path + ".tmp"
|
|
136
|
+
with open(in_path, "r", encoding="utf-8-sig", newline="") as src, open(
|
|
137
|
+
tmp, "w", encoding="utf-8", newline=""
|
|
138
|
+
) as dst:
|
|
139
|
+
reader = csv.DictReader(src)
|
|
140
|
+
if reader.fieldnames is None: # empty input
|
|
141
|
+
os.replace(tmp, out_path)
|
|
142
|
+
return 0
|
|
143
|
+
# extrasaction="ignore" guards against ragged rows with extra columns.
|
|
144
|
+
writer = csv.DictWriter(
|
|
145
|
+
dst, fieldnames=reader.fieldnames, extrasaction="ignore"
|
|
146
|
+
)
|
|
147
|
+
writer.writeheader()
|
|
148
|
+
for row in reader:
|
|
149
|
+
redacted = redact_payload_sync(dict(row), rules, fields_to_redact=fields)
|
|
150
|
+
writer.writerow(redacted)
|
|
151
|
+
count += 1
|
|
152
|
+
os.replace(tmp, out_path)
|
|
153
|
+
return count
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def redact_file(
|
|
157
|
+
in_path: str,
|
|
158
|
+
out_path: str,
|
|
159
|
+
rules: List[RedactionRule],
|
|
160
|
+
fields: FieldsArg = REDACT_ALL,
|
|
161
|
+
fmt: Optional[str] = None,
|
|
162
|
+
indent: Optional[int] = 2,
|
|
163
|
+
) -> Tuple[str, int]:
|
|
164
|
+
"""Dispatch to the JSON, JSONL or CSV adapter based on the detected format.
|
|
165
|
+
|
|
166
|
+
Returns ``(resolved_format, record_count)``.
|
|
167
|
+
"""
|
|
168
|
+
resolved = detect_format(in_path, fmt)
|
|
169
|
+
if resolved == "csv":
|
|
170
|
+
return "csv", redact_csv_file(in_path, out_path, rules, fields)
|
|
171
|
+
if resolved == "jsonl":
|
|
172
|
+
return "jsonl", redact_jsonl_file(in_path, out_path, rules, fields)
|
|
173
|
+
return "json", redact_json_file(in_path, out_path, rules, fields, indent)
|
chatsee_redact/rules.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Redaction-rule fetching.
|
|
3
|
+
|
|
4
|
+
This is the ONLY part of the tool that touches the network, and it is a
|
|
5
|
+
read-only pull of the regex classifiers from ChatSee. No log content is ever
|
|
6
|
+
sent. Rule loading + the in-process TTL cache + fail-open behaviour are reused
|
|
7
|
+
verbatim from the ChatSee SDK.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Any, List, Optional, Tuple
|
|
12
|
+
|
|
13
|
+
import requests
|
|
14
|
+
|
|
15
|
+
from chatsee.env import resolve_api_base_url
|
|
16
|
+
from chatsee.redaction import RedactionRule, load_redaction_rules_sync
|
|
17
|
+
|
|
18
|
+
CLASSIFIERS_PATH = "/v1/redaction/fetch_classifiers"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def build_classifiers_url(
|
|
22
|
+
env: Optional[str] = None, classifiers_url: Optional[str] = None
|
|
23
|
+
) -> str:
|
|
24
|
+
"""Resolve the classifiers endpoint.
|
|
25
|
+
|
|
26
|
+
``classifiers_url`` (a full URL) wins; otherwise the environment alias
|
|
27
|
+
(``dev``/``qa``/``demo``/``poc`` or a full base URL) is resolved via the
|
|
28
|
+
SDK and the standard redaction path is appended.
|
|
29
|
+
"""
|
|
30
|
+
if classifiers_url:
|
|
31
|
+
return classifiers_url
|
|
32
|
+
base = resolve_api_base_url(env).rstrip("/")
|
|
33
|
+
return f"{base}{CLASSIFIERS_PATH}"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def fetch_rules(
|
|
37
|
+
env: Optional[str] = None,
|
|
38
|
+
classifiers_url: Optional[str] = None,
|
|
39
|
+
cache_ttl_seconds: int = 300,
|
|
40
|
+
timeout_seconds: float = 5.0,
|
|
41
|
+
verify_ssl: bool = True,
|
|
42
|
+
) -> Tuple[str, List[RedactionRule]]:
|
|
43
|
+
"""Fetch + compile the redaction rules. Returns ``(url, rules)``.
|
|
44
|
+
|
|
45
|
+
Fails open: on any fetch/parse error the SDK loader returns ``[]`` and we
|
|
46
|
+
propagate that (the caller decides whether to warn-and-copy or abort).
|
|
47
|
+
"""
|
|
48
|
+
url = build_classifiers_url(env, classifiers_url)
|
|
49
|
+
rules = load_redaction_rules_sync(
|
|
50
|
+
classifiers_url=url,
|
|
51
|
+
cache_ttl_seconds=cache_ttl_seconds,
|
|
52
|
+
timeout_seconds=timeout_seconds,
|
|
53
|
+
verify_ssl=verify_ssl,
|
|
54
|
+
)
|
|
55
|
+
return url, rules
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def fetch_raw_classifiers(
|
|
59
|
+
env: Optional[str] = None,
|
|
60
|
+
classifiers_url: Optional[str] = None,
|
|
61
|
+
timeout_seconds: float = 5.0,
|
|
62
|
+
verify_ssl: bool = True,
|
|
63
|
+
) -> Tuple[str, List[Any]]:
|
|
64
|
+
"""Fetch the raw classifier documents exactly as the API serves them, for
|
|
65
|
+
inspection/export (``dump-rules``). Returns ``(url, docs)``.
|
|
66
|
+
|
|
67
|
+
Unlike :func:`fetch_rules`, this returns the un-compiled JSON docs (regex
|
|
68
|
+
as a string, etc.) so a human can audit precisely which patterns run.
|
|
69
|
+
Raises on HTTP/parse errors (the caller reports them).
|
|
70
|
+
"""
|
|
71
|
+
url = build_classifiers_url(env, classifiers_url)
|
|
72
|
+
resp = requests.post(
|
|
73
|
+
url,
|
|
74
|
+
json={},
|
|
75
|
+
timeout=timeout_seconds,
|
|
76
|
+
verify=verify_ssl,
|
|
77
|
+
headers={"accept": "application/json", "content-type": "application/json"},
|
|
78
|
+
)
|
|
79
|
+
resp.raise_for_status()
|
|
80
|
+
payload = resp.json()
|
|
81
|
+
docs = payload.get("data", []) if isinstance(payload, dict) else []
|
|
82
|
+
if not isinstance(docs, list):
|
|
83
|
+
docs = []
|
|
84
|
+
return url, docs
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: chatsee-redact
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Local JSON/JSONL/CSV log redaction using ChatSee's redaction rules.
|
|
5
|
+
Maintainer: ChatSee
|
|
6
|
+
Maintainer-email: contact@chatsee.ai
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Requires-Python: >=3.7
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
Requires-Dist: chatsee-ai>=0.9.5
|
|
13
|
+
Dynamic: classifier
|
|
14
|
+
Dynamic: description
|
|
15
|
+
Dynamic: description-content-type
|
|
16
|
+
Dynamic: maintainer
|
|
17
|
+
Dynamic: maintainer-email
|
|
18
|
+
Dynamic: requires-dist
|
|
19
|
+
Dynamic: requires-python
|
|
20
|
+
Dynamic: summary
|
|
21
|
+
|
|
22
|
+
# chatsee-redact
|
|
23
|
+
|
|
24
|
+
Redact PII in your own **JSON / JSONL / CSV log files** locally, using ChatSee's
|
|
25
|
+
redaction rules.
|
|
26
|
+
|
|
27
|
+
**Your log content never leaves your machine.** The only network call this
|
|
28
|
+
tool makes is a *read-only* fetch of the regex classifier rules from ChatSee.
|
|
29
|
+
The raw and redacted content is read from, and written to, local files only —
|
|
30
|
+
there is no telemetry and nothing is uploaded.
|
|
31
|
+
|
|
32
|
+
Because it reuses the exact redaction engine shipped in the ChatSee SDK, the
|
|
33
|
+
masking you get here is identical to what the ChatSee pipeline would apply.
|
|
34
|
+
|
|
35
|
+
## Install
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install chatsee-redact
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
This pulls in `chatsee-ai` (the SDK), whose base install depends only on
|
|
42
|
+
`requests`.
|
|
43
|
+
|
|
44
|
+
## Usage
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
# Redact one file; writes ./redacted/app.json
|
|
48
|
+
chatsee-redact app.json --out ./redacted/
|
|
49
|
+
|
|
50
|
+
# Default output is alongside the input: app.redacted.json
|
|
51
|
+
chatsee-redact app.json
|
|
52
|
+
|
|
53
|
+
# Multiple files into one folder
|
|
54
|
+
chatsee-redact logs/*.jsonl --out ./redacted/
|
|
55
|
+
|
|
56
|
+
# Pick the environment whose rules to use (default: qa)
|
|
57
|
+
chatsee-redact app.json --env demo --out ./redacted/
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Input → output:
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
app.json --> ./redacted/app.json (all string values masked)
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## What gets redacted
|
|
67
|
+
|
|
68
|
+
By default **every string value** in the document is scanned and masked
|
|
69
|
+
(recursively, through nested objects and arrays). Non-string values
|
|
70
|
+
(numbers, booleans, null) are left untouched.
|
|
71
|
+
|
|
72
|
+
To restrict redaction to specific top-level keys:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
chatsee-redact app.json --fields user_message,bot_message,email
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Formats
|
|
79
|
+
|
|
80
|
+
| Input | Handling |
|
|
81
|
+
|-------|----------|
|
|
82
|
+
| `.json` | Whole-document: JSON array, object, or scalar. |
|
|
83
|
+
| `.jsonl` / `.ndjson` | JSON Lines — streamed one record per line (safe for large logs). |
|
|
84
|
+
| `.csv` | Per-row, streamed; header preserved. `--fields` selects columns. |
|
|
85
|
+
| other extension | Treated as JSONL. Override with `--format`. |
|
|
86
|
+
|
|
87
|
+
Force a format with `--format {auto,json,jsonl,csv}`.
|
|
88
|
+
|
|
89
|
+
## Inspecting the rules (`dump-rules`)
|
|
90
|
+
|
|
91
|
+
Fetch the raw classifier documents (the exact regexes that will run) so they
|
|
92
|
+
can be audited before you trust the tool:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
chatsee-redact dump-rules --env qa # prints the rules JSON to stdout
|
|
96
|
+
chatsee-redact dump-rules --env qa -o rules.json
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## Options
|
|
100
|
+
|
|
101
|
+
| Flag | Default | Description |
|
|
102
|
+
|------|---------|-------------|
|
|
103
|
+
| `-o, --out DIR` | alongside input | Output directory (created if missing). |
|
|
104
|
+
| `--out-file PATH` | — | Explicit output path (single input only). |
|
|
105
|
+
| `--env` | `qa` | Rule source environment: `dev`/`qa`/`demo`/`poc`. |
|
|
106
|
+
| `--classifiers-url` | — | Full classifiers URL (overrides `--env`). |
|
|
107
|
+
| `--fields A,B,C` | all strings | Restrict redaction to these top-level keys / CSV columns. |
|
|
108
|
+
| `--format` | `auto` | `auto` \| `json` \| `jsonl` \| `csv`. |
|
|
109
|
+
| `--indent N` | `2` | JSON output indent (`-1` = compact). |
|
|
110
|
+
| `--timeout` | `5` | Rule-fetch HTTP timeout (seconds). |
|
|
111
|
+
| `--cache-ttl` | `300` | In-process rule cache TTL (seconds). |
|
|
112
|
+
| `--no-verify-ssl` | off | Disable TLS verification for the rule fetch. |
|
|
113
|
+
| `--strict` | off | Abort (write nothing) if no rules could be fetched, instead of copying unredacted. |
|
|
114
|
+
| `-q, --quiet` | off | Only log warnings/errors. |
|
|
115
|
+
|
|
116
|
+
## Behaviour when rules can't be fetched
|
|
117
|
+
|
|
118
|
+
By default the tool **fails open** (matching the SDK): it warns loudly and
|
|
119
|
+
writes an *unredacted* copy so a transient outage doesn't break your job. Pass
|
|
120
|
+
`--strict` to instead **abort and write nothing** — recommended when the output
|
|
121
|
+
must be guaranteed redacted.
|
|
122
|
+
|
|
123
|
+
## Exit codes
|
|
124
|
+
|
|
125
|
+
| Code | Meaning |
|
|
126
|
+
|------|---------|
|
|
127
|
+
| `0` | Success. |
|
|
128
|
+
| `1` | One or more files failed to redact (I/O or parse error). |
|
|
129
|
+
| `2` | Bad invocation (missing input, refused source overwrite, `--out-file` with multiple inputs). |
|
|
130
|
+
| `3` | `--strict` and no rules available. |
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
chatsee_redact/__init__.py,sha256=xSv59TLRxew_sM7e2MLJrrzqvAllli2ee_pa5f49X-0,800
|
|
2
|
+
chatsee_redact/cli.py,sha256=e0BNlecGL2KK8Qavs4Zuy3AIGkofZ9O5QJehXENdUZE,8379
|
|
3
|
+
chatsee_redact/files.py,sha256=wBwKptat1CFx0CPSeGtrcf3X_-TMn1hyQxuwFLCh3TU,5764
|
|
4
|
+
chatsee_redact/rules.py,sha256=2L_G4OkL0ksm7MVdWd2lPHgrNIx-zur-0O78CVEqY0g,2813
|
|
5
|
+
chatsee_redact-0.2.0.dist-info/METADATA,sha256=RliR8SK8qPkdIRR08pxLV-uUBG-msEcqjEJcCjMHiLY,4674
|
|
6
|
+
chatsee_redact-0.2.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
7
|
+
chatsee_redact-0.2.0.dist-info/entry_points.txt,sha256=Ey36XbOA96d0XIY_Z4GeAk9t8FOdX2zCYD_0WUzRmPM,59
|
|
8
|
+
chatsee_redact-0.2.0.dist-info/top_level.txt,sha256=SBakjZGCAsBFtxE36aGhW3maOC5qZsL6Q-K9D4vUqlE,15
|
|
9
|
+
chatsee_redact-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
chatsee_redact
|