ciphergate-cli 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ciphergate/__init__.py +95 -0
- ciphergate/__main__.py +6 -0
- ciphergate/cli.py +832 -0
- ciphergate/config.py +50 -0
- ciphergate/constants.py +274 -0
- ciphergate/detectors/__init__.py +5 -0
- ciphergate/detectors/base.py +53 -0
- ciphergate/detectors/custom_regex.py +37 -0
- ciphergate/detectors/dates.py +92 -0
- ciphergate/detectors/email.py +56 -0
- ciphergate/detectors/field_names.py +220 -0
- ciphergate/detectors/ip_address.py +85 -0
- ciphergate/detectors/network.py +42 -0
- ciphergate/detectors/payment_card.py +77 -0
- ciphergate/detectors/phone.py +67 -0
- ciphergate/detectors/presidio_ner.py +112 -0
- ciphergate/detectors/regex_safety.py +40 -0
- ciphergate/detectors/registry.py +194 -0
- ciphergate/detectors/secrets.py +130 -0
- ciphergate/errors.py +119 -0
- ciphergate/models/__init__.py +1 -0
- ciphergate/models/context.py +69 -0
- ciphergate/models/finding.py +108 -0
- ciphergate/models/policy.py +402 -0
- ciphergate/models/report.py +115 -0
- ciphergate/pipeline/__init__.py +1 -0
- ciphergate/pipeline/atomic_output.py +143 -0
- ciphergate/pipeline/batch.py +533 -0
- ciphergate/pipeline/inspector.py +282 -0
- ciphergate/pipeline/paths.py +98 -0
- ciphergate/pipeline/planner.py +192 -0
- ciphergate/pipeline/rescan.py +120 -0
- ciphergate/pipeline/sanitizer.py +392 -0
- ciphergate/pipeline/stdio.py +173 -0
- ciphergate/pipeline/streams.py +152 -0
- ciphergate/policy/__init__.py +26 -0
- ciphergate/policy/context.py +203 -0
- ciphergate/policy/engine.py +194 -0
- ciphergate/policy/loader.py +116 -0
- ciphergate/policy/precedence.py +149 -0
- ciphergate/policy/validator.py +632 -0
- ciphergate/py.typed +0 -0
- ciphergate/readers/__init__.py +113 -0
- ciphergate/readers/accesslog_reader.py +244 -0
- ciphergate/readers/base.py +131 -0
- ciphergate/readers/csv_reader.py +241 -0
- ciphergate/readers/json_reader.py +184 -0
- ciphergate/readers/jsonl_reader.py +97 -0
- ciphergate/readers/jsonlog_reader.py +178 -0
- ciphergate/readers/line_structure.py +125 -0
- ciphergate/readers/logfmt_reader.py +100 -0
- ciphergate/readers/stacktrace_reader.py +103 -0
- ciphergate/readers/syslog_reader.py +309 -0
- ciphergate/readers/text_reader.py +59 -0
- ciphergate/reporting/__init__.py +1 -0
- ciphergate/reporting/audit.py +85 -0
- ciphergate/reporting/safe_text.py +71 -0
- ciphergate/reporting/serializer.py +44 -0
- ciphergate/reporting/terminal.py +147 -0
- ciphergate/templates/starter_policy.yml +77 -0
- ciphergate/transforms/__init__.py +6 -0
- ciphergate/transforms/date_shift.py +79 -0
- ciphergate/transforms/generalize.py +107 -0
- ciphergate/transforms/keys.py +177 -0
- ciphergate/transforms/pseudonymize.py +113 -0
- ciphergate/transforms/redact.py +9 -0
- ciphergate/transforms/registry.py +201 -0
- ciphergate/transforms/replace.py +16 -0
- ciphergate_cli-0.7.0.dist-info/METADATA +531 -0
- ciphergate_cli-0.7.0.dist-info/RECORD +74 -0
- ciphergate_cli-0.7.0.dist-info/WHEEL +4 -0
- ciphergate_cli-0.7.0.dist-info/entry_points.txt +2 -0
- ciphergate_cli-0.7.0.dist-info/licenses/LICENSE +202 -0
- ciphergate_cli-0.7.0.dist-info/licenses/NOTICE +4 -0
ciphergate/cli.py
ADDED
|
@@ -0,0 +1,832 @@
|
|
|
1
|
+
"""``ciphergate`` command-line interface.
|
|
2
|
+
|
|
3
|
+
Output contract: human summaries go to stdout (suppressed by ``--quiet``); when ``--report-format
|
|
4
|
+
json`` is used without ``--report``, stdout carries only the JSON report. Errors go to stderr as
|
|
5
|
+
``error[<code>]: <safe message>``. Exit codes are documented in README.md and ``ExitCode``.
|
|
6
|
+
|
|
7
|
+
``-`` as INPUT reads stdin (``--format`` required); ``sanitize --output -`` writes the sanitized
|
|
8
|
+
data to stdout only after the run succeeded, and the summary then goes to stderr so stdout
|
|
9
|
+
carries data only. See ``ciphergate.pipeline.stdio``.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import contextlib
|
|
15
|
+
import json
|
|
16
|
+
import logging
|
|
17
|
+
import os
|
|
18
|
+
import signal
|
|
19
|
+
import sys
|
|
20
|
+
import traceback
|
|
21
|
+
from collections.abc import Callable
|
|
22
|
+
from enum import StrEnum
|
|
23
|
+
from importlib import resources
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from types import FrameType
|
|
26
|
+
from typing import Annotated, Any, TextIO
|
|
27
|
+
|
|
28
|
+
import typer
|
|
29
|
+
|
|
30
|
+
from ciphergate import __version__
|
|
31
|
+
from ciphergate.config import RuntimeSettings
|
|
32
|
+
from ciphergate.constants import (
|
|
33
|
+
Destination,
|
|
34
|
+
DomainProfile,
|
|
35
|
+
Environment,
|
|
36
|
+
ExitCode,
|
|
37
|
+
InputFormat,
|
|
38
|
+
TargetEnvironment,
|
|
39
|
+
UseCase,
|
|
40
|
+
)
|
|
41
|
+
from ciphergate.errors import (
|
|
42
|
+
CipherGateError,
|
|
43
|
+
OutputWriteError,
|
|
44
|
+
PolicyValidationError,
|
|
45
|
+
UnsafePathError,
|
|
46
|
+
UsageError,
|
|
47
|
+
)
|
|
48
|
+
from ciphergate.models.context import ContextInput
|
|
49
|
+
from ciphergate.models.finding import Finding, Location
|
|
50
|
+
from ciphergate.models.report import RunReport
|
|
51
|
+
from ciphergate.pipeline.atomic_output import AtomicOutput
|
|
52
|
+
from ciphergate.pipeline.batch import (
|
|
53
|
+
BatchReport,
|
|
54
|
+
analyze_directory,
|
|
55
|
+
is_inside,
|
|
56
|
+
sanitize_directory,
|
|
57
|
+
)
|
|
58
|
+
from ciphergate.pipeline.paths import validate_output_path
|
|
59
|
+
from ciphergate.pipeline.sanitizer import (
|
|
60
|
+
SanitizeOptions,
|
|
61
|
+
inspect_file,
|
|
62
|
+
plan_file,
|
|
63
|
+
rescan_file,
|
|
64
|
+
sanitize_file,
|
|
65
|
+
)
|
|
66
|
+
from ciphergate.pipeline.stdio import (
|
|
67
|
+
STDIN_DISPLAY,
|
|
68
|
+
STDOUT_DISPLAY,
|
|
69
|
+
StdioWorkspace,
|
|
70
|
+
copy_to_stdout,
|
|
71
|
+
is_stdio,
|
|
72
|
+
require_stdin_format,
|
|
73
|
+
silence_stdout,
|
|
74
|
+
)
|
|
75
|
+
from ciphergate.policy.context import evaluate_gates, resolve_context
|
|
76
|
+
from ciphergate.policy.engine import PolicyEngine
|
|
77
|
+
from ciphergate.policy.validator import load_policy, validate_policy
|
|
78
|
+
from ciphergate.reporting.safe_text import escape_control
|
|
79
|
+
from ciphergate.reporting.serializer import (
|
|
80
|
+
batch_report_to_json,
|
|
81
|
+
report_to_json,
|
|
82
|
+
write_batch_report_file,
|
|
83
|
+
write_report_file,
|
|
84
|
+
)
|
|
85
|
+
from ciphergate.reporting.terminal import render_batch_summary, render_summary
|
|
86
|
+
|
|
87
|
+
logger = logging.getLogger("ciphergate")
|
|
88
|
+
|
|
89
|
+
app = typer.Typer(
|
|
90
|
+
name="ciphergate",
|
|
91
|
+
help="Inspect and sanitize logs and test datasets before they leave their environment.\n\n"
|
|
92
|
+
"Workflow: validate-policy -> inspect -> plan -> sanitize --apply -> rescan. "
|
|
93
|
+
"A passing run is not proof that no sensitive data remains.",
|
|
94
|
+
add_completion=False,
|
|
95
|
+
no_args_is_help=True,
|
|
96
|
+
# Rich tracebacks can print local variables (i.e. data). Never enable them.
|
|
97
|
+
pretty_exceptions_enable=False,
|
|
98
|
+
rich_markup_mode=None,
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class ReportFormat(StrEnum):
|
|
103
|
+
JSON = "json"
|
|
104
|
+
TEXT = "text"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class PathDisplayChoice(StrEnum):
|
|
108
|
+
BASENAME = "basename"
|
|
109
|
+
REDACTED = "redacted"
|
|
110
|
+
FULL = "full"
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# --------------------------------------------------------------------------- shared options
|
|
114
|
+
|
|
115
|
+
PolicyOpt = Annotated[Path, typer.Option("--policy", "-p", help="Policy YAML file.")]
|
|
116
|
+
FormatOpt = Annotated[
|
|
117
|
+
InputFormat | None, typer.Option("--format", help="Input format (default: from extension).")
|
|
118
|
+
]
|
|
119
|
+
ReportOpt = Annotated[
|
|
120
|
+
Path | None, typer.Option("--report", help="Write the privacy-safe report to this file.")
|
|
121
|
+
]
|
|
122
|
+
ReportFormatOpt = Annotated[
|
|
123
|
+
ReportFormat | None,
|
|
124
|
+
typer.Option("--report-format", help="Report format: json or text (default json for files)."),
|
|
125
|
+
]
|
|
126
|
+
EnvOpt = Annotated[
|
|
127
|
+
Environment | None,
|
|
128
|
+
typer.Option(
|
|
129
|
+
"--environment",
|
|
130
|
+
"--source-environment",
|
|
131
|
+
help="Where the source data came from (dev|stg|prd|unknown). Declared, not verified.",
|
|
132
|
+
),
|
|
133
|
+
]
|
|
134
|
+
TargetOpt = Annotated[
|
|
135
|
+
TargetEnvironment | None,
|
|
136
|
+
typer.Option("--target-environment", help="Environment receiving the output."),
|
|
137
|
+
]
|
|
138
|
+
DestOpt = Annotated[
|
|
139
|
+
Destination | None,
|
|
140
|
+
typer.Option("--destination", help="Recipient/channel of the output (distinct from env)."),
|
|
141
|
+
]
|
|
142
|
+
UseCaseOpt = Annotated[UseCase | None, typer.Option("--use-case", help="Purpose of the output.")]
|
|
143
|
+
ProfileOpt = Annotated[
|
|
144
|
+
DomainProfile | None, typer.Option("--profile", help="Domain profile of the data.")
|
|
145
|
+
]
|
|
146
|
+
MaxBytesOpt = Annotated[
|
|
147
|
+
int | None, typer.Option("--max-bytes", min=1, help="Maximum input size in bytes.")
|
|
148
|
+
]
|
|
149
|
+
EncodingOpt = Annotated[
|
|
150
|
+
str | None, typer.Option("--encoding", help="Input encoding (ASCII-compatible; default utf-8).")
|
|
151
|
+
]
|
|
152
|
+
PathDisplayOpt = Annotated[
|
|
153
|
+
PathDisplayChoice | None,
|
|
154
|
+
typer.Option("--path-display", help="How file paths appear in reports (default basename)."),
|
|
155
|
+
]
|
|
156
|
+
QuietOpt = Annotated[bool, typer.Option("--quiet", "-q", help="No summary on stdout.")]
|
|
157
|
+
VerboseOpt = Annotated[bool, typer.Option("--verbose", "-v", help="List every reported item.")]
|
|
158
|
+
NoColorOpt = Annotated[bool, typer.Option("--no-color", help="Accepted for CI; output is plain.")]
|
|
159
|
+
DebugOpt = Annotated[
|
|
160
|
+
bool, typer.Option("--debug", help="On internal errors show frames (never values/messages).")
|
|
161
|
+
]
|
|
162
|
+
ForceOpt = Annotated[bool, typer.Option("--force", help="Replace an existing output/report file.")]
|
|
163
|
+
RecursiveOpt = Annotated[
|
|
164
|
+
bool,
|
|
165
|
+
typer.Option("--recursive", "-r", help="With a directory INPUT, also process subdirectories."),
|
|
166
|
+
]
|
|
167
|
+
MaxFilesOpt = Annotated[
|
|
168
|
+
int | None,
|
|
169
|
+
typer.Option("--max-files", min=1, help="Maximum files in a directory run (default 1000)."),
|
|
170
|
+
]
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _context(
|
|
174
|
+
environment: Environment | None,
|
|
175
|
+
target: TargetEnvironment | None,
|
|
176
|
+
destination: Destination | None,
|
|
177
|
+
use_case: UseCase | None,
|
|
178
|
+
profile: DomainProfile | None,
|
|
179
|
+
) -> ContextInput:
|
|
180
|
+
return ContextInput(
|
|
181
|
+
environment=environment,
|
|
182
|
+
target_environment=target,
|
|
183
|
+
destination=destination,
|
|
184
|
+
domain_profile=profile,
|
|
185
|
+
use_case=use_case,
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _runtime(
|
|
190
|
+
max_bytes: int | None,
|
|
191
|
+
encoding: str | None,
|
|
192
|
+
path_display: PathDisplayChoice | None,
|
|
193
|
+
max_files: int | None = None,
|
|
194
|
+
) -> RuntimeSettings:
|
|
195
|
+
try:
|
|
196
|
+
return RuntimeSettings().with_overrides(
|
|
197
|
+
max_input_bytes=max_bytes,
|
|
198
|
+
encoding=encoding,
|
|
199
|
+
path_display=str(path_display) if path_display else None,
|
|
200
|
+
max_batch_files=max_files,
|
|
201
|
+
)
|
|
202
|
+
except ValueError as exc:
|
|
203
|
+
raise UsageError(str(exc), code="invalid_runtime_setting") from None
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _err(message: str) -> None:
|
|
207
|
+
sys.stderr.write(escape_control(message)[:2000] + "\n")
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _safe_traceback(exc: BaseException) -> str:
|
|
211
|
+
frames = traceback.extract_tb(exc.__traceback__)
|
|
212
|
+
lines = [f" {Path(f.filename).name}:{f.lineno} in {f.name}" for f in frames]
|
|
213
|
+
return f"{type(exc).__name__} (message suppressed)\n" + "\n".join(lines)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _guard(debug: bool, body: Callable[[], int]) -> None:
|
|
217
|
+
try:
|
|
218
|
+
code = body()
|
|
219
|
+
except PolicyValidationError as exc:
|
|
220
|
+
_err(f"error[{exc.code}]: {exc.message}")
|
|
221
|
+
for issue in exc.issues:
|
|
222
|
+
_err(f" - {issue.render()}")
|
|
223
|
+
raise typer.Exit(int(exc.exit_code)) from None
|
|
224
|
+
except CipherGateError as exc:
|
|
225
|
+
_err(f"error[{exc.code}]: {exc.message}")
|
|
226
|
+
raise typer.Exit(int(exc.exit_code)) from None
|
|
227
|
+
except KeyboardInterrupt:
|
|
228
|
+
_err("error[interrupted]: operation interrupted; no output was published")
|
|
229
|
+
raise typer.Exit(int(ExitCode.INTERRUPTED)) from None
|
|
230
|
+
except BrokenPipeError:
|
|
231
|
+
silence_stdout()
|
|
232
|
+
_err("error[stdout_broken_pipe]: stdout was closed before all output was written")
|
|
233
|
+
raise typer.Exit(int(ExitCode.OUTPUT_ERROR)) from None
|
|
234
|
+
except (typer.Exit, typer.Abort):
|
|
235
|
+
raise
|
|
236
|
+
except Exception as exc: # noqa: BLE001 - last-resort safe rendering
|
|
237
|
+
_err("error[internal_error]: unexpected internal error (no data was published)")
|
|
238
|
+
if debug:
|
|
239
|
+
_err(_safe_traceback(exc))
|
|
240
|
+
raise typer.Exit(int(ExitCode.INTERNAL)) from None
|
|
241
|
+
raise typer.Exit(code)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _emit(
|
|
245
|
+
report: RunReport,
|
|
246
|
+
*,
|
|
247
|
+
report_path: Path | None,
|
|
248
|
+
report_format: ReportFormat | None,
|
|
249
|
+
quiet: bool,
|
|
250
|
+
verbose: bool,
|
|
251
|
+
force: bool,
|
|
252
|
+
summary_stream: TextIO | None = None,
|
|
253
|
+
) -> int:
|
|
254
|
+
summary = summary_stream or sys.stdout
|
|
255
|
+
if report_path is not None:
|
|
256
|
+
write_report_file(report, report_path, str(report_format or ReportFormat.JSON), force=force)
|
|
257
|
+
if not quiet:
|
|
258
|
+
summary.write(render_summary(report, verbose=verbose))
|
|
259
|
+
elif report_format == ReportFormat.JSON:
|
|
260
|
+
sys.stdout.write(report_to_json(report))
|
|
261
|
+
elif not quiet:
|
|
262
|
+
summary.write(render_summary(report, verbose=verbose))
|
|
263
|
+
return report.exit_code
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _emit_batch(
|
|
267
|
+
report: BatchReport,
|
|
268
|
+
*,
|
|
269
|
+
report_path: Path | None,
|
|
270
|
+
report_format: ReportFormat | None,
|
|
271
|
+
quiet: bool,
|
|
272
|
+
verbose: bool,
|
|
273
|
+
force: bool,
|
|
274
|
+
) -> int:
|
|
275
|
+
if report_path is not None:
|
|
276
|
+
write_batch_report_file(
|
|
277
|
+
report, report_path, str(report_format or ReportFormat.JSON), force=force
|
|
278
|
+
)
|
|
279
|
+
if not quiet:
|
|
280
|
+
sys.stdout.write(render_batch_summary(report, verbose=verbose))
|
|
281
|
+
elif report_format == ReportFormat.JSON:
|
|
282
|
+
sys.stdout.write(batch_report_to_json(report))
|
|
283
|
+
elif not quiet:
|
|
284
|
+
sys.stdout.write(render_batch_summary(report, verbose=verbose))
|
|
285
|
+
return report.exit_code
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _is_directory_input(path: Path) -> bool:
|
|
289
|
+
return not is_stdio(path) and os.path.isdir(path)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _check_batch_report_path(report: Path | None, directories: list[Path], force: bool) -> None:
|
|
293
|
+
_check_report_path(report, [], force)
|
|
294
|
+
if report is not None and any(is_inside(report, d) for d in directories):
|
|
295
|
+
raise UnsafePathError(
|
|
296
|
+
"--report must not be inside the input or output directory",
|
|
297
|
+
code="report_inside_batch_directory",
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _no_recursive(recursive: bool) -> None:
|
|
302
|
+
if recursive:
|
|
303
|
+
raise UsageError("--recursive needs a directory INPUT", code="recursive_needs_directory")
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _check_report_path(report: Path | None, protected: list[Path], force: bool) -> None:
|
|
307
|
+
if report is None:
|
|
308
|
+
return
|
|
309
|
+
if is_stdio(report):
|
|
310
|
+
raise UsageError(
|
|
311
|
+
"--report - is not supported; omit --report and pass --report-format json "
|
|
312
|
+
"to print the JSON report on stdout",
|
|
313
|
+
code="report_stdout_unsupported",
|
|
314
|
+
)
|
|
315
|
+
validate_output_path(report, protected=protected, force=force, label="report")
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _configure_logging(verbose: bool, debug: bool) -> None:
|
|
319
|
+
level = logging.DEBUG if debug else (logging.INFO if verbose else logging.WARNING)
|
|
320
|
+
logging.basicConfig(
|
|
321
|
+
level=level, format="ciphergate: %(levelname)s: %(message)s", stream=sys.stderr
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
# --------------------------------------------------------------------------- commands
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _version_callback(value: bool) -> None:
|
|
329
|
+
if value:
|
|
330
|
+
sys.stdout.write(f"ciphergate {__version__}\n")
|
|
331
|
+
raise typer.Exit(0)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
@app.callback()
|
|
335
|
+
def _root(
|
|
336
|
+
version: Annotated[
|
|
337
|
+
bool,
|
|
338
|
+
typer.Option(
|
|
339
|
+
"--version",
|
|
340
|
+
callback=_version_callback,
|
|
341
|
+
is_eager=True,
|
|
342
|
+
help="Show the version and exit.",
|
|
343
|
+
),
|
|
344
|
+
] = False,
|
|
345
|
+
) -> None:
|
|
346
|
+
"""CipherGate CLI."""
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
@app.command("init")
|
|
350
|
+
def init_command(
|
|
351
|
+
output: Annotated[
|
|
352
|
+
Path, typer.Option("--output", "-o", help="Where to write the policy.")
|
|
353
|
+
] = Path("ciphergate-policy.yml"),
|
|
354
|
+
force: ForceOpt = False,
|
|
355
|
+
quiet: QuietOpt = False,
|
|
356
|
+
debug: DebugOpt = False,
|
|
357
|
+
) -> None:
|
|
358
|
+
"""Write a starter policy template (it MUST be reviewed before real use)."""
|
|
359
|
+
|
|
360
|
+
def body() -> int:
|
|
361
|
+
validate_output_path(output, protected=[], force=force, label="policy")
|
|
362
|
+
text = (
|
|
363
|
+
resources.files("ciphergate")
|
|
364
|
+
.joinpath("templates/starter_policy.yml")
|
|
365
|
+
.read_text(encoding="utf-8")
|
|
366
|
+
)
|
|
367
|
+
with AtomicOutput(output, force=force) as out:
|
|
368
|
+
out.stream.write(text)
|
|
369
|
+
out.publish()
|
|
370
|
+
if not quiet:
|
|
371
|
+
sys.stdout.write(
|
|
372
|
+
f"wrote starter policy to {escape_control(output.name)}\n"
|
|
373
|
+
"WARNING: this is a template. Review every rule, context value and destination "
|
|
374
|
+
"for your data before relying on it. It is not legal or compliance advice.\n"
|
|
375
|
+
)
|
|
376
|
+
return 0
|
|
377
|
+
|
|
378
|
+
_guard(debug, body)
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
@app.command("validate-policy")
|
|
382
|
+
def validate_policy_command(
|
|
383
|
+
policy: Annotated[Path, typer.Argument(help="Policy YAML file.")],
|
|
384
|
+
report_format: ReportFormatOpt = None,
|
|
385
|
+
quiet: QuietOpt = False,
|
|
386
|
+
debug: DebugOpt = False,
|
|
387
|
+
) -> None:
|
|
388
|
+
"""Validate a policy's schema and semantics. Never reads any dataset."""
|
|
389
|
+
|
|
390
|
+
def body() -> int:
|
|
391
|
+
result = validate_policy(policy)
|
|
392
|
+
if report_format == ReportFormat.JSON:
|
|
393
|
+
payload: dict[str, Any] = {
|
|
394
|
+
"valid": result.valid,
|
|
395
|
+
"policy": result.policy.name if result.policy else None,
|
|
396
|
+
"digest": result.policy.digest if result.policy else None,
|
|
397
|
+
"errors": [
|
|
398
|
+
{"location": i.location, "message": i.message, "code": i.code}
|
|
399
|
+
for i in result.errors
|
|
400
|
+
],
|
|
401
|
+
"warnings": list(result.warnings),
|
|
402
|
+
}
|
|
403
|
+
sys.stdout.write(json.dumps(payload, indent=2) + "\n")
|
|
404
|
+
elif result.valid and result.policy is not None:
|
|
405
|
+
if not quiet:
|
|
406
|
+
sys.stdout.write(
|
|
407
|
+
f"policy '{escape_control(result.policy.name)}' is valid "
|
|
408
|
+
f"({result.policy.digest[:19]}...)\n"
|
|
409
|
+
)
|
|
410
|
+
for warning in result.warnings:
|
|
411
|
+
sys.stdout.write(f" warning: {escape_control(warning)}\n")
|
|
412
|
+
else:
|
|
413
|
+
_err("error[policy_invalid]: policy failed validation")
|
|
414
|
+
for issue in result.errors:
|
|
415
|
+
_err(f" - {issue.render()}")
|
|
416
|
+
return 0 if result.valid else int(ExitCode.POLICY_INVALID)
|
|
417
|
+
|
|
418
|
+
_guard(debug, body)
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _run_analysis_command(
|
|
422
|
+
command: str,
|
|
423
|
+
input_path: Path,
|
|
424
|
+
policy: Path,
|
|
425
|
+
fmt: InputFormat | None,
|
|
426
|
+
report: Path | None,
|
|
427
|
+
report_format: ReportFormat | None,
|
|
428
|
+
ctx: ContextInput,
|
|
429
|
+
runtime: RuntimeSettings,
|
|
430
|
+
quiet: bool,
|
|
431
|
+
verbose: bool,
|
|
432
|
+
force: bool,
|
|
433
|
+
strict: bool | None = None,
|
|
434
|
+
recursive: bool = False,
|
|
435
|
+
) -> int:
|
|
436
|
+
if _is_directory_input(input_path):
|
|
437
|
+
_check_batch_report_path(report, [input_path], force)
|
|
438
|
+
batch = analyze_directory(
|
|
439
|
+
command,
|
|
440
|
+
input_path,
|
|
441
|
+
load_policy(policy),
|
|
442
|
+
ctx,
|
|
443
|
+
recursive=recursive,
|
|
444
|
+
runtime=runtime,
|
|
445
|
+
fmt=fmt,
|
|
446
|
+
strict=strict,
|
|
447
|
+
)
|
|
448
|
+
return _emit_batch(
|
|
449
|
+
batch,
|
|
450
|
+
report_path=report,
|
|
451
|
+
report_format=report_format,
|
|
452
|
+
quiet=quiet,
|
|
453
|
+
verbose=verbose,
|
|
454
|
+
force=force,
|
|
455
|
+
)
|
|
456
|
+
_no_recursive(recursive)
|
|
457
|
+
stdin = is_stdio(input_path)
|
|
458
|
+
if stdin:
|
|
459
|
+
fmt = require_stdin_format(fmt, sys.stdin.buffer)
|
|
460
|
+
_check_report_path(report, [] if stdin else [input_path], force)
|
|
461
|
+
loaded = load_policy(policy)
|
|
462
|
+
with StdioWorkspace() if stdin else contextlib.nullcontext() as ws:
|
|
463
|
+
path = input_path
|
|
464
|
+
if ws is not None and fmt is not None:
|
|
465
|
+
path = ws.spool_input(sys.stdin.buffer, fmt, runtime.max_input_bytes)
|
|
466
|
+
if command == "inspect":
|
|
467
|
+
result = inspect_file(path, loaded, ctx, runtime=runtime, fmt=fmt)
|
|
468
|
+
elif command == "plan":
|
|
469
|
+
result = plan_file(path, loaded, ctx, runtime=runtime, fmt=fmt)
|
|
470
|
+
else:
|
|
471
|
+
result = rescan_file(path, loaded, ctx, strict=strict, runtime=runtime, fmt=fmt)
|
|
472
|
+
if stdin:
|
|
473
|
+
result.input["path"] = STDIN_DISPLAY
|
|
474
|
+
return _emit(
|
|
475
|
+
result,
|
|
476
|
+
report_path=report,
|
|
477
|
+
report_format=report_format,
|
|
478
|
+
quiet=quiet,
|
|
479
|
+
verbose=verbose,
|
|
480
|
+
force=force,
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
@app.command("inspect")
|
|
485
|
+
def inspect_command(
|
|
486
|
+
input_path: Annotated[
|
|
487
|
+
Path, typer.Argument(metavar="INPUT", help="File to inspect, a directory, or - for stdin.")
|
|
488
|
+
],
|
|
489
|
+
policy: PolicyOpt,
|
|
490
|
+
fmt: FormatOpt = None,
|
|
491
|
+
report: ReportOpt = None,
|
|
492
|
+
report_format: ReportFormatOpt = None,
|
|
493
|
+
environment: EnvOpt = None,
|
|
494
|
+
target_environment: TargetOpt = None,
|
|
495
|
+
destination: DestOpt = None,
|
|
496
|
+
use_case: UseCaseOpt = None,
|
|
497
|
+
profile: ProfileOpt = None,
|
|
498
|
+
max_bytes: MaxBytesOpt = None,
|
|
499
|
+
encoding: EncodingOpt = None,
|
|
500
|
+
path_display: PathDisplayOpt = None,
|
|
501
|
+
recursive: RecursiveOpt = False,
|
|
502
|
+
max_files: MaxFilesOpt = None,
|
|
503
|
+
force: ForceOpt = False,
|
|
504
|
+
quiet: QuietOpt = False,
|
|
505
|
+
verbose: VerboseOpt = False,
|
|
506
|
+
no_color: NoColorOpt = False,
|
|
507
|
+
debug: DebugOpt = False,
|
|
508
|
+
) -> None:
|
|
509
|
+
"""Detect and report findings. Never modifies the input or writes sanitized data."""
|
|
510
|
+
_configure_logging(verbose, debug)
|
|
511
|
+
_guard(
|
|
512
|
+
debug,
|
|
513
|
+
lambda: _run_analysis_command(
|
|
514
|
+
"inspect",
|
|
515
|
+
input_path,
|
|
516
|
+
policy,
|
|
517
|
+
fmt,
|
|
518
|
+
report,
|
|
519
|
+
report_format,
|
|
520
|
+
_context(environment, target_environment, destination, use_case, profile),
|
|
521
|
+
_runtime(max_bytes, encoding, path_display, max_files),
|
|
522
|
+
quiet,
|
|
523
|
+
verbose,
|
|
524
|
+
force,
|
|
525
|
+
recursive=recursive,
|
|
526
|
+
),
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
@app.command("plan")
|
|
531
|
+
def plan_command(
|
|
532
|
+
input_path: Annotated[
|
|
533
|
+
Path, typer.Argument(metavar="INPUT", help="File to plan for, a directory, or - for stdin.")
|
|
534
|
+
],
|
|
535
|
+
policy: PolicyOpt,
|
|
536
|
+
fmt: FormatOpt = None,
|
|
537
|
+
report: ReportOpt = None,
|
|
538
|
+
report_format: ReportFormatOpt = None,
|
|
539
|
+
environment: EnvOpt = None,
|
|
540
|
+
target_environment: TargetOpt = None,
|
|
541
|
+
destination: DestOpt = None,
|
|
542
|
+
use_case: UseCaseOpt = None,
|
|
543
|
+
profile: ProfileOpt = None,
|
|
544
|
+
max_bytes: MaxBytesOpt = None,
|
|
545
|
+
encoding: EncodingOpt = None,
|
|
546
|
+
path_display: PathDisplayOpt = None,
|
|
547
|
+
recursive: RecursiveOpt = False,
|
|
548
|
+
max_files: MaxFilesOpt = None,
|
|
549
|
+
force: ForceOpt = False,
|
|
550
|
+
quiet: QuietOpt = False,
|
|
551
|
+
verbose: VerboseOpt = False,
|
|
552
|
+
no_color: NoColorOpt = False,
|
|
553
|
+
debug: DebugOpt = False,
|
|
554
|
+
) -> None:
|
|
555
|
+
"""Preview the decisions sanitize --apply would make. Writes no data file."""
|
|
556
|
+
_configure_logging(verbose, debug)
|
|
557
|
+
_guard(
|
|
558
|
+
debug,
|
|
559
|
+
lambda: _run_analysis_command(
|
|
560
|
+
"plan",
|
|
561
|
+
input_path,
|
|
562
|
+
policy,
|
|
563
|
+
fmt,
|
|
564
|
+
report,
|
|
565
|
+
report_format,
|
|
566
|
+
_context(environment, target_environment, destination, use_case, profile),
|
|
567
|
+
_runtime(max_bytes, encoding, path_display, max_files),
|
|
568
|
+
quiet,
|
|
569
|
+
verbose,
|
|
570
|
+
force,
|
|
571
|
+
recursive=recursive,
|
|
572
|
+
),
|
|
573
|
+
)
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
@app.command("sanitize")
|
|
577
|
+
def sanitize_command(
|
|
578
|
+
input_path: Annotated[
|
|
579
|
+
Path, typer.Argument(metavar="INPUT", help="File or directory to sanitize, or - for stdin.")
|
|
580
|
+
],
|
|
581
|
+
policy: PolicyOpt,
|
|
582
|
+
output: Annotated[
|
|
583
|
+
Path,
|
|
584
|
+
typer.Option(
|
|
585
|
+
"--output",
|
|
586
|
+
"-o",
|
|
587
|
+
help="Output file (a new directory for a directory INPUT), or - for stdout.",
|
|
588
|
+
),
|
|
589
|
+
],
|
|
590
|
+
apply: Annotated[
|
|
591
|
+
bool, typer.Option("--apply", help="Actually write output (else plan).")
|
|
592
|
+
] = False,
|
|
593
|
+
fmt: FormatOpt = None,
|
|
594
|
+
report: ReportOpt = None,
|
|
595
|
+
report_format: ReportFormatOpt = None,
|
|
596
|
+
environment: EnvOpt = None,
|
|
597
|
+
target_environment: TargetOpt = None,
|
|
598
|
+
destination: DestOpt = None,
|
|
599
|
+
use_case: UseCaseOpt = None,
|
|
600
|
+
profile: ProfileOpt = None,
|
|
601
|
+
max_bytes: MaxBytesOpt = None,
|
|
602
|
+
encoding: EncodingOpt = None,
|
|
603
|
+
path_display: PathDisplayOpt = None,
|
|
604
|
+
recursive: RecursiveOpt = False,
|
|
605
|
+
max_files: MaxFilesOpt = None,
|
|
606
|
+
force: ForceOpt = False,
|
|
607
|
+
quiet: QuietOpt = False,
|
|
608
|
+
verbose: VerboseOpt = False,
|
|
609
|
+
no_color: NoColorOpt = False,
|
|
610
|
+
debug: DebugOpt = False,
|
|
611
|
+
) -> None:
|
|
612
|
+
"""Apply the policy and atomically write OUTPUT. Requires --apply; refuses to overwrite."""
|
|
613
|
+
_configure_logging(verbose, debug)
|
|
614
|
+
|
|
615
|
+
def body() -> int:
|
|
616
|
+
if _is_directory_input(input_path):
|
|
617
|
+
return sanitize_batch()
|
|
618
|
+
_no_recursive(recursive)
|
|
619
|
+
stdin, to_stdout = is_stdio(input_path), is_stdio(output)
|
|
620
|
+
selected_fmt = require_stdin_format(fmt, sys.stdin.buffer) if stdin else fmt
|
|
621
|
+
if to_stdout and report is None and report_format == ReportFormat.JSON:
|
|
622
|
+
raise UsageError(
|
|
623
|
+
"--output - uses stdout for data; write the JSON report with --report PATH",
|
|
624
|
+
code="stdout_conflict",
|
|
625
|
+
)
|
|
626
|
+
runtime = _runtime(max_bytes, encoding, path_display)
|
|
627
|
+
_check_report_path(report, [p for p in (input_path, output) if not is_stdio(p)], force)
|
|
628
|
+
loaded = load_policy(policy)
|
|
629
|
+
options = SanitizeOptions(apply=apply, force=force, format=selected_fmt, runtime=runtime)
|
|
630
|
+
with StdioWorkspace() if stdin or to_stdout else contextlib.nullcontext() as ws:
|
|
631
|
+
in_path, out_path = input_path, output
|
|
632
|
+
if ws is not None and stdin and selected_fmt is not None:
|
|
633
|
+
in_path = ws.spool_input(sys.stdin.buffer, selected_fmt, runtime.max_input_bytes)
|
|
634
|
+
if ws is not None and to_stdout:
|
|
635
|
+
out_path = ws.output_path()
|
|
636
|
+
result = sanitize_file(
|
|
637
|
+
in_path,
|
|
638
|
+
out_path,
|
|
639
|
+
loaded,
|
|
640
|
+
_context(environment, target_environment, destination, use_case, profile),
|
|
641
|
+
options,
|
|
642
|
+
)
|
|
643
|
+
if stdin:
|
|
644
|
+
result.input["path"] = STDIN_DISPLAY
|
|
645
|
+
if to_stdout:
|
|
646
|
+
result.output["path"] = STDOUT_DISPLAY
|
|
647
|
+
if result.output.get("written"):
|
|
648
|
+
copy_to_stdout(out_path, sys.stdout.buffer)
|
|
649
|
+
try:
|
|
650
|
+
code = _emit(
|
|
651
|
+
result,
|
|
652
|
+
report_path=report,
|
|
653
|
+
report_format=report_format,
|
|
654
|
+
quiet=quiet,
|
|
655
|
+
verbose=verbose,
|
|
656
|
+
force=force,
|
|
657
|
+
summary_stream=sys.stderr if to_stdout else None,
|
|
658
|
+
)
|
|
659
|
+
except OutputWriteError as exc:
|
|
660
|
+
if result.output.get("written"):
|
|
661
|
+
# The report is written after publication; never let this look like no output.
|
|
662
|
+
raise OutputWriteError(
|
|
663
|
+
"the sanitized output WAS published, but the report could not be written: "
|
|
664
|
+
+ exc.message,
|
|
665
|
+
code=exc.code,
|
|
666
|
+
) from None
|
|
667
|
+
raise
|
|
668
|
+
if not apply and not quiet and report_format != ReportFormat.JSON:
|
|
669
|
+
(sys.stderr if to_stdout else sys.stdout).write(
|
|
670
|
+
" (dry run: pass --apply to write the sanitized output)\n"
|
|
671
|
+
)
|
|
672
|
+
return code
|
|
673
|
+
|
|
674
|
+
def sanitize_batch() -> int:
|
|
675
|
+
if is_stdio(output):
|
|
676
|
+
raise UsageError(
|
|
677
|
+
"a directory INPUT needs --output DIR, not stdout", code="stdout_unsupported"
|
|
678
|
+
)
|
|
679
|
+
runtime = _runtime(max_bytes, encoding, path_display, max_files)
|
|
680
|
+
_check_batch_report_path(report, [input_path, output], force)
|
|
681
|
+
batch = sanitize_directory(
|
|
682
|
+
input_path,
|
|
683
|
+
output,
|
|
684
|
+
load_policy(policy),
|
|
685
|
+
_context(environment, target_environment, destination, use_case, profile),
|
|
686
|
+
SanitizeOptions(apply=apply, force=force, format=fmt, runtime=runtime),
|
|
687
|
+
recursive=recursive,
|
|
688
|
+
)
|
|
689
|
+
code = _emit_batch(
|
|
690
|
+
batch,
|
|
691
|
+
report_path=report,
|
|
692
|
+
report_format=report_format,
|
|
693
|
+
quiet=quiet,
|
|
694
|
+
verbose=verbose,
|
|
695
|
+
force=force,
|
|
696
|
+
)
|
|
697
|
+
if not apply and not quiet and report_format != ReportFormat.JSON:
|
|
698
|
+
sys.stdout.write(" (dry run: pass --apply to write the sanitized output)\n")
|
|
699
|
+
return code
|
|
700
|
+
|
|
701
|
+
_guard(debug, body)
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
@app.command("rescan")
|
|
705
|
+
def rescan_command(
|
|
706
|
+
input_path: Annotated[
|
|
707
|
+
Path,
|
|
708
|
+
typer.Argument(
|
|
709
|
+
metavar="INPUT", help="Sanitized file to rescan, a directory, or - for stdin."
|
|
710
|
+
),
|
|
711
|
+
],
|
|
712
|
+
policy: PolicyOpt,
|
|
713
|
+
strict: Annotated[
|
|
714
|
+
bool | None,
|
|
715
|
+
typer.Option(
|
|
716
|
+
"--strict/--no-strict",
|
|
717
|
+
help="Fail (exit 7) on gating residuals (default: policy settings.strict).",
|
|
718
|
+
),
|
|
719
|
+
] = None,
|
|
720
|
+
fmt: FormatOpt = None,
|
|
721
|
+
report: ReportOpt = None,
|
|
722
|
+
report_format: ReportFormatOpt = None,
|
|
723
|
+
environment: EnvOpt = None,
|
|
724
|
+
target_environment: TargetOpt = None,
|
|
725
|
+
destination: DestOpt = None,
|
|
726
|
+
use_case: UseCaseOpt = None,
|
|
727
|
+
profile: ProfileOpt = None,
|
|
728
|
+
max_bytes: MaxBytesOpt = None,
|
|
729
|
+
encoding: EncodingOpt = None,
|
|
730
|
+
path_display: PathDisplayOpt = None,
|
|
731
|
+
recursive: RecursiveOpt = False,
|
|
732
|
+
max_files: MaxFilesOpt = None,
|
|
733
|
+
force: ForceOpt = False,
|
|
734
|
+
quiet: QuietOpt = False,
|
|
735
|
+
verbose: VerboseOpt = False,
|
|
736
|
+
no_color: NoColorOpt = False,
|
|
737
|
+
debug: DebugOpt = False,
|
|
738
|
+
) -> None:
|
|
739
|
+
"""Report residual findings in an existing output. Read-only; never alters the file."""
|
|
740
|
+
_configure_logging(verbose, debug)
|
|
741
|
+
_guard(
|
|
742
|
+
debug,
|
|
743
|
+
lambda: _run_analysis_command(
|
|
744
|
+
"rescan",
|
|
745
|
+
input_path,
|
|
746
|
+
policy,
|
|
747
|
+
fmt,
|
|
748
|
+
report,
|
|
749
|
+
report_format,
|
|
750
|
+
_context(environment, target_environment, destination, use_case, profile),
|
|
751
|
+
_runtime(max_bytes, encoding, path_display, max_files),
|
|
752
|
+
quiet,
|
|
753
|
+
verbose,
|
|
754
|
+
force,
|
|
755
|
+
strict,
|
|
756
|
+
recursive=recursive,
|
|
757
|
+
),
|
|
758
|
+
)
|
|
759
|
+
|
|
760
|
+
|
|
761
|
+
@app.command("explain")
|
|
762
|
+
def explain_command(
|
|
763
|
+
policy: PolicyOpt,
|
|
764
|
+
category: Annotated[str, typer.Option("--category", help="Category ID, e.g. pii.email.")],
|
|
765
|
+
field: Annotated[str, typer.Option("--field", help="Field path, e.g. /user/email.")] = "",
|
|
766
|
+
confidence: Annotated[float, typer.Option("--confidence", min=0.0, max=1.0)] = 0.9,
|
|
767
|
+
detector: Annotated[str, typer.Option("--detector")] = "field_name",
|
|
768
|
+
fmt: Annotated[InputFormat, typer.Option("--format")] = InputFormat.JSON,
|
|
769
|
+
environment: EnvOpt = None,
|
|
770
|
+
target_environment: TargetOpt = None,
|
|
771
|
+
destination: DestOpt = None,
|
|
772
|
+
use_case: UseCaseOpt = None,
|
|
773
|
+
profile: ProfileOpt = None,
|
|
774
|
+
debug: DebugOpt = False,
|
|
775
|
+
) -> None:
|
|
776
|
+
"""Explain the decision for a hypothetical finding (no data needed)."""
|
|
777
|
+
|
|
778
|
+
def body() -> int:
|
|
779
|
+
loaded = load_policy(policy)
|
|
780
|
+
if category not in loaded.categories:
|
|
781
|
+
raise UsageError("unknown category for this policy", code="unknown_category")
|
|
782
|
+
ctx = resolve_context(
|
|
783
|
+
loaded, _context(environment, target_environment, destination, use_case, profile)
|
|
784
|
+
)
|
|
785
|
+
field_name = field.rstrip("/").rsplit("/", 1)[-1] if field else None
|
|
786
|
+
finding = Finding(
|
|
787
|
+
category=category,
|
|
788
|
+
detector=detector,
|
|
789
|
+
confidence=confidence,
|
|
790
|
+
evidence=("hypothetical",),
|
|
791
|
+
field_name=field_name or None,
|
|
792
|
+
value_type="string",
|
|
793
|
+
location=Location(record_index=0, line=None, field_path=field),
|
|
794
|
+
)
|
|
795
|
+
decision = PolicyEngine(loaded).decide(finding, ctx, fmt)
|
|
796
|
+
gates, approvals = evaluate_gates(loaded, ctx)
|
|
797
|
+
payload = {
|
|
798
|
+
"policy": loaded.name,
|
|
799
|
+
"context": ctx.to_report(),
|
|
800
|
+
"finding": {
|
|
801
|
+
"category": category,
|
|
802
|
+
"field_path": field,
|
|
803
|
+
"confidence": confidence,
|
|
804
|
+
"detector": detector,
|
|
805
|
+
"format": str(fmt),
|
|
806
|
+
},
|
|
807
|
+
"decision": decision.to_dict(),
|
|
808
|
+
"run_gates": [g.to_dict() for g in gates],
|
|
809
|
+
"transfer_approvals": approvals,
|
|
810
|
+
}
|
|
811
|
+
# json.dumps escapes every control character (ensure_ascii), so this is terminal-safe.
|
|
812
|
+
sys.stdout.write(json.dumps(payload, indent=2) + "\n")
|
|
813
|
+
return 0
|
|
814
|
+
|
|
815
|
+
_guard(debug, body)
|
|
816
|
+
|
|
817
|
+
|
|
818
|
+
def _sigterm_handler(signum: int, frame: FrameType | None) -> None:
|
|
819
|
+
raise KeyboardInterrupt
|
|
820
|
+
|
|
821
|
+
|
|
822
|
+
def main() -> None:
|
|
823
|
+
for stream in (sys.stdout, sys.stderr):
|
|
824
|
+
with contextlib.suppress(AttributeError, ValueError):
|
|
825
|
+
stream.reconfigure(errors="backslashreplace") # type: ignore[union-attr]
|
|
826
|
+
with contextlib.suppress(ValueError, OSError, AttributeError):
|
|
827
|
+
signal.signal(signal.SIGTERM, _sigterm_handler)
|
|
828
|
+
app()
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
if __name__ == "__main__": # pragma: no cover
|
|
832
|
+
main()
|