ciphergate-cli 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ciphergate/__init__.py +95 -0
- ciphergate/__main__.py +6 -0
- ciphergate/cli.py +832 -0
- ciphergate/config.py +50 -0
- ciphergate/constants.py +274 -0
- ciphergate/detectors/__init__.py +5 -0
- ciphergate/detectors/base.py +53 -0
- ciphergate/detectors/custom_regex.py +37 -0
- ciphergate/detectors/dates.py +92 -0
- ciphergate/detectors/email.py +56 -0
- ciphergate/detectors/field_names.py +220 -0
- ciphergate/detectors/ip_address.py +85 -0
- ciphergate/detectors/network.py +42 -0
- ciphergate/detectors/payment_card.py +77 -0
- ciphergate/detectors/phone.py +67 -0
- ciphergate/detectors/presidio_ner.py +112 -0
- ciphergate/detectors/regex_safety.py +40 -0
- ciphergate/detectors/registry.py +194 -0
- ciphergate/detectors/secrets.py +130 -0
- ciphergate/errors.py +119 -0
- ciphergate/models/__init__.py +1 -0
- ciphergate/models/context.py +69 -0
- ciphergate/models/finding.py +108 -0
- ciphergate/models/policy.py +402 -0
- ciphergate/models/report.py +115 -0
- ciphergate/pipeline/__init__.py +1 -0
- ciphergate/pipeline/atomic_output.py +143 -0
- ciphergate/pipeline/batch.py +533 -0
- ciphergate/pipeline/inspector.py +282 -0
- ciphergate/pipeline/paths.py +98 -0
- ciphergate/pipeline/planner.py +192 -0
- ciphergate/pipeline/rescan.py +120 -0
- ciphergate/pipeline/sanitizer.py +392 -0
- ciphergate/pipeline/stdio.py +173 -0
- ciphergate/pipeline/streams.py +152 -0
- ciphergate/policy/__init__.py +26 -0
- ciphergate/policy/context.py +203 -0
- ciphergate/policy/engine.py +194 -0
- ciphergate/policy/loader.py +116 -0
- ciphergate/policy/precedence.py +149 -0
- ciphergate/policy/validator.py +632 -0
- ciphergate/py.typed +0 -0
- ciphergate/readers/__init__.py +113 -0
- ciphergate/readers/accesslog_reader.py +244 -0
- ciphergate/readers/base.py +131 -0
- ciphergate/readers/csv_reader.py +241 -0
- ciphergate/readers/json_reader.py +184 -0
- ciphergate/readers/jsonl_reader.py +97 -0
- ciphergate/readers/jsonlog_reader.py +178 -0
- ciphergate/readers/line_structure.py +125 -0
- ciphergate/readers/logfmt_reader.py +100 -0
- ciphergate/readers/stacktrace_reader.py +103 -0
- ciphergate/readers/syslog_reader.py +309 -0
- ciphergate/readers/text_reader.py +59 -0
- ciphergate/reporting/__init__.py +1 -0
- ciphergate/reporting/audit.py +85 -0
- ciphergate/reporting/safe_text.py +71 -0
- ciphergate/reporting/serializer.py +44 -0
- ciphergate/reporting/terminal.py +147 -0
- ciphergate/templates/starter_policy.yml +77 -0
- ciphergate/transforms/__init__.py +6 -0
- ciphergate/transforms/date_shift.py +79 -0
- ciphergate/transforms/generalize.py +107 -0
- ciphergate/transforms/keys.py +177 -0
- ciphergate/transforms/pseudonymize.py +113 -0
- ciphergate/transforms/redact.py +9 -0
- ciphergate/transforms/registry.py +201 -0
- ciphergate/transforms/replace.py +16 -0
- ciphergate_cli-0.7.0.dist-info/METADATA +531 -0
- ciphergate_cli-0.7.0.dist-info/RECORD +74 -0
- ciphergate_cli-0.7.0.dist-info/WHEEL +4 -0
- ciphergate_cli-0.7.0.dist-info/entry_points.txt +2 -0
- ciphergate_cli-0.7.0.dist-info/licenses/LICENSE +202 -0
- ciphergate_cli-0.7.0.dist-info/licenses/NOTICE +4 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Plain text / log adapter: each line is one record, scanned as free text.
|
|
2
|
+
|
|
3
|
+
Lines are never interpreted as structured key/value data. Original line endings (LF, CRLF, CR) are
|
|
4
|
+
preserved. Multi-line private key blocks are tracked across lines by the detector set.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Iterator
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import IO, Any
|
|
12
|
+
|
|
13
|
+
from ciphergate.constants import InputFormat
|
|
14
|
+
from ciphergate.models.finding import ScanItem
|
|
15
|
+
from ciphergate.readers.base import (
|
|
16
|
+
ReaderOptions,
|
|
17
|
+
ReadStats,
|
|
18
|
+
Record,
|
|
19
|
+
decode_line,
|
|
20
|
+
has_utf8_bom,
|
|
21
|
+
read_bounded_lines,
|
|
22
|
+
split_line_ending,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class TextReader:
|
|
27
|
+
format = InputFormat.TEXT
|
|
28
|
+
|
|
29
|
+
def __init__(self, path: Path, options: ReaderOptions) -> None:
|
|
30
|
+
self.path = path
|
|
31
|
+
self.options = options
|
|
32
|
+
self.stats = ReadStats()
|
|
33
|
+
self.has_bom = options.encoding.startswith("utf-8") and has_utf8_bom(path)
|
|
34
|
+
|
|
35
|
+
def records(self) -> Iterator[Record]:
|
|
36
|
+
with self.path.open("rb") as fh:
|
|
37
|
+
for line_no, raw in read_bounded_lines(fh, self.options, self.stats):
|
|
38
|
+
text = decode_line(raw, self.options.encoding, line_no, first=line_no == 1)
|
|
39
|
+
content, ending = split_line_ending(text)
|
|
40
|
+
self.stats.records = line_no
|
|
41
|
+
yield Record(index=line_no - 1, line=line_no, data=content, line_ending=ending)
|
|
42
|
+
|
|
43
|
+
def iter_items(self, record: Record) -> Iterator[ScanItem]:
|
|
44
|
+
yield ScanItem(record.index, record.line, "", None, record.data)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class TextWriter:
|
|
48
|
+
def __init__(self, stream: IO[str], reader: TextReader) -> None:
|
|
49
|
+
self._stream = stream
|
|
50
|
+
self._bom_pending = reader.has_bom
|
|
51
|
+
|
|
52
|
+
def write_record(self, record: Record, data: Any) -> None:
|
|
53
|
+
if self._bom_pending:
|
|
54
|
+
self._stream.write("")
|
|
55
|
+
self._bom_pending = False
|
|
56
|
+
self._stream.write(str(data) + record.line_ending)
|
|
57
|
+
|
|
58
|
+
def close(self) -> None:
|
|
59
|
+
self._stream.flush()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Privacy-safe report building, serialization and terminal rendering."""
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Builds ``RunReport`` objects from analysis results (privacy-safe metadata only)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime as dt
|
|
6
|
+
import secrets
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from ciphergate import __version__
|
|
10
|
+
from ciphergate.constants import PSEUDONYM_VERSION
|
|
11
|
+
from ciphergate.models.report import RescanSummary, RunReport, RunStatus
|
|
12
|
+
from ciphergate.pipeline.paths import display_path
|
|
13
|
+
from ciphergate.pipeline.planner import Analysis
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def new_run_id() -> str:
|
|
17
|
+
return secrets.token_hex(8)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def utc_now() -> str:
|
|
21
|
+
return dt.datetime.now(dt.UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def build_report(
|
|
25
|
+
command: str,
|
|
26
|
+
analysis: Analysis,
|
|
27
|
+
*,
|
|
28
|
+
status: RunStatus | None = None,
|
|
29
|
+
output: dict[str, Any] | None = None,
|
|
30
|
+
changes: dict[str, Any] | None = None,
|
|
31
|
+
rescan: RescanSummary | None = None,
|
|
32
|
+
extra_warnings: list[str] | None = None,
|
|
33
|
+
) -> RunReport:
|
|
34
|
+
setup = analysis.setup
|
|
35
|
+
policy = setup.policy
|
|
36
|
+
path_mode = setup.runtime.path_display
|
|
37
|
+
stats = analysis.stats
|
|
38
|
+
acc = analysis.acc
|
|
39
|
+
warnings = list(policy.warnings) + list(extra_warnings or [])
|
|
40
|
+
if stats.blank_lines:
|
|
41
|
+
action = "preserved" if policy.document.settings.preserve_blank_lines else "removed"
|
|
42
|
+
warnings.append(f"{stats.blank_lines} blank line(s) {action} in output")
|
|
43
|
+
summary = acc.summary()
|
|
44
|
+
summary["dropped_columns"] = [acc.dropped_columns[i] for i in sorted(acc.dropped_columns)]
|
|
45
|
+
return RunReport(
|
|
46
|
+
command=command,
|
|
47
|
+
status=status or analysis.status,
|
|
48
|
+
run_id=new_run_id(),
|
|
49
|
+
timestamp=utc_now(),
|
|
50
|
+
version=__version__,
|
|
51
|
+
input={
|
|
52
|
+
"path": display_path(setup.input_path, path_mode),
|
|
53
|
+
"format": str(setup.fmt),
|
|
54
|
+
"bytes": stats.bytes_read,
|
|
55
|
+
"records": stats.records,
|
|
56
|
+
"blank_lines": stats.blank_lines,
|
|
57
|
+
},
|
|
58
|
+
policy={
|
|
59
|
+
"name": policy.name,
|
|
60
|
+
"policy_version": policy.version,
|
|
61
|
+
"digest": policy.digest,
|
|
62
|
+
"source": policy.source_name if path_mode != "redacted" else None,
|
|
63
|
+
"strict": policy.document.settings.strict,
|
|
64
|
+
},
|
|
65
|
+
context=setup.ctx.to_report(),
|
|
66
|
+
gates=[g.to_dict() for g in analysis.gates],
|
|
67
|
+
transfer_approvals=list(analysis.approvals),
|
|
68
|
+
summary=summary,
|
|
69
|
+
items=acc.items,
|
|
70
|
+
items_truncated=acc.truncated,
|
|
71
|
+
keys=[
|
|
72
|
+
{
|
|
73
|
+
"key_env": name,
|
|
74
|
+
"source": analysis.key_sources.get(name, "env"),
|
|
75
|
+
"status": str(state),
|
|
76
|
+
"transform_version": PSEUDONYM_VERSION,
|
|
77
|
+
}
|
|
78
|
+
for name, state in analysis.key_statuses.items()
|
|
79
|
+
],
|
|
80
|
+
changes=changes or {},
|
|
81
|
+
parse_errors=[{"line": line, "code": code} for line, code in stats.parse_errors[:100]],
|
|
82
|
+
output=output or {"written": False},
|
|
83
|
+
rescan=rescan,
|
|
84
|
+
warnings=warnings,
|
|
85
|
+
)
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Make user-controlled strings (field paths, header names, file names) safe to display/store.
|
|
2
|
+
|
|
3
|
+
* Control characters (including ANSI escape introducers) are escaped as ``\\xNN``/``\\uNNNN``.
|
|
4
|
+
* Path segments that themselves look like sensitive values (emails, secrets, card numbers, phone
|
|
5
|
+
numbers, IP addresses) are replaced with ``<redacted-key>`` — JSON keys and CSV headers can carry
|
|
6
|
+
data too.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from functools import lru_cache
|
|
12
|
+
|
|
13
|
+
from ciphergate.detectors.email import EmailDetector
|
|
14
|
+
from ciphergate.detectors.ip_address import IPAddressDetector
|
|
15
|
+
from ciphergate.detectors.payment_card import PaymentCardDetector
|
|
16
|
+
from ciphergate.detectors.phone import PhoneDetector
|
|
17
|
+
from ciphergate.detectors.secrets import SecretDetector
|
|
18
|
+
from ciphergate.models.finding import ScanItem
|
|
19
|
+
|
|
20
|
+
MAX_DISPLAY = 200
|
|
21
|
+
# Line/paragraph separators and bidi overrides can disguise terminal output.
|
|
22
|
+
_INVISIBLE = frozenset(
|
|
23
|
+
chr(c)
|
|
24
|
+
for c in (
|
|
25
|
+
0x2028,
|
|
26
|
+
0x2029,
|
|
27
|
+
0x202A,
|
|
28
|
+
0x202B,
|
|
29
|
+
0x202C,
|
|
30
|
+
0x202D,
|
|
31
|
+
0x202E,
|
|
32
|
+
0x2066,
|
|
33
|
+
0x2067,
|
|
34
|
+
0x2068,
|
|
35
|
+
0x2069,
|
|
36
|
+
)
|
|
37
|
+
)
|
|
38
|
+
_PROBES = (EmailDetector(), SecretDetector(), PaymentCardDetector(), IPAddressDetector())
|
|
39
|
+
_PHONE = PhoneDetector()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def escape_control(text: str) -> str:
|
|
43
|
+
out: list[str] = []
|
|
44
|
+
for ch in text:
|
|
45
|
+
code = ord(ch)
|
|
46
|
+
if code < 0x20 or code == 0x7F or 0x80 <= code < 0xA0 or ch in _INVISIBLE:
|
|
47
|
+
out.append(f"\\x{code:02x}" if code < 0x100 else f"\\u{code:04x}")
|
|
48
|
+
else:
|
|
49
|
+
out.append(ch)
|
|
50
|
+
result = "".join(out)
|
|
51
|
+
return result if len(result) <= MAX_DISPLAY else result[: MAX_DISPLAY - 3] + "..."
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _segment_is_sensitive(segment: str) -> bool:
|
|
55
|
+
if not segment or segment.isdigit() or segment.startswith("#"):
|
|
56
|
+
return False
|
|
57
|
+
probe = ScanItem(0, None, "", None, segment)
|
|
58
|
+
if any(detector.detect(probe) for detector in _PROBES):
|
|
59
|
+
return True
|
|
60
|
+
return any(f.confidence >= 0.5 for f in _PHONE.detect(probe))
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@lru_cache(maxsize=4096)
|
|
64
|
+
def safe_field_path(path: str) -> str:
|
|
65
|
+
if not path:
|
|
66
|
+
return path
|
|
67
|
+
if path.startswith("/"):
|
|
68
|
+
segments = path.split("/")[1:]
|
|
69
|
+
cleaned = ["<redacted-key>" if _segment_is_sensitive(s) else s for s in segments]
|
|
70
|
+
return escape_control("/" + "/".join(cleaned))
|
|
71
|
+
return escape_control("<redacted-key>" if _segment_is_sensitive(path) else path)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Report serialization and atomic report-file writing."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from ciphergate.models.report import RunReport
|
|
9
|
+
from ciphergate.pipeline.atomic_output import AtomicOutput
|
|
10
|
+
from ciphergate.pipeline.batch import BatchReport
|
|
11
|
+
from ciphergate.reporting.terminal import render_batch_summary, render_summary
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def report_to_json(report: RunReport) -> str:
|
|
15
|
+
return json.dumps(report.to_dict(), indent=2, ensure_ascii=False, allow_nan=False) + "\n"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def render_report(report: RunReport, report_format: str) -> str:
|
|
19
|
+
if report_format == "json":
|
|
20
|
+
return report_to_json(report)
|
|
21
|
+
return render_summary(report, verbose=True)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def write_report_file(report: RunReport, path: Path, report_format: str, *, force: bool) -> None:
|
|
25
|
+
with AtomicOutput(path, force=force) as out:
|
|
26
|
+
out.stream.write(render_report(report, report_format))
|
|
27
|
+
out.publish()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def batch_report_to_json(report: BatchReport) -> str:
|
|
31
|
+
return json.dumps(report.to_dict(), indent=2, ensure_ascii=False, allow_nan=False) + "\n"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def write_batch_report_file(
|
|
35
|
+
report: BatchReport, path: Path, report_format: str, *, force: bool
|
|
36
|
+
) -> None:
|
|
37
|
+
text = (
|
|
38
|
+
batch_report_to_json(report)
|
|
39
|
+
if report_format == "json"
|
|
40
|
+
else render_batch_summary(report, verbose=True)
|
|
41
|
+
)
|
|
42
|
+
with AtomicOutput(path, force=force) as out:
|
|
43
|
+
out.stream.write(text)
|
|
44
|
+
out.publish()
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Plain-text summaries for terminals and CI logs (no colour, no values, control chars escaped)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from ciphergate.constants import DISCLAIMER
|
|
8
|
+
from ciphergate.models.report import RunReport
|
|
9
|
+
from ciphergate.pipeline.batch import BatchReport
|
|
10
|
+
from ciphergate.reporting.safe_text import escape_control
|
|
11
|
+
|
|
12
|
+
SUMMARY_ITEMS = 15
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _counts(data: dict[str, int]) -> str:
|
|
16
|
+
return ", ".join(f"{escape_control(k)}={v}" for k, v in data.items()) or "-"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _location(loc: dict[str, Any]) -> str:
|
|
20
|
+
parts = []
|
|
21
|
+
if loc.get("line") is not None:
|
|
22
|
+
parts.append(f"line {loc['line']}")
|
|
23
|
+
else:
|
|
24
|
+
parts.append(f"record {loc.get('record')}")
|
|
25
|
+
if loc.get("field_path"):
|
|
26
|
+
parts.append(f"field {escape_control(str(loc['field_path']))}")
|
|
27
|
+
if loc.get("span"):
|
|
28
|
+
parts.append(f"chars {loc['span'][0]}-{loc['span'][1]}")
|
|
29
|
+
if loc.get("in_key"):
|
|
30
|
+
parts.append("(key)")
|
|
31
|
+
return ", ".join(parts)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def render_summary(report: RunReport, *, verbose: bool = False) -> str:
|
|
35
|
+
d = report.to_dict()
|
|
36
|
+
lines = [f"CipherGate {d['command']}: {d['status'].upper()} (exit {d['exit_code']})"]
|
|
37
|
+
inp = d["input"]
|
|
38
|
+
lines.append(
|
|
39
|
+
f" input: {escape_control(str(inp['path']))} [{inp['format']}, "
|
|
40
|
+
f"{inp['records']} record(s), {inp['bytes']} bytes]"
|
|
41
|
+
)
|
|
42
|
+
pol = d["policy"]
|
|
43
|
+
lines.append(
|
|
44
|
+
f" policy: {escape_control(pol['name'])} v{pol['policy_version']} "
|
|
45
|
+
f"({pol['digest'][:19]}...)"
|
|
46
|
+
)
|
|
47
|
+
ctx = ", ".join(f"{k}={v['value'] or '-'}[{v['source']}]" for k, v in d["context"].items())
|
|
48
|
+
lines.append(f" context: {ctx}")
|
|
49
|
+
s = d["summary"]
|
|
50
|
+
lines.append(
|
|
51
|
+
f" findings: {s.get('findings_total', 0)} blocked: {s.get('blocked', 0)} "
|
|
52
|
+
f"review: {s.get('review_required', 0)}"
|
|
53
|
+
)
|
|
54
|
+
if s.get("findings_total"):
|
|
55
|
+
lines.append(f" by category: {_counts(s['by_category'])}")
|
|
56
|
+
lines.append(f" by action: {_counts(s['by_action'])}")
|
|
57
|
+
lines.append(f" by reason: {_counts(s['by_reason_code'])}")
|
|
58
|
+
if s.get("dropped_columns"):
|
|
59
|
+
lines.append(f" dropped columns: {', '.join(s['dropped_columns'])}")
|
|
60
|
+
for gate in d["gates"]:
|
|
61
|
+
lines.append(f" gate [{gate['severity']}] {gate['reason_code']}: {gate['message']}")
|
|
62
|
+
for approval in d["transfer_approvals"]:
|
|
63
|
+
lines.append(f" transfer approval used: {escape_control(approval)}")
|
|
64
|
+
for key in d["keys"]:
|
|
65
|
+
lines.append(f" key {key['key_env']}: {key['status']}")
|
|
66
|
+
for err in d["parse_errors"][:10]:
|
|
67
|
+
lines.append(f" parse error at line {err['line']}: {err['code']}")
|
|
68
|
+
items = d["items"]
|
|
69
|
+
important = [i for i in items if i["decision"]["action"] in ("block", "review")]
|
|
70
|
+
shown = items if verbose else (important or items)[:SUMMARY_ITEMS]
|
|
71
|
+
if shown:
|
|
72
|
+
lines.append(" items:" if verbose else " items (first shown; use --verbose or --report):")
|
|
73
|
+
for item in shown:
|
|
74
|
+
dec = item["decision"]
|
|
75
|
+
rule = dec.get("rule_id") or "-"
|
|
76
|
+
lines.append(
|
|
77
|
+
f" {_location(item['location'])}: {item['category']} "
|
|
78
|
+
f"({item['band']}, {item['detector']}) -> {dec['action']} "
|
|
79
|
+
f"[{dec['reason_code']}; rule {escape_control(rule)}]"
|
|
80
|
+
)
|
|
81
|
+
if d["items_truncated"]:
|
|
82
|
+
lines.append(" (item list truncated; counts above are complete)")
|
|
83
|
+
if d["changes"]:
|
|
84
|
+
c = d["changes"]
|
|
85
|
+
lines.append(
|
|
86
|
+
f" changes: records written={c.get('records_written')}, dropped="
|
|
87
|
+
f"{c.get('records_dropped')}, values changed={c.get('values_changed')}, "
|
|
88
|
+
f"type changes={c.get('type_changes')}"
|
|
89
|
+
)
|
|
90
|
+
if d["rescan"]:
|
|
91
|
+
r = d["rescan"]
|
|
92
|
+
verdict = "FAILED" if r["failed"] else "passed"
|
|
93
|
+
lines.append(
|
|
94
|
+
f" rescan: {verdict} (strict={r['strict']}) residuals: {_counts(r['counts'])}"
|
|
95
|
+
)
|
|
96
|
+
if r["gating_by_category"]:
|
|
97
|
+
lines.append(f" gating residuals: {_counts(r['gating_by_category'])}")
|
|
98
|
+
out = d["output"]
|
|
99
|
+
if out:
|
|
100
|
+
if out.get("written"):
|
|
101
|
+
lines.append(f" output: written to {escape_control(str(out.get('path')))}")
|
|
102
|
+
elif out.get("withheld_reason"):
|
|
103
|
+
lines.append(f" output: not written ({out['withheld_reason']})")
|
|
104
|
+
for warning in d["warnings"]:
|
|
105
|
+
lines.append(f" warning: {escape_control(warning)}")
|
|
106
|
+
lines.append(f" note: {DISCLAIMER}")
|
|
107
|
+
return "\n".join(lines) + "\n"
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def render_batch_summary(report: BatchReport, *, verbose: bool = False) -> str:
|
|
111
|
+
d = report.to_dict()
|
|
112
|
+
s = d["summary"]
|
|
113
|
+
lines = [
|
|
114
|
+
f"CipherGate {d['command']} (directory): {d['status'].upper()} (exit {d['exit_code']})"
|
|
115
|
+
]
|
|
116
|
+
recursive = " (recursive)" if d["input"]["recursive"] else ""
|
|
117
|
+
lines.append(f" input: {escape_control(str(d['input']['path']))}{recursive}")
|
|
118
|
+
if d["policy"]:
|
|
119
|
+
lines.append(f" policy: {escape_control(d['policy']['name'])}")
|
|
120
|
+
lines.append(
|
|
121
|
+
f" files: {s['files_total']} ok: {s['files_ok']} failed: {s['files_failed']} "
|
|
122
|
+
f"skipped: {s['files_skipped']}"
|
|
123
|
+
)
|
|
124
|
+
lines.append(
|
|
125
|
+
f" findings: {s['findings_total']} blocked: {s['blocked']} "
|
|
126
|
+
f"review: {s['review_required']}"
|
|
127
|
+
)
|
|
128
|
+
for entry in d["files"]:
|
|
129
|
+
detail = entry["status"]
|
|
130
|
+
if "error" in entry:
|
|
131
|
+
detail = f"error[{entry['error']['code']}]"
|
|
132
|
+
elif verbose:
|
|
133
|
+
detail += f", findings={entry['report']['summary'].get('findings_total', 0)}"
|
|
134
|
+
lines.append(f" {escape_control(entry['path'])}: {detail} (exit {entry['exit_code']})")
|
|
135
|
+
for skip in d["skipped"]:
|
|
136
|
+
lines.append(f" skipped {escape_control(skip['path'])}: {skip['reason']}")
|
|
137
|
+
out = d["output"]
|
|
138
|
+
if out:
|
|
139
|
+
if out.get("written"):
|
|
140
|
+
lines.append(f" output: directory written to {escape_control(str(out['path']))}")
|
|
141
|
+
else:
|
|
142
|
+
lines.append(f" output: not written ({out.get('withheld_reason')})")
|
|
143
|
+
for warning in d["warnings"]:
|
|
144
|
+
lines.append(f" warning: {escape_control(warning)}")
|
|
145
|
+
lines.append(" note: per-file details are in the JSON report (--report).")
|
|
146
|
+
lines.append(f" note: {DISCLAIMER}")
|
|
147
|
+
return "\n".join(lines) + "\n"
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# CipherGate starter policy — TEMPLATE, REVIEW BEFORE USE.
|
|
2
|
+
#
|
|
3
|
+
# Generated by `ciphergate init`. It is a conservative starting point, not a guaranteed-safe or
|
|
4
|
+
# compliant configuration. Review every rule against your data, environments and destinations.
|
|
5
|
+
# Context (environment/destination) is NOT pinned: supply it per run, e.g.
|
|
6
|
+
# ciphergate inspect app.log --policy ciphergate-policy.yml --environment dev --destination local_dev
|
|
7
|
+
policy_version: 1
|
|
8
|
+
name: starter-policy
|
|
9
|
+
metadata:
|
|
10
|
+
description: "Starter template generated by ciphergate init. Template requiring review."
|
|
11
|
+
template: true
|
|
12
|
+
requirements:
|
|
13
|
+
environment: true
|
|
14
|
+
destination: true
|
|
15
|
+
settings:
|
|
16
|
+
strict: true
|
|
17
|
+
scan_json_keys: true # keys can carry data; required for untrusted destinations
|
|
18
|
+
unknown_sensitive_behavior: review
|
|
19
|
+
fail_on_parse_error: true
|
|
20
|
+
report_raw_values: false
|
|
21
|
+
rescan:
|
|
22
|
+
enabled: true
|
|
23
|
+
rules:
|
|
24
|
+
- id: block-credentials-untrusted
|
|
25
|
+
description: Credentials never leave for external, public or unknown destinations.
|
|
26
|
+
hard: true
|
|
27
|
+
priority: 1000
|
|
28
|
+
match:
|
|
29
|
+
categories: [credential.secret]
|
|
30
|
+
destinations: [external_vendor, external_ai, public_repository, unknown]
|
|
31
|
+
action: block
|
|
32
|
+
reason: credentials_not_permitted_for_destination
|
|
33
|
+
|
|
34
|
+
- id: redact-credentials
|
|
35
|
+
description: Elsewhere, remove credentials completely (no partial masking).
|
|
36
|
+
priority: 900
|
|
37
|
+
match:
|
|
38
|
+
categories: [credential.secret]
|
|
39
|
+
action: redact
|
|
40
|
+
reason: remove_credential
|
|
41
|
+
|
|
42
|
+
- id: redact-telemetry-untrusted
|
|
43
|
+
priority: 400
|
|
44
|
+
match:
|
|
45
|
+
categories: [network.telemetry, security.internal]
|
|
46
|
+
destinations: [external_vendor, external_ai, public_repository, unknown]
|
|
47
|
+
action: redact
|
|
48
|
+
reason: hide_internal_infrastructure
|
|
49
|
+
|
|
50
|
+
- id: keep-telemetry-internal
|
|
51
|
+
description: Internal hosts, private IPs and trace IDs are useful for internal debugging.
|
|
52
|
+
priority: 400
|
|
53
|
+
match:
|
|
54
|
+
categories: [network.telemetry]
|
|
55
|
+
destinations: [local_dev, approved_internal, staging_test, production_monitoring]
|
|
56
|
+
action: allow
|
|
57
|
+
reason: keep_debug_context_internally
|
|
58
|
+
|
|
59
|
+
- id: redact-personal-and-sensitive
|
|
60
|
+
priority: 300
|
|
61
|
+
match:
|
|
62
|
+
categories:
|
|
63
|
+
- pii.*
|
|
64
|
+
- health.phi
|
|
65
|
+
- financial.sensitive
|
|
66
|
+
- employment.sensitive
|
|
67
|
+
- security.internal
|
|
68
|
+
action: redact
|
|
69
|
+
reason: remove_identifier
|
|
70
|
+
|
|
71
|
+
defaults:
|
|
72
|
+
date.generic:
|
|
73
|
+
action: allow
|
|
74
|
+
location.coarse:
|
|
75
|
+
action: redact
|
|
76
|
+
identifier.generic:
|
|
77
|
+
action: review
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""Transformations that execute approved policy decisions."""
|
|
2
|
+
|
|
3
|
+
from ciphergate.transforms.keys import KeyProvider, KeyStatus
|
|
4
|
+
from ciphergate.transforms.registry import Transformer, ValueOutcome
|
|
5
|
+
|
|
6
|
+
__all__ = ["KeyProvider", "KeyStatus", "Transformer", "ValueOutcome"]
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Keyed date shifting that preserves intervals within a scope.
|
|
2
|
+
|
|
3
|
+
Offsets are derived with HMAC-SHA-256 from the key and a scope value, in ``[-max_days, -1]`` or
|
|
4
|
+
``[1, max_days]`` (never zero):
|
|
5
|
+
|
|
6
|
+
* ``scope: dataset`` — one offset for the whole run (all relationships preserved).
|
|
7
|
+
* ``scope: entity`` — one offset per value of ``entity_field`` in the same record (per-subject
|
|
8
|
+
consistency; structured formats only).
|
|
9
|
+
|
|
10
|
+
Supported inputs: ``iso`` (``YYYY-MM-DD``, ``YYYY-MM-DD[T ]HH:MM[:SS[.ffffff]][Z|±HH:MM]``) or
|
|
11
|
+
explicit ``strptime`` patterns. Fixed UTC offsets are preserved as written; named time zones and
|
|
12
|
+
epoch integers are not supported. Leap days are handled by calendar arithmetic (Feb 29 + 1 day =
|
|
13
|
+
Mar 1). Unparseable or out-of-range values raise ``TransformError``.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import datetime as dt
|
|
19
|
+
import hashlib
|
|
20
|
+
import hmac
|
|
21
|
+
import re
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
from ciphergate.errors import TransformError
|
|
25
|
+
from ciphergate.models.policy import DateShiftParams
|
|
26
|
+
from ciphergate.transforms.pseudonymize import canonical_value
|
|
27
|
+
|
|
28
|
+
_ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
|
29
|
+
_ISO_DATETIME = re.compile(
|
|
30
|
+
r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d{1,6})?)?(?:Z|[+-]\d{2}:\d{2})?$"
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def derive_offset_days(key: bytes, namespace: str, scope_value: Any, max_days: int) -> int:
|
|
35
|
+
tag, canonical = canonical_value(scope_value, "nfc")
|
|
36
|
+
message = f"ciphergate:dateshift:v1:{namespace}".encode() + b"\x00" + tag + b"\x00" + canonical
|
|
37
|
+
digest = hmac.new(key, message, hashlib.sha256).digest()
|
|
38
|
+
n = int.from_bytes(digest[:8], "big") % (2 * max_days)
|
|
39
|
+
return n - max_days if n < max_days else n - max_days + 1
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _fail() -> TransformError:
|
|
43
|
+
return TransformError(
|
|
44
|
+
"date_shift cannot parse a value with the configured formats", code="date_shift_unparseable"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def shift_date_text(text: str, offset_days: int, formats: list[str]) -> str:
|
|
49
|
+
delta = dt.timedelta(days=offset_days)
|
|
50
|
+
stripped = text.strip()
|
|
51
|
+
for fmt in formats:
|
|
52
|
+
try:
|
|
53
|
+
if fmt == "iso":
|
|
54
|
+
if _ISO_DATE.match(stripped):
|
|
55
|
+
return (dt.date.fromisoformat(stripped) + delta).isoformat()
|
|
56
|
+
if _ISO_DATETIME.match(stripped):
|
|
57
|
+
parsed = dt.datetime.fromisoformat(stripped.replace("Z", "+00:00"))
|
|
58
|
+
sep = "T" if "T" in stripped else " "
|
|
59
|
+
out = (parsed + delta).isoformat(sep=sep)
|
|
60
|
+
if stripped.endswith("Z"):
|
|
61
|
+
out = out.removesuffix("+00:00") + "Z"
|
|
62
|
+
return out
|
|
63
|
+
continue
|
|
64
|
+
parsed = dt.datetime.strptime(stripped, fmt)
|
|
65
|
+
return (parsed + delta).strftime(fmt)
|
|
66
|
+
except ValueError:
|
|
67
|
+
continue
|
|
68
|
+
except OverflowError:
|
|
69
|
+
raise TransformError(
|
|
70
|
+
"date_shift result is outside the supported date range",
|
|
71
|
+
code="date_shift_out_of_range",
|
|
72
|
+
) from None
|
|
73
|
+
raise _fail()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def date_shift(value: Any, params: DateShiftParams, *, offset_days: int) -> str:
|
|
77
|
+
if not isinstance(value, str):
|
|
78
|
+
raise _fail()
|
|
79
|
+
return shift_date_text(value, offset_days, params.formats)
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Precision-reducing generalization strategies.
|
|
2
|
+
|
|
3
|
+
* ``year`` / ``year_month``: ISO date or datetime (``YYYY-MM-DD[...]``) or ``YYYY/MM/DD``.
|
|
4
|
+
* ``age_band``: non-negative number -> ``"30-39"`` (band_size 10); ``>= max_age`` -> ``"120+"``.
|
|
5
|
+
* ``round_coordinates``: number within [-180, 180] rounded to ``decimals`` (type preserved).
|
|
6
|
+
* ``numeric_bin``: number -> ``"<a"``, ``"[a, b)"`` or ``">=z"`` using the configured edges.
|
|
7
|
+
|
|
8
|
+
Values that do not fit the strategy raise ``TransformError`` (fail closed; no partial guessing).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import datetime as dt
|
|
14
|
+
import itertools
|
|
15
|
+
import math
|
|
16
|
+
import re
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from ciphergate.errors import TransformError
|
|
20
|
+
from ciphergate.models.policy import GeneralizeParams
|
|
21
|
+
|
|
22
|
+
_DATE_PREFIX = re.compile(r"^(\d{4})[-/](\d{2})[-/](\d{2})(?:$|[T ])")
|
|
23
|
+
YEAR_RE = re.compile(r"^\d{4}$")
|
|
24
|
+
YEAR_MONTH_RE = re.compile(r"^\d{4}-\d{2}$")
|
|
25
|
+
AGE_BAND_RE = re.compile(r"^\d+-\d+$|^\d+\+$")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _fail(strategy: str) -> TransformError:
|
|
29
|
+
return TransformError(
|
|
30
|
+
f"generalize ({strategy}) cannot be applied to a value of this shape",
|
|
31
|
+
code="generalize_unsupported_value",
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _number(value: Any, strategy: str) -> float:
|
|
36
|
+
if isinstance(value, bool):
|
|
37
|
+
raise _fail(strategy)
|
|
38
|
+
if isinstance(value, int | float):
|
|
39
|
+
number = float(value)
|
|
40
|
+
elif isinstance(value, str):
|
|
41
|
+
try:
|
|
42
|
+
number = float(value.strip())
|
|
43
|
+
except ValueError:
|
|
44
|
+
raise _fail(strategy) from None
|
|
45
|
+
else:
|
|
46
|
+
raise _fail(strategy)
|
|
47
|
+
if not math.isfinite(number):
|
|
48
|
+
raise _fail(strategy)
|
|
49
|
+
return number
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _fmt(number: float) -> str:
|
|
53
|
+
return f"{number:g}"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def generalize(value: Any, params: GeneralizeParams) -> Any:
|
|
57
|
+
strategy = params.strategy
|
|
58
|
+
if strategy in ("year", "year_month"):
|
|
59
|
+
if not isinstance(value, str):
|
|
60
|
+
raise _fail(strategy)
|
|
61
|
+
m = _DATE_PREFIX.match(value.strip())
|
|
62
|
+
if not m:
|
|
63
|
+
raise _fail(strategy)
|
|
64
|
+
try:
|
|
65
|
+
dt.date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
|
66
|
+
except ValueError:
|
|
67
|
+
raise _fail(strategy) from None
|
|
68
|
+
return m.group(1) if strategy == "year" else f"{m.group(1)}-{m.group(2)}"
|
|
69
|
+
if strategy == "age_band":
|
|
70
|
+
age = _number(value, strategy)
|
|
71
|
+
if age < 0:
|
|
72
|
+
raise _fail(strategy)
|
|
73
|
+
if age >= params.max_age:
|
|
74
|
+
return f"{params.max_age}+"
|
|
75
|
+
low = int(age // params.band_size) * params.band_size
|
|
76
|
+
return f"{low}-{min(low + params.band_size - 1, params.max_age - 1)}"
|
|
77
|
+
if strategy == "round_coordinates":
|
|
78
|
+
coordinate = _number(value, strategy)
|
|
79
|
+
if abs(coordinate) > 180:
|
|
80
|
+
raise _fail(strategy)
|
|
81
|
+
rounded = round(coordinate, params.decimals)
|
|
82
|
+
if isinstance(value, str):
|
|
83
|
+
return f"{rounded:.{params.decimals}f}"
|
|
84
|
+
return rounded
|
|
85
|
+
if strategy == "numeric_bin":
|
|
86
|
+
number = _number(value, strategy)
|
|
87
|
+
edges = params.bins or []
|
|
88
|
+
if number < edges[0]:
|
|
89
|
+
return f"<{_fmt(edges[0])}"
|
|
90
|
+
for lo, hi in itertools.pairwise(edges):
|
|
91
|
+
if lo <= number < hi:
|
|
92
|
+
return f"[{_fmt(lo)}, {_fmt(hi)})"
|
|
93
|
+
return f">={_fmt(edges[-1])}"
|
|
94
|
+
raise TransformError("unsupported generalize strategy", code="unsupported_transform")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def matches_generalized_shape(text: str, params: GeneralizeParams) -> bool:
|
|
98
|
+
"""Used by rescan: does this text already look like this strategy's output?"""
|
|
99
|
+
if params.strategy == "year":
|
|
100
|
+
return bool(YEAR_RE.match(text))
|
|
101
|
+
if params.strategy == "year_month":
|
|
102
|
+
return bool(YEAR_MONTH_RE.match(text))
|
|
103
|
+
if params.strategy == "age_band":
|
|
104
|
+
return bool(AGE_BAND_RE.match(text))
|
|
105
|
+
if params.strategy == "numeric_bin":
|
|
106
|
+
return text.startswith(("<", "[", ">="))
|
|
107
|
+
return False
|