ciphergate-cli 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. ciphergate/__init__.py +95 -0
  2. ciphergate/__main__.py +6 -0
  3. ciphergate/cli.py +832 -0
  4. ciphergate/config.py +50 -0
  5. ciphergate/constants.py +274 -0
  6. ciphergate/detectors/__init__.py +5 -0
  7. ciphergate/detectors/base.py +53 -0
  8. ciphergate/detectors/custom_regex.py +37 -0
  9. ciphergate/detectors/dates.py +92 -0
  10. ciphergate/detectors/email.py +56 -0
  11. ciphergate/detectors/field_names.py +220 -0
  12. ciphergate/detectors/ip_address.py +85 -0
  13. ciphergate/detectors/network.py +42 -0
  14. ciphergate/detectors/payment_card.py +77 -0
  15. ciphergate/detectors/phone.py +67 -0
  16. ciphergate/detectors/presidio_ner.py +112 -0
  17. ciphergate/detectors/regex_safety.py +40 -0
  18. ciphergate/detectors/registry.py +194 -0
  19. ciphergate/detectors/secrets.py +130 -0
  20. ciphergate/errors.py +119 -0
  21. ciphergate/models/__init__.py +1 -0
  22. ciphergate/models/context.py +69 -0
  23. ciphergate/models/finding.py +108 -0
  24. ciphergate/models/policy.py +402 -0
  25. ciphergate/models/report.py +115 -0
  26. ciphergate/pipeline/__init__.py +1 -0
  27. ciphergate/pipeline/atomic_output.py +143 -0
  28. ciphergate/pipeline/batch.py +533 -0
  29. ciphergate/pipeline/inspector.py +282 -0
  30. ciphergate/pipeline/paths.py +98 -0
  31. ciphergate/pipeline/planner.py +192 -0
  32. ciphergate/pipeline/rescan.py +120 -0
  33. ciphergate/pipeline/sanitizer.py +392 -0
  34. ciphergate/pipeline/stdio.py +173 -0
  35. ciphergate/pipeline/streams.py +152 -0
  36. ciphergate/policy/__init__.py +26 -0
  37. ciphergate/policy/context.py +203 -0
  38. ciphergate/policy/engine.py +194 -0
  39. ciphergate/policy/loader.py +116 -0
  40. ciphergate/policy/precedence.py +149 -0
  41. ciphergate/policy/validator.py +632 -0
  42. ciphergate/py.typed +0 -0
  43. ciphergate/readers/__init__.py +113 -0
  44. ciphergate/readers/accesslog_reader.py +244 -0
  45. ciphergate/readers/base.py +131 -0
  46. ciphergate/readers/csv_reader.py +241 -0
  47. ciphergate/readers/json_reader.py +184 -0
  48. ciphergate/readers/jsonl_reader.py +97 -0
  49. ciphergate/readers/jsonlog_reader.py +178 -0
  50. ciphergate/readers/line_structure.py +125 -0
  51. ciphergate/readers/logfmt_reader.py +100 -0
  52. ciphergate/readers/stacktrace_reader.py +103 -0
  53. ciphergate/readers/syslog_reader.py +309 -0
  54. ciphergate/readers/text_reader.py +59 -0
  55. ciphergate/reporting/__init__.py +1 -0
  56. ciphergate/reporting/audit.py +85 -0
  57. ciphergate/reporting/safe_text.py +71 -0
  58. ciphergate/reporting/serializer.py +44 -0
  59. ciphergate/reporting/terminal.py +147 -0
  60. ciphergate/templates/starter_policy.yml +77 -0
  61. ciphergate/transforms/__init__.py +6 -0
  62. ciphergate/transforms/date_shift.py +79 -0
  63. ciphergate/transforms/generalize.py +107 -0
  64. ciphergate/transforms/keys.py +177 -0
  65. ciphergate/transforms/pseudonymize.py +113 -0
  66. ciphergate/transforms/redact.py +9 -0
  67. ciphergate/transforms/registry.py +201 -0
  68. ciphergate/transforms/replace.py +16 -0
  69. ciphergate_cli-0.7.0.dist-info/METADATA +531 -0
  70. ciphergate_cli-0.7.0.dist-info/RECORD +74 -0
  71. ciphergate_cli-0.7.0.dist-info/WHEEL +4 -0
  72. ciphergate_cli-0.7.0.dist-info/entry_points.txt +2 -0
  73. ciphergate_cli-0.7.0.dist-info/licenses/LICENSE +202 -0
  74. ciphergate_cli-0.7.0.dist-info/licenses/NOTICE +4 -0
@@ -0,0 +1,59 @@
1
+ """Plain text / log adapter: each line is one record, scanned as free text.
2
+
3
+ Lines are never interpreted as structured key/value data. Original line endings (LF, CRLF, CR) are
4
+ preserved. Multi-line private key blocks are tracked across lines by the detector set.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from collections.abc import Iterator
10
+ from pathlib import Path
11
+ from typing import IO, Any
12
+
13
+ from ciphergate.constants import InputFormat
14
+ from ciphergate.models.finding import ScanItem
15
+ from ciphergate.readers.base import (
16
+ ReaderOptions,
17
+ ReadStats,
18
+ Record,
19
+ decode_line,
20
+ has_utf8_bom,
21
+ read_bounded_lines,
22
+ split_line_ending,
23
+ )
24
+
25
+
26
+ class TextReader:
27
+ format = InputFormat.TEXT
28
+
29
+ def __init__(self, path: Path, options: ReaderOptions) -> None:
30
+ self.path = path
31
+ self.options = options
32
+ self.stats = ReadStats()
33
+ self.has_bom = options.encoding.startswith("utf-8") and has_utf8_bom(path)
34
+
35
+ def records(self) -> Iterator[Record]:
36
+ with self.path.open("rb") as fh:
37
+ for line_no, raw in read_bounded_lines(fh, self.options, self.stats):
38
+ text = decode_line(raw, self.options.encoding, line_no, first=line_no == 1)
39
+ content, ending = split_line_ending(text)
40
+ self.stats.records = line_no
41
+ yield Record(index=line_no - 1, line=line_no, data=content, line_ending=ending)
42
+
43
+ def iter_items(self, record: Record) -> Iterator[ScanItem]:
44
+ yield ScanItem(record.index, record.line, "", None, record.data)
45
+
46
+
47
+ class TextWriter:
48
+ def __init__(self, stream: IO[str], reader: TextReader) -> None:
49
+ self._stream = stream
50
+ self._bom_pending = reader.has_bom
51
+
52
+ def write_record(self, record: Record, data: Any) -> None:
53
+ if self._bom_pending:
54
+ self._stream.write("")
55
+ self._bom_pending = False
56
+ self._stream.write(str(data) + record.line_ending)
57
+
58
+ def close(self) -> None:
59
+ self._stream.flush()
@@ -0,0 +1 @@
1
+ """Privacy-safe report building, serialization and terminal rendering."""
@@ -0,0 +1,85 @@
1
+ """Builds ``RunReport`` objects from analysis results (privacy-safe metadata only)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import datetime as dt
6
+ import secrets
7
+ from typing import Any
8
+
9
+ from ciphergate import __version__
10
+ from ciphergate.constants import PSEUDONYM_VERSION
11
+ from ciphergate.models.report import RescanSummary, RunReport, RunStatus
12
+ from ciphergate.pipeline.paths import display_path
13
+ from ciphergate.pipeline.planner import Analysis
14
+
15
+
16
+ def new_run_id() -> str:
17
+ return secrets.token_hex(8)
18
+
19
+
20
+ def utc_now() -> str:
21
+ return dt.datetime.now(dt.UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
22
+
23
+
24
+ def build_report(
25
+ command: str,
26
+ analysis: Analysis,
27
+ *,
28
+ status: RunStatus | None = None,
29
+ output: dict[str, Any] | None = None,
30
+ changes: dict[str, Any] | None = None,
31
+ rescan: RescanSummary | None = None,
32
+ extra_warnings: list[str] | None = None,
33
+ ) -> RunReport:
34
+ setup = analysis.setup
35
+ policy = setup.policy
36
+ path_mode = setup.runtime.path_display
37
+ stats = analysis.stats
38
+ acc = analysis.acc
39
+ warnings = list(policy.warnings) + list(extra_warnings or [])
40
+ if stats.blank_lines:
41
+ action = "preserved" if policy.document.settings.preserve_blank_lines else "removed"
42
+ warnings.append(f"{stats.blank_lines} blank line(s) {action} in output")
43
+ summary = acc.summary()
44
+ summary["dropped_columns"] = [acc.dropped_columns[i] for i in sorted(acc.dropped_columns)]
45
+ return RunReport(
46
+ command=command,
47
+ status=status or analysis.status,
48
+ run_id=new_run_id(),
49
+ timestamp=utc_now(),
50
+ version=__version__,
51
+ input={
52
+ "path": display_path(setup.input_path, path_mode),
53
+ "format": str(setup.fmt),
54
+ "bytes": stats.bytes_read,
55
+ "records": stats.records,
56
+ "blank_lines": stats.blank_lines,
57
+ },
58
+ policy={
59
+ "name": policy.name,
60
+ "policy_version": policy.version,
61
+ "digest": policy.digest,
62
+ "source": policy.source_name if path_mode != "redacted" else None,
63
+ "strict": policy.document.settings.strict,
64
+ },
65
+ context=setup.ctx.to_report(),
66
+ gates=[g.to_dict() for g in analysis.gates],
67
+ transfer_approvals=list(analysis.approvals),
68
+ summary=summary,
69
+ items=acc.items,
70
+ items_truncated=acc.truncated,
71
+ keys=[
72
+ {
73
+ "key_env": name,
74
+ "source": analysis.key_sources.get(name, "env"),
75
+ "status": str(state),
76
+ "transform_version": PSEUDONYM_VERSION,
77
+ }
78
+ for name, state in analysis.key_statuses.items()
79
+ ],
80
+ changes=changes or {},
81
+ parse_errors=[{"line": line, "code": code} for line, code in stats.parse_errors[:100]],
82
+ output=output or {"written": False},
83
+ rescan=rescan,
84
+ warnings=warnings,
85
+ )
@@ -0,0 +1,71 @@
1
+ """Make user-controlled strings (field paths, header names, file names) safe to display/store.
2
+
3
+ * Control characters (including ANSI escape introducers) are escaped as ``\\xNN``/``\\uNNNN``.
4
+ * Path segments that themselves look like sensitive values (emails, secrets, card numbers, phone
5
+ numbers, IP addresses) are replaced with ``<redacted-key>`` — JSON keys and CSV headers can carry
6
+ data too.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from functools import lru_cache
12
+
13
+ from ciphergate.detectors.email import EmailDetector
14
+ from ciphergate.detectors.ip_address import IPAddressDetector
15
+ from ciphergate.detectors.payment_card import PaymentCardDetector
16
+ from ciphergate.detectors.phone import PhoneDetector
17
+ from ciphergate.detectors.secrets import SecretDetector
18
+ from ciphergate.models.finding import ScanItem
19
+
20
+ MAX_DISPLAY = 200
21
+ # Line/paragraph separators and bidi overrides can disguise terminal output.
22
+ _INVISIBLE = frozenset(
23
+ chr(c)
24
+ for c in (
25
+ 0x2028,
26
+ 0x2029,
27
+ 0x202A,
28
+ 0x202B,
29
+ 0x202C,
30
+ 0x202D,
31
+ 0x202E,
32
+ 0x2066,
33
+ 0x2067,
34
+ 0x2068,
35
+ 0x2069,
36
+ )
37
+ )
38
+ _PROBES = (EmailDetector(), SecretDetector(), PaymentCardDetector(), IPAddressDetector())
39
+ _PHONE = PhoneDetector()
40
+
41
+
42
+ def escape_control(text: str) -> str:
43
+ out: list[str] = []
44
+ for ch in text:
45
+ code = ord(ch)
46
+ if code < 0x20 or code == 0x7F or 0x80 <= code < 0xA0 or ch in _INVISIBLE:
47
+ out.append(f"\\x{code:02x}" if code < 0x100 else f"\\u{code:04x}")
48
+ else:
49
+ out.append(ch)
50
+ result = "".join(out)
51
+ return result if len(result) <= MAX_DISPLAY else result[: MAX_DISPLAY - 3] + "..."
52
+
53
+
54
+ def _segment_is_sensitive(segment: str) -> bool:
55
+ if not segment or segment.isdigit() or segment.startswith("#"):
56
+ return False
57
+ probe = ScanItem(0, None, "", None, segment)
58
+ if any(detector.detect(probe) for detector in _PROBES):
59
+ return True
60
+ return any(f.confidence >= 0.5 for f in _PHONE.detect(probe))
61
+
62
+
63
+ @lru_cache(maxsize=4096)
64
+ def safe_field_path(path: str) -> str:
65
+ if not path:
66
+ return path
67
+ if path.startswith("/"):
68
+ segments = path.split("/")[1:]
69
+ cleaned = ["<redacted-key>" if _segment_is_sensitive(s) else s for s in segments]
70
+ return escape_control("/" + "/".join(cleaned))
71
+ return escape_control("<redacted-key>" if _segment_is_sensitive(path) else path)
@@ -0,0 +1,44 @@
1
+ """Report serialization and atomic report-file writing."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+
8
+ from ciphergate.models.report import RunReport
9
+ from ciphergate.pipeline.atomic_output import AtomicOutput
10
+ from ciphergate.pipeline.batch import BatchReport
11
+ from ciphergate.reporting.terminal import render_batch_summary, render_summary
12
+
13
+
14
+ def report_to_json(report: RunReport) -> str:
15
+ return json.dumps(report.to_dict(), indent=2, ensure_ascii=False, allow_nan=False) + "\n"
16
+
17
+
18
+ def render_report(report: RunReport, report_format: str) -> str:
19
+ if report_format == "json":
20
+ return report_to_json(report)
21
+ return render_summary(report, verbose=True)
22
+
23
+
24
+ def write_report_file(report: RunReport, path: Path, report_format: str, *, force: bool) -> None:
25
+ with AtomicOutput(path, force=force) as out:
26
+ out.stream.write(render_report(report, report_format))
27
+ out.publish()
28
+
29
+
30
+ def batch_report_to_json(report: BatchReport) -> str:
31
+ return json.dumps(report.to_dict(), indent=2, ensure_ascii=False, allow_nan=False) + "\n"
32
+
33
+
34
+ def write_batch_report_file(
35
+ report: BatchReport, path: Path, report_format: str, *, force: bool
36
+ ) -> None:
37
+ text = (
38
+ batch_report_to_json(report)
39
+ if report_format == "json"
40
+ else render_batch_summary(report, verbose=True)
41
+ )
42
+ with AtomicOutput(path, force=force) as out:
43
+ out.stream.write(text)
44
+ out.publish()
@@ -0,0 +1,147 @@
1
+ """Plain-text summaries for terminals and CI logs (no colour, no values, control chars escaped)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ from ciphergate.constants import DISCLAIMER
8
+ from ciphergate.models.report import RunReport
9
+ from ciphergate.pipeline.batch import BatchReport
10
+ from ciphergate.reporting.safe_text import escape_control
11
+
12
+ SUMMARY_ITEMS = 15
13
+
14
+
15
+ def _counts(data: dict[str, int]) -> str:
16
+ return ", ".join(f"{escape_control(k)}={v}" for k, v in data.items()) or "-"
17
+
18
+
19
+ def _location(loc: dict[str, Any]) -> str:
20
+ parts = []
21
+ if loc.get("line") is not None:
22
+ parts.append(f"line {loc['line']}")
23
+ else:
24
+ parts.append(f"record {loc.get('record')}")
25
+ if loc.get("field_path"):
26
+ parts.append(f"field {escape_control(str(loc['field_path']))}")
27
+ if loc.get("span"):
28
+ parts.append(f"chars {loc['span'][0]}-{loc['span'][1]}")
29
+ if loc.get("in_key"):
30
+ parts.append("(key)")
31
+ return ", ".join(parts)
32
+
33
+
34
+ def render_summary(report: RunReport, *, verbose: bool = False) -> str:
35
+ d = report.to_dict()
36
+ lines = [f"CipherGate {d['command']}: {d['status'].upper()} (exit {d['exit_code']})"]
37
+ inp = d["input"]
38
+ lines.append(
39
+ f" input: {escape_control(str(inp['path']))} [{inp['format']}, "
40
+ f"{inp['records']} record(s), {inp['bytes']} bytes]"
41
+ )
42
+ pol = d["policy"]
43
+ lines.append(
44
+ f" policy: {escape_control(pol['name'])} v{pol['policy_version']} "
45
+ f"({pol['digest'][:19]}...)"
46
+ )
47
+ ctx = ", ".join(f"{k}={v['value'] or '-'}[{v['source']}]" for k, v in d["context"].items())
48
+ lines.append(f" context: {ctx}")
49
+ s = d["summary"]
50
+ lines.append(
51
+ f" findings: {s.get('findings_total', 0)} blocked: {s.get('blocked', 0)} "
52
+ f"review: {s.get('review_required', 0)}"
53
+ )
54
+ if s.get("findings_total"):
55
+ lines.append(f" by category: {_counts(s['by_category'])}")
56
+ lines.append(f" by action: {_counts(s['by_action'])}")
57
+ lines.append(f" by reason: {_counts(s['by_reason_code'])}")
58
+ if s.get("dropped_columns"):
59
+ lines.append(f" dropped columns: {', '.join(s['dropped_columns'])}")
60
+ for gate in d["gates"]:
61
+ lines.append(f" gate [{gate['severity']}] {gate['reason_code']}: {gate['message']}")
62
+ for approval in d["transfer_approvals"]:
63
+ lines.append(f" transfer approval used: {escape_control(approval)}")
64
+ for key in d["keys"]:
65
+ lines.append(f" key {key['key_env']}: {key['status']}")
66
+ for err in d["parse_errors"][:10]:
67
+ lines.append(f" parse error at line {err['line']}: {err['code']}")
68
+ items = d["items"]
69
+ important = [i for i in items if i["decision"]["action"] in ("block", "review")]
70
+ shown = items if verbose else (important or items)[:SUMMARY_ITEMS]
71
+ if shown:
72
+ lines.append(" items:" if verbose else " items (first shown; use --verbose or --report):")
73
+ for item in shown:
74
+ dec = item["decision"]
75
+ rule = dec.get("rule_id") or "-"
76
+ lines.append(
77
+ f" {_location(item['location'])}: {item['category']} "
78
+ f"({item['band']}, {item['detector']}) -> {dec['action']} "
79
+ f"[{dec['reason_code']}; rule {escape_control(rule)}]"
80
+ )
81
+ if d["items_truncated"]:
82
+ lines.append(" (item list truncated; counts above are complete)")
83
+ if d["changes"]:
84
+ c = d["changes"]
85
+ lines.append(
86
+ f" changes: records written={c.get('records_written')}, dropped="
87
+ f"{c.get('records_dropped')}, values changed={c.get('values_changed')}, "
88
+ f"type changes={c.get('type_changes')}"
89
+ )
90
+ if d["rescan"]:
91
+ r = d["rescan"]
92
+ verdict = "FAILED" if r["failed"] else "passed"
93
+ lines.append(
94
+ f" rescan: {verdict} (strict={r['strict']}) residuals: {_counts(r['counts'])}"
95
+ )
96
+ if r["gating_by_category"]:
97
+ lines.append(f" gating residuals: {_counts(r['gating_by_category'])}")
98
+ out = d["output"]
99
+ if out:
100
+ if out.get("written"):
101
+ lines.append(f" output: written to {escape_control(str(out.get('path')))}")
102
+ elif out.get("withheld_reason"):
103
+ lines.append(f" output: not written ({out['withheld_reason']})")
104
+ for warning in d["warnings"]:
105
+ lines.append(f" warning: {escape_control(warning)}")
106
+ lines.append(f" note: {DISCLAIMER}")
107
+ return "\n".join(lines) + "\n"
108
+
109
+
110
+ def render_batch_summary(report: BatchReport, *, verbose: bool = False) -> str:
111
+ d = report.to_dict()
112
+ s = d["summary"]
113
+ lines = [
114
+ f"CipherGate {d['command']} (directory): {d['status'].upper()} (exit {d['exit_code']})"
115
+ ]
116
+ recursive = " (recursive)" if d["input"]["recursive"] else ""
117
+ lines.append(f" input: {escape_control(str(d['input']['path']))}{recursive}")
118
+ if d["policy"]:
119
+ lines.append(f" policy: {escape_control(d['policy']['name'])}")
120
+ lines.append(
121
+ f" files: {s['files_total']} ok: {s['files_ok']} failed: {s['files_failed']} "
122
+ f"skipped: {s['files_skipped']}"
123
+ )
124
+ lines.append(
125
+ f" findings: {s['findings_total']} blocked: {s['blocked']} "
126
+ f"review: {s['review_required']}"
127
+ )
128
+ for entry in d["files"]:
129
+ detail = entry["status"]
130
+ if "error" in entry:
131
+ detail = f"error[{entry['error']['code']}]"
132
+ elif verbose:
133
+ detail += f", findings={entry['report']['summary'].get('findings_total', 0)}"
134
+ lines.append(f" {escape_control(entry['path'])}: {detail} (exit {entry['exit_code']})")
135
+ for skip in d["skipped"]:
136
+ lines.append(f" skipped {escape_control(skip['path'])}: {skip['reason']}")
137
+ out = d["output"]
138
+ if out:
139
+ if out.get("written"):
140
+ lines.append(f" output: directory written to {escape_control(str(out['path']))}")
141
+ else:
142
+ lines.append(f" output: not written ({out.get('withheld_reason')})")
143
+ for warning in d["warnings"]:
144
+ lines.append(f" warning: {escape_control(warning)}")
145
+ lines.append(" note: per-file details are in the JSON report (--report).")
146
+ lines.append(f" note: {DISCLAIMER}")
147
+ return "\n".join(lines) + "\n"
@@ -0,0 +1,77 @@
1
+ # CipherGate starter policy — TEMPLATE, REVIEW BEFORE USE.
2
+ #
3
+ # Generated by `ciphergate init`. It is a conservative starting point, not a guaranteed-safe or
4
+ # compliant configuration. Review every rule against your data, environments and destinations.
5
+ # Context (environment/destination) is NOT pinned: supply it per run, e.g.
6
+ # ciphergate inspect app.log --policy ciphergate-policy.yml --environment dev --destination local_dev
7
+ policy_version: 1
8
+ name: starter-policy
9
+ metadata:
10
+ description: "Starter template generated by ciphergate init. Template requiring review."
11
+ template: true
12
+ requirements:
13
+ environment: true
14
+ destination: true
15
+ settings:
16
+ strict: true
17
+ scan_json_keys: true # keys can carry data; required for untrusted destinations
18
+ unknown_sensitive_behavior: review
19
+ fail_on_parse_error: true
20
+ report_raw_values: false
21
+ rescan:
22
+ enabled: true
23
+ rules:
24
+ - id: block-credentials-untrusted
25
+ description: Credentials never leave for external, public or unknown destinations.
26
+ hard: true
27
+ priority: 1000
28
+ match:
29
+ categories: [credential.secret]
30
+ destinations: [external_vendor, external_ai, public_repository, unknown]
31
+ action: block
32
+ reason: credentials_not_permitted_for_destination
33
+
34
+ - id: redact-credentials
35
+ description: Elsewhere, remove credentials completely (no partial masking).
36
+ priority: 900
37
+ match:
38
+ categories: [credential.secret]
39
+ action: redact
40
+ reason: remove_credential
41
+
42
+ - id: redact-telemetry-untrusted
43
+ priority: 400
44
+ match:
45
+ categories: [network.telemetry, security.internal]
46
+ destinations: [external_vendor, external_ai, public_repository, unknown]
47
+ action: redact
48
+ reason: hide_internal_infrastructure
49
+
50
+ - id: keep-telemetry-internal
51
+ description: Internal hosts, private IPs and trace IDs are useful for internal debugging.
52
+ priority: 400
53
+ match:
54
+ categories: [network.telemetry]
55
+ destinations: [local_dev, approved_internal, staging_test, production_monitoring]
56
+ action: allow
57
+ reason: keep_debug_context_internally
58
+
59
+ - id: redact-personal-and-sensitive
60
+ priority: 300
61
+ match:
62
+ categories:
63
+ - pii.*
64
+ - health.phi
65
+ - financial.sensitive
66
+ - employment.sensitive
67
+ - security.internal
68
+ action: redact
69
+ reason: remove_identifier
70
+
71
+ defaults:
72
+ date.generic:
73
+ action: allow
74
+ location.coarse:
75
+ action: redact
76
+ identifier.generic:
77
+ action: review
@@ -0,0 +1,6 @@
1
+ """Transformations that execute approved policy decisions."""
2
+
3
+ from ciphergate.transforms.keys import KeyProvider, KeyStatus
4
+ from ciphergate.transforms.registry import Transformer, ValueOutcome
5
+
6
+ __all__ = ["KeyProvider", "KeyStatus", "Transformer", "ValueOutcome"]
@@ -0,0 +1,79 @@
1
+ """Keyed date shifting that preserves intervals within a scope.
2
+
3
+ Offsets are derived with HMAC-SHA-256 from the key and a scope value, in ``[-max_days, -1]`` or
4
+ ``[1, max_days]`` (never zero):
5
+
6
+ * ``scope: dataset`` — one offset for the whole run (all relationships preserved).
7
+ * ``scope: entity`` — one offset per value of ``entity_field`` in the same record (per-subject
8
+ consistency; structured formats only).
9
+
10
+ Supported inputs: ``iso`` (``YYYY-MM-DD``, ``YYYY-MM-DD[T ]HH:MM[:SS[.ffffff]][Z|±HH:MM]``) or
11
+ explicit ``strptime`` patterns. Fixed UTC offsets are preserved as written; named time zones and
12
+ epoch integers are not supported. Leap days are handled by calendar arithmetic (Feb 29 + 1 day =
13
+ Mar 1). Unparseable or out-of-range values raise ``TransformError``.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import datetime as dt
19
+ import hashlib
20
+ import hmac
21
+ import re
22
+ from typing import Any
23
+
24
+ from ciphergate.errors import TransformError
25
+ from ciphergate.models.policy import DateShiftParams
26
+ from ciphergate.transforms.pseudonymize import canonical_value
27
+
28
+ _ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
29
+ _ISO_DATETIME = re.compile(
30
+ r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d{1,6})?)?(?:Z|[+-]\d{2}:\d{2})?$"
31
+ )
32
+
33
+
34
+ def derive_offset_days(key: bytes, namespace: str, scope_value: Any, max_days: int) -> int:
35
+ tag, canonical = canonical_value(scope_value, "nfc")
36
+ message = f"ciphergate:dateshift:v1:{namespace}".encode() + b"\x00" + tag + b"\x00" + canonical
37
+ digest = hmac.new(key, message, hashlib.sha256).digest()
38
+ n = int.from_bytes(digest[:8], "big") % (2 * max_days)
39
+ return n - max_days if n < max_days else n - max_days + 1
40
+
41
+
42
+ def _fail() -> TransformError:
43
+ return TransformError(
44
+ "date_shift cannot parse a value with the configured formats", code="date_shift_unparseable"
45
+ )
46
+
47
+
48
+ def shift_date_text(text: str, offset_days: int, formats: list[str]) -> str:
49
+ delta = dt.timedelta(days=offset_days)
50
+ stripped = text.strip()
51
+ for fmt in formats:
52
+ try:
53
+ if fmt == "iso":
54
+ if _ISO_DATE.match(stripped):
55
+ return (dt.date.fromisoformat(stripped) + delta).isoformat()
56
+ if _ISO_DATETIME.match(stripped):
57
+ parsed = dt.datetime.fromisoformat(stripped.replace("Z", "+00:00"))
58
+ sep = "T" if "T" in stripped else " "
59
+ out = (parsed + delta).isoformat(sep=sep)
60
+ if stripped.endswith("Z"):
61
+ out = out.removesuffix("+00:00") + "Z"
62
+ return out
63
+ continue
64
+ parsed = dt.datetime.strptime(stripped, fmt)
65
+ return (parsed + delta).strftime(fmt)
66
+ except ValueError:
67
+ continue
68
+ except OverflowError:
69
+ raise TransformError(
70
+ "date_shift result is outside the supported date range",
71
+ code="date_shift_out_of_range",
72
+ ) from None
73
+ raise _fail()
74
+
75
+
76
+ def date_shift(value: Any, params: DateShiftParams, *, offset_days: int) -> str:
77
+ if not isinstance(value, str):
78
+ raise _fail()
79
+ return shift_date_text(value, offset_days, params.formats)
@@ -0,0 +1,107 @@
1
+ """Precision-reducing generalization strategies.
2
+
3
+ * ``year`` / ``year_month``: ISO date or datetime (``YYYY-MM-DD[...]``) or ``YYYY/MM/DD``.
4
+ * ``age_band``: non-negative number -> ``"30-39"`` (band_size 10); ``>= max_age`` -> ``"120+"``.
5
+ * ``round_coordinates``: number within [-180, 180] rounded to ``decimals`` (type preserved).
6
+ * ``numeric_bin``: number -> ``"<a"``, ``"[a, b)"`` or ``">=z"`` using the configured edges.
7
+
8
+ Values that do not fit the strategy raise ``TransformError`` (fail closed; no partial guessing).
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import datetime as dt
14
+ import itertools
15
+ import math
16
+ import re
17
+ from typing import Any
18
+
19
+ from ciphergate.errors import TransformError
20
+ from ciphergate.models.policy import GeneralizeParams
21
+
22
+ _DATE_PREFIX = re.compile(r"^(\d{4})[-/](\d{2})[-/](\d{2})(?:$|[T ])")
23
+ YEAR_RE = re.compile(r"^\d{4}$")
24
+ YEAR_MONTH_RE = re.compile(r"^\d{4}-\d{2}$")
25
+ AGE_BAND_RE = re.compile(r"^\d+-\d+$|^\d+\+$")
26
+
27
+
28
+ def _fail(strategy: str) -> TransformError:
29
+ return TransformError(
30
+ f"generalize ({strategy}) cannot be applied to a value of this shape",
31
+ code="generalize_unsupported_value",
32
+ )
33
+
34
+
35
+ def _number(value: Any, strategy: str) -> float:
36
+ if isinstance(value, bool):
37
+ raise _fail(strategy)
38
+ if isinstance(value, int | float):
39
+ number = float(value)
40
+ elif isinstance(value, str):
41
+ try:
42
+ number = float(value.strip())
43
+ except ValueError:
44
+ raise _fail(strategy) from None
45
+ else:
46
+ raise _fail(strategy)
47
+ if not math.isfinite(number):
48
+ raise _fail(strategy)
49
+ return number
50
+
51
+
52
+ def _fmt(number: float) -> str:
53
+ return f"{number:g}"
54
+
55
+
56
+ def generalize(value: Any, params: GeneralizeParams) -> Any:
57
+ strategy = params.strategy
58
+ if strategy in ("year", "year_month"):
59
+ if not isinstance(value, str):
60
+ raise _fail(strategy)
61
+ m = _DATE_PREFIX.match(value.strip())
62
+ if not m:
63
+ raise _fail(strategy)
64
+ try:
65
+ dt.date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
66
+ except ValueError:
67
+ raise _fail(strategy) from None
68
+ return m.group(1) if strategy == "year" else f"{m.group(1)}-{m.group(2)}"
69
+ if strategy == "age_band":
70
+ age = _number(value, strategy)
71
+ if age < 0:
72
+ raise _fail(strategy)
73
+ if age >= params.max_age:
74
+ return f"{params.max_age}+"
75
+ low = int(age // params.band_size) * params.band_size
76
+ return f"{low}-{min(low + params.band_size - 1, params.max_age - 1)}"
77
+ if strategy == "round_coordinates":
78
+ coordinate = _number(value, strategy)
79
+ if abs(coordinate) > 180:
80
+ raise _fail(strategy)
81
+ rounded = round(coordinate, params.decimals)
82
+ if isinstance(value, str):
83
+ return f"{rounded:.{params.decimals}f}"
84
+ return rounded
85
+ if strategy == "numeric_bin":
86
+ number = _number(value, strategy)
87
+ edges = params.bins or []
88
+ if number < edges[0]:
89
+ return f"<{_fmt(edges[0])}"
90
+ for lo, hi in itertools.pairwise(edges):
91
+ if lo <= number < hi:
92
+ return f"[{_fmt(lo)}, {_fmt(hi)})"
93
+ return f">={_fmt(edges[-1])}"
94
+ raise TransformError("unsupported generalize strategy", code="unsupported_transform")
95
+
96
+
97
+ def matches_generalized_shape(text: str, params: GeneralizeParams) -> bool:
98
+ """Used by rescan: does this text already look like this strategy's output?"""
99
+ if params.strategy == "year":
100
+ return bool(YEAR_RE.match(text))
101
+ if params.strategy == "year_month":
102
+ return bool(YEAR_MONTH_RE.match(text))
103
+ if params.strategy == "age_band":
104
+ return bool(AGE_BAND_RE.match(text))
105
+ if params.strategy == "numeric_bin":
106
+ return text.startswith(("<", "[", ">="))
107
+ return False