ciphergate-cli 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ciphergate/__init__.py +95 -0
- ciphergate/__main__.py +6 -0
- ciphergate/cli.py +832 -0
- ciphergate/config.py +50 -0
- ciphergate/constants.py +274 -0
- ciphergate/detectors/__init__.py +5 -0
- ciphergate/detectors/base.py +53 -0
- ciphergate/detectors/custom_regex.py +37 -0
- ciphergate/detectors/dates.py +92 -0
- ciphergate/detectors/email.py +56 -0
- ciphergate/detectors/field_names.py +220 -0
- ciphergate/detectors/ip_address.py +85 -0
- ciphergate/detectors/network.py +42 -0
- ciphergate/detectors/payment_card.py +77 -0
- ciphergate/detectors/phone.py +67 -0
- ciphergate/detectors/presidio_ner.py +112 -0
- ciphergate/detectors/regex_safety.py +40 -0
- ciphergate/detectors/registry.py +194 -0
- ciphergate/detectors/secrets.py +130 -0
- ciphergate/errors.py +119 -0
- ciphergate/models/__init__.py +1 -0
- ciphergate/models/context.py +69 -0
- ciphergate/models/finding.py +108 -0
- ciphergate/models/policy.py +402 -0
- ciphergate/models/report.py +115 -0
- ciphergate/pipeline/__init__.py +1 -0
- ciphergate/pipeline/atomic_output.py +143 -0
- ciphergate/pipeline/batch.py +533 -0
- ciphergate/pipeline/inspector.py +282 -0
- ciphergate/pipeline/paths.py +98 -0
- ciphergate/pipeline/planner.py +192 -0
- ciphergate/pipeline/rescan.py +120 -0
- ciphergate/pipeline/sanitizer.py +392 -0
- ciphergate/pipeline/stdio.py +173 -0
- ciphergate/pipeline/streams.py +152 -0
- ciphergate/policy/__init__.py +26 -0
- ciphergate/policy/context.py +203 -0
- ciphergate/policy/engine.py +194 -0
- ciphergate/policy/loader.py +116 -0
- ciphergate/policy/precedence.py +149 -0
- ciphergate/policy/validator.py +632 -0
- ciphergate/py.typed +0 -0
- ciphergate/readers/__init__.py +113 -0
- ciphergate/readers/accesslog_reader.py +244 -0
- ciphergate/readers/base.py +131 -0
- ciphergate/readers/csv_reader.py +241 -0
- ciphergate/readers/json_reader.py +184 -0
- ciphergate/readers/jsonl_reader.py +97 -0
- ciphergate/readers/jsonlog_reader.py +178 -0
- ciphergate/readers/line_structure.py +125 -0
- ciphergate/readers/logfmt_reader.py +100 -0
- ciphergate/readers/stacktrace_reader.py +103 -0
- ciphergate/readers/syslog_reader.py +309 -0
- ciphergate/readers/text_reader.py +59 -0
- ciphergate/reporting/__init__.py +1 -0
- ciphergate/reporting/audit.py +85 -0
- ciphergate/reporting/safe_text.py +71 -0
- ciphergate/reporting/serializer.py +44 -0
- ciphergate/reporting/terminal.py +147 -0
- ciphergate/templates/starter_policy.yml +77 -0
- ciphergate/transforms/__init__.py +6 -0
- ciphergate/transforms/date_shift.py +79 -0
- ciphergate/transforms/generalize.py +107 -0
- ciphergate/transforms/keys.py +177 -0
- ciphergate/transforms/pseudonymize.py +113 -0
- ciphergate/transforms/redact.py +9 -0
- ciphergate/transforms/registry.py +201 -0
- ciphergate/transforms/replace.py +16 -0
- ciphergate_cli-0.7.0.dist-info/METADATA +531 -0
- ciphergate_cli-0.7.0.dist-info/RECORD +74 -0
- ciphergate_cli-0.7.0.dist-info/WHEEL +4 -0
- ciphergate_cli-0.7.0.dist-info/entry_points.txt +2 -0
- ciphergate_cli-0.7.0.dist-info/licenses/LICENSE +202 -0
- ciphergate_cli-0.7.0.dist-info/licenses/NOTICE +4 -0
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
"""Typed, strict schema for human-authored YAML policies (``policy_version: 1``).
|
|
2
|
+
|
|
3
|
+
Unknown keys are rejected everywhere except inside ``metadata``. Semantic checks that span
|
|
4
|
+
several fields (conflicts, risky allowances, key references) live in
|
|
5
|
+
``ciphergate.policy.validator``.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import datetime as dt
|
|
11
|
+
import re
|
|
12
|
+
from typing import Annotated, Any, Literal
|
|
13
|
+
|
|
14
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
15
|
+
|
|
16
|
+
from ciphergate.constants import (
|
|
17
|
+
DEFAULT_KEY_ENV,
|
|
18
|
+
Action,
|
|
19
|
+
Destination,
|
|
20
|
+
DomainProfile,
|
|
21
|
+
Environment,
|
|
22
|
+
InputFormat,
|
|
23
|
+
TargetEnvironment,
|
|
24
|
+
UseCase,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
SLUG_RE = r"^[a-z0-9][a-z0-9_.-]{0,63}$"
|
|
28
|
+
CATEGORY_RE = re.compile(r"^[a-z][a-z0-9_]*\.[a-z0-9_]+$")
|
|
29
|
+
CATEGORY_SELECTOR_RE = re.compile(r"^(\*|[a-z][a-z0-9_]*\.\*|[a-z][a-z0-9_]*\.[a-z0-9_]+)$")
|
|
30
|
+
ENV_VAR_RE = r"^[A-Z_][A-Z0-9_]{0,127}$"
|
|
31
|
+
MARKER_RE = re.compile(r"^\[[A-Z][A-Z0-9_]{0,31}(?::(?:\{category\}|[a-z0-9_.]{1,48}))?\]$")
|
|
32
|
+
PREFIX_RE = r"^([a-z][a-z0-9_]{0,31})?$"
|
|
33
|
+
|
|
34
|
+
Slug = Annotated[str, Field(pattern=SLUG_RE)]
|
|
35
|
+
EnvVarName = Annotated[str, Field(pattern=ENV_VAR_RE)]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class StrictModel(BaseModel):
|
|
39
|
+
model_config = ConfigDict(extra="forbid", str_strip_whitespace=False)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# --------------------------------------------------------------------------- action parameters
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class EmptyParams(StrictModel):
|
|
46
|
+
"""Actions that take no parameters."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class RedactParams(StrictModel):
|
|
50
|
+
marker: str = "[REDACTED:{category}]"
|
|
51
|
+
|
|
52
|
+
@field_validator("marker")
|
|
53
|
+
@classmethod
|
|
54
|
+
def _marker_shape(cls, value: str) -> str:
|
|
55
|
+
if not MARKER_RE.match(value):
|
|
56
|
+
raise ValueError(
|
|
57
|
+
"marker must look like [WORD] or [WORD:{category}] / [WORD:label] "
|
|
58
|
+
"(uppercase word, ASCII only) so rescans can recognise it"
|
|
59
|
+
)
|
|
60
|
+
return value
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class ReplaceParams(StrictModel):
|
|
64
|
+
value: str | int | float | bool | None
|
|
65
|
+
|
|
66
|
+
@field_validator("value")
|
|
67
|
+
@classmethod
|
|
68
|
+
def _bounded(cls, value: str | int | float | bool | None) -> str | int | float | bool | None:
|
|
69
|
+
if isinstance(value, str) and (len(value) > 256 or any(c in value for c in "\r\n\x00")):
|
|
70
|
+
raise ValueError("replacement strings must be at most 256 characters without newlines")
|
|
71
|
+
return value
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class PseudonymizeParams(StrictModel):
|
|
75
|
+
key_env: EnvVarName = DEFAULT_KEY_ENV
|
|
76
|
+
scope: Literal["field", "category", "entity"]
|
|
77
|
+
entity: Slug | None = None
|
|
78
|
+
prefix: Annotated[str, Field(pattern=PREFIX_RE)] = ""
|
|
79
|
+
normalize: Literal["nfc", "casefold_nfc", "none"] = "nfc"
|
|
80
|
+
namespace: Slug | None = None
|
|
81
|
+
|
|
82
|
+
@model_validator(mode="after")
|
|
83
|
+
def _entity_scope(self) -> PseudonymizeParams:
|
|
84
|
+
if self.scope == "entity" and not self.entity:
|
|
85
|
+
raise ValueError("scope 'entity' requires an 'entity' name")
|
|
86
|
+
if self.scope != "entity" and self.entity:
|
|
87
|
+
raise ValueError("'entity' is only valid with scope 'entity'")
|
|
88
|
+
return self
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class GeneralizeParams(StrictModel):
|
|
92
|
+
strategy: Literal["year", "year_month", "age_band", "round_coordinates", "numeric_bin"]
|
|
93
|
+
band_size: Annotated[int, Field(ge=1, le=100)] = 10
|
|
94
|
+
max_age: Annotated[int, Field(ge=1, le=200)] = 120
|
|
95
|
+
decimals: Annotated[int, Field(ge=0, le=4)] = 1
|
|
96
|
+
bins: list[float] | None = None
|
|
97
|
+
|
|
98
|
+
@model_validator(mode="after")
|
|
99
|
+
def _bins(self) -> GeneralizeParams:
|
|
100
|
+
if self.strategy == "numeric_bin":
|
|
101
|
+
if not self.bins or len(self.bins) < 2:
|
|
102
|
+
raise ValueError("numeric_bin requires 'bins' with at least two increasing edges")
|
|
103
|
+
if any(b >= a for a, b in zip(self.bins[1:], self.bins, strict=False)):
|
|
104
|
+
raise ValueError("'bins' edges must be strictly increasing")
|
|
105
|
+
elif self.bins is not None:
|
|
106
|
+
raise ValueError("'bins' is only valid with strategy numeric_bin")
|
|
107
|
+
return self
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class DateShiftParams(StrictModel):
|
|
111
|
+
key_env: EnvVarName = DEFAULT_KEY_ENV
|
|
112
|
+
max_days: Annotated[int, Field(ge=1, le=3650)]
|
|
113
|
+
scope: Literal["dataset", "entity"] = "dataset"
|
|
114
|
+
entity_field: str | None = None
|
|
115
|
+
formats: list[str] = Field(default_factory=lambda: ["iso"])
|
|
116
|
+
namespace: Slug | None = None
|
|
117
|
+
|
|
118
|
+
@model_validator(mode="after")
|
|
119
|
+
def _entity(self) -> DateShiftParams:
|
|
120
|
+
if self.scope == "entity" and not self.entity_field:
|
|
121
|
+
raise ValueError("scope 'entity' requires 'entity_field'")
|
|
122
|
+
if self.scope != "entity" and self.entity_field:
|
|
123
|
+
raise ValueError("'entity_field' is only valid with scope 'entity'")
|
|
124
|
+
if not self.formats:
|
|
125
|
+
raise ValueError("'formats' must not be empty")
|
|
126
|
+
for fmt in self.formats:
|
|
127
|
+
if fmt != "iso" and ("%" not in fmt or len(fmt) > 64):
|
|
128
|
+
raise ValueError("formats must be 'iso' or an explicit strptime pattern")
|
|
129
|
+
return self
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
ActionParams = (
|
|
133
|
+
EmptyParams
|
|
134
|
+
| RedactParams
|
|
135
|
+
| ReplaceParams
|
|
136
|
+
| PseudonymizeParams
|
|
137
|
+
| GeneralizeParams
|
|
138
|
+
| DateShiftParams
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
PARAM_MODELS: dict[Action, type[StrictModel]] = {
|
|
142
|
+
Action.ALLOW: EmptyParams,
|
|
143
|
+
Action.REDACT: RedactParams,
|
|
144
|
+
Action.REPLACE: ReplaceParams,
|
|
145
|
+
Action.PSEUDONYMIZE: PseudonymizeParams,
|
|
146
|
+
Action.GENERALIZE: GeneralizeParams,
|
|
147
|
+
Action.DATE_SHIFT: DateShiftParams,
|
|
148
|
+
Action.DROP_FIELD: EmptyParams,
|
|
149
|
+
Action.DROP_RECORD: EmptyParams,
|
|
150
|
+
Action.BLOCK: EmptyParams,
|
|
151
|
+
Action.REVIEW: EmptyParams,
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
_REJECTED_ACTIONS = {
|
|
155
|
+
"tokenize": "reversible tokenization is not supported in this release (no token vault); "
|
|
156
|
+
"use 'pseudonymize' for one-way keyed pseudonyms",
|
|
157
|
+
"detokenize": "reversible tokenization is not supported in this release",
|
|
158
|
+
"mask": "partial masking is not supported; use 'redact' or 'replace'",
|
|
159
|
+
"encrypt": "encryption is not supported in this release",
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _check_action(value: Any) -> Any:
|
|
164
|
+
if isinstance(value, str) and value in _REJECTED_ACTIONS:
|
|
165
|
+
raise ValueError(_REJECTED_ACTIONS[value])
|
|
166
|
+
return value
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
# --------------------------------------------------------------------------- policy sections
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
ContextOverrideField = Literal[
|
|
173
|
+
"environment", "target_environment", "destination", "domain_profile", "use_case"
|
|
174
|
+
]
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
class PolicyContext(StrictModel):
|
|
178
|
+
environment: Environment | None = None
|
|
179
|
+
target_environment: TargetEnvironment | None = None
|
|
180
|
+
destination: Destination | None = None
|
|
181
|
+
domain_profile: DomainProfile | None = None
|
|
182
|
+
use_case: UseCase | None = None
|
|
183
|
+
allow_cli_override: list[ContextOverrideField] = Field(default_factory=list)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class Requirements(StrictModel):
|
|
187
|
+
environment: bool = True
|
|
188
|
+
target_environment: bool = False
|
|
189
|
+
destination: bool = True
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class RescanSettings(StrictModel):
|
|
193
|
+
enabled: bool = True
|
|
194
|
+
fail_on_categories: list[str] | None = None
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
class PolicySettings(StrictModel):
|
|
198
|
+
strict: bool = True
|
|
199
|
+
unknown_sensitive_behavior: Literal["review", "block"] = "review"
|
|
200
|
+
fail_on_parse_error: bool = True
|
|
201
|
+
malformed_record_action: Literal["drop_record"] | None = None
|
|
202
|
+
report_raw_values: Literal[False] = False
|
|
203
|
+
scan_json_keys: bool = False
|
|
204
|
+
csv_duplicate_headers: Literal["reject", "disambiguate"] = "reject"
|
|
205
|
+
preserve_blank_lines: bool = False
|
|
206
|
+
rescan: RescanSettings = Field(default_factory=RescanSettings)
|
|
207
|
+
|
|
208
|
+
@field_validator("report_raw_values", mode="before")
|
|
209
|
+
@classmethod
|
|
210
|
+
def _no_raw(cls, value: Any) -> Any:
|
|
211
|
+
if value is True:
|
|
212
|
+
raise ValueError("reporting raw values is not supported; reports never contain values")
|
|
213
|
+
return value
|
|
214
|
+
|
|
215
|
+
@model_validator(mode="after")
|
|
216
|
+
def _parse_errors(self) -> PolicySettings:
|
|
217
|
+
if not self.fail_on_parse_error and self.malformed_record_action is None:
|
|
218
|
+
raise ValueError(
|
|
219
|
+
"fail_on_parse_error: false requires an explicit malformed_record_action "
|
|
220
|
+
"(currently only 'drop_record')"
|
|
221
|
+
)
|
|
222
|
+
if self.fail_on_parse_error and self.malformed_record_action is not None:
|
|
223
|
+
raise ValueError(
|
|
224
|
+
"malformed_record_action is only valid when fail_on_parse_error is false"
|
|
225
|
+
)
|
|
226
|
+
return self
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
class CustomDetector(StrictModel):
|
|
230
|
+
id: Slug
|
|
231
|
+
category: str
|
|
232
|
+
pattern: Annotated[str, Field(min_length=1, max_length=512)]
|
|
233
|
+
confidence: Annotated[float, Field(ge=0.0, le=1.0)] = 0.8
|
|
234
|
+
ignore_case: bool = False
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
PRESIDIO_ENTITY_RE = r"^[A-Z][A-Z0-9_]{1,63}$"
|
|
238
|
+
SPACY_MODEL_RE = r"^[a-z][a-z0-9_]{1,63}$"
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _default_presidio_entities() -> dict[str, str]:
|
|
242
|
+
return {"PERSON": "pii.person_name", "LOCATION": "location.coarse"}
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
class PresidioSettings(StrictModel):
|
|
246
|
+
"""Optional local NER (Presidio + an already-installed spaCy model). Off by default."""
|
|
247
|
+
|
|
248
|
+
enabled: bool = False
|
|
249
|
+
language: Annotated[str, Field(pattern=r"^[a-z]{2}$")] = "en"
|
|
250
|
+
model: Annotated[str, Field(pattern=SPACY_MODEL_RE)] = "en_core_web_lg"
|
|
251
|
+
min_score: Annotated[float, Field(ge=0.0, le=1.0)] = 0.6
|
|
252
|
+
entities: dict[Annotated[str, Field(pattern=PRESIDIO_ENTITY_RE)], str] = Field(
|
|
253
|
+
default_factory=_default_presidio_entities, min_length=1
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
class DetectorSettings(StrictModel):
|
|
258
|
+
disabled: list[str] = Field(default_factory=list)
|
|
259
|
+
dates_emit_generic: bool = False
|
|
260
|
+
internal_host_suffixes: list[str] = Field(
|
|
261
|
+
default_factory=lambda: [
|
|
262
|
+
".internal",
|
|
263
|
+
".local",
|
|
264
|
+
".corp",
|
|
265
|
+
".lan",
|
|
266
|
+
".intranet",
|
|
267
|
+
".localdomain",
|
|
268
|
+
]
|
|
269
|
+
)
|
|
270
|
+
custom: list[CustomDetector] = Field(default_factory=list)
|
|
271
|
+
reject_risky_regex: bool = True
|
|
272
|
+
presidio: PresidioSettings = Field(default_factory=PresidioSettings)
|
|
273
|
+
|
|
274
|
+
@field_validator("internal_host_suffixes")
|
|
275
|
+
@classmethod
|
|
276
|
+
def _suffixes(cls, value: list[str]) -> list[str]:
|
|
277
|
+
for item in value:
|
|
278
|
+
if not re.fullmatch(r"\.[a-z0-9-]{1,63}(\.[a-z0-9-]{1,63}){0,3}", item):
|
|
279
|
+
raise ValueError("internal_host_suffixes entries must look like '.internal'")
|
|
280
|
+
return value
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
class Classification(StrictModel):
|
|
284
|
+
"""Policy-declared field classification: tells the field detector what a field contains."""
|
|
285
|
+
|
|
286
|
+
id: Slug
|
|
287
|
+
field_names: list[str] = Field(default_factory=list)
|
|
288
|
+
field_paths: list[str] = Field(default_factory=list)
|
|
289
|
+
category: str
|
|
290
|
+
confidence: Annotated[float, Field(ge=0.0, le=1.0)] = 0.9
|
|
291
|
+
|
|
292
|
+
@model_validator(mode="after")
|
|
293
|
+
def _has_selector(self) -> Classification:
|
|
294
|
+
if not self.field_names and not self.field_paths:
|
|
295
|
+
raise ValueError("a classification needs field_names or field_paths")
|
|
296
|
+
return self
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
class RuleMatch(StrictModel):
|
|
300
|
+
categories: list[str] | None = None
|
|
301
|
+
detectors: list[str] | None = None
|
|
302
|
+
field_names: list[str] | None = None
|
|
303
|
+
field_paths: list[str] | None = None
|
|
304
|
+
field_regex: Annotated[str, Field(min_length=1, max_length=256)] | None = None
|
|
305
|
+
environments: list[Environment] | None = None
|
|
306
|
+
target_environments: list[TargetEnvironment] | None = None
|
|
307
|
+
destinations: list[Destination] | None = None
|
|
308
|
+
domain_profiles: list[DomainProfile] | None = None
|
|
309
|
+
use_cases: list[UseCase] | None = None
|
|
310
|
+
formats: list[InputFormat] | None = None
|
|
311
|
+
min_confidence: Annotated[float, Field(ge=0.0, le=1.0)] | None = None
|
|
312
|
+
max_confidence: Annotated[float, Field(ge=0.0, le=1.0)] | None = None
|
|
313
|
+
|
|
314
|
+
@model_validator(mode="after")
|
|
315
|
+
def _non_empty(self) -> RuleMatch:
|
|
316
|
+
for name, value in self:
|
|
317
|
+
if isinstance(value, list) and not value:
|
|
318
|
+
raise ValueError(f"selector '{name}' must not be an empty list (omit it instead)")
|
|
319
|
+
if all(value is None for _, value in self):
|
|
320
|
+
raise ValueError(
|
|
321
|
+
"a rule must declare at least one selector; match-everything is unsafe"
|
|
322
|
+
)
|
|
323
|
+
if (
|
|
324
|
+
self.min_confidence is not None
|
|
325
|
+
and self.max_confidence is not None
|
|
326
|
+
and self.min_confidence > self.max_confidence
|
|
327
|
+
):
|
|
328
|
+
raise ValueError("min_confidence must not exceed max_confidence")
|
|
329
|
+
return self
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
class Rule(StrictModel):
|
|
333
|
+
id: Slug
|
|
334
|
+
description: Annotated[str, Field(max_length=500)] | None = None
|
|
335
|
+
priority: Annotated[int, Field(ge=0, le=10_000)] = 100
|
|
336
|
+
hard: bool = False
|
|
337
|
+
guardrail: bool = False
|
|
338
|
+
match: RuleMatch
|
|
339
|
+
action: Action
|
|
340
|
+
params: dict[str, Any] = Field(default_factory=dict)
|
|
341
|
+
reason: Slug | None = None
|
|
342
|
+
acknowledge_risk: bool = False
|
|
343
|
+
|
|
344
|
+
@field_validator("action", mode="before")
|
|
345
|
+
@classmethod
|
|
346
|
+
def _rejected_actions(cls, value: Any) -> Any:
|
|
347
|
+
return _check_action(value)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
class DefaultAction(StrictModel):
|
|
351
|
+
action: Action
|
|
352
|
+
params: dict[str, Any] = Field(default_factory=dict)
|
|
353
|
+
reason: Slug | None = None
|
|
354
|
+
|
|
355
|
+
@field_validator("action", mode="before")
|
|
356
|
+
@classmethod
|
|
357
|
+
def _rejected_actions(cls, value: Any) -> Any:
|
|
358
|
+
return _check_action(value)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
class TransferApproval(StrictModel):
|
|
362
|
+
reviewed_by: Annotated[str, Field(min_length=1, max_length=200)]
|
|
363
|
+
reference: Annotated[str, Field(min_length=1, max_length=200)]
|
|
364
|
+
approved_on: dt.date
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
class Transfer(StrictModel):
|
|
368
|
+
"""Explicit, reviewed approval lifting the built-in PRD -> non-production transfer block."""
|
|
369
|
+
|
|
370
|
+
id: Slug
|
|
371
|
+
from_environments: Annotated[list[Environment], Field(min_length=1)]
|
|
372
|
+
to_target_environments: list[TargetEnvironment] = Field(default_factory=list)
|
|
373
|
+
to_destinations: list[Destination] = Field(default_factory=list)
|
|
374
|
+
approval: TransferApproval
|
|
375
|
+
|
|
376
|
+
@model_validator(mode="after")
|
|
377
|
+
def _has_target(self) -> Transfer:
|
|
378
|
+
if not self.to_target_environments and not self.to_destinations:
|
|
379
|
+
raise ValueError("a transfer needs to_target_environments or to_destinations")
|
|
380
|
+
return self
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
class PolicyDocument(StrictModel):
|
|
384
|
+
policy_version: Literal[1]
|
|
385
|
+
name: Slug
|
|
386
|
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
387
|
+
context: PolicyContext = Field(default_factory=PolicyContext)
|
|
388
|
+
requirements: Requirements = Field(default_factory=Requirements)
|
|
389
|
+
settings: PolicySettings = Field(default_factory=PolicySettings)
|
|
390
|
+
detectors: DetectorSettings = Field(default_factory=DetectorSettings)
|
|
391
|
+
custom_categories: list[str] = Field(default_factory=list)
|
|
392
|
+
classifications: list[Classification] = Field(default_factory=list)
|
|
393
|
+
rules: list[Rule] = Field(default_factory=list)
|
|
394
|
+
defaults: dict[str, DefaultAction] = Field(default_factory=dict)
|
|
395
|
+
transfers: list[Transfer] = Field(default_factory=list)
|
|
396
|
+
|
|
397
|
+
@field_validator("policy_version", mode="before")
|
|
398
|
+
@classmethod
|
|
399
|
+
def _version(cls, value: Any) -> Any:
|
|
400
|
+
if value != 1:
|
|
401
|
+
raise ValueError("unsupported policy_version; this release supports policy_version: 1")
|
|
402
|
+
return value
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Privacy-safe run report (schema ``ciphergate.report/v1``).
|
|
2
|
+
|
|
3
|
+
A report contains counts, safe structural locations, rule IDs and reason codes. It never contains
|
|
4
|
+
raw values, snippets, keys, key derivations or value fingerprints.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from enum import StrEnum
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from ciphergate.constants import DISCLAIMER, REPORT_SCHEMA, ExitCode
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class RunStatus(StrEnum):
|
|
17
|
+
READY = "ready"
|
|
18
|
+
BLOCKED = "blocked"
|
|
19
|
+
REVIEW_REQUIRED = "review_required"
|
|
20
|
+
CONFIGURATION_REQUIRED = "configuration_required"
|
|
21
|
+
SANITIZED = "sanitized"
|
|
22
|
+
RESCAN_FAILED = "rescan_failed"
|
|
23
|
+
PASSED = "passed"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
STATUS_EXIT_CODES: dict[RunStatus, ExitCode] = {
|
|
27
|
+
RunStatus.READY: ExitCode.OK,
|
|
28
|
+
RunStatus.SANITIZED: ExitCode.OK,
|
|
29
|
+
RunStatus.PASSED: ExitCode.OK,
|
|
30
|
+
RunStatus.BLOCKED: ExitCode.BLOCKED,
|
|
31
|
+
RunStatus.REVIEW_REQUIRED: ExitCode.BLOCKED,
|
|
32
|
+
RunStatus.CONFIGURATION_REQUIRED: ExitCode.KEY_OR_TRANSFORM_CONFIG,
|
|
33
|
+
RunStatus.RESCAN_FAILED: ExitCode.RESCAN_FAILED,
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class RescanSummary:
|
|
39
|
+
strict: bool
|
|
40
|
+
failed: bool
|
|
41
|
+
fail_on_categories: list[str]
|
|
42
|
+
counts: dict[str, int]
|
|
43
|
+
unexpected_by_category: dict[str, int]
|
|
44
|
+
gating_by_category: dict[str, int]
|
|
45
|
+
items: list[dict[str, Any]]
|
|
46
|
+
items_truncated: bool = False
|
|
47
|
+
|
|
48
|
+
def to_dict(self) -> dict[str, Any]:
|
|
49
|
+
return {
|
|
50
|
+
"strict": self.strict,
|
|
51
|
+
"failed": self.failed,
|
|
52
|
+
"fail_on_categories": self.fail_on_categories,
|
|
53
|
+
"counts": self.counts,
|
|
54
|
+
"unexpected_by_category": self.unexpected_by_category,
|
|
55
|
+
"gating_by_category": self.gating_by_category,
|
|
56
|
+
"items": self.items,
|
|
57
|
+
"items_truncated": self.items_truncated,
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass
|
|
62
|
+
class RunReport:
|
|
63
|
+
command: str
|
|
64
|
+
status: RunStatus
|
|
65
|
+
run_id: str
|
|
66
|
+
timestamp: str
|
|
67
|
+
version: str
|
|
68
|
+
input: dict[str, Any]
|
|
69
|
+
policy: dict[str, Any]
|
|
70
|
+
context: dict[str, Any]
|
|
71
|
+
gates: list[dict[str, Any]] = field(default_factory=list)
|
|
72
|
+
transfer_approvals: list[str] = field(default_factory=list)
|
|
73
|
+
summary: dict[str, Any] = field(default_factory=dict)
|
|
74
|
+
items: list[dict[str, Any]] = field(default_factory=list)
|
|
75
|
+
items_truncated: bool = False
|
|
76
|
+
keys: list[dict[str, str]] = field(default_factory=list)
|
|
77
|
+
changes: dict[str, Any] = field(default_factory=dict)
|
|
78
|
+
parse_errors: list[dict[str, Any]] = field(default_factory=list)
|
|
79
|
+
output: dict[str, Any] = field(default_factory=dict)
|
|
80
|
+
rescan: RescanSummary | None = None
|
|
81
|
+
warnings: list[str] = field(default_factory=list)
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def exit_code(self) -> int:
|
|
85
|
+
return int(STATUS_EXIT_CODES[self.status])
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def ok(self) -> bool:
|
|
89
|
+
return self.exit_code == 0
|
|
90
|
+
|
|
91
|
+
def to_dict(self) -> dict[str, Any]:
|
|
92
|
+
return {
|
|
93
|
+
"schema": REPORT_SCHEMA,
|
|
94
|
+
"command": self.command,
|
|
95
|
+
"status": str(self.status),
|
|
96
|
+
"exit_code": self.exit_code,
|
|
97
|
+
"run_id": self.run_id,
|
|
98
|
+
"timestamp": self.timestamp,
|
|
99
|
+
"ciphergate_version": self.version,
|
|
100
|
+
"input": self.input,
|
|
101
|
+
"policy": self.policy,
|
|
102
|
+
"context": self.context,
|
|
103
|
+
"gates": self.gates,
|
|
104
|
+
"transfer_approvals": self.transfer_approvals,
|
|
105
|
+
"summary": self.summary,
|
|
106
|
+
"items": self.items,
|
|
107
|
+
"items_truncated": self.items_truncated,
|
|
108
|
+
"keys": self.keys,
|
|
109
|
+
"changes": self.changes,
|
|
110
|
+
"parse_errors": self.parse_errors,
|
|
111
|
+
"output": self.output,
|
|
112
|
+
"rescan": self.rescan.to_dict() if self.rescan else None,
|
|
113
|
+
"warnings": self.warnings,
|
|
114
|
+
"disclaimer": DISCLAIMER,
|
|
115
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Inspect -> plan -> sanitize -> rescan orchestration and safe output handling."""
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""Atomic, no-overwrite-by-default output publishing.
|
|
2
|
+
|
|
3
|
+
Contract:
|
|
4
|
+
|
|
5
|
+
1. A temp file is created with ``mkstemp`` in the destination directory (same filesystem; owner-only
|
|
6
|
+
``0600`` permissions on POSIX; on Windows the file inherits the directory ACL).
|
|
7
|
+
2. Content is written, flushed and ``fsync``-ed, then the temp file is closed.
|
|
8
|
+
3. Only after every check passes, ``publish()`` moves it into place:
|
|
9
|
+
* without ``force``: ``os.link`` + unlink on POSIX (fails if the target appeared meanwhile),
|
|
10
|
+
``os.rename`` on Windows (fails if the target exists) — never replaces an existing file;
|
|
11
|
+
* with ``force``: ``os.replace`` (atomic replacement; the previous file stays intact until then).
|
|
12
|
+
4. On any exception, ``KeyboardInterrupt`` or ``abort()``, the temp file is removed. If removal
|
|
13
|
+
fails, a warning naming the temp path is logged to stderr (the name is CipherGate-generated and
|
|
14
|
+
contains no data) and ``cleanup_failed_path`` is set.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import contextlib
|
|
20
|
+
import logging
|
|
21
|
+
import os
|
|
22
|
+
import tempfile
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from types import TracebackType
|
|
25
|
+
from typing import IO
|
|
26
|
+
|
|
27
|
+
from ciphergate.errors import OutputWriteError
|
|
28
|
+
from ciphergate.reporting.safe_text import escape_control
|
|
29
|
+
|
|
30
|
+
logger = logging.getLogger("ciphergate")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class AtomicOutput:
|
|
34
|
+
def __init__(self, final_path: Path, *, force: bool, encoding: str = "utf-8") -> None:
|
|
35
|
+
self.final_path = final_path
|
|
36
|
+
self.force = force
|
|
37
|
+
self.encoding = encoding
|
|
38
|
+
self.temp_path: Path | None = None
|
|
39
|
+
self._stream: IO[str] | None = None
|
|
40
|
+
self.published = False
|
|
41
|
+
self.cleanup_failed_path: Path | None = None
|
|
42
|
+
|
|
43
|
+
def __enter__(self) -> AtomicOutput:
|
|
44
|
+
parent = self.final_path.parent if str(self.final_path.parent) else Path(".")
|
|
45
|
+
try:
|
|
46
|
+
parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
47
|
+
fd, name = tempfile.mkstemp(prefix=".ciphergate-", suffix=".tmp", dir=parent)
|
|
48
|
+
except OSError:
|
|
49
|
+
raise OutputWriteError(
|
|
50
|
+
"cannot create a temporary output file in the destination directory",
|
|
51
|
+
code="output_temp_failed",
|
|
52
|
+
) from None
|
|
53
|
+
self.temp_path = Path(name)
|
|
54
|
+
with contextlib.suppress(OSError):
|
|
55
|
+
os.chmod(name, 0o600)
|
|
56
|
+
self._stream = os.fdopen(fd, "w", encoding=self.encoding, newline="", errors="strict")
|
|
57
|
+
return self
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def stream(self) -> IO[str]:
|
|
61
|
+
if self._stream is None:
|
|
62
|
+
raise OutputWriteError("output stream is not open", code="output_closed")
|
|
63
|
+
return self._stream
|
|
64
|
+
|
|
65
|
+
def finish_writing(self) -> None:
|
|
66
|
+
"""Flush, fsync and close the temp file (content complete, not yet published)."""
|
|
67
|
+
if self._stream is None:
|
|
68
|
+
return
|
|
69
|
+
try:
|
|
70
|
+
self._stream.flush()
|
|
71
|
+
os.fsync(self._stream.fileno())
|
|
72
|
+
self._stream.close()
|
|
73
|
+
except UnicodeEncodeError:
|
|
74
|
+
raise OutputWriteError(
|
|
75
|
+
"output contains characters not representable in the output encoding",
|
|
76
|
+
code="output_encoding",
|
|
77
|
+
) from None
|
|
78
|
+
except OSError:
|
|
79
|
+
raise OutputWriteError(
|
|
80
|
+
"failed to flush output to disk", code="output_flush_failed"
|
|
81
|
+
) from None
|
|
82
|
+
finally:
|
|
83
|
+
self._stream = None
|
|
84
|
+
|
|
85
|
+
def publish(self) -> None:
|
|
86
|
+
self.finish_writing()
|
|
87
|
+
if self.temp_path is None:
|
|
88
|
+
raise OutputWriteError("output was never opened", code="output_closed")
|
|
89
|
+
try:
|
|
90
|
+
if self.force:
|
|
91
|
+
os.replace(self.temp_path, self.final_path)
|
|
92
|
+
elif os.name == "nt":
|
|
93
|
+
os.rename(self.temp_path, self.final_path)
|
|
94
|
+
else:
|
|
95
|
+
os.link(self.temp_path, self.final_path)
|
|
96
|
+
os.unlink(self.temp_path)
|
|
97
|
+
except FileExistsError:
|
|
98
|
+
raise OutputWriteError(
|
|
99
|
+
"output file appeared while processing; not replaced (use --force)",
|
|
100
|
+
code="output_exists",
|
|
101
|
+
) from None
|
|
102
|
+
except OSError:
|
|
103
|
+
raise OutputWriteError(
|
|
104
|
+
"failed to publish output atomically", code="output_publish_failed"
|
|
105
|
+
) from None
|
|
106
|
+
self.published = True
|
|
107
|
+
if os.name != "nt":
|
|
108
|
+
with contextlib.suppress(OSError):
|
|
109
|
+
dir_fd = os.open(self.final_path.parent or Path("."), os.O_RDONLY)
|
|
110
|
+
try:
|
|
111
|
+
os.fsync(dir_fd)
|
|
112
|
+
finally:
|
|
113
|
+
os.close(dir_fd)
|
|
114
|
+
|
|
115
|
+
def abort(self) -> None:
|
|
116
|
+
if self._stream is not None:
|
|
117
|
+
with contextlib.suppress(OSError, ValueError, UnicodeEncodeError):
|
|
118
|
+
self._stream.close()
|
|
119
|
+
self._stream = None
|
|
120
|
+
if self.temp_path is not None and not self.published:
|
|
121
|
+
try:
|
|
122
|
+
if self.temp_path.exists():
|
|
123
|
+
self.temp_path.unlink()
|
|
124
|
+
except OSError:
|
|
125
|
+
self.cleanup_failed_path = self.temp_path
|
|
126
|
+
logger.warning(
|
|
127
|
+
"could not remove temporary output file %s; delete it manually",
|
|
128
|
+
escape_control(str(self.temp_path)),
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
def __exit__(
|
|
132
|
+
self,
|
|
133
|
+
exc_type: type[BaseException] | None,
|
|
134
|
+
exc: BaseException | None,
|
|
135
|
+
tb: TracebackType | None,
|
|
136
|
+
) -> None:
|
|
137
|
+
if exc_type is not None or not self.published:
|
|
138
|
+
self.abort()
|
|
139
|
+
if exc_type is not None and issubclass(exc_type, UnicodeEncodeError):
|
|
140
|
+
raise OutputWriteError(
|
|
141
|
+
"output contains characters not representable in the output encoding",
|
|
142
|
+
code="output_encoding",
|
|
143
|
+
) from None
|