ciphergate-cli 0.9.0__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/CHANGELOG.md +35 -1
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/PKG-INFO +3 -2
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/README.md +2 -1
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/docs/detector-coverage.md +3 -2
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/docs/limitations.md +3 -1
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/docs/policy-schema.md +7 -1
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/pyproject.toml +2 -0
- ciphergate_cli-0.11.0/scripts/benchmark/README.md +39 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/__init__.py +1 -1
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/cli.py +4 -1
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/constants.py +4 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/payment_card.py +17 -2
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/phone.py +5 -1
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/registry.py +14 -1
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/secrets.py +62 -2
- ciphergate_cli-0.11.0/src/ciphergate/detectors/ssn.py +45 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/policy/validator.py +45 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/reporting/terminal.py +20 -4
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/integration/test_cli_e2e.py +22 -0
- ciphergate_cli-0.11.0/tests/tooling/test_benchmark.py +76 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/test_detectors.py +139 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/test_policy_validation.py +66 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/.gitignore +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/LICENSE +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/NOTICE +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/SECURITY.md +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/docs/architecture.md +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/docs/claude-code-tooling.md +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/docs/examples.md +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/docs/security-and-privacy.md +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/docs/threat-model.md +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/examples/expected/sample.dev_local_dev.csv +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/examples/expected/sample.dev_local_dev.json +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/examples/expected/sample.dev_local_dev.jsonl +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/examples/expected/sample.dev_local_dev.log +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/examples/sample.csv +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/examples/sample.json +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/examples/sample.jsonl +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/examples/sample.log +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/policies/dev_local.yml +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/policies/developer_logs.yml +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/policies/external_ai.yml +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/policies/financial_template.yml +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/policies/healthcare_template.yml +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/policies/local_debug.yml +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/policies/multi_environment.yml +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/policies/prd_external_ai.yml +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/policies/stg_test.yml +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/policies/test_dataset.yml +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/__main__.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/config.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/__init__.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/base.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/custom_regex.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/dates.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/email.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/field_names.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/iban.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/ip_address.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/mac_address.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/network.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/presidio_ner.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/detectors/regex_safety.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/errors.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/models/__init__.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/models/context.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/models/finding.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/models/policy.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/models/report.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/pipeline/__init__.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/pipeline/atomic_output.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/pipeline/batch.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/pipeline/inspector.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/pipeline/paths.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/pipeline/planner.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/pipeline/rescan.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/pipeline/sanitizer.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/pipeline/stdio.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/pipeline/streams.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/policy/__init__.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/policy/context.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/policy/engine.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/policy/loader.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/policy/precedence.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/policy/templates.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/py.typed +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/readers/__init__.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/readers/accesslog_reader.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/readers/base.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/readers/csv_reader.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/readers/json_reader.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/readers/jsonl_reader.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/readers/jsonlog_reader.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/readers/line_structure.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/readers/logfmt_reader.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/readers/stacktrace_reader.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/readers/syslog_reader.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/readers/text_reader.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/reporting/__init__.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/reporting/audit.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/reporting/safe_text.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/reporting/serializer.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/templates/starter_policy.yml +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/transforms/__init__.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/transforms/date_shift.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/transforms/generalize.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/transforms/keys.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/transforms/pseudonymize.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/transforms/redact.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/transforms/registry.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/src/ciphergate/transforms/replace.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/__init__.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/conftest.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/integration/__init__.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/integration/test_api_batch_streams.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/integration/test_atomic_output.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/integration/test_batch.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/integration/test_domain_templates.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/integration/test_environment_matrix.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/integration/test_pipelines.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/integration/test_stdio.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/regression/__init__.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/regression/test_known_cases.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/security/__init__.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/security/test_key_file.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/security/test_no_network.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/security/test_no_raw_value_leaks.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/security/test_path_safety.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/security/test_policy_fail_closed.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/security/test_review_findings.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/tooling/__init__.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/tooling/test_ci_supply_chain.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/tooling/test_claude_tooling.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/tooling/test_git_tooling.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/tooling/test_publish_workflow.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/__init__.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/test_accesslog.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/test_jsonlog.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/test_logfmt.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/test_policy_engine.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/test_policy_precedence.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/test_policy_templates.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/test_presidio_ner.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/test_properties.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/test_readers.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/test_stacktrace.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/test_syslog.py +0 -0
- {ciphergate_cli-0.9.0 → ciphergate_cli-0.11.0}/tests/unit/test_transforms.py +0 -0
|
@@ -6,6 +6,38 @@ versioning: [PEP 440](https://peps.python.org/pep-0440/) with SemVer meaning (pr
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.11.0] - 2026-10-31
|
|
10
|
+
|
|
11
|
+
Makes a clean rescan harder to mistake for proof of safety. Scripts that read `PASSED` from the
|
|
12
|
+
`rescan` terminal output need updating; exit codes and the JSON report do not change.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- The `rescan` terminal summary no longer prints `PASSED`. It says `NO GATING FINDINGS`, or
|
|
17
|
+
`NOT ENFORCED` when gating findings remain with strict off, and states what rescan cannot see.
|
|
18
|
+
Exit codes and the JSON report's `status` are unchanged.
|
|
19
|
+
- `validate-policy` warns when a rule or default handles `credential.secret` with anything other
|
|
20
|
+
than `block` or `review` toward a possibly untrusted destination.
|
|
21
|
+
|
|
22
|
+
## [0.10.0] - 2026-10-30
|
|
23
|
+
|
|
24
|
+
Fixes a password leak in nested values and detects more credentials and identifiers. A run that
|
|
25
|
+
passed before can now redact, review or block, so re-check your policies before upgrading a gate.
|
|
26
|
+
|
|
27
|
+
### Added
|
|
28
|
+
|
|
29
|
+
- `ssn_pattern` detector for dashed US Social Security numbers (`pii.government_id`). They were
|
|
30
|
+
labelled `pii.phone`, so government-ID rules did not apply and a strict template can now block.
|
|
31
|
+
|
|
32
|
+
### Fixed
|
|
33
|
+
|
|
34
|
+
- `secret_pattern` no longer stops at `{` for nested values such as Actuator dumps, which left
|
|
35
|
+
the real value in the output.
|
|
36
|
+
- `secret_pattern` now finds camelCase password names, `Cookie` and `Set-Cookie` headers, and
|
|
37
|
+
session and CSRF cookie values. `luhn_card` finds labelled CVV and CVC values.
|
|
38
|
+
- Fewer false alarms: shell `PWD` paths, code such as `settings.X` or `request.Y` on the right of
|
|
39
|
+
an assignment, and an HTTP status followed by a byte count.
|
|
40
|
+
|
|
9
41
|
## [0.9.0] - 2026-10-30
|
|
10
42
|
|
|
11
43
|
Detects more values. A run that passed before can now redact, review or block, so re-check your
|
|
@@ -296,7 +328,9 @@ Pre-release (alpha). Licensed under Apache-2.0. Not a final release: the securit
|
|
|
296
328
|
See `docs/limitations.md`. Deferred: optional NER/Presidio detectors, stdin/stdout streaming,
|
|
297
329
|
directory processing, structured log parsers, key files, reversible token vault, lossy decoding.
|
|
298
330
|
|
|
299
|
-
[Unreleased]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.
|
|
331
|
+
[Unreleased]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.11.0...HEAD
|
|
332
|
+
[0.11.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.10.0...v0.11.0
|
|
333
|
+
[0.10.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.9.0...v0.10.0
|
|
300
334
|
[0.9.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.8.0...v0.9.0
|
|
301
335
|
[0.8.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.7.1...v0.8.0
|
|
302
336
|
[0.7.1]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.7.0...v0.7.1
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ciphergate-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Local-first, environment- and destination-aware inspection and sanitization of logs and test datasets.
|
|
5
5
|
Project-URL: Homepage, https://github.com/aashishkumar-tech/ciphergate
|
|
6
6
|
Project-URL: Source, https://github.com/aashishkumar-tech/ciphergate
|
|
@@ -50,7 +50,7 @@ vendor ticket. CipherGate replaces that with a local, explainable, scriptable ga
|
|
|
50
50
|
inspect -> plan -> sanitize --apply -> rescan
|
|
51
51
|
```
|
|
52
52
|
|
|
53
|
-
> **Status: 0.
|
|
53
|
+
> **Status: 0.11.0 (alpha).** No detector finds every sensitive value. A passing run means the
|
|
54
54
|
> configured processing completed under the selected policy; it is **not** proof that the output
|
|
55
55
|
> contains no sensitive information, and it is not a compliance (GDPR, HIPAA, PCI DSS, ...) or
|
|
56
56
|
> anonymization guarantee. Policy templates must be reviewed before use.
|
|
@@ -154,6 +154,7 @@ ciphergate sanitize examples/sample.csv --policy policies/prd_external_ai.yml \
|
|
|
154
154
|
--output build/sample.sanitized.csv --report build/sample.report.json --apply
|
|
155
155
|
|
|
156
156
|
# 5. Rescan the output (read-only). Exit 7 if gating residual findings remain.
|
|
157
|
+
# A clean result only means no gating findings. It does not show the output is safe.
|
|
157
158
|
ciphergate rescan build/sample.sanitized.csv --policy policies/prd_external_ai.yml
|
|
158
159
|
```
|
|
159
160
|
|
|
@@ -11,7 +11,7 @@ vendor ticket. CipherGate replaces that with a local, explainable, scriptable ga
|
|
|
11
11
|
inspect -> plan -> sanitize --apply -> rescan
|
|
12
12
|
```
|
|
13
13
|
|
|
14
|
-
> **Status: 0.
|
|
14
|
+
> **Status: 0.11.0 (alpha).** No detector finds every sensitive value. A passing run means the
|
|
15
15
|
> configured processing completed under the selected policy; it is **not** proof that the output
|
|
16
16
|
> contains no sensitive information, and it is not a compliance (GDPR, HIPAA, PCI DSS, ...) or
|
|
17
17
|
> anonymization guarantee. Policy templates must be reviewed before use.
|
|
@@ -115,6 +115,7 @@ ciphergate sanitize examples/sample.csv --policy policies/prd_external_ai.yml \
|
|
|
115
115
|
--output build/sample.sanitized.csv --report build/sample.report.json --apply
|
|
116
116
|
|
|
117
117
|
# 5. Rescan the output (read-only). Exit 7 if gating residual findings remain.
|
|
118
|
+
# A clean result only means no gating findings. It does not show the output is safe.
|
|
118
119
|
ciphergate rescan build/sample.sanitized.csv --policy policies/prd_external_ai.yml
|
|
119
120
|
```
|
|
120
121
|
|
|
@@ -12,11 +12,12 @@ are engineering estimates mapped to bands (`low` < 0.5 <= `medium` < 0.8 <= `hig
|
|
|
12
12
|
| `email_regex` | `pii.email` | `alice.test@example.com`, `alice.test%40example.com` | `a..b@example.com`, `user@localhost` | 0.9 | ASCII only; no IDN; obfuscated forms (`alice at example dot com`) missed; `%40` is the only percent-encoding recognised outside `accesslog` |
|
|
13
13
|
| `phone_pattern` | `pii.phone` | `+1 202 555 0143` (high), `(202) 555-0143` (medium) | dates, IPs, card numbers, repeated digits, < 7 digits | 0.3 bare digits, 0.55–0.8 prefixed/formatted, +0.2 keyword | no numbering-plan validation; IDs can look like phones (low band) |
|
|
14
14
|
| `ip_address` | `pii.ip_address` (global), `network.telemetry` (private/loopback/link-local) | `8.8.8.8`, `10.0.0.1`, `2606:4700::1111`, `fe80::1%eth0` | `256.1.1.1`, `std::vector`, `12:30:45` | 0.85 / 0.8 | version strings like `1.2.3.4` match; documentation ranges flagged with evidence |
|
|
15
|
-
| `luhn_card` | `pii.payment_card` | `4111 1111 1111 1111` | Luhn-invalid, mixed separators, repeated digits | 0.4 Luhn-only, 0.8 issuer prefix, +0.15 keyword | random IDs pass Luhn ~10% (low band); cards split across fields missed |
|
|
15
|
+
| `luhn_card` | `pii.payment_card` | `4111 1111 1111 1111` | Luhn-invalid, mixed separators, repeated digits | 0.4 Luhn-only, 0.8 issuer prefix, +0.15 keyword; labelled CVV/CVC values 0.85 | random IDs pass Luhn ~10% (low band); cards split across fields missed |
|
|
16
16
|
| `mac_address` | `pii.device_identifier` | `00:1A:2B:3C:4D:5E`, `00-1a-2b-3c-4d-5e`, `001a.2b3c.4d5e` | all-zero, broadcast, mixed separators, `12:30:45` | 0.85, 0.6 when all digits | bare 12-hex values are missed; digit-only IDs in this shape are flagged |
|
|
17
17
|
| `iban` | `pii.bank_account` | a mod-97-valid IBAN, compact or space-grouped | wrong check digits, lowercase | 0.9 | country length and national format are not checked; about 1 in 97 random strings of this shape pass; lowercase forms are missed |
|
|
18
|
+
| `ssn_pattern` | `pii.government_id` | `219-09-9999` | never-issued areas (`000`, `666`, `9xx`), group `00`, serial `0000`, undashed | 0.8, 0.95 with a keyword | US only; undashed numbers are missed; other dashed IDs in this shape are flagged |
|
|
18
19
|
| `date_pattern` | `pii.date_of_birth` with field/keyword evidence; `date.generic` if enabled | `dob: 1990-04-12`, field `birth_date` | `created 2026-01-05` (no DOB evidence), invalid dates | 0.85 / 0.6 | `DD/MM` vs `MM/DD` ambiguity flagged; free-text DOBs without keywords missed |
|
|
19
|
-
| `secret_pattern` | `credential.secret` | PEM blocks, `Bearer ...`, `AKIA...`, `ghp_...`, `xox?-`, `sk_test_...`, `AIza...`, JWTs, `scheme://u:pw@`, `Password=...;`, `password=`, assignments to `PGPASSWORD` or `
|
|
20
|
+
| `secret_pattern` | `credential.secret` | PEM blocks, `Bearer ...`, `AKIA...`, `ghp_...`, `xox?-`, `sk_test_...`, `AIza...`, JWTs, `scheme://u:pw@`, `Password=...;`, `password=`, assignments to `PGPASSWORD`, `MYSQL_PWD` or camelCase `...Password` names, `Cookie` and `Set-Cookie` headers, session and CSRF cookie values, `api_key:`, `token=` | `password=`, `password: null`, markers, `******` | 0.8–0.99 | base64/hex-wrapped, split, custom-format or high-entropy secrets without keywords are **missed**; prose like `password: required` is flagged |
|
|
20
21
|
| `internal_host` | `network.telemetry` | `api-1.prod.internal`, `db.corp`, `localhost` | `example.com`, `a.internalx` | 0.75 | internal names without listed suffixes missed; configurable |
|
|
21
22
|
| `custom:<id>` | configured | | | configured | regex-safety checks are heuristics |
|
|
22
23
|
| `presidio_ner` (optional, off by default) | Presidio entities -> mapped categories (default `PERSON` -> `pii.person_name`, `LOCATION` -> `location.coarse`) | `ticket opened by <a person's name>` | numbers, single characters, CipherGate markers/pseudonyms | Presidio score, filtered by `min_score` | statistical model: misses and false positives, language/model dependent, slow; requires `ciphergate-cli[ner]` plus a locally installed spaCy model |
|
|
@@ -5,7 +5,9 @@
|
|
|
5
5
|
- Detection is incomplete. Names, addresses, government IDs and bank accounts are not detected by
|
|
6
6
|
value in free text; secrets in custom formats or encoded forms are missed. See
|
|
7
7
|
[detector-coverage.md](detector-coverage.md).
|
|
8
|
-
- A clean rescan is a quality gate, not proof of safety.
|
|
8
|
+
- A clean rescan is a quality gate, not proof of safety. The terminal summary says "no gating
|
|
9
|
+
findings" for this reason, while the JSON report keeps `status: passed` for CI. Names and
|
|
10
|
+
addresses in free text are often missed. Rescan cannot verify date-shifted values
|
|
9
11
|
(reported as `unverifiable`) and recognises CipherGate's own markers/pseudonyms/replacement
|
|
10
12
|
constants as transformed output.
|
|
11
13
|
- Confidence values are uncalibrated engineering estimates.
|
|
@@ -52,7 +52,7 @@ A pinned value combined with a different caller value fails with
|
|
|
52
52
|
## Detectors
|
|
53
53
|
|
|
54
54
|
`disabled` (built-in IDs: `field_name`, `secret_pattern`, `luhn_card`, `ip_address`,
|
|
55
|
-
`date_pattern`, `email_regex`, `internal_host`, `phone_pattern`, `mac_address`, `iban`),
|
|
55
|
+
`date_pattern`, `email_regex`, `internal_host`, `phone_pattern`, `mac_address`, `iban`, `ssn_pattern`),
|
|
56
56
|
`dates_emit_generic` (report
|
|
57
57
|
non-DOB dates as `date.generic`), `internal_host_suffixes` (default `.internal .local .corp .lan
|
|
58
58
|
.intranet .localdomain`), `custom` (list of `{id, category, pattern, confidence, ignore_case}`;
|
|
@@ -122,6 +122,12 @@ A context selector never matches missing context.
|
|
|
122
122
|
| `review` | — | run needs human decision, nothing written |
|
|
123
123
|
| `tokenize` | — | **rejected**: no token vault in this release |
|
|
124
124
|
|
|
125
|
+
A rule or default that applies `redact`, `replace`, or any other non-`block`, non-`review` action
|
|
126
|
+
to `credential.secret` toward a possibly untrusted destination validates with a warning, because
|
|
127
|
+
a detector can match only part of a secret. A destination that is not pinned counts as possibly
|
|
128
|
+
untrusted. A `hard: true` block that covers every untrusted destination, with no selectors other
|
|
129
|
+
than `categories` and `destinations`, removes the warning.
|
|
130
|
+
|
|
125
131
|
## Precedence (deterministic)
|
|
126
132
|
|
|
127
133
|
1. Engine hard constraints: missing/invalid context gates; credential never allowed (forced
|
|
@@ -110,6 +110,8 @@ ignore = [
|
|
|
110
110
|
[tool.ruff.lint.per-file-ignores]
|
|
111
111
|
"tests/**" = ["S101", "S105", "S106", "S108", "S311", "PLC0415", "S607"]
|
|
112
112
|
"scripts/**" = ["S607", "T201"]
|
|
113
|
+
# Reproducible synthetic data, and optional tools that may not be installed.
|
|
114
|
+
"scripts/benchmark/**" = ["S311", "PLC0415", "S607", "T201"]
|
|
113
115
|
|
|
114
116
|
[tool.mypy]
|
|
115
117
|
python_version = "3.11"
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Detection benchmark
|
|
2
|
+
|
|
3
|
+
Measures what CipherGate catches and flags wrongly on generated, labelled log lines, next to other
|
|
4
|
+
local tools. It is an engineering aid for finding gaps. It is not part of the package, CI or the
|
|
5
|
+
test suite, and it never contacts a service.
|
|
6
|
+
|
|
7
|
+
## Do not use the results as a public claim
|
|
8
|
+
|
|
9
|
+
- The corpus is written by the people who wrote the detectors, so it leans toward CipherGate.
|
|
10
|
+
- The data is synthetic. Real logs behave differently.
|
|
11
|
+
- Other tools run on default settings, and their results depend on versions and models.
|
|
12
|
+
- A high score here does not show that sensitive data is gone from any file.
|
|
13
|
+
|
|
14
|
+
## Run it
|
|
15
|
+
|
|
16
|
+
Use a separate virtual environment, because these tools are not project dependencies:
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
python -m venv /path/outside/repo/venv
|
|
20
|
+
# Activate it, then:
|
|
21
|
+
python -m pip install . detect-secrets scrubadub presidio-analyzer
|
|
22
|
+
python -m pip install en_core_web_sm # see the spaCy docs for the model install command
|
|
23
|
+
python scripts/benchmark/compare_tools.py --out /path/outside/repo/results
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Only CipherGate is required. Any other tool that is not installed is skipped and listed under
|
|
27
|
+
`not run`. To include gitleaks, install it yourself and pass `--gitleaks PATH`.
|
|
28
|
+
|
|
29
|
+
`--out` must be outside the repository, because `corpus.jsonl` holds format-shaped fake
|
|
30
|
+
credentials that must not be committed. The tool refuses a path inside the repository. Results
|
|
31
|
+
are repeatable for the same `--seed` and tool versions.
|
|
32
|
+
|
|
33
|
+
## Reading the output
|
|
34
|
+
|
|
35
|
+
Each row is a data class, and each cell is caught/total. A positive counts only if the finding
|
|
36
|
+
overlaps the planted value; for CipherGate, the value must be gone from the output. `NEGATIVE`
|
|
37
|
+
counts false alarms on clean lines. `presidio@0.5` is Presidio with a 0.5 score threshold.
|
|
38
|
+
|
|
39
|
+
Random 16-digit IDs pass the card checksum about 10% of the time, so `NEGATIVE` varies by seed.
|
|
@@ -24,7 +24,7 @@ Supported public API (everything else is internal and may change without notice)
|
|
|
24
24
|
rescan_directory(root, policy, context=None, *, recursive=False, strict=None) -> BatchReport
|
|
25
25
|
"""
|
|
26
26
|
|
|
27
|
-
__version__ = "0.
|
|
27
|
+
__version__ = "0.11.0"
|
|
28
28
|
|
|
29
29
|
from ciphergate.config import RuntimeSettings
|
|
30
30
|
from ciphergate.constants import (
|
|
@@ -749,7 +749,10 @@ def rescan_command(
|
|
|
749
749
|
no_color: NoColorOpt = False,
|
|
750
750
|
debug: DebugOpt = False,
|
|
751
751
|
) -> None:
|
|
752
|
-
"""Report residual findings in an existing output. Read-only; never alters the file.
|
|
752
|
+
"""Report residual findings in an existing output. Read-only; never alters the file.
|
|
753
|
+
|
|
754
|
+
No gating findings does not mean the output is safe: rescan only sees what detectors match.
|
|
755
|
+
"""
|
|
753
756
|
_configure_logging(verbose, debug)
|
|
754
757
|
_guard(
|
|
755
758
|
debug,
|
|
@@ -272,3 +272,7 @@ DISCLAIMER = (
|
|
|
272
272
|
"It does not prove that the output contains no sensitive information; "
|
|
273
273
|
"detectors can miss values."
|
|
274
274
|
)
|
|
275
|
+
RESCAN_LIMIT = (
|
|
276
|
+
"rescan only sees what the enabled detectors match. No gating findings does not mean the "
|
|
277
|
+
"output is safe; names and addresses in free text are often missed."
|
|
278
|
+
)
|
|
@@ -12,6 +12,9 @@ from ciphergate.detectors.base import make_finding, preceding_context
|
|
|
12
12
|
from ciphergate.models.finding import Finding, ScanItem
|
|
13
13
|
|
|
14
14
|
_CARD_RE = re.compile(r"(?<![0-9A-Za-z_])(\d(?:[ -]?\d){12,18})(?![0-9A-Za-z_])")
|
|
15
|
+
_CVV_RE = re.compile(
|
|
16
|
+
r"(?i)(?<![a-z0-9])(?:cvv2?|cvc2?)(?:\\?[\"'])?\s{0,3}[:=]\s{0,3}(?:\\?[\"'])?(\d{3,4})(?!\d)"
|
|
17
|
+
)
|
|
15
18
|
_KEYWORDS = ("card", "visa", "mastercard", "amex", "pan", "cc", "credit", "debit", "payment")
|
|
16
19
|
|
|
17
20
|
|
|
@@ -45,9 +48,21 @@ class PaymentCardDetector:
|
|
|
45
48
|
|
|
46
49
|
def detect(self, item: ScanItem) -> list[Finding]:
|
|
47
50
|
text = item.text
|
|
48
|
-
if not text
|
|
51
|
+
if not text:
|
|
49
52
|
return []
|
|
50
|
-
out
|
|
53
|
+
out = [
|
|
54
|
+
make_finding(
|
|
55
|
+
item,
|
|
56
|
+
category="pii.payment_card",
|
|
57
|
+
detector=self.id,
|
|
58
|
+
confidence=0.85,
|
|
59
|
+
evidence=("pattern_match", "context_keyword"),
|
|
60
|
+
span=(m.start(1), m.end(1)),
|
|
61
|
+
)
|
|
62
|
+
for m in _CVV_RE.finditer(text)
|
|
63
|
+
]
|
|
64
|
+
if sum(c.isdigit() for c in text) < 13:
|
|
65
|
+
return out
|
|
51
66
|
for m in _CARD_RE.finditer(text):
|
|
52
67
|
raw = m.group(1)
|
|
53
68
|
digits = raw.replace(" ", "").replace("-", "")
|
|
@@ -21,6 +21,7 @@ _PHONE_RE = re.compile(
|
|
|
21
21
|
)
|
|
22
22
|
_DATE_LIKE = re.compile(r"^\d{4}[-./]\d{1,2}[-./]\d{1,2}$|^\d{1,2}[-./]\d{1,2}[-./]\d{2,4}$")
|
|
23
23
|
_SEPARATED = re.compile(r"^\(?\d{2,4}\)?[ .-]\d{2,5}[ .-]\d{2,5}(?:[ .-]\d{2,5})?$")
|
|
24
|
+
_HTTP_STATUS_BYTES = re.compile(r"^[1-5]\d{2} \d{1,12}$")
|
|
24
25
|
_KEYWORDS = ("phone", "tel", "mobile", "cell", "call", "fax", "sms", "whatsapp", "contact")
|
|
25
26
|
|
|
26
27
|
|
|
@@ -40,6 +41,9 @@ class PhoneDetector:
|
|
|
40
41
|
continue
|
|
41
42
|
if _DATE_LIKE.match(core.strip()):
|
|
42
43
|
continue
|
|
44
|
+
context = preceding_context(text, m.start(), _KEYWORDS)
|
|
45
|
+
if _HTTP_STATUS_BYTES.match(core.strip()) and not context:
|
|
46
|
+
continue
|
|
43
47
|
evidence = ["pattern_match"]
|
|
44
48
|
if core.startswith("+"):
|
|
45
49
|
confidence = 0.8 if len(digits) >= 8 else 0.55
|
|
@@ -49,7 +53,7 @@ class PhoneDetector:
|
|
|
49
53
|
evidence.append("separator_layout")
|
|
50
54
|
else:
|
|
51
55
|
confidence = 0.3
|
|
52
|
-
if
|
|
56
|
+
if context:
|
|
53
57
|
confidence += 0.2
|
|
54
58
|
evidence.append("context_keyword")
|
|
55
59
|
if "x" in candidate.lower() or "ext" in candidate.lower():
|
|
@@ -16,6 +16,7 @@ from ciphergate.detectors.network import InternalHostDetector
|
|
|
16
16
|
from ciphergate.detectors.payment_card import PaymentCardDetector
|
|
17
17
|
from ciphergate.detectors.phone import PhoneDetector
|
|
18
18
|
from ciphergate.detectors.secrets import PEM_BEGIN_RE, PEM_END_RE, SecretDetector
|
|
19
|
+
from ciphergate.detectors.ssn import SsnDetector
|
|
19
20
|
from ciphergate.models.finding import Finding, ScanItem
|
|
20
21
|
from ciphergate.models.policy import DetectorSettings, PolicyDocument
|
|
21
22
|
|
|
@@ -30,11 +31,20 @@ BUILTIN_DETECTOR_IDS: tuple[str, ...] = (
|
|
|
30
31
|
"phone_pattern",
|
|
31
32
|
"mac_address",
|
|
32
33
|
"iban",
|
|
34
|
+
"ssn_pattern",
|
|
33
35
|
)
|
|
34
36
|
|
|
35
37
|
# Phone matching is the least specific pattern; drop phone spans that overlap these detectors.
|
|
36
38
|
_PHONE_SUPPRESSORS = frozenset(
|
|
37
|
-
{
|
|
39
|
+
{
|
|
40
|
+
"luhn_card",
|
|
41
|
+
"ip_address",
|
|
42
|
+
"date_pattern",
|
|
43
|
+
"secret_pattern",
|
|
44
|
+
"mac_address",
|
|
45
|
+
"iban",
|
|
46
|
+
"ssn_pattern",
|
|
47
|
+
}
|
|
38
48
|
)
|
|
39
49
|
|
|
40
50
|
|
|
@@ -67,6 +77,7 @@ class DetectorSet:
|
|
|
67
77
|
"phone_pattern": PhoneDetector(),
|
|
68
78
|
"mac_address": MacAddressDetector(),
|
|
69
79
|
"iban": IbanDetector(),
|
|
80
|
+
"ssn_pattern": SsnDetector(),
|
|
70
81
|
}
|
|
71
82
|
for detector_id in BUILTIN_DETECTOR_IDS:
|
|
72
83
|
if detector_id not in disabled:
|
|
@@ -185,6 +196,8 @@ def unsafe_replacement_reason(value: str) -> str | None:
|
|
|
185
196
|
return "looks like a real IBAN"
|
|
186
197
|
if MacAddressDetector().detect(probe):
|
|
187
198
|
return "looks like a real MAC address"
|
|
199
|
+
if SsnDetector().detect(probe):
|
|
200
|
+
return "looks like a real government identifier"
|
|
188
201
|
for f in EmailDetector().detect(probe):
|
|
189
202
|
if "reserved_domain" not in f.evidence:
|
|
190
203
|
return "email replacements must use a reserved domain such as example.invalid"
|
|
@@ -69,17 +69,68 @@ _PATTERNS: tuple[_Pattern, ...] = (
|
|
|
69
69
|
r"api[_-]?key|apikey|access[_-]?token|"
|
|
70
70
|
r"refresh[_-]?token|auth[_-]?token|client[_-]?secret|secret[_-]?key|private[_-]?key|"
|
|
71
71
|
r"access[_-]?key|token|credentials?)"
|
|
72
|
+
r"(?:\\?[\"'])?\s{0,3}[:=]\s{0,3}"
|
|
73
|
+
# Spring Actuator and similar dumps nest the value in an object.
|
|
74
|
+
r"(?:\{\s{0,3}\\?[\"']value\\?[\"']\s{0,3}:\s{0,3})?(?:\\?[\"'])?"
|
|
75
|
+
r"([^\s\"'\\,;&}]{1,512})"
|
|
76
|
+
),
|
|
77
|
+
1,
|
|
78
|
+
0.8,
|
|
79
|
+
),
|
|
80
|
+
_Pattern(
|
|
81
|
+
"camel_case_password",
|
|
82
|
+
re.compile(
|
|
83
|
+
r"(?<![A-Za-z0-9])[a-z][A-Za-z0-9]{0,40}Password"
|
|
72
84
|
r"(?:\\?[\"'])?\s{0,3}[:=]\s{0,3}(?:\\?[\"'])?"
|
|
73
85
|
r"([^\s\"'\\,;&}]{1,512})"
|
|
74
86
|
),
|
|
75
87
|
1,
|
|
76
88
|
0.8,
|
|
77
89
|
),
|
|
90
|
+
_Pattern(
|
|
91
|
+
"cookie_header",
|
|
92
|
+
re.compile(
|
|
93
|
+
r"(?i)(?<![a-z0-9_-])(?:set-)?cookie(?:\\?[\"'])?\s{0,3}:\s{0,3}(?:\\?[\"'])?"
|
|
94
|
+
r"([^\s=;\"']{1,128}=[^\r\n\"']{1,2048})"
|
|
95
|
+
),
|
|
96
|
+
1,
|
|
97
|
+
0.85,
|
|
98
|
+
),
|
|
99
|
+
_Pattern(
|
|
100
|
+
"session_cookie",
|
|
101
|
+
re.compile(
|
|
102
|
+
r"(?i)(?<![a-z0-9_-])(?:(?:sessionid|jsessionid|phpsessid|connect\.sid|csrftoken|"
|
|
103
|
+
r"csrfmiddlewaretoken|xsrf-token)(?:\\?[\"'])?\s{0,3}[:=]\s{0,3}|session=)"
|
|
104
|
+
r"(?:\\?[\"'])?([^\s\"'\\,;&}]{8,512})"
|
|
105
|
+
),
|
|
106
|
+
1,
|
|
107
|
+
0.8,
|
|
108
|
+
),
|
|
78
109
|
)
|
|
79
110
|
|
|
80
111
|
_PLACEHOLDER_VALUES = frozenset(
|
|
81
|
-
{
|
|
112
|
+
{
|
|
113
|
+
"",
|
|
114
|
+
"null",
|
|
115
|
+
"none",
|
|
116
|
+
"nil",
|
|
117
|
+
"undefined",
|
|
118
|
+
"true",
|
|
119
|
+
"false",
|
|
120
|
+
"redacted",
|
|
121
|
+
"***",
|
|
122
|
+
"******",
|
|
123
|
+
"xxx",
|
|
124
|
+
"{",
|
|
125
|
+
"[",
|
|
126
|
+
}
|
|
82
127
|
)
|
|
128
|
+
# Code that reads a secret, not the secret. Only well-known names, because a dotted value can
|
|
129
|
+
# itself be a password.
|
|
130
|
+
_CODE_REFERENCE_RE = re.compile(
|
|
131
|
+
r"(?:settings|config|request|self|this|args|params|env|os\.environ|process\.env)[.\[]"
|
|
132
|
+
)
|
|
133
|
+
_PATH_VALUE_RE = re.compile(r"(?:/|~/|[A-Za-z]:\\)")
|
|
83
134
|
|
|
84
135
|
|
|
85
136
|
def _is_placeholder(value: str) -> bool:
|
|
@@ -110,7 +161,7 @@ class SecretDetector:
|
|
|
110
161
|
for m in pattern.regex.finditer(text):
|
|
111
162
|
start, end = m.start(pattern.group), m.end(pattern.group)
|
|
112
163
|
value = m.group(pattern.group)
|
|
113
|
-
if _is_placeholder(value):
|
|
164
|
+
if _is_placeholder(value) or self._is_reference(pattern.name, m, value):
|
|
114
165
|
continue
|
|
115
166
|
if any(s <= start and end <= e for s, e in covered):
|
|
116
167
|
continue
|
|
@@ -118,6 +169,15 @@ class SecretDetector:
|
|
|
118
169
|
out.append(self._finding(item, (start, end), pattern.name, pattern.confidence))
|
|
119
170
|
return out
|
|
120
171
|
|
|
172
|
+
@staticmethod
|
|
173
|
+
def _is_reference(family: str, match: re.Match[str], value: str) -> bool:
|
|
174
|
+
if family not in ("assignment", "connection_string_password"):
|
|
175
|
+
return False
|
|
176
|
+
if _CODE_REFERENCE_RE.match(value):
|
|
177
|
+
return True
|
|
178
|
+
# PWD is also the shell's working directory.
|
|
179
|
+
return match.group(0)[:3].lower() == "pwd" and bool(_PATH_VALUE_RE.match(value))
|
|
180
|
+
|
|
121
181
|
def _finding(
|
|
122
182
|
self, item: ScanItem, span: tuple[int, int], family: str, confidence: float
|
|
123
183
|
) -> Finding:
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""US Social Security number detector: dashed form, with the never-issued ranges excluded.
|
|
2
|
+
|
|
3
|
+
Bare nine-digit runs are not matched because they are indistinguishable from other identifiers.
|
|
4
|
+
Other countries' identifiers are not covered.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
from ciphergate.detectors.base import make_finding, preceding_context
|
|
12
|
+
from ciphergate.models.finding import Finding, ScanItem
|
|
13
|
+
|
|
14
|
+
_SSN_RE = re.compile(r"(?<![\d-])(\d{3})-(\d{2})-(\d{4})(?![\d-])")
|
|
15
|
+
_KEYWORDS = ("ssn", "social", "tax", "tin")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class SsnDetector:
|
|
19
|
+
id = "ssn_pattern"
|
|
20
|
+
|
|
21
|
+
def detect(self, item: ScanItem) -> list[Finding]:
|
|
22
|
+
text = item.text
|
|
23
|
+
if not text or len(text) < 11:
|
|
24
|
+
return []
|
|
25
|
+
out: list[Finding] = []
|
|
26
|
+
for m in _SSN_RE.finditer(text):
|
|
27
|
+
area, group, serial = m.groups()
|
|
28
|
+
if area in ("000", "666") or area[0] == "9" or group == "00" or serial == "0000":
|
|
29
|
+
continue
|
|
30
|
+
evidence = ["pattern_match"]
|
|
31
|
+
confidence = 0.8
|
|
32
|
+
if preceding_context(text, m.start(), _KEYWORDS):
|
|
33
|
+
confidence = 0.95
|
|
34
|
+
evidence.append("context_keyword")
|
|
35
|
+
out.append(
|
|
36
|
+
make_finding(
|
|
37
|
+
item,
|
|
38
|
+
category="pii.government_id",
|
|
39
|
+
detector=self.id,
|
|
40
|
+
confidence=confidence,
|
|
41
|
+
evidence=tuple(evidence),
|
|
42
|
+
span=(m.start(), m.end()),
|
|
43
|
+
)
|
|
44
|
+
)
|
|
45
|
+
return out
|
|
@@ -225,6 +225,34 @@ def _effective_destinations(doc: PolicyDocument, rule_dests: list[Destination] |
|
|
|
225
225
|
return {str(d) for d in Destination}
|
|
226
226
|
|
|
227
227
|
|
|
228
|
+
def _hard_blocked_destinations(doc: PolicyDocument, categories: frozenset[str]) -> set[str]:
|
|
229
|
+
"""Destinations where a hard rule blocks every credential finding, whatever its field."""
|
|
230
|
+
blocked: set[str] = set()
|
|
231
|
+
for rule in doc.rules:
|
|
232
|
+
m = rule.match
|
|
233
|
+
if (
|
|
234
|
+
rule.hard
|
|
235
|
+
and rule.action == Action.BLOCK
|
|
236
|
+
and m.categories is not None
|
|
237
|
+
and CREDENTIAL_CATEGORY in _selector_can_cover(m.categories, categories)
|
|
238
|
+
and set(m.model_dump(exclude_none=True)) <= {"categories", "destinations"}
|
|
239
|
+
):
|
|
240
|
+
blocked |= _effective_destinations(doc, m.destinations)
|
|
241
|
+
return blocked
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _relaxes_credentials(
|
|
245
|
+
doc: PolicyDocument,
|
|
246
|
+
action: Action,
|
|
247
|
+
rule_dests: list[Destination] | None,
|
|
248
|
+
hard_blocked: set[str],
|
|
249
|
+
) -> bool:
|
|
250
|
+
if action in (Action.BLOCK, Action.REVIEW):
|
|
251
|
+
return False
|
|
252
|
+
reachable = _effective_destinations(doc, rule_dests) - hard_blocked
|
|
253
|
+
return bool(reachable & {str(d) for d in UNTRUSTED_DESTINATIONS})
|
|
254
|
+
|
|
255
|
+
|
|
228
256
|
def _selector_can_cover(selectors: list[str] | None, categories: frozenset[str]) -> set[str]:
|
|
229
257
|
if selectors is None:
|
|
230
258
|
return set(categories)
|
|
@@ -335,6 +363,7 @@ def compile_policy(raw: dict[str, Any], *, digest: str, source_name: str | None
|
|
|
335
363
|
required_keys: set[str] = set()
|
|
336
364
|
replacement_values: set[str] = set()
|
|
337
365
|
seen_rules: set[str] = set()
|
|
366
|
+
hard_blocked = _hard_blocked_destinations(doc, categories)
|
|
338
367
|
for i, rule in enumerate(doc.rules):
|
|
339
368
|
where = f"rules[{i}]"
|
|
340
369
|
if rule.id in seen_rules:
|
|
@@ -394,6 +423,15 @@ def compile_policy(raw: dict[str, Any], *, digest: str, source_name: str | None
|
|
|
394
423
|
"credential_unsafe_action",
|
|
395
424
|
)
|
|
396
425
|
)
|
|
426
|
+
if (
|
|
427
|
+
CREDENTIAL_CATEGORY in covered
|
|
428
|
+
and m.categories is not None
|
|
429
|
+
and _relaxes_credentials(doc, rule.action, m.destinations, hard_blocked)
|
|
430
|
+
):
|
|
431
|
+
warnings.append(
|
|
432
|
+
f"rule '{rule.id}' applies '{rule.action}' to credential.secret toward a possibly "
|
|
433
|
+
"untrusted destination instead of blocking; detection can miss part of a secret"
|
|
434
|
+
)
|
|
397
435
|
if rule.action == Action.ALLOW:
|
|
398
436
|
dests = _effective_destinations(doc, m.destinations)
|
|
399
437
|
untrusted = dests & {str(d) for d in UNTRUSTED_DESTINATIONS}
|
|
@@ -492,6 +530,13 @@ def compile_policy(raw: dict[str, Any], *, digest: str, source_name: str | None
|
|
|
492
530
|
"credential_unsafe_action",
|
|
493
531
|
)
|
|
494
532
|
)
|
|
533
|
+
if CREDENTIAL_CATEGORY in covered and _relaxes_credentials(
|
|
534
|
+
doc, spec.action, None, hard_blocked
|
|
535
|
+
):
|
|
536
|
+
warnings.append(
|
|
537
|
+
f"default '{sel}' applies '{spec.action}' to credential.secret toward a possibly "
|
|
538
|
+
"untrusted destination instead of blocking; detection can miss part of a secret"
|
|
539
|
+
)
|
|
495
540
|
if isinstance(params, PseudonymizeParams | DateShiftParams):
|
|
496
541
|
required_keys.add(params.key_env)
|
|
497
542
|
if isinstance(params, ReplaceParams) and params.value is not None:
|
|
@@ -4,7 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
from typing import Any
|
|
6
6
|
|
|
7
|
-
from ciphergate.constants import DISCLAIMER
|
|
7
|
+
from ciphergate.constants import DISCLAIMER, RESCAN_LIMIT
|
|
8
8
|
from ciphergate.models.report import RunReport
|
|
9
9
|
from ciphergate.pipeline.batch import BatchReport
|
|
10
10
|
from ciphergate.reporting.safe_text import escape_control
|
|
@@ -31,9 +31,24 @@ def _location(loc: dict[str, Any]) -> str:
|
|
|
31
31
|
return ", ".join(parts)
|
|
32
32
|
|
|
33
33
|
|
|
34
|
+
def _rescan_verdict(rescan: dict[str, Any]) -> str:
|
|
35
|
+
if rescan["failed"]:
|
|
36
|
+
return "FAILED, gating findings remain"
|
|
37
|
+
if rescan["gating_by_category"]:
|
|
38
|
+
return "NOT ENFORCED, gating findings remain but strict is off"
|
|
39
|
+
return "no gating findings"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _headline(d: dict[str, Any]) -> str:
|
|
43
|
+
# Never print a bare PASSED for rescan: it reads as "this output is safe".
|
|
44
|
+
if d["command"] == "rescan" and d["rescan"] and not d["rescan"]["failed"]:
|
|
45
|
+
return _rescan_verdict(d["rescan"]).split(",")[0].upper()
|
|
46
|
+
return str(d["status"]).upper()
|
|
47
|
+
|
|
48
|
+
|
|
34
49
|
def render_summary(report: RunReport, *, verbose: bool = False) -> str:
|
|
35
50
|
d = report.to_dict()
|
|
36
|
-
lines = [f"CipherGate {d['command']}: {d
|
|
51
|
+
lines = [f"CipherGate {d['command']}: {_headline(d)} (exit {d['exit_code']})"]
|
|
37
52
|
inp = d["input"]
|
|
38
53
|
lines.append(
|
|
39
54
|
f" input: {escape_control(str(inp['path']))} [{inp['format']}, "
|
|
@@ -89,12 +104,13 @@ def render_summary(report: RunReport, *, verbose: bool = False) -> str:
|
|
|
89
104
|
)
|
|
90
105
|
if d["rescan"]:
|
|
91
106
|
r = d["rescan"]
|
|
92
|
-
verdict = "FAILED" if r["failed"] else "passed"
|
|
93
107
|
lines.append(
|
|
94
|
-
f" rescan: {
|
|
108
|
+
f" rescan: {_rescan_verdict(r)} (strict={r['strict']}) "
|
|
109
|
+
f"residuals: {_counts(r['counts'])}"
|
|
95
110
|
)
|
|
96
111
|
if r["gating_by_category"]:
|
|
97
112
|
lines.append(f" gating residuals: {_counts(r['gating_by_category'])}")
|
|
113
|
+
lines.append(f" rescan limit: {RESCAN_LIMIT}")
|
|
98
114
|
out = d["output"]
|
|
99
115
|
if out:
|
|
100
116
|
if out.get("written"):
|
|
@@ -197,6 +197,28 @@ def test_sanitize_missing_key_exit_6(tmp_path: Path, no_key_env: None) -> None:
|
|
|
197
197
|
assert r.exit_code == 6 and not out.exists() and "missing" in r.stdout
|
|
198
198
|
|
|
199
199
|
|
|
200
|
+
def test_rescan_summary_never_says_a_bare_pass(tmp_path: Path) -> None:
|
|
201
|
+
policy = POLICIES / "prd_external_ai.yml"
|
|
202
|
+
clean = tmp_path / "clean.log"
|
|
203
|
+
clean.write_text("service started\n", encoding="utf-8")
|
|
204
|
+
r = invoke("rescan", clean, "--policy", policy)
|
|
205
|
+
assert r.exit_code == 0
|
|
206
|
+
assert "NO GATING FINDINGS" in r.stdout and "does not mean the output is safe" in r.stdout
|
|
207
|
+
assert "PASSED" not in r.stdout and "passed" not in r.stdout.replace("--", "")
|
|
208
|
+
|
|
209
|
+
report = tmp_path / "report.json"
|
|
210
|
+
assert invoke("rescan", clean, "--policy", policy, "--report", report).exit_code == 0
|
|
211
|
+
assert json.loads(report.read_text(encoding="utf-8"))["status"] == "passed"
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def test_rescan_summary_says_when_gating_findings_are_not_enforced(tmp_path: Path) -> None:
|
|
215
|
+
src = tmp_path / "raw.csv"
|
|
216
|
+
shutil.copy(EXAMPLES / "sample.csv", src)
|
|
217
|
+
r = invoke("rescan", src, "--policy", POLICIES / "prd_external_ai.yml", "--no-strict")
|
|
218
|
+
assert r.exit_code == 0
|
|
219
|
+
assert "NOT ENFORCED" in r.stdout and "PASSED" not in r.stdout
|
|
220
|
+
|
|
221
|
+
|
|
200
222
|
def test_rescan_exit_7_and_read_only(tmp_path: Path) -> None:
|
|
201
223
|
src = tmp_path / "raw.csv"
|
|
202
224
|
shutil.copy(EXAMPLES / "sample.csv", src)
|