ciphergate-cli 0.11.0__tar.gz → 0.13.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/CHANGELOG.md +38 -1
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/PKG-INFO +8 -4
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/README.md +7 -3
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/docs/detector-coverage.md +1 -1
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/docs/policy-schema.md +13 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/__init__.py +1 -1
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/payment_card.py +26 -5
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/pipeline/batch.py +48 -2
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/readers/__init__.py +2 -0
- ciphergate_cli-0.13.0/src/ciphergate/reporting/next_steps.py +106 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/reporting/terminal.py +10 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/integration/test_batch.py +51 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/integration/test_cli_e2e.py +29 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/test_detectors.py +50 -4
- ciphergate_cli-0.13.0/tests/unit/test_next_steps.py +62 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/.gitignore +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/LICENSE +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/NOTICE +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/SECURITY.md +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/docs/architecture.md +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/docs/claude-code-tooling.md +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/docs/examples.md +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/docs/limitations.md +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/docs/security-and-privacy.md +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/docs/threat-model.md +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/examples/expected/sample.dev_local_dev.csv +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/examples/expected/sample.dev_local_dev.json +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/examples/expected/sample.dev_local_dev.jsonl +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/examples/expected/sample.dev_local_dev.log +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/examples/sample.csv +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/examples/sample.json +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/examples/sample.jsonl +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/examples/sample.log +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/policies/dev_local.yml +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/policies/developer_logs.yml +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/policies/external_ai.yml +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/policies/financial_template.yml +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/policies/healthcare_template.yml +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/policies/local_debug.yml +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/policies/multi_environment.yml +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/policies/prd_external_ai.yml +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/policies/stg_test.yml +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/policies/test_dataset.yml +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/pyproject.toml +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/scripts/benchmark/README.md +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/__main__.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/cli.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/config.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/constants.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/__init__.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/base.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/custom_regex.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/dates.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/email.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/field_names.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/iban.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/ip_address.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/mac_address.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/network.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/phone.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/presidio_ner.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/regex_safety.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/registry.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/secrets.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/detectors/ssn.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/errors.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/models/__init__.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/models/context.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/models/finding.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/models/policy.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/models/report.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/pipeline/__init__.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/pipeline/atomic_output.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/pipeline/inspector.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/pipeline/paths.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/pipeline/planner.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/pipeline/rescan.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/pipeline/sanitizer.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/pipeline/stdio.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/pipeline/streams.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/policy/__init__.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/policy/context.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/policy/engine.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/policy/loader.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/policy/precedence.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/policy/templates.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/policy/validator.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/py.typed +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/readers/accesslog_reader.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/readers/base.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/readers/csv_reader.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/readers/json_reader.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/readers/jsonl_reader.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/readers/jsonlog_reader.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/readers/line_structure.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/readers/logfmt_reader.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/readers/stacktrace_reader.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/readers/syslog_reader.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/readers/text_reader.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/reporting/__init__.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/reporting/audit.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/reporting/safe_text.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/reporting/serializer.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/templates/starter_policy.yml +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/transforms/__init__.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/transforms/date_shift.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/transforms/generalize.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/transforms/keys.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/transforms/pseudonymize.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/transforms/redact.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/transforms/registry.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/src/ciphergate/transforms/replace.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/__init__.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/conftest.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/integration/__init__.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/integration/test_api_batch_streams.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/integration/test_atomic_output.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/integration/test_domain_templates.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/integration/test_environment_matrix.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/integration/test_pipelines.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/integration/test_stdio.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/regression/__init__.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/regression/test_known_cases.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/security/__init__.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/security/test_key_file.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/security/test_no_network.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/security/test_no_raw_value_leaks.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/security/test_path_safety.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/security/test_policy_fail_closed.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/security/test_review_findings.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/tooling/__init__.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/tooling/test_benchmark.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/tooling/test_ci_supply_chain.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/tooling/test_claude_tooling.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/tooling/test_git_tooling.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/tooling/test_publish_workflow.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/__init__.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/test_accesslog.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/test_jsonlog.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/test_logfmt.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/test_policy_engine.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/test_policy_precedence.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/test_policy_templates.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/test_policy_validation.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/test_presidio_ner.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/test_properties.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/test_readers.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/test_stacktrace.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/test_syslog.py +0 -0
- {ciphergate_cli-0.11.0 → ciphergate_cli-0.13.0}/tests/unit/test_transforms.py +0 -0
|
@@ -6,6 +6,41 @@ versioning: [PEP 440](https://peps.python.org/pep-0440/) with SemVer meaning (pr
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.13.0] - 2026-11-02
|
|
10
|
+
|
|
11
|
+
Fewer random IDs are reported as payment cards. Cards with an unrecognised prefix and no card
|
|
12
|
+
keyword nearby are no longer reported, so check that your data is still covered.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- `luhn_card` reports fewer random IDs as cards. A Luhn-valid number now needs a known
|
|
17
|
+
issuer prefix or a card keyword nearby, and an issuer-prefixed number right after an ID
|
|
18
|
+
word such as `invoice` or `batch id` needs a card keyword. JCB, UnionPay, Diners Club,
|
|
19
|
+
Mir and Discover 644-649 prefixes are now recognised. Cards with other prefixes and no
|
|
20
|
+
keyword are no longer reported.
|
|
21
|
+
|
|
22
|
+
## [0.12.0] - 2026-11-02
|
|
23
|
+
|
|
24
|
+
Blocked runs now say what to do next, and directory runs no longer hide files they never
|
|
25
|
+
scanned. A strict directory `rescan` that skips files with unrecognised extensions now exits 7
|
|
26
|
+
instead of 0.
|
|
27
|
+
|
|
28
|
+
### Added
|
|
29
|
+
|
|
30
|
+
- Blocked, review and failed runs print `next:` lines in the terminal summary: what to pass,
|
|
31
|
+
what to change in the policy, or what to remove from the input. The hints come from reason
|
|
32
|
+
codes and never include input values. The JSON report is unchanged.
|
|
33
|
+
- `docs/policy-schema.md` explains `hard` and `guardrail` rules.
|
|
34
|
+
- `.out` and `.err` files are read as plain text, in directories and as single files.
|
|
35
|
+
|
|
36
|
+
### Fixed
|
|
37
|
+
|
|
38
|
+
- Directory runs warn when files were skipped because their extension is not recognised, so a
|
|
39
|
+
`COMPLETED` run no longer hides files that were never scanned. A strict directory `rescan` now
|
|
40
|
+
fails (exit 7) when it skipped such files, and `sanitize` says they are not in the output.
|
|
41
|
+
- Directory runs warn when `--format` is applied to files whose extension means another format.
|
|
42
|
+
- Directory summaries show the same `next:` hints as single-file runs.
|
|
43
|
+
|
|
9
44
|
## [0.11.0] - 2026-10-31
|
|
10
45
|
|
|
11
46
|
Makes a clean rescan harder to mistake for proof of safety. Scripts that read `PASSED` from the
|
|
@@ -328,7 +363,9 @@ Pre-release (alpha). Licensed under Apache-2.0. Not a final release: the securit
|
|
|
328
363
|
See `docs/limitations.md`. Deferred: optional NER/Presidio detectors, stdin/stdout streaming,
|
|
329
364
|
directory processing, structured log parsers, key files, reversible token vault, lossy decoding.
|
|
330
365
|
|
|
331
|
-
[Unreleased]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.
|
|
366
|
+
[Unreleased]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.13.0...HEAD
|
|
367
|
+
[0.13.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.12.0...v0.13.0
|
|
368
|
+
[0.12.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.11.0...v0.12.0
|
|
332
369
|
[0.11.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.10.0...v0.11.0
|
|
333
370
|
[0.10.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.9.0...v0.10.0
|
|
334
371
|
[0.9.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.8.0...v0.9.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ciphergate-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.13.0
|
|
4
4
|
Summary: Local-first, environment- and destination-aware inspection and sanitization of logs and test datasets.
|
|
5
5
|
Project-URL: Homepage, https://github.com/aashishkumar-tech/ciphergate
|
|
6
6
|
Project-URL: Source, https://github.com/aashishkumar-tech/ciphergate
|
|
@@ -50,7 +50,7 @@ vendor ticket. CipherGate replaces that with a local, explainable, scriptable ga
|
|
|
50
50
|
inspect -> plan -> sanitize --apply -> rescan
|
|
51
51
|
```
|
|
52
52
|
|
|
53
|
-
> **Status: 0.
|
|
53
|
+
> **Status: 0.13.0 (alpha).** No detector finds every sensitive value. A passing run means the
|
|
54
54
|
> configured processing completed under the selected policy; it is **not** proof that the output
|
|
55
55
|
> contains no sensitive information, and it is not a compliance (GDPR, HIPAA, PCI DSS, ...) or
|
|
56
56
|
> anonymization guarantee. Policy templates must be reviewed before use.
|
|
@@ -287,10 +287,14 @@ ciphergate inspect logs/ --policy policies/developer_logs.yml --environment dev
|
|
|
287
287
|
ciphergate sanitize logs/ --policy ... --output build/logs.sanitized --apply
|
|
288
288
|
```
|
|
289
289
|
|
|
290
|
-
- Files are chosen by extension (`.csv`, `.json`, `.jsonl`, `.ndjson`, `.txt`, `.log
|
|
291
|
-
regular files when `--format` is given. Subdirectories are processed only with
|
|
290
|
+
- Files are chosen by extension (`.csv`, `.json`, `.jsonl`, `.ndjson`, `.txt`, `.log`, `.out`,
|
|
291
|
+
`.err`), or all regular files when `--format` is given. Subdirectories are processed only with
|
|
292
292
|
`--recursive`/`-r` (at most 32 levels). Symlinks, junctions, devices, FIFOs and other
|
|
293
293
|
extensions are **skipped and listed** in the report with a reason; links are never followed.
|
|
294
|
+
Skipped extensions also produce a warning, because those files were never scanned; a strict
|
|
295
|
+
`rescan` fails (exit 7) when it skipped any, and `sanitize` leaves them out of the output.
|
|
296
|
+
`--format` on a directory applies to every file, and the run warns when that contradicts an
|
|
297
|
+
extension (for example a CSV read as text loses its column names).
|
|
294
298
|
At most 1000 files by default (`--max-files N`); more fails the run (exit 4) before anything
|
|
295
299
|
is processed. An empty selection fails with `batch_empty` (exit 4).
|
|
296
300
|
- Each file is processed exactly like a single-file run, in sorted path order; a failing file
|
|
@@ -11,7 +11,7 @@ vendor ticket. CipherGate replaces that with a local, explainable, scriptable ga
|
|
|
11
11
|
inspect -> plan -> sanitize --apply -> rescan
|
|
12
12
|
```
|
|
13
13
|
|
|
14
|
-
> **Status: 0.
|
|
14
|
+
> **Status: 0.13.0 (alpha).** No detector finds every sensitive value. A passing run means the
|
|
15
15
|
> configured processing completed under the selected policy; it is **not** proof that the output
|
|
16
16
|
> contains no sensitive information, and it is not a compliance (GDPR, HIPAA, PCI DSS, ...) or
|
|
17
17
|
> anonymization guarantee. Policy templates must be reviewed before use.
|
|
@@ -248,10 +248,14 @@ ciphergate inspect logs/ --policy policies/developer_logs.yml --environment dev
|
|
|
248
248
|
ciphergate sanitize logs/ --policy ... --output build/logs.sanitized --apply
|
|
249
249
|
```
|
|
250
250
|
|
|
251
|
-
- Files are chosen by extension (`.csv`, `.json`, `.jsonl`, `.ndjson`, `.txt`, `.log
|
|
252
|
-
regular files when `--format` is given. Subdirectories are processed only with
|
|
251
|
+
- Files are chosen by extension (`.csv`, `.json`, `.jsonl`, `.ndjson`, `.txt`, `.log`, `.out`,
|
|
252
|
+
`.err`), or all regular files when `--format` is given. Subdirectories are processed only with
|
|
253
253
|
`--recursive`/`-r` (at most 32 levels). Symlinks, junctions, devices, FIFOs and other
|
|
254
254
|
extensions are **skipped and listed** in the report with a reason; links are never followed.
|
|
255
|
+
Skipped extensions also produce a warning, because those files were never scanned; a strict
|
|
256
|
+
`rescan` fails (exit 7) when it skipped any, and `sanitize` leaves them out of the output.
|
|
257
|
+
`--format` on a directory applies to every file, and the run warns when that contradicts an
|
|
258
|
+
extension (for example a CSV read as text loses its column names).
|
|
255
259
|
At most 1000 files by default (`--max-files N`); more fails the run (exit 4) before anything
|
|
256
260
|
is processed. An empty selection fails with `batch_empty` (exit 4).
|
|
257
261
|
- Each file is processed exactly like a single-file run, in sorted path order; a failing file
|
|
@@ -12,7 +12,7 @@ are engineering estimates mapped to bands (`low` < 0.5 <= `medium` < 0.8 <= `hig
|
|
|
12
12
|
| `email_regex` | `pii.email` | `alice.test@example.com`, `alice.test%40example.com` | `a..b@example.com`, `user@localhost` | 0.9 | ASCII only; no IDN; obfuscated forms (`alice at example dot com`) missed; `%40` is the only percent-encoding recognised outside `accesslog` |
|
|
13
13
|
| `phone_pattern` | `pii.phone` | `+1 202 555 0143` (high), `(202) 555-0143` (medium) | dates, IPs, card numbers, repeated digits, < 7 digits | 0.3 bare digits, 0.55–0.8 prefixed/formatted, +0.2 keyword | no numbering-plan validation; IDs can look like phones (low band) |
|
|
14
14
|
| `ip_address` | `pii.ip_address` (global), `network.telemetry` (private/loopback/link-local) | `8.8.8.8`, `10.0.0.1`, `2606:4700::1111`, `fe80::1%eth0` | `256.1.1.1`, `std::vector`, `12:30:45` | 0.85 / 0.8 | version strings like `1.2.3.4` match; documentation ranges flagged with evidence |
|
|
15
|
-
| `luhn_card` | `pii.payment_card` | `4111 1111 1111 1111` | Luhn-invalid, mixed separators, repeated digits | 0.
|
|
15
|
+
| `luhn_card` | `pii.payment_card` | `4111 1111 1111 1111` | Luhn-invalid, mixed separators, repeated digits | 0.8 issuer prefix (Visa, Mastercard, Amex, Discover, JCB, UnionPay, Diners, Mir), +0.15 keyword; 0.55 keyword without prefix; labelled CVV/CVC values 0.85 | a Luhn-valid number with no issuer prefix and no card keyword is not reported, nor is one right after an ID word (`invoice`, `batch id`, `order`) without a card keyword, so such cards are missed (for example Maestro without a keyword); cards split across fields missed |
|
|
16
16
|
| `mac_address` | `pii.device_identifier` | `00:1A:2B:3C:4D:5E`, `00-1a-2b-3c-4d-5e`, `001a.2b3c.4d5e` | all-zero, broadcast, mixed separators, `12:30:45` | 0.85, 0.6 when all digits | bare 12-hex values are missed; digit-only IDs in this shape are flagged |
|
|
17
17
|
| `iban` | `pii.bank_account` | a mod-97-valid IBAN, compact or space-grouped | wrong check digits, lowercase | 0.9 | country length and national format are not checked; about 1 in 97 random strings of this shape pass; lowercase forms are missed |
|
|
18
18
|
| `ssn_pattern` | `pii.government_id` | `219-09-9999` | never-issued areas (`000`, `666`, `9xx`), group `00`, serial `0000`, undashed | 0.8, 0.95 with a keyword | US only; undashed numbers are missed; other dashed IDs in this shape are flagged |
|
|
@@ -150,6 +150,19 @@ actions/params are **rejected** (`policy_conflict`, naming both rule IDs).
|
|
|
150
150
|
Several findings on one value combine: `drop_record` > `drop_field` > whole-value transform
|
|
151
151
|
(most protective by rank) > span transforms (overlaps merged, most protective wins).
|
|
152
152
|
|
|
153
|
+
## Hard rules and guardrails
|
|
154
|
+
|
|
155
|
+
`hard: true` puts a rule at level 2, above guardrails, field rules and everything else except the
|
|
156
|
+
engine's own constraints. Use it for "never, whatever other rules say", such as blocking
|
|
157
|
+
credentials for untrusted destinations. A hard rule cannot `allow`.
|
|
158
|
+
|
|
159
|
+
`guardrail: true` puts a rule at level 3. It must name `environments` and either
|
|
160
|
+
`target_environments` or `destinations`, so it only fires for a specific environment-to-recipient
|
|
161
|
+
move (for example PRD to `external_ai`). A guardrail cannot `allow`, and a rule cannot be both.
|
|
162
|
+
|
|
163
|
+
A blocked run prints `next:` lines in the terminal summary with what to change, based on the
|
|
164
|
+
reason codes. They never include input values.
|
|
165
|
+
|
|
153
166
|
## Transfers
|
|
154
167
|
|
|
155
168
|
```yaml
|
|
@@ -24,7 +24,7 @@ Supported public API (everything else is internal and may change without notice)
|
|
|
24
24
|
rescan_directory(root, policy, context=None, *, recursive=False, strict=None) -> BatchReport
|
|
25
25
|
"""
|
|
26
26
|
|
|
27
|
-
__version__ = "0.
|
|
27
|
+
__version__ = "0.13.0"
|
|
28
28
|
|
|
29
29
|
from ciphergate.config import RuntimeSettings
|
|
30
30
|
from ciphergate.constants import (
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"""Payment-card-like detector: 13-19 digits, Luhn-valid, scored by issuer prefix and context.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
Long numeric identifiers pass Luhn roughly 10% of the time by chance, so a Luhn-valid run is
|
|
4
|
+
reported only with a known issuer prefix or a card keyword nearby. An issuer-prefixed run right
|
|
5
|
+
after an identifier word such as "invoice" or "batch id" is also skipped unless a card keyword
|
|
6
|
+
is present. Cards with other prefixes (for example Maestro) and no keyword are missed.
|
|
5
7
|
"""
|
|
6
8
|
|
|
7
9
|
from __future__ import annotations
|
|
@@ -16,6 +18,12 @@ _CVV_RE = re.compile(
|
|
|
16
18
|
r"(?i)(?<![a-z0-9])(?:cvv2?|cvc2?)(?:\\?[\"'])?\s{0,3}[:=]\s{0,3}(?:\\?[\"'])?(\d{3,4})(?!\d)"
|
|
17
19
|
)
|
|
18
20
|
_KEYWORDS = ("card", "visa", "mastercard", "amex", "pan", "cc", "credit", "debit", "payment")
|
|
21
|
+
# Words that label a number as some other identifier. Whole words only: a substring match
|
|
22
|
+
# would let "paid" (which contains "id") hide a real card.
|
|
23
|
+
_ID_WORD_RE = re.compile(
|
|
24
|
+
r"(?i)(?<![a-z])(?:id|invoice|order|batch|total|trace|request|ref|seq|sequence|ticket|"
|
|
25
|
+
r"build|job|txn|transaction)(?![a-z])"
|
|
26
|
+
)
|
|
19
27
|
|
|
20
28
|
|
|
21
29
|
def luhn_valid(digits: str) -> bool:
|
|
@@ -38,8 +46,15 @@ def known_issuer_prefix(digits: str) -> bool:
|
|
|
38
46
|
return len(digits) == 16
|
|
39
47
|
if two in (34, 37):
|
|
40
48
|
return len(digits) == 15
|
|
41
|
-
|
|
49
|
+
three = int(digits[:3])
|
|
50
|
+
if digits.startswith("6011") or two == 65 or 644 <= three <= 649:
|
|
42
51
|
return len(digits) in (16, 19)
|
|
52
|
+
if 3528 <= four <= 3589 or two == 62: # JCB, UnionPay
|
|
53
|
+
return 16 <= len(digits) <= 19
|
|
54
|
+
if 300 <= three <= 305 or two in (36, 38, 39): # Diners Club
|
|
55
|
+
return 14 <= len(digits) <= 19
|
|
56
|
+
if 2200 <= four <= 2204: # Mir
|
|
57
|
+
return 16 <= len(digits) <= 19
|
|
43
58
|
return False
|
|
44
59
|
|
|
45
60
|
|
|
@@ -71,12 +86,18 @@ class PaymentCardDetector:
|
|
|
71
86
|
seps = {c for c in raw if c in " -"}
|
|
72
87
|
if len(seps) > 1:
|
|
73
88
|
continue
|
|
89
|
+
issuer = known_issuer_prefix(digits)
|
|
90
|
+
keyword = preceding_context(text, m.start(), _KEYWORDS, window=32)
|
|
91
|
+
if not keyword and (
|
|
92
|
+
not issuer or _ID_WORD_RE.search(text[max(0, m.start() - 16) : m.start()])
|
|
93
|
+
):
|
|
94
|
+
continue
|
|
74
95
|
evidence = ["pattern_match", "checksum_valid"]
|
|
75
96
|
confidence = 0.4
|
|
76
|
-
if
|
|
97
|
+
if issuer:
|
|
77
98
|
confidence = 0.8
|
|
78
99
|
evidence.append("issuer_prefix")
|
|
79
|
-
if
|
|
100
|
+
if keyword:
|
|
80
101
|
confidence += 0.15
|
|
81
102
|
evidence.append("context_keyword")
|
|
82
103
|
out.append(
|
|
@@ -96,6 +96,9 @@ class BatchReport:
|
|
|
96
96
|
skipped: list[dict[str, str]]
|
|
97
97
|
output: dict[str, Any] = field(default_factory=dict)
|
|
98
98
|
warnings: list[str] = field(default_factory=list)
|
|
99
|
+
# Exit code when every processed file passed but the run still cannot count as
|
|
100
|
+
# complete, e.g. a strict rescan that skipped files it could not read.
|
|
101
|
+
incomplete_exit: int = 0
|
|
99
102
|
run_id: str = field(default_factory=new_run_id)
|
|
100
103
|
timestamp: str = field(default_factory=utc_now)
|
|
101
104
|
|
|
@@ -104,7 +107,7 @@ class BatchReport:
|
|
|
104
107
|
for result in self.files:
|
|
105
108
|
if result.exit_code != 0:
|
|
106
109
|
return result.exit_code
|
|
107
|
-
return
|
|
110
|
+
return self.incomplete_exit
|
|
108
111
|
|
|
109
112
|
@property
|
|
110
113
|
def status(self) -> str:
|
|
@@ -277,6 +280,34 @@ def _run_each(
|
|
|
277
280
|
return results
|
|
278
281
|
|
|
279
282
|
|
|
283
|
+
def coverage_warnings(
|
|
284
|
+
discovery: Discovery, fmt: InputFormat | None, *, sanitize: bool = False
|
|
285
|
+
) -> list[str]:
|
|
286
|
+
"""Say which files were not scanned, or were read as a format their extension contradicts."""
|
|
287
|
+
warnings: list[str] = []
|
|
288
|
+
unknown = sum(1 for _, reason in discovery.skipped if reason == "unsupported_extension")
|
|
289
|
+
if unknown:
|
|
290
|
+
warnings.append(
|
|
291
|
+
f"{unknown} file(s) were skipped and not scanned because their extension is not "
|
|
292
|
+
"recognised"
|
|
293
|
+
+ ("; they are not in the output" if sanitize else "")
|
|
294
|
+
+ "; rename them, or pass --format to read every file as one format"
|
|
295
|
+
)
|
|
296
|
+
if fmt is not None:
|
|
297
|
+
mismatched = sum(
|
|
298
|
+
1
|
|
299
|
+
for rel in discovery.files
|
|
300
|
+
if EXTENSION_FORMATS.get(rel.suffix.lower()) not in (None, fmt)
|
|
301
|
+
)
|
|
302
|
+
if mismatched:
|
|
303
|
+
warnings.append(
|
|
304
|
+
f"--format {fmt} was applied to {mismatched} file(s) whose extension means "
|
|
305
|
+
"another format; structure such as CSV columns or JSON keys may be lost. "
|
|
306
|
+
"Run those files separately without --format"
|
|
307
|
+
)
|
|
308
|
+
return warnings
|
|
309
|
+
|
|
310
|
+
|
|
280
311
|
def _require_files(discovery: Discovery) -> None:
|
|
281
312
|
if not discovery.files:
|
|
282
313
|
raise InputFormatError(
|
|
@@ -312,6 +343,8 @@ def analyze_directory(
|
|
|
312
343
|
return rescan_file(path, policy, context, strict=strict, runtime=runtime, fmt=fmt)
|
|
313
344
|
|
|
314
345
|
mode = runtime.path_display
|
|
346
|
+
effective_strict = policy.document.settings.strict if strict is None else strict
|
|
347
|
+
unscanned = any(reason == "unsupported_extension" for _, reason in discovery.skipped)
|
|
315
348
|
return BatchReport(
|
|
316
349
|
command=command,
|
|
317
350
|
input_display=display_relative(Path(root.name), 0, root.parent, mode)
|
|
@@ -320,6 +353,11 @@ def analyze_directory(
|
|
|
320
353
|
recursive=recursive,
|
|
321
354
|
files=_run_each(discovery, mode, run),
|
|
322
355
|
skipped=_skipped(discovery, mode),
|
|
356
|
+
warnings=coverage_warnings(discovery, fmt),
|
|
357
|
+
# A strict rescan is a gate: files it could not read must not count as passed.
|
|
358
|
+
incomplete_exit=int(ExitCode.RESCAN_FAILED)
|
|
359
|
+
if command == "rescan" and effective_strict and unscanned
|
|
360
|
+
else 0,
|
|
323
361
|
)
|
|
324
362
|
|
|
325
363
|
|
|
@@ -472,6 +510,7 @@ def sanitize_directory(
|
|
|
472
510
|
results,
|
|
473
511
|
_skipped(discovery, mode),
|
|
474
512
|
output={"written": False, "path": shown_out, "withheld_reason": "apply_not_set"},
|
|
513
|
+
warnings=coverage_warnings(discovery, options.format, sanitize=True),
|
|
475
514
|
)
|
|
476
515
|
|
|
477
516
|
parent = output.parent if str(output.parent) else Path(".")
|
|
@@ -491,7 +530,14 @@ def sanitize_directory(
|
|
|
491
530
|
mode,
|
|
492
531
|
lambda path, rel: sanitize_file(path, staging / rel, policy, context, file_options),
|
|
493
532
|
)
|
|
494
|
-
report = BatchReport(
|
|
533
|
+
report = BatchReport(
|
|
534
|
+
"sanitize",
|
|
535
|
+
shown_root,
|
|
536
|
+
recursive,
|
|
537
|
+
results,
|
|
538
|
+
_skipped(discovery, mode),
|
|
539
|
+
warnings=coverage_warnings(discovery, options.format, sanitize=True),
|
|
540
|
+
)
|
|
495
541
|
for result in results:
|
|
496
542
|
if result.report is not None:
|
|
497
543
|
result.report.output["path"] = f"{shown_out}/{result.display}"
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Short next-step hints for runs that did not complete. Built from reason codes only."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from ciphergate.constants import ReasonCode
|
|
8
|
+
|
|
9
|
+
_HINTS: dict[str, str] = {
|
|
10
|
+
ReasonCode.CREDENTIAL_DETECTED_HARD_BLOCK: (
|
|
11
|
+
"a credential was found: remove it from the input and rotate it if it is real; "
|
|
12
|
+
"no policy can let credentials through"
|
|
13
|
+
),
|
|
14
|
+
ReasonCode.EXPLICIT_HARD_BLOCK: (
|
|
15
|
+
"a hard rule blocks these values for this destination: remove them from the input, or "
|
|
16
|
+
"use a policy written for a destination allowed to receive them; "
|
|
17
|
+
"'ciphergate explain' shows which rule wins"
|
|
18
|
+
),
|
|
19
|
+
ReasonCode.ENVIRONMENT_GUARDRAIL: (
|
|
20
|
+
"a guardrail rule blocks this environment-to-destination move; check the context, or "
|
|
21
|
+
"change the guardrail in the policy after review"
|
|
22
|
+
),
|
|
23
|
+
ReasonCode.PRD_TO_NONPRODUCTION_BLOCKED: (
|
|
24
|
+
"production data cannot go to a non-production target without a reviewed 'transfers' "
|
|
25
|
+
"approval in the policy (docs/policy-schema.md#transfers)"
|
|
26
|
+
),
|
|
27
|
+
ReasonCode.NO_APPLICABLE_RULE_REQUIRES_REVIEW: (
|
|
28
|
+
"no rule covers some findings: add rules for the categories listed above, then run "
|
|
29
|
+
"'ciphergate plan' again"
|
|
30
|
+
),
|
|
31
|
+
ReasonCode.NO_APPLICABLE_RULE_BLOCKS: (
|
|
32
|
+
"no rule covers some findings: add rules for the categories listed above, then run "
|
|
33
|
+
"'ciphergate plan' again"
|
|
34
|
+
),
|
|
35
|
+
ReasonCode.UNKNOWN_CONTEXT_REQUIRES_REVIEW: (
|
|
36
|
+
"the environment or destination is 'unknown': pass the real values with --environment "
|
|
37
|
+
"and --destination"
|
|
38
|
+
),
|
|
39
|
+
ReasonCode.HIGH_RISK_ALLOW_REQUIRES_ACKNOWLEDGEMENT: (
|
|
40
|
+
"an allow rule reaches high-risk data toward an untrusted destination: narrow the rule, "
|
|
41
|
+
"or set acknowledge_risk: true after review"
|
|
42
|
+
),
|
|
43
|
+
ReasonCode.UNSUPPORTED_TRANSFORM: (
|
|
44
|
+
"a rule's action cannot run on this format or location: choose another action for it"
|
|
45
|
+
),
|
|
46
|
+
ReasonCode.ENVIRONMENT_REQUIRED_BUT_MISSING: (
|
|
47
|
+
"pass --environment (dev, stg, prd) or pin context.environment in the policy"
|
|
48
|
+
),
|
|
49
|
+
ReasonCode.DESTINATION_REQUIRED_BUT_MISSING: (
|
|
50
|
+
"pass --destination (for example local_dev or external_ai) or pin context.destination "
|
|
51
|
+
"in the policy"
|
|
52
|
+
),
|
|
53
|
+
ReasonCode.TARGET_ENVIRONMENT_REQUIRED_BUT_MISSING: (
|
|
54
|
+
"pass --target-environment or pin context.target_environment in the policy"
|
|
55
|
+
),
|
|
56
|
+
ReasonCode.ENVIRONMENT_CONTEXT_CONFLICT: (
|
|
57
|
+
"a CLI flag contradicts the policy's pinned context: drop the flag, or use a policy for "
|
|
58
|
+
"that context"
|
|
59
|
+
),
|
|
60
|
+
ReasonCode.UNSUPPORTED_ENVIRONMENT: "use one of the documented environment values",
|
|
61
|
+
ReasonCode.STRICT_POLICY_REQUIRED: (
|
|
62
|
+
"this destination needs a strict policy: set settings.strict: true"
|
|
63
|
+
),
|
|
64
|
+
ReasonCode.RESCAN_REQUIRED: (
|
|
65
|
+
"this destination needs the rescan gate: do not disable settings.rescan in the policy"
|
|
66
|
+
),
|
|
67
|
+
ReasonCode.CREDENTIAL_DETECTION_REQUIRED: (
|
|
68
|
+
"this destination needs credential detection: remove secret_pattern and field_name from "
|
|
69
|
+
"detectors.disabled"
|
|
70
|
+
),
|
|
71
|
+
ReasonCode.KEY_SCAN_REQUIRED: (
|
|
72
|
+
"this destination needs JSON key scanning: set settings.scan_json_keys: true"
|
|
73
|
+
),
|
|
74
|
+
ReasonCode.RESIDUAL_FINDING: (
|
|
75
|
+
"the rescan still found gating categories in the output: add rules for them, then run again"
|
|
76
|
+
),
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
_KEY_HINT = (
|
|
80
|
+
"set {env} to at least 32 random bytes in your environment (never on the command line); "
|
|
81
|
+
"see 'Pseudonymization key' in the README"
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def next_steps(report: dict[str, Any]) -> list[str]:
|
|
86
|
+
"""Hints for a run that ended with a non-zero exit code; empty otherwise."""
|
|
87
|
+
if not report.get("exit_code"):
|
|
88
|
+
return []
|
|
89
|
+
codes = [gate["reason_code"] for gate in report.get("gates", [])]
|
|
90
|
+
if report.get("command") != "rescan":
|
|
91
|
+
# A rescan's policy decisions describe the output, not what to fix in the input.
|
|
92
|
+
codes += list(report.get("summary", {}).get("by_reason_code", {}))
|
|
93
|
+
rescan = report.get("rescan")
|
|
94
|
+
if rescan and rescan.get("failed"):
|
|
95
|
+
codes.append(ReasonCode.RESIDUAL_FINDING)
|
|
96
|
+
hints: list[str] = []
|
|
97
|
+
for code in codes:
|
|
98
|
+
hint = _HINTS.get(code)
|
|
99
|
+
if hint and hint not in hints:
|
|
100
|
+
hints.append(hint)
|
|
101
|
+
for key in report.get("keys", []):
|
|
102
|
+
if key.get("status") != "available":
|
|
103
|
+
hint = _KEY_HINT.format(env=key["key_env"])
|
|
104
|
+
if hint not in hints:
|
|
105
|
+
hints.append(hint)
|
|
106
|
+
return hints
|
|
@@ -7,6 +7,7 @@ from typing import Any
|
|
|
7
7
|
from ciphergate.constants import DISCLAIMER, RESCAN_LIMIT
|
|
8
8
|
from ciphergate.models.report import RunReport
|
|
9
9
|
from ciphergate.pipeline.batch import BatchReport
|
|
10
|
+
from ciphergate.reporting.next_steps import next_steps
|
|
10
11
|
from ciphergate.reporting.safe_text import escape_control
|
|
11
12
|
|
|
12
13
|
SUMMARY_ITEMS = 15
|
|
@@ -119,6 +120,8 @@ def render_summary(report: RunReport, *, verbose: bool = False) -> str:
|
|
|
119
120
|
lines.append(f" output: not written ({out['withheld_reason']})")
|
|
120
121
|
for warning in d["warnings"]:
|
|
121
122
|
lines.append(f" warning: {escape_control(warning)}")
|
|
123
|
+
for hint in next_steps(d):
|
|
124
|
+
lines.append(f" next: {hint}")
|
|
122
125
|
lines.append(f" note: {DISCLAIMER}")
|
|
123
126
|
return "\n".join(lines) + "\n"
|
|
124
127
|
|
|
@@ -158,6 +161,13 @@ def render_batch_summary(report: BatchReport, *, verbose: bool = False) -> str:
|
|
|
158
161
|
lines.append(f" output: not written ({out.get('withheld_reason')})")
|
|
159
162
|
for warning in d["warnings"]:
|
|
160
163
|
lines.append(f" warning: {escape_control(warning)}")
|
|
164
|
+
hints: list[str] = []
|
|
165
|
+
for entry in d["files"]:
|
|
166
|
+
for hint in next_steps(entry.get("report") or {}):
|
|
167
|
+
if hint not in hints:
|
|
168
|
+
hints.append(hint)
|
|
169
|
+
for hint in hints:
|
|
170
|
+
lines.append(f" next: {hint}")
|
|
161
171
|
lines.append(" note: per-file details are in the JSON report (--report).")
|
|
162
172
|
lines.append(f" note: {DISCLAIMER}")
|
|
163
173
|
return "\n".join(lines) + "\n"
|
|
@@ -354,3 +354,54 @@ def test_entry_replaced_after_discovery_is_not_processed(
|
|
|
354
354
|
)
|
|
355
355
|
assert calls == []
|
|
356
356
|
assert results[0].error_code == "input_changed_type" and results[0].exit_code != 0
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
# --------------------------------------------------------------------------- coverage
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def test_out_and_err_files_are_read_as_text(tmp_path: Path) -> None:
|
|
363
|
+
root = tmp_path / "in"
|
|
364
|
+
root.mkdir()
|
|
365
|
+
(root / "app.out").write_text("started\n", encoding="utf-8")
|
|
366
|
+
(root / "app.err").write_text("failed\n", encoding="utf-8")
|
|
367
|
+
d = discover(root, recursive=False, fmt=None, max_files=10)
|
|
368
|
+
assert [p.as_posix() for p in d.files] == ["app.err", "app.out"] and not d.skipped
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def test_skipped_files_are_a_warning_and_fail_a_strict_rescan(tmp_path: Path, env) -> None:
|
|
372
|
+
root = tmp_path / "in"
|
|
373
|
+
root.mkdir()
|
|
374
|
+
(root / "clean.log").write_text("service started\n", encoding="utf-8")
|
|
375
|
+
(root / "notes.md").write_text("not scanned\n", encoding="utf-8")
|
|
376
|
+
r = invoke("inspect", root, *PRD_EXTERNAL, env=env)
|
|
377
|
+
assert r.exit_code == 0 and "1 file(s) were skipped and not scanned" in r.stdout
|
|
378
|
+
strict = invoke("rescan", root, *PRD_EXTERNAL, env=env)
|
|
379
|
+
assert strict.exit_code == 7 and "not scanned" in strict.stdout
|
|
380
|
+
assert invoke("rescan", root, *PRD_EXTERNAL, "--no-strict", env=env).exit_code == 0
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def test_sanitize_warns_that_skipped_files_are_not_in_the_output(tmp_path: Path, env) -> None:
|
|
384
|
+
root = tmp_path / "in"
|
|
385
|
+
root.mkdir()
|
|
386
|
+
(root / "clean.log").write_text("service started\n", encoding="utf-8")
|
|
387
|
+
(root / "notes.md").write_text("not scanned\n", encoding="utf-8")
|
|
388
|
+
out = tmp_path / "out"
|
|
389
|
+
r = invoke("sanitize", root, *PRD_EXTERNAL, "-o", out, "--apply", env=env)
|
|
390
|
+
assert r.exit_code == 0 and "they are not in the output" in r.stdout
|
|
391
|
+
assert sorted(p.name for p in out.iterdir()) == ["clean.log"]
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def test_forced_format_that_contradicts_an_extension_warns(tmp_path: Path, env) -> None:
|
|
395
|
+
root = _tree(tmp_path)
|
|
396
|
+
report = tmp_path / "report.json"
|
|
397
|
+
r = invoke("inspect", root, "--format", "text", *PRD_EXTERNAL, "--report", report, env=env)
|
|
398
|
+
assert "--format text was applied to 1 file(s)" in r.stdout
|
|
399
|
+
assert any("--format text" in w for w in _report(report)["warnings"])
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def test_directory_summary_carries_next_steps(tmp_path: Path) -> None:
|
|
403
|
+
root = tmp_path / "in"
|
|
404
|
+
root.mkdir()
|
|
405
|
+
(root / "app.log").write_text(SECRET_LINE, encoding="utf-8")
|
|
406
|
+
r = invoke("inspect", root, "--policy", POLICIES / "developer_logs.yml")
|
|
407
|
+
assert r.exit_code == 1 and "next: pass --environment" in r.stdout
|
|
@@ -197,6 +197,35 @@ def test_sanitize_missing_key_exit_6(tmp_path: Path, no_key_env: None) -> None:
|
|
|
197
197
|
assert r.exit_code == 6 and not out.exists() and "missing" in r.stdout
|
|
198
198
|
|
|
199
199
|
|
|
200
|
+
def test_blocked_run_says_what_to_do_next(tmp_path: Path) -> None:
|
|
201
|
+
src = tmp_path / "raw.csv"
|
|
202
|
+
shutil.copy(EXAMPLES / "sample.csv", src)
|
|
203
|
+
out = tmp_path / "out.csv"
|
|
204
|
+
r = invoke("sanitize", src, "--policy", POLICIES / "developer_logs.yml", "-o", out, "--apply")
|
|
205
|
+
assert r.exit_code == 1 and not out.exists()
|
|
206
|
+
assert "next: pass --environment" in r.stdout and "next: pass --destination" in r.stdout
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def test_successful_run_has_no_next_steps(tmp_path: Path) -> None:
|
|
210
|
+
clean = tmp_path / "clean.log"
|
|
211
|
+
clean.write_text("service started\n", encoding="utf-8")
|
|
212
|
+
out = tmp_path / "out.log"
|
|
213
|
+
r = invoke(
|
|
214
|
+
"sanitize",
|
|
215
|
+
clean,
|
|
216
|
+
"--policy",
|
|
217
|
+
POLICIES / "developer_logs.yml",
|
|
218
|
+
"--environment",
|
|
219
|
+
"dev",
|
|
220
|
+
"--destination",
|
|
221
|
+
"local_dev",
|
|
222
|
+
"-o",
|
|
223
|
+
out,
|
|
224
|
+
"--apply",
|
|
225
|
+
)
|
|
226
|
+
assert r.exit_code == 0 and "next:" not in r.stdout
|
|
227
|
+
|
|
228
|
+
|
|
200
229
|
def test_rescan_summary_never_says_a_bare_pass(tmp_path: Path) -> None:
|
|
201
230
|
policy = POLICIES / "prd_external_ai.yml"
|
|
202
231
|
clean = tmp_path / "clean.log"
|
|
@@ -158,12 +158,58 @@ def test_card_luhn_invalid_ignored() -> None:
|
|
|
158
158
|
assert PaymentCardDetector().detect(item("4111 1111 1111 1112")) == []
|
|
159
159
|
|
|
160
160
|
|
|
161
|
-
def
|
|
162
|
-
# 18-digit Luhn-valid number without issuer prefix or context.
|
|
161
|
+
def test_luhn_valid_identifier_without_prefix_or_keyword_is_ignored() -> None:
|
|
162
|
+
# 18-digit Luhn-valid number without issuer prefix or context: a random ID, not a card.
|
|
163
163
|
digits = "100000000000000008"
|
|
164
164
|
assert luhn_valid(digits)
|
|
165
|
-
|
|
166
|
-
|
|
165
|
+
assert PaymentCardDetector().detect(item(f"order {digits}")) == []
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def test_card_keyword_still_finds_a_number_without_issuer_prefix() -> None:
|
|
169
|
+
f = PaymentCardDetector().detect(item("card 100000000000000008"))
|
|
170
|
+
assert f and f[0].band == "medium" and "context_keyword" in f[0].evidence
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@pytest.mark.parametrize(
|
|
174
|
+
"text",
|
|
175
|
+
["invoice total 4111111111111111 units", "batch id 4111111111111111 queued"],
|
|
176
|
+
)
|
|
177
|
+
def test_issuer_prefixed_identifier_after_an_id_word_is_ignored(text: str) -> None:
|
|
178
|
+
assert PaymentCardDetector().detect(item(text)) == []
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
@pytest.mark.parametrize(
|
|
182
|
+
"text",
|
|
183
|
+
[
|
|
184
|
+
"user paid with 4111 1111 1111 1111",
|
|
185
|
+
"valid 4111111111111111",
|
|
186
|
+
"orderly 4111111111111111",
|
|
187
|
+
"rebuilds 4111111111111111",
|
|
188
|
+
],
|
|
189
|
+
)
|
|
190
|
+
def test_id_words_match_whole_words_only(text: str) -> None:
|
|
191
|
+
# "paid" contains "id": a substring match hid real cards (caught by the leak test).
|
|
192
|
+
assert PaymentCardDetector().detect(item(text))
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def test_card_keyword_wins_over_an_id_word() -> None:
|
|
196
|
+
assert PaymentCardDetector().detect(item("order paid by card 4111111111111111"))
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
@pytest.mark.parametrize(
|
|
200
|
+
"digits",
|
|
201
|
+
[
|
|
202
|
+
"3530111333300000", # JCB
|
|
203
|
+
"6200000000000005", # UnionPay
|
|
204
|
+
"36227206271667", # Diners Club
|
|
205
|
+
"6445644564456445", # Discover 644-649
|
|
206
|
+
"2200000000000004", # Mir
|
|
207
|
+
],
|
|
208
|
+
)
|
|
209
|
+
def test_more_issuer_prefixes(digits: str) -> None:
|
|
210
|
+
assert luhn_valid(digits)
|
|
211
|
+
f = PaymentCardDetector().detect(item(f"charged {digits}"))
|
|
212
|
+
assert f and "issuer_prefix" in f[0].evidence
|
|
167
213
|
|
|
168
214
|
|
|
169
215
|
def test_card_mixed_separators_and_repeated_digits_ignored() -> None:
|