ciphergate-cli 0.13.0__tar.gz → 0.15.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/CHANGELOG.md +35 -1
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/PKG-INFO +8 -2
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/README.md +7 -1
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/docs/detector-coverage.md +6 -2
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/docs/security-and-privacy.md +2 -1
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/__init__.py +1 -1
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/cli.py +54 -5
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/config.py +13 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/models/report.py +6 -2
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/pipeline/batch.py +35 -8
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/pipeline/inspector.py +1 -1
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/readers/__init__.py +12 -3
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/reporting/next_steps.py +28 -9
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/reporting/terminal.py +4 -10
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/integration/test_batch.py +104 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/integration/test_cli_e2e.py +14 -0
- ciphergate_cli-0.15.0/tests/integration/test_presidio_real.py +138 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_next_steps.py +33 -1
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/.gitignore +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/LICENSE +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/NOTICE +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/SECURITY.md +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/docs/architecture.md +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/docs/claude-code-tooling.md +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/docs/examples.md +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/docs/limitations.md +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/docs/policy-schema.md +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/docs/threat-model.md +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/examples/expected/sample.dev_local_dev.csv +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/examples/expected/sample.dev_local_dev.json +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/examples/expected/sample.dev_local_dev.jsonl +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/examples/expected/sample.dev_local_dev.log +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/examples/sample.csv +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/examples/sample.json +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/examples/sample.jsonl +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/examples/sample.log +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/policies/dev_local.yml +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/policies/developer_logs.yml +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/policies/external_ai.yml +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/policies/financial_template.yml +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/policies/healthcare_template.yml +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/policies/local_debug.yml +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/policies/multi_environment.yml +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/policies/prd_external_ai.yml +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/policies/stg_test.yml +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/policies/test_dataset.yml +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/pyproject.toml +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/scripts/benchmark/README.md +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/__main__.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/constants.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/__init__.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/base.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/custom_regex.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/dates.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/email.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/field_names.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/iban.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/ip_address.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/mac_address.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/network.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/payment_card.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/phone.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/presidio_ner.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/regex_safety.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/registry.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/secrets.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/detectors/ssn.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/errors.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/models/__init__.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/models/context.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/models/finding.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/models/policy.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/pipeline/__init__.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/pipeline/atomic_output.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/pipeline/paths.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/pipeline/planner.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/pipeline/rescan.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/pipeline/sanitizer.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/pipeline/stdio.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/pipeline/streams.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/policy/__init__.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/policy/context.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/policy/engine.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/policy/loader.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/policy/precedence.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/policy/templates.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/policy/validator.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/py.typed +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/readers/accesslog_reader.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/readers/base.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/readers/csv_reader.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/readers/json_reader.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/readers/jsonl_reader.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/readers/jsonlog_reader.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/readers/line_structure.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/readers/logfmt_reader.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/readers/stacktrace_reader.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/readers/syslog_reader.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/readers/text_reader.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/reporting/__init__.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/reporting/audit.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/reporting/safe_text.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/reporting/serializer.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/templates/starter_policy.yml +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/transforms/__init__.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/transforms/date_shift.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/transforms/generalize.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/transforms/keys.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/transforms/pseudonymize.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/transforms/redact.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/transforms/registry.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/src/ciphergate/transforms/replace.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/__init__.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/conftest.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/integration/__init__.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/integration/test_api_batch_streams.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/integration/test_atomic_output.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/integration/test_domain_templates.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/integration/test_environment_matrix.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/integration/test_pipelines.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/integration/test_stdio.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/regression/__init__.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/regression/test_known_cases.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/security/__init__.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/security/test_key_file.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/security/test_no_network.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/security/test_no_raw_value_leaks.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/security/test_path_safety.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/security/test_policy_fail_closed.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/security/test_review_findings.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/tooling/__init__.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/tooling/test_benchmark.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/tooling/test_ci_supply_chain.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/tooling/test_claude_tooling.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/tooling/test_git_tooling.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/tooling/test_publish_workflow.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/__init__.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_accesslog.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_detectors.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_jsonlog.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_logfmt.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_policy_engine.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_policy_precedence.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_policy_templates.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_policy_validation.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_presidio_ner.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_properties.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_readers.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_stacktrace.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_syslog.py +0 -0
- {ciphergate_cli-0.13.0 → ciphergate_cli-0.15.0}/tests/unit/test_transforms.py +0 -0
|
@@ -6,6 +6,38 @@ versioning: [PEP 440](https://peps.python.org/pep-0440/) with SemVer meaning (pr
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.15.0] - 2026-11-02
|
|
10
|
+
|
|
11
|
+
JSON reports now include the next-step hints, so CI pipelines can act on them. The field is new;
|
|
12
|
+
nothing existing changes.
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- JSON reports (single-file and directory) include `next_steps`: the hints from the terminal
|
|
17
|
+
summary as `{"code", "message"}` pairs, empty for a successful run. The field is new and
|
|
18
|
+
optional to read; the schema names are unchanged.
|
|
19
|
+
|
|
20
|
+
### Fixed
|
|
21
|
+
|
|
22
|
+
- The `batch_empty` error for a directory with no recognised files now mentions
|
|
23
|
+
`--extension-format`.
|
|
24
|
+
|
|
25
|
+
## [0.14.0] - 2026-11-02
|
|
26
|
+
|
|
27
|
+
Adds `--extension-format` for mixed directories and tests the optional NER detector against a
|
|
28
|
+
real Presidio install. No existing behaviour changes.
|
|
29
|
+
|
|
30
|
+
### Added
|
|
31
|
+
|
|
32
|
+
- `--extension-format EXT=FORMAT` on `inspect`, `plan`, `sanitize` and `rescan` (repeatable) reads
|
|
33
|
+
files with that extension as FORMAT, for directories and single files. It adds unknown
|
|
34
|
+
extensions or replaces built-in ones, and cannot be combined with `--format`. The Python API
|
|
35
|
+
takes `RuntimeSettings(extension_formats=...)`.
|
|
36
|
+
- Tests that run the optional NER detector against a real Presidio install with the
|
|
37
|
+
`en_core_web_sm` model, including a `sanitize` run with network access disabled. They skip
|
|
38
|
+
when Presidio is not installed. A new non-required CI job, `ner`, installs the extra and a
|
|
39
|
+
hash-pinned model and runs them.
|
|
40
|
+
|
|
9
41
|
## [0.13.0] - 2026-11-02
|
|
10
42
|
|
|
11
43
|
Fewer random IDs are reported as payment cards. Cards with an unrecognised prefix and no card
|
|
@@ -363,7 +395,9 @@ Pre-release (alpha). Licensed under Apache-2.0. Not a final release: the securit
|
|
|
363
395
|
See `docs/limitations.md`. Deferred: optional NER/Presidio detectors, stdin/stdout streaming,
|
|
364
396
|
directory processing, structured log parsers, key files, reversible token vault, lossy decoding.
|
|
365
397
|
|
|
366
|
-
[Unreleased]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.
|
|
398
|
+
[Unreleased]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.15.0...HEAD
|
|
399
|
+
[0.15.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.14.0...v0.15.0
|
|
400
|
+
[0.14.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.13.0...v0.14.0
|
|
367
401
|
[0.13.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.12.0...v0.13.0
|
|
368
402
|
[0.12.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.11.0...v0.12.0
|
|
369
403
|
[0.11.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.10.0...v0.11.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ciphergate-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.15.0
|
|
4
4
|
Summary: Local-first, environment- and destination-aware inspection and sanitization of logs and test datasets.
|
|
5
5
|
Project-URL: Homepage, https://github.com/aashishkumar-tech/ciphergate
|
|
6
6
|
Project-URL: Source, https://github.com/aashishkumar-tech/ciphergate
|
|
@@ -50,7 +50,7 @@ vendor ticket. CipherGate replaces that with a local, explainable, scriptable ga
|
|
|
50
50
|
inspect -> plan -> sanitize --apply -> rescan
|
|
51
51
|
```
|
|
52
52
|
|
|
53
|
-
> **Status: 0.
|
|
53
|
+
> **Status: 0.15.0 (alpha).** No detector finds every sensitive value. A passing run means the
|
|
54
54
|
> configured processing completed under the selected policy; it is **not** proof that the output
|
|
55
55
|
> contains no sensitive information, and it is not a compliance (GDPR, HIPAA, PCI DSS, ...) or
|
|
56
56
|
> anonymization guarantee. Policy templates must be reviewed before use.
|
|
@@ -89,6 +89,8 @@ inspect -> plan -> sanitize --apply -> rescan
|
|
|
89
89
|
`--force`, input/output alias protection, input-change detection, cleanup on failure/Ctrl-C.
|
|
90
90
|
- **Reports:** counts by category/detector/action/reason/rule, safe locations, effective context
|
|
91
91
|
and its source, gates, key *status* (never key material), no raw values or fingerprints.
|
|
92
|
+
A failed run's report lists `next_steps` as `{"code", "message"}` pairs, the same hints the
|
|
93
|
+
terminal prints as `next:` lines, so CI can act on them.
|
|
92
94
|
|
|
93
95
|
## Install
|
|
94
96
|
|
|
@@ -295,6 +297,10 @@ ciphergate sanitize logs/ --policy ... --output build/logs.sanitized --apply
|
|
|
295
297
|
`rescan` fails (exit 7) when it skipped any, and `sanitize` leaves them out of the output.
|
|
296
298
|
`--format` on a directory applies to every file, and the run warns when that contradicts an
|
|
297
299
|
extension (for example a CSV read as text loses its column names).
|
|
300
|
+
- `--extension-format EXT=FORMAT` (repeatable) reads files with that extension as FORMAT, for
|
|
301
|
+
example `--extension-format .dat=csv --extension-format .log=jsonlog`. It adds unknown extensions
|
|
302
|
+
and replaces built-in ones, works for single files too, and cannot be combined with `--format`.
|
|
303
|
+
From Python, pass `RuntimeSettings(extension_formats={".dat": InputFormat.CSV})`.
|
|
298
304
|
At most 1000 files by default (`--max-files N`); more fails the run (exit 4) before anything
|
|
299
305
|
is processed. An empty selection fails with `batch_empty` (exit 4).
|
|
300
306
|
- Each file is processed exactly like a single-file run, in sorted path order; a failing file
|
|
@@ -11,7 +11,7 @@ vendor ticket. CipherGate replaces that with a local, explainable, scriptable ga
|
|
|
11
11
|
inspect -> plan -> sanitize --apply -> rescan
|
|
12
12
|
```
|
|
13
13
|
|
|
14
|
-
> **Status: 0.
|
|
14
|
+
> **Status: 0.15.0 (alpha).** No detector finds every sensitive value. A passing run means the
|
|
15
15
|
> configured processing completed under the selected policy; it is **not** proof that the output
|
|
16
16
|
> contains no sensitive information, and it is not a compliance (GDPR, HIPAA, PCI DSS, ...) or
|
|
17
17
|
> anonymization guarantee. Policy templates must be reviewed before use.
|
|
@@ -50,6 +50,8 @@ inspect -> plan -> sanitize --apply -> rescan
|
|
|
50
50
|
`--force`, input/output alias protection, input-change detection, cleanup on failure/Ctrl-C.
|
|
51
51
|
- **Reports:** counts by category/detector/action/reason/rule, safe locations, effective context
|
|
52
52
|
and its source, gates, key *status* (never key material), no raw values or fingerprints.
|
|
53
|
+
A failed run's report lists `next_steps` as `{"code", "message"}` pairs, the same hints the
|
|
54
|
+
terminal prints as `next:` lines, so CI can act on them.
|
|
53
55
|
|
|
54
56
|
## Install
|
|
55
57
|
|
|
@@ -256,6 +258,10 @@ ciphergate sanitize logs/ --policy ... --output build/logs.sanitized --apply
|
|
|
256
258
|
`rescan` fails (exit 7) when it skipped any, and `sanitize` leaves them out of the output.
|
|
257
259
|
`--format` on a directory applies to every file, and the run warns when that contradicts an
|
|
258
260
|
extension (for example a CSV read as text loses its column names).
|
|
261
|
+
- `--extension-format EXT=FORMAT` (repeatable) reads files with that extension as FORMAT, for
|
|
262
|
+
example `--extension-format .dat=csv --extension-format .log=jsonlog`. It adds unknown extensions
|
|
263
|
+
and replaces built-in ones, works for single files too, and cannot be combined with `--format`.
|
|
264
|
+
From Python, pass `RuntimeSettings(extension_formats={".dat": InputFormat.CSV})`.
|
|
259
265
|
At most 1000 files by default (`--max-files N`); more fails the run (exit 4) before anything
|
|
260
266
|
is processed. An empty selection fails with `batch_empty` (exit 4).
|
|
261
267
|
- Each file is processed exactly like a single-file run, in sorted path order; a failing file
|
|
@@ -28,8 +28,12 @@ Needs `pip install 'ciphergate-cli[ner]'` and a spaCy model installed by the use
|
|
|
28
28
|
`python -m spacy download en_core_web_lg`). CipherGate checks that the model package is importable
|
|
29
29
|
before importing Presidio and fails with `ner_model_missing` instead of letting Presidio download
|
|
30
30
|
it. Strings with no letters or under two characters are not sent to the model. Tests
|
|
31
|
-
|
|
32
|
-
|
|
31
|
+
in `tests/unit/test_presidio_ner.py` use a fake engine. `tests/integration/test_presidio_real.py`
|
|
32
|
+
runs the real Presidio with `en_core_web_sm`: it finds a common synthetic name and place, ignores
|
|
33
|
+
text with no entities, removes them in a `sanitize` run with network access disabled, and fails
|
|
34
|
+
closed when the model is missing. CI runs it in the `ner` job (not a required check). These are
|
|
35
|
+
smoke tests on a few sentences: accuracy on real text has not been measured by this project,
|
|
36
|
+
and the small model misses names that the larger `en_core_web_lg` may find.
|
|
33
37
|
|
|
34
38
|
## Text/log specifics
|
|
35
39
|
|
|
@@ -67,7 +67,8 @@ Reports contain: run ID, UTC time, version, input basename (or `<redacted>`), fo
|
|
|
67
67
|
record counts, policy name/version/SHA-256 digest of the policy file, effective context with
|
|
68
68
|
sources, gates, counts by category/detector/action/band/reason/rule, safe locations (record,
|
|
69
69
|
line, field path, character span), rule IDs and reason codes, key status, changes, parse error
|
|
70
|
-
line numbers and codes, rescan summary, warnings
|
|
70
|
+
line numbers and codes, rescan summary, warnings, `next_steps` (fixed hint text with a reason
|
|
71
|
+
code, for failed runs only) and a disclaimer. They never contain values,
|
|
71
72
|
snippets, keys, mappings, or hashes of values. Field paths whose segments look like sensitive
|
|
72
73
|
values are replaced with `<redacted-key>`; control and bidi characters are escaped.
|
|
73
74
|
|
|
@@ -24,7 +24,7 @@ Supported public API (everything else is internal and may change without notice)
|
|
|
24
24
|
rescan_directory(root, policy, context=None, *, recursive=False, strict=None) -> BatchReport
|
|
25
25
|
"""
|
|
26
26
|
|
|
27
|
-
__version__ = "0.
|
|
27
|
+
__version__ = "0.15.0"
|
|
28
28
|
|
|
29
29
|
from ciphergate.config import RuntimeSettings
|
|
30
30
|
from ciphergate.constants import (
|
|
@@ -117,6 +117,15 @@ PolicyOpt = Annotated[Path, typer.Option("--policy", "-p", help="Policy YAML fil
|
|
|
117
117
|
FormatOpt = Annotated[
|
|
118
118
|
InputFormat | None, typer.Option("--format", help="Input format (default: from extension).")
|
|
119
119
|
]
|
|
120
|
+
ExtFormatOpt = Annotated[
|
|
121
|
+
list[str] | None,
|
|
122
|
+
typer.Option(
|
|
123
|
+
"--extension-format",
|
|
124
|
+
metavar="EXT=FORMAT",
|
|
125
|
+
help="Read files with this extension as FORMAT, e.g. .dat=csv or .log=jsonlog. "
|
|
126
|
+
"Repeatable. Cannot be combined with --format.",
|
|
127
|
+
),
|
|
128
|
+
]
|
|
120
129
|
ReportOpt = Annotated[
|
|
121
130
|
Path | None, typer.Option("--report", help="Write the privacy-safe report to this file.")
|
|
122
131
|
]
|
|
@@ -192,18 +201,52 @@ def _runtime(
|
|
|
192
201
|
encoding: str | None,
|
|
193
202
|
path_display: PathDisplayChoice | None,
|
|
194
203
|
max_files: int | None = None,
|
|
204
|
+
extension_formats: list[str] | None = None,
|
|
205
|
+
fmt: InputFormat | None = None,
|
|
195
206
|
) -> RuntimeSettings:
|
|
207
|
+
mapping = _parse_extension_formats(extension_formats or [])
|
|
208
|
+
if mapping and fmt is not None:
|
|
209
|
+
raise UsageError(
|
|
210
|
+
"use either --format (one format for every file) or --extension-format, not both",
|
|
211
|
+
code="format_conflict",
|
|
212
|
+
)
|
|
196
213
|
try:
|
|
197
214
|
return RuntimeSettings().with_overrides(
|
|
198
215
|
max_input_bytes=max_bytes,
|
|
199
216
|
encoding=encoding,
|
|
200
217
|
path_display=str(path_display) if path_display else None,
|
|
201
218
|
max_batch_files=max_files,
|
|
219
|
+
extension_formats=mapping or None,
|
|
202
220
|
)
|
|
203
221
|
except ValueError as exc:
|
|
204
222
|
raise UsageError(str(exc), code="invalid_runtime_setting") from None
|
|
205
223
|
|
|
206
224
|
|
|
225
|
+
def _parse_extension_formats(values: list[str]) -> dict[str, InputFormat]:
|
|
226
|
+
mapping: dict[str, InputFormat] = {}
|
|
227
|
+
for value in values:
|
|
228
|
+
ext, sep, name = value.partition("=")
|
|
229
|
+
ext = ext.strip().lower()
|
|
230
|
+
if not ext.startswith("."):
|
|
231
|
+
ext = "." + ext
|
|
232
|
+
try:
|
|
233
|
+
fmt = InputFormat(name.strip().lower())
|
|
234
|
+
except ValueError:
|
|
235
|
+
fmt = None
|
|
236
|
+
if not sep or fmt is None:
|
|
237
|
+
raise UsageError(
|
|
238
|
+
"--extension-format needs EXT=FORMAT with FORMAT one of " + "|".join(InputFormat),
|
|
239
|
+
code="invalid_extension_format",
|
|
240
|
+
)
|
|
241
|
+
if ext in mapping and mapping[ext] != fmt:
|
|
242
|
+
raise UsageError(
|
|
243
|
+
"--extension-format maps the same extension to two formats",
|
|
244
|
+
code="invalid_extension_format",
|
|
245
|
+
)
|
|
246
|
+
mapping[ext] = fmt
|
|
247
|
+
return mapping
|
|
248
|
+
|
|
249
|
+
|
|
207
250
|
def _err(message: str) -> None:
|
|
208
251
|
sys.stderr.write(escape_control(message)[:2000] + "\n")
|
|
209
252
|
|
|
@@ -501,6 +544,7 @@ def inspect_command(
|
|
|
501
544
|
],
|
|
502
545
|
policy: PolicyOpt,
|
|
503
546
|
fmt: FormatOpt = None,
|
|
547
|
+
extension_format: ExtFormatOpt = None,
|
|
504
548
|
report: ReportOpt = None,
|
|
505
549
|
report_format: ReportFormatOpt = None,
|
|
506
550
|
environment: EnvOpt = None,
|
|
@@ -531,7 +575,7 @@ def inspect_command(
|
|
|
531
575
|
report,
|
|
532
576
|
report_format,
|
|
533
577
|
_context(environment, target_environment, destination, use_case, profile),
|
|
534
|
-
_runtime(max_bytes, encoding, path_display, max_files),
|
|
578
|
+
_runtime(max_bytes, encoding, path_display, max_files, extension_format, fmt),
|
|
535
579
|
quiet,
|
|
536
580
|
verbose,
|
|
537
581
|
force,
|
|
@@ -547,6 +591,7 @@ def plan_command(
|
|
|
547
591
|
],
|
|
548
592
|
policy: PolicyOpt,
|
|
549
593
|
fmt: FormatOpt = None,
|
|
594
|
+
extension_format: ExtFormatOpt = None,
|
|
550
595
|
report: ReportOpt = None,
|
|
551
596
|
report_format: ReportFormatOpt = None,
|
|
552
597
|
environment: EnvOpt = None,
|
|
@@ -577,7 +622,7 @@ def plan_command(
|
|
|
577
622
|
report,
|
|
578
623
|
report_format,
|
|
579
624
|
_context(environment, target_environment, destination, use_case, profile),
|
|
580
|
-
_runtime(max_bytes, encoding, path_display, max_files),
|
|
625
|
+
_runtime(max_bytes, encoding, path_display, max_files, extension_format, fmt),
|
|
581
626
|
quiet,
|
|
582
627
|
verbose,
|
|
583
628
|
force,
|
|
@@ -604,6 +649,7 @@ def sanitize_command(
|
|
|
604
649
|
bool, typer.Option("--apply", help="Actually write output (else plan).")
|
|
605
650
|
] = False,
|
|
606
651
|
fmt: FormatOpt = None,
|
|
652
|
+
extension_format: ExtFormatOpt = None,
|
|
607
653
|
report: ReportOpt = None,
|
|
608
654
|
report_format: ReportFormatOpt = None,
|
|
609
655
|
environment: EnvOpt = None,
|
|
@@ -636,7 +682,9 @@ def sanitize_command(
|
|
|
636
682
|
"--output - uses stdout for data; write the JSON report with --report PATH",
|
|
637
683
|
code="stdout_conflict",
|
|
638
684
|
)
|
|
639
|
-
runtime = _runtime(
|
|
685
|
+
runtime = _runtime(
|
|
686
|
+
max_bytes, encoding, path_display, extension_formats=extension_format, fmt=fmt
|
|
687
|
+
)
|
|
640
688
|
_check_report_path(report, [p for p in (input_path, output) if not is_stdio(p)], force)
|
|
641
689
|
loaded = load_policy(policy)
|
|
642
690
|
options = SanitizeOptions(apply=apply, force=force, format=selected_fmt, runtime=runtime)
|
|
@@ -689,7 +737,7 @@ def sanitize_command(
|
|
|
689
737
|
raise UsageError(
|
|
690
738
|
"a directory INPUT needs --output DIR, not stdout", code="stdout_unsupported"
|
|
691
739
|
)
|
|
692
|
-
runtime = _runtime(max_bytes, encoding, path_display, max_files)
|
|
740
|
+
runtime = _runtime(max_bytes, encoding, path_display, max_files, extension_format, fmt)
|
|
693
741
|
_check_batch_report_path(report, [input_path, output], force)
|
|
694
742
|
batch = sanitize_directory(
|
|
695
743
|
input_path,
|
|
@@ -731,6 +779,7 @@ def rescan_command(
|
|
|
731
779
|
),
|
|
732
780
|
] = None,
|
|
733
781
|
fmt: FormatOpt = None,
|
|
782
|
+
extension_format: ExtFormatOpt = None,
|
|
734
783
|
report: ReportOpt = None,
|
|
735
784
|
report_format: ReportFormatOpt = None,
|
|
736
785
|
environment: EnvOpt = None,
|
|
@@ -764,7 +813,7 @@ def rescan_command(
|
|
|
764
813
|
report,
|
|
765
814
|
report_format,
|
|
766
815
|
_context(environment, target_environment, destination, use_case, profile),
|
|
767
|
-
_runtime(max_bytes, encoding, path_display, max_files),
|
|
816
|
+
_runtime(max_bytes, encoding, path_display, max_files, extension_format, fmt),
|
|
768
817
|
quiet,
|
|
769
818
|
verbose,
|
|
770
819
|
force,
|
|
@@ -2,10 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import re
|
|
5
6
|
from dataclasses import dataclass, field, replace
|
|
6
7
|
from typing import Literal
|
|
7
8
|
|
|
9
|
+
from ciphergate.constants import InputFormat
|
|
10
|
+
|
|
8
11
|
PathDisplay = Literal["basename", "redacted", "full"]
|
|
12
|
+
_EXTENSION_RE = re.compile(r"^\.[a-z0-9_-]{1,16}$")
|
|
9
13
|
|
|
10
14
|
MIB = 1024 * 1024
|
|
11
15
|
|
|
@@ -27,6 +31,9 @@ class RuntimeSettings:
|
|
|
27
31
|
encoding: str = "utf-8"
|
|
28
32
|
csv_delimiter: str = ","
|
|
29
33
|
path_display: PathDisplay = "basename"
|
|
34
|
+
# Extra or replacement extension -> format mappings, e.g. {".dat": csv}. Keys are
|
|
35
|
+
# lower-case single suffixes. An explicit --format still wins for every file.
|
|
36
|
+
extension_formats: dict[str, InputFormat] = field(default_factory=dict)
|
|
30
37
|
# Stored for completeness of the run report; never changes decisions.
|
|
31
38
|
extra: dict[str, str] = field(default_factory=dict)
|
|
32
39
|
|
|
@@ -44,6 +51,12 @@ class RuntimeSettings:
|
|
|
44
51
|
raise ValueError(f"{name} must be a positive integer")
|
|
45
52
|
if len(self.csv_delimiter) != 1 or self.csv_delimiter in '\r\n"':
|
|
46
53
|
raise ValueError("csv_delimiter must be a single character other than quote or newline")
|
|
54
|
+
for ext in self.extension_formats:
|
|
55
|
+
if not _EXTENSION_RE.match(ext):
|
|
56
|
+
raise ValueError(
|
|
57
|
+
"extension_formats keys must look like '.dat' (a dot, then 1-16 "
|
|
58
|
+
"lower-case letters, digits, '_' or '-')"
|
|
59
|
+
)
|
|
47
60
|
|
|
48
61
|
def with_overrides(self, **changes: object) -> RuntimeSettings:
|
|
49
62
|
clean = {k: v for k, v in changes.items() if v is not None}
|
|
@@ -11,6 +11,7 @@ from enum import StrEnum
|
|
|
11
11
|
from typing import Any
|
|
12
12
|
|
|
13
13
|
from ciphergate.constants import DISCLAIMER, REPORT_SCHEMA, ExitCode
|
|
14
|
+
from ciphergate.reporting.next_steps import next_step_items
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
class RunStatus(StrEnum):
|
|
@@ -89,7 +90,7 @@ class RunReport:
|
|
|
89
90
|
return self.exit_code == 0
|
|
90
91
|
|
|
91
92
|
def to_dict(self) -> dict[str, Any]:
|
|
92
|
-
|
|
93
|
+
data: dict[str, Any] = {
|
|
93
94
|
"schema": REPORT_SCHEMA,
|
|
94
95
|
"command": self.command,
|
|
95
96
|
"status": str(self.status),
|
|
@@ -111,5 +112,8 @@ class RunReport:
|
|
|
111
112
|
"output": self.output,
|
|
112
113
|
"rescan": self.rescan.to_dict() if self.rescan else None,
|
|
113
114
|
"warnings": self.warnings,
|
|
114
|
-
"disclaimer": DISCLAIMER,
|
|
115
115
|
}
|
|
116
|
+
# Fixed hint text chosen by reason code: never contains input values.
|
|
117
|
+
data["next_steps"] = next_step_items(data)
|
|
118
|
+
data["disclaimer"] = DISCLAIMER
|
|
119
|
+
return data
|
|
@@ -24,7 +24,7 @@ import shutil
|
|
|
24
24
|
import stat
|
|
25
25
|
import sys
|
|
26
26
|
import tempfile
|
|
27
|
-
from collections.abc import Callable
|
|
27
|
+
from collections.abc import Callable, Mapping
|
|
28
28
|
from dataclasses import dataclass, field
|
|
29
29
|
from pathlib import Path
|
|
30
30
|
from typing import Any
|
|
@@ -52,6 +52,7 @@ from ciphergate.pipeline.sanitizer import (
|
|
|
52
52
|
from ciphergate.policy.validator import Policy
|
|
53
53
|
from ciphergate.readers import EXTENSION_FORMATS
|
|
54
54
|
from ciphergate.reporting.audit import new_run_id, utc_now
|
|
55
|
+
from ciphergate.reporting.next_steps import merge_next_steps
|
|
55
56
|
|
|
56
57
|
logger = logging.getLogger("ciphergate")
|
|
57
58
|
|
|
@@ -136,6 +137,7 @@ class BatchReport:
|
|
|
136
137
|
|
|
137
138
|
def to_dict(self) -> dict[str, Any]:
|
|
138
139
|
first = next((r.report for r in self.files if r.report is not None), None)
|
|
140
|
+
files = [r.to_dict() for r in self.files]
|
|
139
141
|
return {
|
|
140
142
|
"schema": BATCH_REPORT_SCHEMA,
|
|
141
143
|
"command": self.command,
|
|
@@ -148,10 +150,11 @@ class BatchReport:
|
|
|
148
150
|
"policy": first.policy if first is not None else None,
|
|
149
151
|
"context": first.context if first is not None else None,
|
|
150
152
|
"summary": self.summary(),
|
|
151
|
-
"files":
|
|
153
|
+
"files": files,
|
|
152
154
|
"skipped": self.skipped,
|
|
153
155
|
"output": self.output,
|
|
154
156
|
"warnings": self.warnings,
|
|
157
|
+
"next_steps": merge_next_steps([f["report"] for f in files if "report" in f]),
|
|
155
158
|
"disclaimer": DISCLAIMER,
|
|
156
159
|
}
|
|
157
160
|
|
|
@@ -171,7 +174,14 @@ def _is_link(entry: os.DirEntry[str]) -> bool:
|
|
|
171
174
|
return False
|
|
172
175
|
|
|
173
176
|
|
|
174
|
-
def discover(
|
|
177
|
+
def discover(
|
|
178
|
+
root: Path,
|
|
179
|
+
*,
|
|
180
|
+
recursive: bool,
|
|
181
|
+
fmt: InputFormat | None,
|
|
182
|
+
max_files: int,
|
|
183
|
+
extensions: Mapping[str, InputFormat] | None = None,
|
|
184
|
+
) -> Discovery:
|
|
175
185
|
"""List the files a batch run will process. Raises on limits; never follows links."""
|
|
176
186
|
try:
|
|
177
187
|
st = os.lstat(root)
|
|
@@ -210,7 +220,7 @@ def discover(root: Path, *, recursive: bool, fmt: InputFormat | None, max_files:
|
|
|
210
220
|
walk(rel, depth + 1)
|
|
211
221
|
elif not entry.is_file(follow_symlinks=False):
|
|
212
222
|
skipped.append((rel, "not_regular_file"))
|
|
213
|
-
elif fmt is None and
|
|
223
|
+
elif fmt is None and not _known_extension(rel, extensions):
|
|
214
224
|
skipped.append((rel, "unsupported_extension"))
|
|
215
225
|
else:
|
|
216
226
|
files.append(rel)
|
|
@@ -226,6 +236,11 @@ def discover(root: Path, *, recursive: bool, fmt: InputFormat | None, max_files:
|
|
|
226
236
|
return Discovery(root, sorted(files, key=lambda p: p.as_posix()), skipped)
|
|
227
237
|
|
|
228
238
|
|
|
239
|
+
def _known_extension(rel: Path, extensions: Mapping[str, InputFormat] | None) -> bool:
|
|
240
|
+
suffix = rel.suffix.lower()
|
|
241
|
+
return suffix in EXTENSION_FORMATS or suffix in (extensions or {})
|
|
242
|
+
|
|
243
|
+
|
|
229
244
|
def display_relative(rel: Path, index: int, root: Path, mode: str) -> str:
|
|
230
245
|
if mode == "redacted":
|
|
231
246
|
return f"<redacted:{index:04d}>{rel.suffix}"
|
|
@@ -291,7 +306,8 @@ def coverage_warnings(
|
|
|
291
306
|
f"{unknown} file(s) were skipped and not scanned because their extension is not "
|
|
292
307
|
"recognised"
|
|
293
308
|
+ ("; they are not in the output" if sanitize else "")
|
|
294
|
-
+ ";
|
|
309
|
+
+ "; map them with --extension-format EXT=FORMAT, or pass --format to read every "
|
|
310
|
+
"file as one format"
|
|
295
311
|
)
|
|
296
312
|
if fmt is not None:
|
|
297
313
|
mismatched = sum(
|
|
@@ -311,7 +327,8 @@ def coverage_warnings(
|
|
|
311
327
|
def _require_files(discovery: Discovery) -> None:
|
|
312
328
|
if not discovery.files:
|
|
313
329
|
raise InputFormatError(
|
|
314
|
-
"no supported input files found in the directory (see --format, --
|
|
330
|
+
"no supported input files found in the directory (see --extension-format, --format, "
|
|
331
|
+
"--recursive)",
|
|
315
332
|
code="batch_empty",
|
|
316
333
|
)
|
|
317
334
|
|
|
@@ -332,7 +349,13 @@ def analyze_directory(
|
|
|
332
349
|
) -> BatchReport:
|
|
333
350
|
"""Batch ``inspect`` / ``plan`` / ``rescan``. Read-only, like the single-file commands."""
|
|
334
351
|
root = Path(root)
|
|
335
|
-
discovery = discover(
|
|
352
|
+
discovery = discover(
|
|
353
|
+
root,
|
|
354
|
+
recursive=recursive,
|
|
355
|
+
fmt=fmt,
|
|
356
|
+
max_files=runtime.max_batch_files,
|
|
357
|
+
extensions=runtime.extension_formats,
|
|
358
|
+
)
|
|
336
359
|
_require_files(discovery)
|
|
337
360
|
|
|
338
361
|
def run(path: Path, rel: Path) -> RunReport:
|
|
@@ -471,7 +494,11 @@ def sanitize_directory(
|
|
|
471
494
|
runtime = options.runtime or RuntimeSettings()
|
|
472
495
|
validate_output_directory(root, output)
|
|
473
496
|
discovery = discover(
|
|
474
|
-
root,
|
|
497
|
+
root,
|
|
498
|
+
recursive=recursive,
|
|
499
|
+
fmt=options.format,
|
|
500
|
+
max_files=runtime.max_batch_files,
|
|
501
|
+
extensions=runtime.extension_formats,
|
|
475
502
|
)
|
|
476
503
|
_require_files(discovery)
|
|
477
504
|
mode = runtime.path_display
|
|
@@ -85,7 +85,7 @@ def prepare_run(
|
|
|
85
85
|
ctx = resolve_context(policy, context)
|
|
86
86
|
path = Path(input_path)
|
|
87
87
|
validate_input_path(path, runtime.max_input_bytes)
|
|
88
|
-
detected = detect_format(path, fmt)
|
|
88
|
+
detected = detect_format(path, fmt, runtime.extension_formats)
|
|
89
89
|
options = ReaderOptions(
|
|
90
90
|
runtime=runtime,
|
|
91
91
|
policy_settings=policy.document.settings,
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
from collections.abc import Mapping
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
from typing import IO
|
|
7
8
|
|
|
@@ -49,11 +50,19 @@ _JSON_PREFIXES = (
|
|
|
49
50
|
)
|
|
50
51
|
|
|
51
52
|
|
|
52
|
-
def detect_format(
|
|
53
|
-
|
|
53
|
+
def detect_format(
|
|
54
|
+
path: Path,
|
|
55
|
+
explicit: InputFormat | None = None,
|
|
56
|
+
extensions: Mapping[str, InputFormat] | None = None,
|
|
57
|
+
) -> InputFormat:
|
|
58
|
+
"""Explicit ``--format`` wins; then ``extensions``, then the built-in extension table.
|
|
59
|
+
|
|
60
|
+
Content that contradicts a JSON format is refused either way.
|
|
61
|
+
"""
|
|
54
62
|
if explicit is not None:
|
|
55
63
|
return explicit
|
|
56
|
-
|
|
64
|
+
suffix = path.suffix.lower()
|
|
65
|
+
fmt = (extensions or {}).get(suffix) or EXTENSION_FORMATS.get(suffix)
|
|
57
66
|
if fmt is None:
|
|
58
67
|
raise UsageError(
|
|
59
68
|
"cannot infer the input format from this file extension; pass --format "
|
|
@@ -83,7 +83,12 @@ _KEY_HINT = (
|
|
|
83
83
|
|
|
84
84
|
|
|
85
85
|
def next_steps(report: dict[str, Any]) -> list[str]:
|
|
86
|
-
"""
|
|
86
|
+
"""Hint messages for a run that ended with a non-zero exit code; empty otherwise."""
|
|
87
|
+
return [step["message"] for step in next_step_items(report)]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def next_step_items(report: dict[str, Any]) -> list[dict[str, str]]:
|
|
91
|
+
"""``{"code", "message"}`` pairs for a failed run; ``code`` is a stable reason code."""
|
|
87
92
|
if not report.get("exit_code"):
|
|
88
93
|
return []
|
|
89
94
|
codes = [gate["reason_code"] for gate in report.get("gates", [])]
|
|
@@ -93,14 +98,28 @@ def next_steps(report: dict[str, Any]) -> list[str]:
|
|
|
93
98
|
rescan = report.get("rescan")
|
|
94
99
|
if rescan and rescan.get("failed"):
|
|
95
100
|
codes.append(ReasonCode.RESIDUAL_FINDING)
|
|
96
|
-
|
|
101
|
+
items: list[dict[str, str]] = []
|
|
97
102
|
for code in codes:
|
|
98
103
|
hint = _HINTS.get(code)
|
|
99
|
-
if hint
|
|
100
|
-
|
|
104
|
+
if hint:
|
|
105
|
+
_add(items, str(code), hint)
|
|
101
106
|
for key in report.get("keys", []):
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
if
|
|
105
|
-
|
|
106
|
-
return
|
|
107
|
+
status = key.get("status")
|
|
108
|
+
if status != "available":
|
|
109
|
+
code = ReasonCode.KEY_MISSING if status == "missing" else ReasonCode.KEY_INVALID
|
|
110
|
+
_add(items, str(code), _KEY_HINT.format(env=key["key_env"]))
|
|
111
|
+
return items
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def merge_next_steps(reports: list[dict[str, Any]]) -> list[dict[str, str]]:
|
|
115
|
+
"""Combine the hints of several reports (a directory run), first occurrence first."""
|
|
116
|
+
items: list[dict[str, str]] = []
|
|
117
|
+
for report in reports:
|
|
118
|
+
for step in report.get("next_steps") or next_step_items(report):
|
|
119
|
+
_add(items, step["code"], step["message"])
|
|
120
|
+
return items
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _add(items: list[dict[str, str]], code: str, message: str) -> None:
|
|
124
|
+
if all(step["message"] != message for step in items):
|
|
125
|
+
items.append({"code": code, "message": message})
|
|
@@ -7,7 +7,6 @@ from typing import Any
|
|
|
7
7
|
from ciphergate.constants import DISCLAIMER, RESCAN_LIMIT
|
|
8
8
|
from ciphergate.models.report import RunReport
|
|
9
9
|
from ciphergate.pipeline.batch import BatchReport
|
|
10
|
-
from ciphergate.reporting.next_steps import next_steps
|
|
11
10
|
from ciphergate.reporting.safe_text import escape_control
|
|
12
11
|
|
|
13
12
|
SUMMARY_ITEMS = 15
|
|
@@ -120,8 +119,8 @@ def render_summary(report: RunReport, *, verbose: bool = False) -> str:
|
|
|
120
119
|
lines.append(f" output: not written ({out['withheld_reason']})")
|
|
121
120
|
for warning in d["warnings"]:
|
|
122
121
|
lines.append(f" warning: {escape_control(warning)}")
|
|
123
|
-
for
|
|
124
|
-
lines.append(f" next: {
|
|
122
|
+
for step in d["next_steps"]:
|
|
123
|
+
lines.append(f" next: {step['message']}")
|
|
125
124
|
lines.append(f" note: {DISCLAIMER}")
|
|
126
125
|
return "\n".join(lines) + "\n"
|
|
127
126
|
|
|
@@ -161,13 +160,8 @@ def render_batch_summary(report: BatchReport, *, verbose: bool = False) -> str:
|
|
|
161
160
|
lines.append(f" output: not written ({out.get('withheld_reason')})")
|
|
162
161
|
for warning in d["warnings"]:
|
|
163
162
|
lines.append(f" warning: {escape_control(warning)}")
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
for hint in next_steps(entry.get("report") or {}):
|
|
167
|
-
if hint not in hints:
|
|
168
|
-
hints.append(hint)
|
|
169
|
-
for hint in hints:
|
|
170
|
-
lines.append(f" next: {hint}")
|
|
163
|
+
for step in d["next_steps"]:
|
|
164
|
+
lines.append(f" next: {step['message']}")
|
|
171
165
|
lines.append(" note: per-file details are in the JSON report (--report).")
|
|
172
166
|
lines.append(f" note: {DISCLAIMER}")
|
|
173
167
|
return "\n".join(lines) + "\n"
|