ciphergate-cli 0.7.1__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/CHANGELOG.md +29 -1
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/PKG-INFO +11 -2
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/README.md +10 -1
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/docs/detector-coverage.md +4 -2
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/docs/policy-schema.md +2 -1
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/__init__.py +6 -1
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/detectors/email.py +8 -2
- ciphergate_cli-0.9.0/src/ciphergate/detectors/iban.py +53 -0
- ciphergate_cli-0.9.0/src/ciphergate/detectors/mac_address.py +47 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/detectors/registry.py +13 -1
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/detectors/secrets.py +2 -1
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/policy/templates.py +6 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/unit/test_accesslog.py +2 -1
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/unit/test_detectors.py +103 -0
- ciphergate_cli-0.9.0/tests/unit/test_policy_templates.py +55 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/.gitignore +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/LICENSE +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/NOTICE +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/SECURITY.md +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/docs/architecture.md +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/docs/claude-code-tooling.md +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/docs/examples.md +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/docs/limitations.md +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/docs/security-and-privacy.md +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/docs/threat-model.md +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/examples/expected/sample.dev_local_dev.csv +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/examples/expected/sample.dev_local_dev.json +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/examples/expected/sample.dev_local_dev.jsonl +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/examples/expected/sample.dev_local_dev.log +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/examples/sample.csv +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/examples/sample.json +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/examples/sample.jsonl +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/examples/sample.log +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/policies/dev_local.yml +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/policies/developer_logs.yml +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/policies/external_ai.yml +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/policies/financial_template.yml +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/policies/healthcare_template.yml +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/policies/local_debug.yml +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/policies/multi_environment.yml +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/policies/prd_external_ai.yml +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/policies/stg_test.yml +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/policies/test_dataset.yml +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/pyproject.toml +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/__main__.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/cli.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/config.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/constants.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/detectors/__init__.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/detectors/base.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/detectors/custom_regex.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/detectors/dates.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/detectors/field_names.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/detectors/ip_address.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/detectors/network.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/detectors/payment_card.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/detectors/phone.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/detectors/presidio_ner.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/detectors/regex_safety.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/errors.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/models/__init__.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/models/context.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/models/finding.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/models/policy.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/models/report.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/pipeline/__init__.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/pipeline/atomic_output.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/pipeline/batch.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/pipeline/inspector.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/pipeline/paths.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/pipeline/planner.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/pipeline/rescan.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/pipeline/sanitizer.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/pipeline/stdio.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/pipeline/streams.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/policy/__init__.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/policy/context.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/policy/engine.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/policy/loader.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/policy/precedence.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/policy/validator.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/py.typed +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/readers/__init__.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/readers/accesslog_reader.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/readers/base.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/readers/csv_reader.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/readers/json_reader.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/readers/jsonl_reader.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/readers/jsonlog_reader.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/readers/line_structure.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/readers/logfmt_reader.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/readers/stacktrace_reader.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/readers/syslog_reader.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/readers/text_reader.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/reporting/__init__.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/reporting/audit.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/reporting/safe_text.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/reporting/serializer.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/reporting/terminal.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/templates/starter_policy.yml +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/transforms/__init__.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/transforms/date_shift.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/transforms/generalize.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/transforms/keys.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/transforms/pseudonymize.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/transforms/redact.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/transforms/registry.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/src/ciphergate/transforms/replace.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/__init__.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/conftest.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/integration/__init__.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/integration/test_api_batch_streams.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/integration/test_atomic_output.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/integration/test_batch.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/integration/test_cli_e2e.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/integration/test_domain_templates.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/integration/test_environment_matrix.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/integration/test_pipelines.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/integration/test_stdio.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/regression/__init__.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/regression/test_known_cases.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/security/__init__.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/security/test_key_file.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/security/test_no_network.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/security/test_no_raw_value_leaks.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/security/test_path_safety.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/security/test_policy_fail_closed.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/security/test_review_findings.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/tooling/__init__.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/tooling/test_ci_supply_chain.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/tooling/test_claude_tooling.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/tooling/test_git_tooling.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/tooling/test_publish_workflow.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/unit/__init__.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/unit/test_jsonlog.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/unit/test_logfmt.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/unit/test_policy_engine.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/unit/test_policy_precedence.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/unit/test_policy_validation.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/unit/test_presidio_ner.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/unit/test_properties.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/unit/test_readers.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/unit/test_stacktrace.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/unit/test_syslog.py +0 -0
- {ciphergate_cli-0.7.1 → ciphergate_cli-0.9.0}/tests/unit/test_transforms.py +0 -0
|
@@ -6,6 +6,32 @@ versioning: [PEP 440](https://peps.python.org/pep-0440/) with SemVer meaning (pr
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.9.0] - 2026-10-30
|
|
10
|
+
|
|
11
|
+
Detects more values. A run that passed before can now redact, review or block, so re-check your
|
|
12
|
+
policies before upgrading a gate.
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- `mac_address` detector (`pii.device_identifier`) and `iban` detector (`pii.bank_account`, mod-97
|
|
17
|
+
check). Both are on by default and can be turned off with `detectors.disabled`.
|
|
18
|
+
|
|
19
|
+
### Changed
|
|
20
|
+
|
|
21
|
+
- `email_regex` also finds percent-encoded addresses (`%40`) in text, and `secret_pattern` finds
|
|
22
|
+
values assigned to `PGPASSWORD`, `MYSQL_PWD` and similar names (`DBPASSWORD`, `ROOTPASSWORD`).
|
|
23
|
+
These values now follow your policy rules, so a run can redact, review or block content it
|
|
24
|
+
used to pass.
|
|
25
|
+
|
|
26
|
+
## [0.8.0] - 2026-10-30
|
|
27
|
+
|
|
28
|
+
Adds the packaged policy templates to the Python API. Existing behaviour is unchanged.
|
|
29
|
+
|
|
30
|
+
### Added
|
|
31
|
+
|
|
32
|
+
- Python API for the packaged policy templates: `load_template(name)` returns a `Policy` and
|
|
33
|
+
`template_names()` lists the names. An unknown name raises `UsageError`.
|
|
34
|
+
|
|
9
35
|
## [0.7.1] - 2026-10-30
|
|
10
36
|
|
|
11
37
|
Makes the starter policy templates available after `pip install`. Existing commands and
|
|
@@ -270,7 +296,9 @@ Pre-release (alpha). Licensed under Apache-2.0. Not a final release: the securit
|
|
|
270
296
|
See `docs/limitations.md`. Deferred: optional NER/Presidio detectors, stdin/stdout streaming,
|
|
271
297
|
directory processing, structured log parsers, key files, reversible token vault, lossy decoding.
|
|
272
298
|
|
|
273
|
-
[Unreleased]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.
|
|
299
|
+
[Unreleased]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.9.0...HEAD
|
|
300
|
+
[0.9.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.8.0...v0.9.0
|
|
301
|
+
[0.8.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.7.1...v0.8.0
|
|
274
302
|
[0.7.1]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.7.0...v0.7.1
|
|
275
303
|
[0.7.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.6.1...v0.7.0
|
|
276
304
|
[0.6.1]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.6.0...v0.6.1
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ciphergate-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Local-first, environment- and destination-aware inspection and sanitization of logs and test datasets.
|
|
5
5
|
Project-URL: Homepage, https://github.com/aashishkumar-tech/ciphergate
|
|
6
6
|
Project-URL: Source, https://github.com/aashishkumar-tech/ciphergate
|
|
@@ -50,7 +50,7 @@ vendor ticket. CipherGate replaces that with a local, explainable, scriptable ga
|
|
|
50
50
|
inspect -> plan -> sanitize --apply -> rescan
|
|
51
51
|
```
|
|
52
52
|
|
|
53
|
-
> **Status: 0.
|
|
53
|
+
> **Status: 0.9.0 (alpha).** No detector finds every sensitive value. A passing run means the
|
|
54
54
|
> configured processing completed under the selected policy; it is **not** proof that the output
|
|
55
55
|
> contains no sensitive information, and it is not a compliance (GDPR, HIPAA, PCI DSS, ...) or
|
|
56
56
|
> anonymization guarantee. Policy templates must be reviewed before use.
|
|
@@ -494,6 +494,15 @@ report = sanitize_file("app.log", "app.clean.log", policy,
|
|
|
494
494
|
print(report.status, report.exit_code)
|
|
495
495
|
```
|
|
496
496
|
|
|
497
|
+
After `pip install`, there is no `policies/` folder. Load a packaged template by name instead;
|
|
498
|
+
`template_names()` lists them. Templates require review for your data and destination.
|
|
499
|
+
|
|
500
|
+
```python
|
|
501
|
+
from ciphergate import load_template, template_names
|
|
502
|
+
print(template_names())
|
|
503
|
+
policy = load_template("developer_logs")
|
|
504
|
+
```
|
|
505
|
+
|
|
497
506
|
Streams and directories use the same pipeline and the same safety rules as the CLI:
|
|
498
507
|
|
|
499
508
|
```python
|
|
@@ -11,7 +11,7 @@ vendor ticket. CipherGate replaces that with a local, explainable, scriptable ga
|
|
|
11
11
|
inspect -> plan -> sanitize --apply -> rescan
|
|
12
12
|
```
|
|
13
13
|
|
|
14
|
-
> **Status: 0.
|
|
14
|
+
> **Status: 0.9.0 (alpha).** No detector finds every sensitive value. A passing run means the
|
|
15
15
|
> configured processing completed under the selected policy; it is **not** proof that the output
|
|
16
16
|
> contains no sensitive information, and it is not a compliance (GDPR, HIPAA, PCI DSS, ...) or
|
|
17
17
|
> anonymization guarantee. Policy templates must be reviewed before use.
|
|
@@ -455,6 +455,15 @@ report = sanitize_file("app.log", "app.clean.log", policy,
|
|
|
455
455
|
print(report.status, report.exit_code)
|
|
456
456
|
```
|
|
457
457
|
|
|
458
|
+
After `pip install`, there is no `policies/` folder. Load a packaged template by name instead;
|
|
459
|
+
`template_names()` lists them. Templates require review for your data and destination.
|
|
460
|
+
|
|
461
|
+
```python
|
|
462
|
+
from ciphergate import load_template, template_names
|
|
463
|
+
print(template_names())
|
|
464
|
+
policy = load_template("developer_logs")
|
|
465
|
+
```
|
|
466
|
+
|
|
458
467
|
Streams and directories use the same pipeline and the same safety rules as the CLI:
|
|
459
468
|
|
|
460
469
|
```python
|
|
@@ -9,12 +9,14 @@ are engineering estimates mapped to bands (`low` < 0.5 <= `medium` < 0.8 <= `hig
|
|
|
9
9
|
|---|---|---|---|---|---|
|
|
10
10
|
| `field_name` | field names -> many categories | `email`, `contactEmail`, `api_key`, `date-of-birth`, `userId`, `trace_id` | `token_count`, `key_name`, `monkey`, `product_name`, `email_verified` | 0.45–0.85 per term | heuristic dictionary; benign values in sensitive-looking fields are flagged; sensitive data in generic fields (`notes`, `value`) is missed unless content detectors match |
|
|
11
11
|
| `policy_classification` | policy-declared fields | `admission_date` (healthcare template) | | configured | only as good as the policy |
|
|
12
|
-
| `email_regex` | `pii.email` | `alice.test@example.com` | `a..b@example.com`, `user@localhost` | 0.9 | ASCII only; no IDN; obfuscated forms (`alice at example dot com`) missed |
|
|
12
|
+
| `email_regex` | `pii.email` | `alice.test@example.com`, `alice.test%40example.com` | `a..b@example.com`, `user@localhost` | 0.9 | ASCII only; no IDN; obfuscated forms (`alice at example dot com`) missed; `%40` is the only percent-encoding recognised outside `accesslog` |
|
|
13
13
|
| `phone_pattern` | `pii.phone` | `+1 202 555 0143` (high), `(202) 555-0143` (medium) | dates, IPs, card numbers, repeated digits, < 7 digits | 0.3 bare digits, 0.55–0.8 prefixed/formatted, +0.2 keyword | no numbering-plan validation; IDs can look like phones (low band) |
|
|
14
14
|
| `ip_address` | `pii.ip_address` (global), `network.telemetry` (private/loopback/link-local) | `8.8.8.8`, `10.0.0.1`, `2606:4700::1111`, `fe80::1%eth0` | `256.1.1.1`, `std::vector`, `12:30:45` | 0.85 / 0.8 | version strings like `1.2.3.4` match; documentation ranges flagged with evidence |
|
|
15
15
|
| `luhn_card` | `pii.payment_card` | `4111 1111 1111 1111` | Luhn-invalid, mixed separators, repeated digits | 0.4 Luhn-only, 0.8 issuer prefix, +0.15 keyword | random IDs pass Luhn ~10% (low band); cards split across fields missed |
|
|
16
|
+
| `mac_address` | `pii.device_identifier` | `00:1A:2B:3C:4D:5E`, `00-1a-2b-3c-4d-5e`, `001a.2b3c.4d5e` | all-zero, broadcast, mixed separators, `12:30:45` | 0.85, 0.6 when all digits | bare 12-hex values are missed; digit-only IDs in this shape are flagged |
|
|
17
|
+
| `iban` | `pii.bank_account` | a mod-97-valid IBAN, compact or space-grouped | wrong check digits, lowercase | 0.9 | country length and national format are not checked; about 1 in 97 random strings of this shape pass; lowercase forms are missed |
|
|
16
18
|
| `date_pattern` | `pii.date_of_birth` with field/keyword evidence; `date.generic` if enabled | `dob: 1990-04-12`, field `birth_date` | `created 2026-01-05` (no DOB evidence), invalid dates | 0.85 / 0.6 | `DD/MM` vs `MM/DD` ambiguity flagged; free-text DOBs without keywords missed |
|
|
17
|
-
| `secret_pattern` | `credential.secret` | PEM blocks, `Bearer ...`, `AKIA...`, `ghp_...`, `xox?-`, `sk_test_...`, `AIza...`, JWTs, `scheme://u:pw@`, `Password=...;`, `password=`, `api_key:`, `token=` | `password=`, `password: null`, markers, `******` | 0.8–0.99 | base64/hex-wrapped, split, custom-format or high-entropy secrets without keywords are **missed**; prose like `password: required` is flagged |
|
|
19
|
+
| `secret_pattern` | `credential.secret` | PEM blocks, `Bearer ...`, `AKIA...`, `ghp_...`, `xox?-`, `sk_test_...`, `AIza...`, JWTs, `scheme://u:pw@`, `Password=...;`, `password=`, assignments to `PGPASSWORD` or `MYSQL_PWD`, `api_key:`, `token=` | `password=`, `password: null`, markers, `******` | 0.8–0.99 | base64/hex-wrapped, split, custom-format or high-entropy secrets without keywords are **missed**; prose like `password: required` is flagged |
|
|
18
20
|
| `internal_host` | `network.telemetry` | `api-1.prod.internal`, `db.corp`, `localhost` | `example.com`, `a.internalx` | 0.75 | internal names without listed suffixes missed; configurable |
|
|
19
21
|
| `custom:<id>` | configured | | | configured | regex-safety checks are heuristics |
|
|
20
22
|
| `presidio_ner` (optional, off by default) | Presidio entities -> mapped categories (default `PERSON` -> `pii.person_name`, `LOCATION` -> `location.coarse`) | `ticket opened by <a person's name>` | numbers, single characters, CipherGate markers/pseudonyms | Presidio score, filtered by `min_score` | statistical model: misses and false positives, language/model dependent, slow; requires `ciphergate-cli[ner]` plus a locally installed spaCy model |
|
|
@@ -52,7 +52,8 @@ A pinned value combined with a different caller value fails with
|
|
|
52
52
|
## Detectors
|
|
53
53
|
|
|
54
54
|
`disabled` (built-in IDs: `field_name`, `secret_pattern`, `luhn_card`, `ip_address`,
|
|
55
|
-
`date_pattern`, `email_regex`, `internal_host`, `phone_pattern`
|
|
55
|
+
`date_pattern`, `email_regex`, `internal_host`, `phone_pattern`, `mac_address`, `iban`),
|
|
56
|
+
`dates_emit_generic` (report
|
|
56
57
|
non-DOB dates as `date.generic`), `internal_host_suffixes` (default `.internal .local .corp .lan
|
|
57
58
|
.intranet .localdomain`), `custom` (list of `{id, category, pattern, confidence, ignore_case}`;
|
|
58
59
|
patterns are checked for compile errors, empty matches, nested quantifiers, backreferences and
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
Supported public API (everything else is internal and may change without notice)::
|
|
4
4
|
|
|
5
5
|
load_policy(path) -> Policy
|
|
6
|
+
load_template(name) -> Policy # a packaged template; review it before relying on it
|
|
7
|
+
template_names() -> list[str]
|
|
6
8
|
validate_policy(path_or_policy) -> ValidationResult
|
|
7
9
|
inspect_file(path, policy, context=None) -> RunReport
|
|
8
10
|
plan_file(path, policy, context=None) -> RunReport
|
|
@@ -22,7 +24,7 @@ Supported public API (everything else is internal and may change without notice)
|
|
|
22
24
|
rescan_directory(root, policy, context=None, *, recursive=False, strict=None) -> BatchReport
|
|
23
25
|
"""
|
|
24
26
|
|
|
25
|
-
__version__ = "0.
|
|
27
|
+
__version__ = "0.9.0"
|
|
26
28
|
|
|
27
29
|
from ciphergate.config import RuntimeSettings
|
|
28
30
|
from ciphergate.constants import (
|
|
@@ -55,6 +57,7 @@ from ciphergate.pipeline.streams import (
|
|
|
55
57
|
rescan_stream,
|
|
56
58
|
sanitize_stream,
|
|
57
59
|
)
|
|
60
|
+
from ciphergate.policy.templates import load_template, template_names
|
|
58
61
|
from ciphergate.policy.validator import (
|
|
59
62
|
Policy,
|
|
60
63
|
ValidationResult,
|
|
@@ -82,6 +85,7 @@ __all__ = [
|
|
|
82
85
|
"inspect_file",
|
|
83
86
|
"inspect_stream",
|
|
84
87
|
"load_policy",
|
|
88
|
+
"load_template",
|
|
85
89
|
"plan_directory",
|
|
86
90
|
"plan_file",
|
|
87
91
|
"plan_stream",
|
|
@@ -91,5 +95,6 @@ __all__ = [
|
|
|
91
95
|
"sanitize_directory",
|
|
92
96
|
"sanitize_file",
|
|
93
97
|
"sanitize_stream",
|
|
98
|
+
"template_names",
|
|
94
99
|
"validate_policy",
|
|
95
100
|
]
|
|
@@ -12,6 +12,12 @@ _EMAIL_RE = re.compile(
|
|
|
12
12
|
r"([A-Za-z0-9._%+-]{1,64})@((?:[A-Za-z0-9-]{1,63}\.){1,8}[A-Za-z]{2,63})"
|
|
13
13
|
r"(?![A-Za-z0-9-])"
|
|
14
14
|
)
|
|
15
|
+
# Percent-encoded form in URLs and query strings: the "@" is written as %40.
|
|
16
|
+
_ENCODED_EMAIL_RE = re.compile(
|
|
17
|
+
r"(?<![A-Za-z0-9._%+-])"
|
|
18
|
+
r"([A-Za-z0-9._%+-]{1,64})%40((?:[A-Za-z0-9-]{1,63}\.){1,8}[A-Za-z]{2,63})"
|
|
19
|
+
r"(?![A-Za-z0-9-])"
|
|
20
|
+
)
|
|
15
21
|
|
|
16
22
|
RESERVED_EMAIL_DOMAINS = ("example.com", "example.org", "example.net")
|
|
17
23
|
RESERVED_EMAIL_TLDS = (".invalid", ".example", ".test", ".localhost")
|
|
@@ -31,10 +37,10 @@ class EmailDetector:
|
|
|
31
37
|
|
|
32
38
|
def detect(self, item: ScanItem) -> list[Finding]:
|
|
33
39
|
text = item.text
|
|
34
|
-
if not text or "@" not in text:
|
|
40
|
+
if not text or ("@" not in text and "%40" not in text):
|
|
35
41
|
return []
|
|
36
42
|
out: list[Finding] = []
|
|
37
|
-
for m in _EMAIL_RE.finditer(text):
|
|
43
|
+
for m in (*_EMAIL_RE.finditer(text), *_ENCODED_EMAIL_RE.finditer(text)):
|
|
38
44
|
local, domain = m.group(1), m.group(2)
|
|
39
45
|
if local.startswith(".") or local.endswith(".") or ".." in local or ".." in domain:
|
|
40
46
|
continue
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""IBAN detector: compact or space-grouped, uppercase, validated with the ISO 7064 mod-97 check.
|
|
2
|
+
|
|
3
|
+
Country-specific lengths and national formats are not checked. Lowercase forms are not matched.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
|
|
10
|
+
from ciphergate.detectors.base import make_finding
|
|
11
|
+
from ciphergate.models.finding import Finding, ScanItem
|
|
12
|
+
|
|
13
|
+
_IBAN_RE = re.compile(
|
|
14
|
+
r"(?<![0-9A-Za-z])"
|
|
15
|
+
r"([A-Z]{2}\d{2}(?:[A-Z0-9]{11,30}|(?: [A-Z0-9]{4}){2,7}(?: [A-Z0-9]{1,3})?))"
|
|
16
|
+
r"(?![0-9A-Za-z])"
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def iban_valid(compact: str) -> bool:
|
|
21
|
+
if not 15 <= len(compact) <= 34:
|
|
22
|
+
return False
|
|
23
|
+
rearranged = compact[4:] + compact[:4]
|
|
24
|
+
return int("".join(str(int(c, 36)) for c in rearranged)) % 97 == 1
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class IbanDetector:
|
|
28
|
+
id = "iban"
|
|
29
|
+
|
|
30
|
+
def detect(self, item: ScanItem) -> list[Finding]:
|
|
31
|
+
text = item.text
|
|
32
|
+
if not text or len(text) < 15:
|
|
33
|
+
return []
|
|
34
|
+
out: list[Finding] = []
|
|
35
|
+
for m in _IBAN_RE.finditer(text):
|
|
36
|
+
# A following uppercase word can join a spaced match, so retry without trailing groups.
|
|
37
|
+
groups = m.group(1).split(" ")
|
|
38
|
+
while groups:
|
|
39
|
+
if iban_valid("".join(groups)):
|
|
40
|
+
end = m.start(1) + len(" ".join(groups))
|
|
41
|
+
out.append(
|
|
42
|
+
make_finding(
|
|
43
|
+
item,
|
|
44
|
+
category="pii.bank_account",
|
|
45
|
+
detector=self.id,
|
|
46
|
+
confidence=0.9,
|
|
47
|
+
evidence=("pattern_match", "checksum_valid"),
|
|
48
|
+
span=(m.start(1), end),
|
|
49
|
+
)
|
|
50
|
+
)
|
|
51
|
+
break
|
|
52
|
+
groups.pop()
|
|
53
|
+
return out
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""MAC address detector: six hex pairs with one separator, or Cisco dotted form.
|
|
2
|
+
|
|
3
|
+
All-digit forms are medium confidence because they can be other identifiers. The all-zero and
|
|
4
|
+
broadcast addresses identify no device and are skipped.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
from ciphergate.detectors.base import make_finding
|
|
12
|
+
from ciphergate.models.finding import Finding, ScanItem
|
|
13
|
+
|
|
14
|
+
_MAC_RE = re.compile(
|
|
15
|
+
r"(?<![0-9A-Fa-f:.-])"
|
|
16
|
+
r"([0-9A-Fa-f]{2}(?:[:-][0-9A-Fa-f]{2}){5}|[0-9A-Fa-f]{4}(?:\.[0-9A-Fa-f]{4}){2})"
|
|
17
|
+
r"(?![0-9A-Za-z:.-])"
|
|
18
|
+
)
|
|
19
|
+
_NON_DEVICE = frozenset({"0" * 12, "f" * 12})
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class MacAddressDetector:
|
|
23
|
+
id = "mac_address"
|
|
24
|
+
|
|
25
|
+
def detect(self, item: ScanItem) -> list[Finding]:
|
|
26
|
+
text = item.text
|
|
27
|
+
if not text or len(text) < 14:
|
|
28
|
+
return []
|
|
29
|
+
out: list[Finding] = []
|
|
30
|
+
for m in _MAC_RE.finditer(text):
|
|
31
|
+
raw = m.group(1)
|
|
32
|
+
if len({c for c in raw if c in ":-."}) > 1:
|
|
33
|
+
continue
|
|
34
|
+
compact = re.sub(r"[:.-]", "", raw).lower()
|
|
35
|
+
if compact in _NON_DEVICE:
|
|
36
|
+
continue
|
|
37
|
+
out.append(
|
|
38
|
+
make_finding(
|
|
39
|
+
item,
|
|
40
|
+
category="pii.device_identifier",
|
|
41
|
+
detector=self.id,
|
|
42
|
+
confidence=0.85 if not compact.isdigit() else 0.6,
|
|
43
|
+
evidence=("pattern_match",),
|
|
44
|
+
span=(m.start(1), m.end(1)),
|
|
45
|
+
)
|
|
46
|
+
)
|
|
47
|
+
return out
|
|
@@ -9,7 +9,9 @@ from ciphergate.detectors.custom_regex import CustomRegexDetector
|
|
|
9
9
|
from ciphergate.detectors.dates import DateDetector
|
|
10
10
|
from ciphergate.detectors.email import EmailDetector, is_reserved_email_domain
|
|
11
11
|
from ciphergate.detectors.field_names import FieldNameDetector, PolicyClassificationDetector
|
|
12
|
+
from ciphergate.detectors.iban import IbanDetector
|
|
12
13
|
from ciphergate.detectors.ip_address import IPAddressDetector
|
|
14
|
+
from ciphergate.detectors.mac_address import MacAddressDetector
|
|
13
15
|
from ciphergate.detectors.network import InternalHostDetector
|
|
14
16
|
from ciphergate.detectors.payment_card import PaymentCardDetector
|
|
15
17
|
from ciphergate.detectors.phone import PhoneDetector
|
|
@@ -26,10 +28,14 @@ BUILTIN_DETECTOR_IDS: tuple[str, ...] = (
|
|
|
26
28
|
"email_regex",
|
|
27
29
|
"internal_host",
|
|
28
30
|
"phone_pattern",
|
|
31
|
+
"mac_address",
|
|
32
|
+
"iban",
|
|
29
33
|
)
|
|
30
34
|
|
|
31
35
|
# Phone matching is the least specific pattern; drop phone spans that overlap these detectors.
|
|
32
|
-
_PHONE_SUPPRESSORS = frozenset(
|
|
36
|
+
_PHONE_SUPPRESSORS = frozenset(
|
|
37
|
+
{"luhn_card", "ip_address", "date_pattern", "secret_pattern", "mac_address", "iban"}
|
|
38
|
+
)
|
|
33
39
|
|
|
34
40
|
|
|
35
41
|
FIELD_CONTEXT_DETECTORS = frozenset({"field_name", "policy_classification"})
|
|
@@ -59,6 +65,8 @@ class DetectorSet:
|
|
|
59
65
|
"email_regex": EmailDetector(),
|
|
60
66
|
"internal_host": InternalHostDetector(settings.internal_host_suffixes),
|
|
61
67
|
"phone_pattern": PhoneDetector(),
|
|
68
|
+
"mac_address": MacAddressDetector(),
|
|
69
|
+
"iban": IbanDetector(),
|
|
62
70
|
}
|
|
63
71
|
for detector_id in BUILTIN_DETECTOR_IDS:
|
|
64
72
|
if detector_id not in disabled:
|
|
@@ -173,6 +181,10 @@ def unsafe_replacement_reason(value: str) -> str | None:
|
|
|
173
181
|
return f"looks like a credential ({secrets[0].evidence[-1]})"
|
|
174
182
|
if PaymentCardDetector().detect(probe):
|
|
175
183
|
return "looks like a payment card number"
|
|
184
|
+
if IbanDetector().detect(probe):
|
|
185
|
+
return "looks like a real IBAN"
|
|
186
|
+
if MacAddressDetector().detect(probe):
|
|
187
|
+
return "looks like a real MAC address"
|
|
176
188
|
for f in EmailDetector().detect(probe):
|
|
177
189
|
if "reserved_domain" not in f.evidence:
|
|
178
190
|
return "email replacements must use a reserved domain such as example.invalid"
|
|
@@ -65,7 +65,8 @@ _PATTERNS: tuple[_Pattern, ...] = (
|
|
|
65
65
|
"assignment",
|
|
66
66
|
re.compile(
|
|
67
67
|
r"(?i)(?<![a-z0-9])"
|
|
68
|
-
r"(?:password|passwd|passphrase|secret|
|
|
68
|
+
r"(?:(?:pg|db|mysql|admin|root|user)?password|mysql_pwd|passwd|passphrase|secret|"
|
|
69
|
+
r"api[_-]?key|apikey|access[_-]?token|"
|
|
69
70
|
r"refresh[_-]?token|auth[_-]?token|client[_-]?secret|secret[_-]?key|private[_-]?key|"
|
|
70
71
|
r"access[_-]?key|token|credentials?)"
|
|
71
72
|
r"(?:\\?[\"'])?\s{0,3}[:=]\s{0,3}(?:\\?[\"'])?"
|
|
@@ -7,6 +7,7 @@ from importlib.resources.abc import Traversable
|
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
|
|
9
9
|
from ciphergate.errors import UsageError
|
|
10
|
+
from ciphergate.policy.validator import Policy, load_policy_text
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
def _templates_dir() -> Traversable:
|
|
@@ -34,3 +35,8 @@ def read_template(name: str) -> str:
|
|
|
34
35
|
code="unknown_policy_template",
|
|
35
36
|
)
|
|
36
37
|
return _templates_dir().joinpath(f"{name}.yml").read_text(encoding="utf-8")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def load_template(name: str) -> Policy:
|
|
41
|
+
"""Load a named template as a `Policy`. It is a starting point that requires review."""
|
|
42
|
+
return load_policy_text(read_template(name), source_name=f"{name}.yml")
|
|
@@ -161,7 +161,8 @@ def test_field_context_and_decoded_values(
|
|
|
161
161
|
assert ("pii.ip_address", "/remote_addr") in found
|
|
162
162
|
assert ("pii.email", "/http_user_agent") in found # only visible after \x40 decoding
|
|
163
163
|
text_cats = [i["category"] for i in _items(tmp_path, line, InputFormat.TEXT, policy)]
|
|
164
|
-
|
|
164
|
+
# Text mode sees the %40 form but does not decode log escapes such as \x40.
|
|
165
|
+
assert text_cats.count("pii.email") == 1
|
|
165
166
|
|
|
166
167
|
|
|
167
168
|
def test_not_inferred_from_extension(tmp_path: Path, policy_text: Callable[[str], Policy]) -> None:
|
|
@@ -13,7 +13,9 @@ from ciphergate.detectors.field_names import (
|
|
|
13
13
|
classify_field_name,
|
|
14
14
|
normalize_field_name,
|
|
15
15
|
)
|
|
16
|
+
from ciphergate.detectors.iban import IbanDetector, iban_valid
|
|
16
17
|
from ciphergate.detectors.ip_address import IPAddressDetector
|
|
18
|
+
from ciphergate.detectors.mac_address import MacAddressDetector
|
|
17
19
|
from ciphergate.detectors.network import InternalHostDetector
|
|
18
20
|
from ciphergate.detectors.payment_card import PaymentCardDetector, luhn_valid
|
|
19
21
|
from ciphergate.detectors.phone import PhoneDetector
|
|
@@ -390,3 +392,104 @@ def test_findings_never_contain_raw_values() -> None:
|
|
|
390
392
|
rendered = repr(f) + str(f.location.to_dict()) + str(f.evidence)
|
|
391
393
|
for secret in ("alice.test", "TEST_ONLY_NOT", "4111"):
|
|
392
394
|
assert secret not in rendered
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
# ----------------------------------------------------------------- mac address and iban
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
@pytest.mark.parametrize(
|
|
401
|
+
("text", "expected"),
|
|
402
|
+
[
|
|
403
|
+
("dhcp lease for 00:1A:2B:3C:4D:5E done", ["00:1A:2B:3C:4D:5E"]),
|
|
404
|
+
("device 00-1a-2b-3c-4d-5e offline", ["00-1a-2b-3c-4d-5e"]),
|
|
405
|
+
("port 001a.2b3c.4d5e up", ["001a.2b3c.4d5e"]),
|
|
406
|
+
("mixed 00:1a-2b:3c:4d:5e", []),
|
|
407
|
+
("null 00:00:00:00:00:00 broadcast ff:ff:ff:ff:ff:ff", []),
|
|
408
|
+
("at 12:30:45 today", []),
|
|
409
|
+
("seven 00:1a:2b:3c:4d:5e:6f groups", []),
|
|
410
|
+
("host 2001:db8::1 up", []),
|
|
411
|
+
],
|
|
412
|
+
)
|
|
413
|
+
def test_mac_address(text: str, expected: list[str]) -> None:
|
|
414
|
+
findings = MacAddressDetector().detect(item(text))
|
|
415
|
+
assert spans(findings, text) == expected
|
|
416
|
+
assert all(f.category == "pii.device_identifier" for f in findings)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def test_mac_address_confidence_is_lower_when_all_digits() -> None:
|
|
420
|
+
assert MacAddressDetector().detect(item("id 12-34-56-78-90-12"))[0].confidence == 0.6
|
|
421
|
+
assert MacAddressDetector().detect(item("id 12-34-56-78-90-ab"))[0].confidence == 0.85
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
IBAN = "GB82" + "WEST" + "12345698765432" # the ISO 13616 example number
|
|
425
|
+
IBAN_SPACED = "GB82 WEST 1234 5698 7654 32"
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
@pytest.mark.parametrize(
|
|
429
|
+
("text", "expected"),
|
|
430
|
+
[
|
|
431
|
+
(f"payout to {IBAN} failed", [IBAN]),
|
|
432
|
+
(f"payout to {IBAN_SPACED} failed", [IBAN_SPACED]),
|
|
433
|
+
(f"payout to {IBAN_SPACED} DONE", [IBAN_SPACED]),
|
|
434
|
+
("payout to GB83" + "WEST12345698765432 failed", []),
|
|
435
|
+
("payout to " + IBAN.lower(), []),
|
|
436
|
+
("code AB12 short", []),
|
|
437
|
+
],
|
|
438
|
+
)
|
|
439
|
+
def test_iban(text: str, expected: list[str]) -> None:
|
|
440
|
+
findings = IbanDetector().detect(item(text))
|
|
441
|
+
assert spans(findings, text) == expected
|
|
442
|
+
assert all(f.category == "pii.bank_account" for f in findings)
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def test_iban_valid_rejects_bad_lengths() -> None:
|
|
446
|
+
assert iban_valid(IBAN)
|
|
447
|
+
assert not iban_valid("GB82WEST")
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def test_new_detectors_suppress_phone_matches() -> None:
|
|
451
|
+
detectors = DetectorSet(DetectorSettings())
|
|
452
|
+
text = f"account {IBAN_SPACED} and 12-34-56-78-90-12"
|
|
453
|
+
ids = {f.detector for f in detectors.scan(item(text))}
|
|
454
|
+
assert "phone_pattern" not in ids and {"iban", "mac_address"} <= ids
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def test_new_detectors_can_be_disabled() -> None:
|
|
458
|
+
detectors = DetectorSet(DetectorSettings(disabled=["mac_address", "iban"]))
|
|
459
|
+
assert not detectors.scan(item(f"{IBAN} 00:1A:2B:3C:4D:5E"))
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
@pytest.mark.parametrize("value", [IBAN, "00:1A:2B:3C:4D:5E"])
|
|
463
|
+
def test_replacement_must_not_look_real(value: str) -> None:
|
|
464
|
+
assert unsafe_replacement_reason(value) is not None
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
# ------------------------------------------------------------- encoded and env-style values
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
@pytest.mark.parametrize(
|
|
471
|
+
("text", "expected"),
|
|
472
|
+
[
|
|
473
|
+
("GET /users?mail=alice.test%40example.com 404", "alice.test%40example.com"),
|
|
474
|
+
("redirect /v?a=Alice%40EXAMPLE.org&b=1", "Alice%40EXAMPLE.org"),
|
|
475
|
+
],
|
|
476
|
+
)
|
|
477
|
+
def test_percent_encoded_email(text: str, expected: str) -> None:
|
|
478
|
+
assert spans(EmailDetector().detect(item(text)), text) == [expected]
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def test_percent_40_without_an_address_is_ignored() -> None:
|
|
482
|
+
assert not EmailDetector().detect(item("progress 100%40 done"))
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
@pytest.mark.parametrize(
|
|
486
|
+
"name", ["PGPASSWORD", "pgpassword", "MYSQL_PWD", "DBPASSWORD", "ROOTPASSWORD", "UserPassword"]
|
|
487
|
+
)
|
|
488
|
+
def test_env_style_password_names(name: str) -> None:
|
|
489
|
+
text = f"{name}=TEST_ONLY_NOT_A_CREDENTIAL psql"
|
|
490
|
+
assert spans(SecretDetector().detect(item(text)), text) == ["TEST_ONLY_NOT_A_CREDENTIAL"]
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
@pytest.mark.parametrize("name", ["nopassword", "haspassword", "password_reset_enabled"])
|
|
494
|
+
def test_other_password_words_stay_unmatched(name: str) -> None:
|
|
495
|
+
assert not SecretDetector().detect(item(f"{name}=enabled"))
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Packaged policy templates through the public Python API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
import ciphergate
|
|
10
|
+
from ciphergate import (
|
|
11
|
+
ContextInput,
|
|
12
|
+
RunStatus,
|
|
13
|
+
load_policy,
|
|
14
|
+
load_template,
|
|
15
|
+
sanitize_file,
|
|
16
|
+
template_names,
|
|
17
|
+
)
|
|
18
|
+
from ciphergate.errors import CipherGateError, UsageError
|
|
19
|
+
from tests.conftest import EXAMPLES, POLICIES
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_public_exports() -> None:
|
|
23
|
+
for name in ("load_template", "template_names"):
|
|
24
|
+
assert name in ciphergate.__all__ and hasattr(ciphergate, name)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_names_match_the_policy_files() -> None:
|
|
28
|
+
assert template_names() == sorted(p.stem for p in POLICIES.glob("*.yml"))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.mark.parametrize("name", template_names())
|
|
32
|
+
def test_load_template_matches_the_policy_file(name: str) -> None:
|
|
33
|
+
policy = load_template(name)
|
|
34
|
+
assert policy.name == load_policy(POLICIES / f"{name}.yml").name
|
|
35
|
+
assert policy.source_name == f"{name}.yml"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@pytest.mark.parametrize("name", ["nope", "../pyproject", "dev_local.yml", ""])
|
|
39
|
+
def test_unknown_template_is_rejected(name: str) -> None:
|
|
40
|
+
with pytest.raises(UsageError) as err:
|
|
41
|
+
load_template(name)
|
|
42
|
+
assert err.value.code == "unknown_policy_template"
|
|
43
|
+
assert isinstance(err.value, CipherGateError)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_loaded_template_still_blocks_and_publishes_nothing(tmp_path: Path) -> None:
|
|
47
|
+
output = tmp_path / "out.log"
|
|
48
|
+
report = sanitize_file(
|
|
49
|
+
EXAMPLES / "sample.log",
|
|
50
|
+
output,
|
|
51
|
+
load_template("prd_external_ai"),
|
|
52
|
+
ContextInput(),
|
|
53
|
+
)
|
|
54
|
+
assert report.status == RunStatus.BLOCKED and report.exit_code == 1
|
|
55
|
+
assert not output.exists()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|