ciphergate-cli 0.14.0__tar.gz → 0.16.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/CHANGELOG.md +40 -1
  2. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/PKG-INFO +53 -4
  3. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/README.md +52 -3
  4. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/docs/architecture.md +1 -0
  5. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/docs/detector-coverage.md +25 -2
  6. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/docs/security-and-privacy.md +17 -1
  7. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/scripts/benchmark/README.md +8 -0
  8. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/__init__.py +1 -1
  9. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/cli.py +81 -0
  10. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/models/report.py +6 -2
  11. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/pipeline/batch.py +6 -2
  12. ciphergate_cli-0.16.0/src/ciphergate/pipeline/review.py +521 -0
  13. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/pipeline/sanitizer.py +14 -1
  14. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/reporting/next_steps.py +28 -9
  15. ciphergate_cli-0.16.0/src/ciphergate/reporting/review_page.py +871 -0
  16. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/reporting/safe_text.py +10 -3
  17. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/reporting/terminal.py +4 -10
  18. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/integration/test_batch.py +13 -0
  19. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/integration/test_cli_e2e.py +24 -1
  20. ciphergate_cli-0.16.0/tests/integration/test_review.py +255 -0
  21. ciphergate_cli-0.16.0/tests/security/test_review_page.py +48 -0
  22. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/tooling/test_benchmark.py +55 -0
  23. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_next_steps.py +33 -1
  24. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/.gitignore +0 -0
  25. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/LICENSE +0 -0
  26. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/NOTICE +0 -0
  27. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/SECURITY.md +0 -0
  28. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/docs/claude-code-tooling.md +0 -0
  29. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/docs/examples.md +0 -0
  30. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/docs/limitations.md +0 -0
  31. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/docs/policy-schema.md +0 -0
  32. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/docs/threat-model.md +0 -0
  33. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/examples/expected/sample.dev_local_dev.csv +0 -0
  34. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/examples/expected/sample.dev_local_dev.json +0 -0
  35. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/examples/expected/sample.dev_local_dev.jsonl +0 -0
  36. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/examples/expected/sample.dev_local_dev.log +0 -0
  37. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/examples/sample.csv +0 -0
  38. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/examples/sample.json +0 -0
  39. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/examples/sample.jsonl +0 -0
  40. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/examples/sample.log +0 -0
  41. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/policies/dev_local.yml +0 -0
  42. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/policies/developer_logs.yml +0 -0
  43. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/policies/external_ai.yml +0 -0
  44. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/policies/financial_template.yml +0 -0
  45. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/policies/healthcare_template.yml +0 -0
  46. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/policies/local_debug.yml +0 -0
  47. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/policies/multi_environment.yml +0 -0
  48. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/policies/prd_external_ai.yml +0 -0
  49. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/policies/stg_test.yml +0 -0
  50. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/policies/test_dataset.yml +0 -0
  51. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/pyproject.toml +0 -0
  52. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/__main__.py +0 -0
  53. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/config.py +0 -0
  54. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/constants.py +0 -0
  55. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/__init__.py +0 -0
  56. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/base.py +0 -0
  57. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/custom_regex.py +0 -0
  58. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/dates.py +0 -0
  59. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/email.py +0 -0
  60. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/field_names.py +0 -0
  61. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/iban.py +0 -0
  62. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/ip_address.py +0 -0
  63. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/mac_address.py +0 -0
  64. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/network.py +0 -0
  65. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/payment_card.py +0 -0
  66. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/phone.py +0 -0
  67. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/presidio_ner.py +0 -0
  68. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/regex_safety.py +0 -0
  69. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/registry.py +0 -0
  70. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/secrets.py +0 -0
  71. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/detectors/ssn.py +0 -0
  72. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/errors.py +0 -0
  73. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/models/__init__.py +0 -0
  74. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/models/context.py +0 -0
  75. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/models/finding.py +0 -0
  76. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/models/policy.py +0 -0
  77. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/pipeline/__init__.py +0 -0
  78. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/pipeline/atomic_output.py +0 -0
  79. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/pipeline/inspector.py +0 -0
  80. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/pipeline/paths.py +0 -0
  81. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/pipeline/planner.py +0 -0
  82. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/pipeline/rescan.py +0 -0
  83. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/pipeline/stdio.py +0 -0
  84. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/pipeline/streams.py +0 -0
  85. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/policy/__init__.py +0 -0
  86. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/policy/context.py +0 -0
  87. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/policy/engine.py +0 -0
  88. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/policy/loader.py +0 -0
  89. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/policy/precedence.py +0 -0
  90. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/policy/templates.py +0 -0
  91. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/policy/validator.py +0 -0
  92. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/py.typed +0 -0
  93. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/readers/__init__.py +0 -0
  94. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/readers/accesslog_reader.py +0 -0
  95. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/readers/base.py +0 -0
  96. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/readers/csv_reader.py +0 -0
  97. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/readers/json_reader.py +0 -0
  98. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/readers/jsonl_reader.py +0 -0
  99. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/readers/jsonlog_reader.py +0 -0
  100. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/readers/line_structure.py +0 -0
  101. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/readers/logfmt_reader.py +0 -0
  102. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/readers/stacktrace_reader.py +0 -0
  103. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/readers/syslog_reader.py +0 -0
  104. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/readers/text_reader.py +0 -0
  105. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/reporting/__init__.py +0 -0
  106. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/reporting/audit.py +0 -0
  107. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/reporting/serializer.py +0 -0
  108. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/templates/starter_policy.yml +0 -0
  109. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/transforms/__init__.py +0 -0
  110. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/transforms/date_shift.py +0 -0
  111. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/transforms/generalize.py +0 -0
  112. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/transforms/keys.py +0 -0
  113. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/transforms/pseudonymize.py +0 -0
  114. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/transforms/redact.py +0 -0
  115. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/transforms/registry.py +0 -0
  116. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/src/ciphergate/transforms/replace.py +0 -0
  117. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/__init__.py +0 -0
  118. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/conftest.py +0 -0
  119. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/integration/__init__.py +0 -0
  120. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/integration/test_api_batch_streams.py +0 -0
  121. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/integration/test_atomic_output.py +0 -0
  122. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/integration/test_domain_templates.py +0 -0
  123. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/integration/test_environment_matrix.py +0 -0
  124. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/integration/test_pipelines.py +0 -0
  125. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/integration/test_presidio_real.py +0 -0
  126. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/integration/test_stdio.py +0 -0
  127. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/regression/__init__.py +0 -0
  128. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/regression/test_known_cases.py +0 -0
  129. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/security/__init__.py +0 -0
  130. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/security/test_key_file.py +0 -0
  131. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/security/test_no_network.py +0 -0
  132. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/security/test_no_raw_value_leaks.py +0 -0
  133. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/security/test_path_safety.py +0 -0
  134. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/security/test_policy_fail_closed.py +0 -0
  135. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/security/test_review_findings.py +0 -0
  136. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/tooling/__init__.py +0 -0
  137. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/tooling/test_ci_supply_chain.py +0 -0
  138. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/tooling/test_claude_tooling.py +0 -0
  139. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/tooling/test_git_tooling.py +0 -0
  140. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/tooling/test_publish_workflow.py +0 -0
  141. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/__init__.py +0 -0
  142. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_accesslog.py +0 -0
  143. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_detectors.py +0 -0
  144. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_jsonlog.py +0 -0
  145. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_logfmt.py +0 -0
  146. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_policy_engine.py +0 -0
  147. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_policy_precedence.py +0 -0
  148. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_policy_templates.py +0 -0
  149. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_policy_validation.py +0 -0
  150. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_presidio_ner.py +0 -0
  151. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_properties.py +0 -0
  152. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_readers.py +0 -0
  153. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_stacktrace.py +0 -0
  154. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_syslog.py +0 -0
  155. {ciphergate_cli-0.14.0 → ciphergate_cli-0.16.0}/tests/unit/test_transforms.py +0 -0
@@ -6,6 +6,43 @@ versioning: [PEP 440](https://peps.python.org/pep-0440/) with SemVer meaning (pr
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [0.16.0] - 2026-11-02
10
+
11
+ `ciphergate review` writes a local HTML page that shows, as a trace, what sanitize would change
12
+ and how the run went. The CLI's other commands and the report format are unchanged.
13
+
14
+ ### Added
15
+
16
+ - `ciphergate review INPUT --policy P -o page.html` writes one local HTML page that shows
17
+ what sanitize would change. It covers findings, how many are transformed, kept, blocked
18
+ or waiting for review, a per-category table, rescan results, possible partial finds,
19
+ and each record with its changes highlighted and labelled with category, action, rule
20
+ and reason code. No sanitized file is written. The page holds the would-be output; the
21
+ raw input is added only with `--include-input`. The page has no scripts, loads nothing,
22
+ is written atomically, and is not overwritten without `--force`. The exit code matches
23
+ `plan`. The page is laid out as a trace: a tree of the run, its steps with timings, and the
24
+ records; each record opens as Input and Output panels.
25
+ - `scripts/benchmark/ner_names.py` measures the optional NER detector on generated log lines.
26
+ With `en_core_web_sm` it found any part of 57-59% of names and the whole name in 41%, missed
27
+ most lowercase and upper-case names, and flagged 3 of 16 name-free lines. Details and limits
28
+ are in `docs/detector-coverage.md`. The detector itself is unchanged.
29
+
30
+ ## [0.15.0] - 2026-11-02
31
+
32
+ JSON reports now include the next-step hints, so CI pipelines can act on them. The field is new;
33
+ nothing existing changes.
34
+
35
+ ### Added
36
+
37
+ - JSON reports (single-file and directory) include `next_steps`: the hints from the terminal
38
+ summary as `{"code", "message"}` pairs, empty for a successful run. The field is new and
39
+ optional to read; the schema names are unchanged.
40
+
41
+ ### Fixed
42
+
43
+ - The `batch_empty` error for a directory with no recognised files now mentions
44
+ `--extension-format`.
45
+
9
46
  ## [0.14.0] - 2026-11-02
10
47
 
11
48
  Adds `--extension-format` for mixed directories and tests the optional NER detector against a
@@ -379,7 +416,9 @@ Pre-release (alpha). Licensed under Apache-2.0. Not a final release: the securit
379
416
  See `docs/limitations.md`. Deferred: optional NER/Presidio detectors, stdin/stdout streaming,
380
417
  directory processing, structured log parsers, key files, reversible token vault, lossy decoding.
381
418
 
382
- [Unreleased]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.14.0...HEAD
419
+ [Unreleased]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.16.0...HEAD
420
+ [0.16.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.15.0...v0.16.0
421
+ [0.15.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.14.0...v0.15.0
383
422
  [0.14.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.13.0...v0.14.0
384
423
  [0.13.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.12.0...v0.13.0
385
424
  [0.12.0]: https://github.com/aashishkumar-tech/ciphergate/compare/v0.11.0...v0.12.0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: ciphergate-cli
3
- Version: 0.14.0
3
+ Version: 0.16.0
4
4
  Summary: Local-first, environment- and destination-aware inspection and sanitization of logs and test datasets.
5
5
  Project-URL: Homepage, https://github.com/aashishkumar-tech/ciphergate
6
6
  Project-URL: Source, https://github.com/aashishkumar-tech/ciphergate
@@ -50,7 +50,7 @@ vendor ticket. CipherGate replaces that with a local, explainable, scriptable ga
50
50
  inspect -> plan -> sanitize --apply -> rescan
51
51
  ```
52
52
 
53
- > **Status: 0.14.0 (alpha).** No detector finds every sensitive value. A passing run means the
53
+ > **Status: 0.16.0 (alpha).** No detector finds every sensitive value. A passing run means the
54
54
  > configured processing completed under the selected policy; it is **not** proof that the output
55
55
  > contains no sensitive information, and it is not a compliance (GDPR, HIPAA, PCI DSS, ...) or
56
56
  > anonymization guarantee. Policy templates must be reviewed before use.
@@ -89,6 +89,10 @@ inspect -> plan -> sanitize --apply -> rescan
89
89
  `--force`, input/output alias protection, input-change detection, cleanup on failure/Ctrl-C.
90
90
  - **Reports:** counts by category/detector/action/reason/rule, safe locations, effective context
91
91
  and its source, gates, key *status* (never key material), no raw values or fingerprints.
92
+ A failed run's report lists `next_steps` as `{"code", "message"}` pairs, the same hints the
93
+ terminal prints as `next:` lines, so CI can act on them.
94
+ - **Review page:** `ciphergate review` writes one local HTML file that shows what sanitize
95
+ would change, record by record, with run metrics. See [Review page](#review-page).
92
96
 
93
97
  ## Install
94
98
 
@@ -234,8 +238,9 @@ healthcare or finance are starting points, **not** HIPAA/PCI controls.
234
238
 
235
239
  ## Commands and exit codes
236
240
 
237
- `init`, `validate-policy`, `inspect`, `plan`, `sanitize` (needs `--apply`), `rescan`, `explain`
238
- (decision for a hypothetical finding — no data needed). All support `--help`.
241
+ `init`, `validate-policy`, `inspect`, `plan`, `sanitize` (needs `--apply`), `rescan`, `review`
242
+ (local HTML page, see below), `explain` (decision for a hypothetical finding — no data
243
+ needed). All support `--help`.
239
244
 
240
245
  Common options: `--policy`, `--format csv|json|jsonl|text|logfmt|jsonlog|accesslog|syslog|stacktrace`, `--report PATH`,
241
246
  `--report-format json|text` (JSON without `--report` prints only JSON on stdout),
@@ -483,6 +488,50 @@ kubectl logs my-pod | ciphergate sanitize - --format text --policy policies/prd_
483
488
  | 8 | Unexpected internal error (safe message) |
484
489
  | 130 | Interrupted (no output published) |
485
490
 
491
+ ### Review page
492
+
493
+ `ciphergate review` runs the same steps as `sanitize --apply`, including the rescan, but writes
494
+ no sanitized file. It writes one HTML page instead:
495
+
496
+ ```bash
497
+ ciphergate review app.log --policy policies/developer_logs.yml --environment dev --destination local_dev -o build/review.html
498
+ ```
499
+
500
+ Open the file in a browser. It is laid out like a trace: a tree on the left, details of the
501
+ selected item on the right.
502
+
503
+ - **review run:** a verdict, then metrics. These include findings and how many are
504
+ transformed, kept by an `allow` rule, need review or are blocked (with percentages), records
505
+ with findings, values changed, records dropped, rescan results and possible partial finds.
506
+ - **Steps**, with measured timings: the plan pass (read input, detect, decide), transform,
507
+ rescan and output.
508
+ - *detect* shows counts by category, detector and confidence.
509
+ - *decide* shows the policy and context, what happened to each category, counts by action,
510
+ rule and reason code, plus gates and key status.
511
+ - **Records with findings:** each one opens as Input and Output panels with a decisions table.
512
+ Changed parts are highlighted, and hovering over one shows its category, action, rule and
513
+ reason code. Values kept by an `allow` rule are underlined. A dashed outline marks a
514
+ *possible partial find*: a replaced name or address next to a capitalised word that stayed.
515
+ This is a hint, not a check.
516
+
517
+ What the page contains, and how to handle it:
518
+
519
+ - By default it contains the would-be **output**, not the input. Treat it like the sanitized
520
+ file: detectors can miss values.
521
+ - `--include-input` also adds the **raw input** next to the output, with a warning banner. A
522
+ page made with it must be handled like the original file. Do not share, upload or commit it.
523
+ - If the run is blocked, needs review or a key is missing, there is no output preview. The page
524
+ shows the plan and the decisions for each record. Planned transforms are labelled as not run.
525
+ - There are no default locations: `-o` is required. `build/` is in `.gitignore`. The page is not
526
+ overwritten without `--force`, can't be the input or the policy file, and is written
527
+ atomically (0600 on POSIX). Directories and stdin/stdout are not supported.
528
+ - The page has no scripts and a Content-Security-Policy that blocks all loads, so opening it
529
+ makes no network request. All text is escaped.
530
+ - The exit code is the plan's: 0 ready, 1 blocked/review, 6 key/config, 7 rescan found gating
531
+ findings. Use `sanitize` and `rescan` as the CI gate. Up to 500 records are shown, each cut
532
+ to 4,000 characters; the counts include all records. It takes a little longer than
533
+ `sanitize` (about 15% on a 40,000-line synthetic log).
534
+
486
535
  ### Troubleshooting
487
536
 
488
537
  - `destination_required_but_missing` — pass `--destination` (and `--environment`).
@@ -11,7 +11,7 @@ vendor ticket. CipherGate replaces that with a local, explainable, scriptable ga
11
11
  inspect -> plan -> sanitize --apply -> rescan
12
12
  ```
13
13
 
14
- > **Status: 0.14.0 (alpha).** No detector finds every sensitive value. A passing run means the
14
+ > **Status: 0.16.0 (alpha).** No detector finds every sensitive value. A passing run means the
15
15
  > configured processing completed under the selected policy; it is **not** proof that the output
16
16
  > contains no sensitive information, and it is not a compliance (GDPR, HIPAA, PCI DSS, ...) or
17
17
  > anonymization guarantee. Policy templates must be reviewed before use.
@@ -50,6 +50,10 @@ inspect -> plan -> sanitize --apply -> rescan
50
50
  `--force`, input/output alias protection, input-change detection, cleanup on failure/Ctrl-C.
51
51
  - **Reports:** counts by category/detector/action/reason/rule, safe locations, effective context
52
52
  and its source, gates, key *status* (never key material), no raw values or fingerprints.
53
+ A failed run's report lists `next_steps` as `{"code", "message"}` pairs, the same hints the
54
+ terminal prints as `next:` lines, so CI can act on them.
55
+ - **Review page:** `ciphergate review` writes one local HTML file that shows what sanitize
56
+ would change, record by record, with run metrics. See [Review page](#review-page).
53
57
 
54
58
  ## Install
55
59
 
@@ -195,8 +199,9 @@ healthcare or finance are starting points, **not** HIPAA/PCI controls.
195
199
 
196
200
  ## Commands and exit codes
197
201
 
198
- `init`, `validate-policy`, `inspect`, `plan`, `sanitize` (needs `--apply`), `rescan`, `explain`
199
- (decision for a hypothetical finding — no data needed). All support `--help`.
202
+ `init`, `validate-policy`, `inspect`, `plan`, `sanitize` (needs `--apply`), `rescan`, `review`
203
+ (local HTML page, see below), `explain` (decision for a hypothetical finding — no data
204
+ needed). All support `--help`.
200
205
 
201
206
  Common options: `--policy`, `--format csv|json|jsonl|text|logfmt|jsonlog|accesslog|syslog|stacktrace`, `--report PATH`,
202
207
  `--report-format json|text` (JSON without `--report` prints only JSON on stdout),
@@ -444,6 +449,50 @@ kubectl logs my-pod | ciphergate sanitize - --format text --policy policies/prd_
444
449
  | 8 | Unexpected internal error (safe message) |
445
450
  | 130 | Interrupted (no output published) |
446
451
 
452
+ ### Review page
453
+
454
+ `ciphergate review` runs the same steps as `sanitize --apply`, including the rescan, but writes
455
+ no sanitized file. It writes one HTML page instead:
456
+
457
+ ```bash
458
+ ciphergate review app.log --policy policies/developer_logs.yml --environment dev --destination local_dev -o build/review.html
459
+ ```
460
+
461
+ Open the file in a browser. It is laid out like a trace: a tree on the left, details of the
462
+ selected item on the right.
463
+
464
+ - **review run:** a verdict, then metrics. These include findings and how many are
465
+ transformed, kept by an `allow` rule, need review or are blocked (with percentages), records
466
+ with findings, values changed, records dropped, rescan results and possible partial finds.
467
+ - **Steps**, with measured timings: the plan pass (read input, detect, decide), transform,
468
+ rescan and output.
469
+ - *detect* shows counts by category, detector and confidence.
470
+ - *decide* shows the policy and context, what happened to each category, counts by action,
471
+ rule and reason code, plus gates and key status.
472
+ - **Records with findings:** each one opens as Input and Output panels with a decisions table.
473
+ Changed parts are highlighted, and hovering over one shows its category, action, rule and
474
+ reason code. Values kept by an `allow` rule are underlined. A dashed outline marks a
475
+ *possible partial find*: a replaced name or address next to a capitalised word that stayed.
476
+ This is a hint, not a check.
477
+
478
+ What the page contains, and how to handle it:
479
+
480
+ - By default it contains the would-be **output**, not the input. Treat it like the sanitized
481
+ file: detectors can miss values.
482
+ - `--include-input` also adds the **raw input** next to the output, with a warning banner. A
483
+ page made with it must be handled like the original file. Do not share, upload or commit it.
484
+ - If the run is blocked, needs review or a key is missing, there is no output preview. The page
485
+ shows the plan and the decisions for each record. Planned transforms are labelled as not run.
486
+ - There are no default locations: `-o` is required. `build/` is in `.gitignore`. The page is not
487
+ overwritten without `--force`, can't be the input or the policy file, and is written
488
+ atomically (0600 on POSIX). Directories and stdin/stdout are not supported.
489
+ - The page has no scripts and a Content-Security-Policy that blocks all loads, so opening it
490
+ makes no network request. All text is escaped.
491
+ - The exit code is the plan's: 0 ready, 1 blocked/review, 6 key/config, 7 rescan found gating
492
+ findings. Use `sanitize` and `rescan` as the CI gate. Up to 500 records are shown, each cut
493
+ to 4,000 characters; the counts include all records. It takes a little longer than
494
+ `sanitize` (about 15% on a 40,000-line synthetic log).
495
+
447
496
  ### Troubleshooting
448
497
 
449
498
  - `destination_required_but_missing` — pass `--destination` (and `--environment`).
@@ -40,6 +40,7 @@ Report (reporting/) privacy-safe JSON / text summary
40
40
  | `transforms/` | Execute approved actions; HMAC pseudonyms; key lookup | log values or keys |
41
41
  | `pipeline/` | Orchestration, two passes, rescan, atomic output, path safety | publish on any failure |
42
42
  | `reporting/` | Report building, control-char escaping, field-path masking, rendering | include values/fingerprints |
43
+ | `pipeline/review.py`, `reporting/review_page.py` | `review`: sanitize steps without publishing, per-record changes, local HTML page | publish output, add input without `--include-input`, load anything |
43
44
  | `cli.py` | Typer CLI, exit codes, safe error rendering, SIGTERM -> interrupt | enable rich tracebacks (they print locals) |
44
45
 
45
46
  ## Why two passes
@@ -32,8 +32,31 @@ in `tests/unit/test_presidio_ner.py` use a fake engine. `tests/integration/test_
32
32
  runs the real Presidio with `en_core_web_sm`: it finds a common synthetic name and place, ignores
33
33
  text with no entities, removes them in a `sanitize` run with network access disabled, and fails
34
34
  closed when the model is missing. CI runs it in the `ner` job (not a required check). These are
35
- smoke tests on a few sentences: accuracy on real text has not been measured by this project,
36
- and the small model misses names that the larger `en_core_web_lg` may find.
35
+ smoke tests on a few sentences.
36
+
37
+ `scripts/benchmark/ner_names.py` measures name detection on 320 generated log lines (8 regions x
38
+ 5 ways of writing a name x 8 lines) plus 16 name-free lines. With `en_core_web_sm` (spaCy 3.8,
39
+ Presidio 2.2.366), seeds 7 and 11:
40
+
41
+ | | seed 7 | seed 11 |
42
+ |---|---|---|
43
+ | names with any part found | 184/320 (57%) | 190/320 (59%) |
44
+ | names found in full | 130/320 (41%) | 132/320 (41%) |
45
+ | name-free lines flagged | 3/16 | 3/16 |
46
+
47
+ - When only part of a name is found, the rest stays in the output (for example the family name).
48
+ - `First Last` and `Last, First` are found most often (50-61 of 64). Lowercase (27/64) and
49
+ upper-case (15/64) names are mostly missed.
50
+ - English, Chinese and German names were found more often (27-35 of 40) than Indian and West
51
+ African names (12-17 of 40).
52
+ - The false alarms were product names (`Kafka`, `Jenkins`, `Grafana`).
53
+ - `min_score` from 0.4 to 0.85 changed nothing: Presidio gives these spaCy entities one fixed
54
+ score, so the threshold cannot trade misses against false alarms with this model.
55
+ - The default `en_core_web_lg` was not measured. Run the script with `--model en_core_web_lg`
56
+ after installing it.
57
+
58
+ These are synthetic lines written by the project, not a public accuracy claim. Treat the NER
59
+ detector as an extra layer: do not rely on it alone to remove names, especially from free text.
37
60
 
38
61
  ## Text/log specifics
39
62
 
@@ -67,13 +67,29 @@ Reports contain: run ID, UTC time, version, input basename (or `<redacted>`), fo
67
67
  record counts, policy name/version/SHA-256 digest of the policy file, effective context with
68
68
  sources, gates, counts by category/detector/action/band/reason/rule, safe locations (record,
69
69
  line, field path, character span), rule IDs and reason codes, key status, changes, parse error
70
- line numbers and codes, rescan summary, warnings and a disclaimer. They never contain values,
70
+ line numbers and codes, rescan summary, warnings, `next_steps` (fixed hint text with a reason
71
+ code, for failed runs only) and a disclaimer. They never contain values,
71
72
  snippets, keys, mappings, or hashes of values. Field paths whose segments look like sensitive
72
73
  values are replaced with `<redacted-key>`; control and bidi characters are escaped.
73
74
 
74
75
  Logging uses the standard library and never receives values. `--verbose` lists more items;
75
76
  `--debug` adds traceback frame locations for internal errors without exception messages.
76
77
 
78
+ ## Review page
79
+
80
+ `ciphergate review` is the only command that puts data into a file other than the output. By
81
+ default the page holds the would-be sanitized output, so it is as sensitive as that output.
82
+ With `--include-input` it also holds the raw input and must be handled like the input; the
83
+ page and the command's stderr both say so. The JSON report never contains either.
84
+
85
+ The would-be output is written to a `mkstemp` file in the page's directory, rescanned, and
86
+ always removed; it is never published. The page itself uses the same atomic,
87
+ no-overwrite-by-default writer as outputs and refuses the input and policy paths. It contains
88
+ no scripts. A Content-Security-Policy (`default-src 'none'`) and `no-referrer` stop any load
89
+ or request, and every text is HTML-escaped with control and bidi characters shown as escapes.
90
+ Field paths in the decision tables use the same masking as reports. The possible-partial-find
91
+ mark is a heuristic and can miss partial finds or flag whole ones.
92
+
77
93
  ## Files
78
94
 
79
95
  Inputs must be regular files (FIFOs/devices refused). A directory INPUT starts a batch run:
@@ -30,6 +30,14 @@ Only CipherGate is required. Any other tool that is not installed is skipped and
30
30
  credentials that must not be committed. The tool refuses a path inside the repository. Results
31
31
  are repeatable for the same `--seed` and tool versions.
32
32
 
33
+ ## Name detection (`ner_names.py`)
34
+
35
+ `python scripts/benchmark/ner_names.py --model en_core_web_sm` measures the optional NER detector
36
+ on generated log lines: names from eight regions written five ways (`First Last`, `Last, First`,
37
+ lowercase, first name only, upper case) and name-free lines. It needs `presidio-analyzer` and the
38
+ spaCy model installed in the same environment, and prints only counts. Results for
39
+ `en_core_web_sm` are in [docs/detector-coverage.md](../../docs/detector-coverage.md).
40
+
33
41
  ## Reading the output
34
42
 
35
43
  Each row is a data class, and each cell is caught/total. A positive counts only if the finding
@@ -24,7 +24,7 @@ Supported public API (everything else is internal and may change without notice)
24
24
  rescan_directory(root, policy, context=None, *, recursive=False, strict=None) -> BatchReport
25
25
  """
26
26
 
27
- __version__ = "0.14.0"
27
+ __version__ = "0.16.0"
28
28
 
29
29
  from ciphergate.config import RuntimeSettings
30
30
  from ciphergate.constants import (
@@ -56,6 +56,7 @@ from ciphergate.pipeline.batch import (
56
56
  sanitize_directory,
57
57
  )
58
58
  from ciphergate.pipeline.paths import validate_output_path
59
+ from ciphergate.pipeline.review import ReviewOptions, review_file
59
60
  from ciphergate.pipeline.sanitizer import (
60
61
  SanitizeOptions,
61
62
  inspect_file,
@@ -823,6 +824,86 @@ def rescan_command(
823
824
  )
824
825
 
825
826
 
827
+ @app.command("review")
828
+ def review_command(
829
+ input_path: Annotated[Path, typer.Argument(metavar="INPUT", help="File to review.")],
830
+ policy: PolicyOpt,
831
+ output: Annotated[
832
+ Path,
833
+ typer.Option("--output", "-o", help="HTML page to write, e.g. build/review.html."),
834
+ ],
835
+ include_input: Annotated[
836
+ bool,
837
+ typer.Option(
838
+ "--include-input",
839
+ help="Also put the RAW input on the page; handle the page like the input.",
840
+ ),
841
+ ] = False,
842
+ fmt: FormatOpt = None,
843
+ extension_format: ExtFormatOpt = None,
844
+ environment: EnvOpt = None,
845
+ target_environment: TargetOpt = None,
846
+ destination: DestOpt = None,
847
+ use_case: UseCaseOpt = None,
848
+ profile: ProfileOpt = None,
849
+ max_bytes: MaxBytesOpt = None,
850
+ encoding: EncodingOpt = None,
851
+ path_display: PathDisplayOpt = None,
852
+ force: ForceOpt = False,
853
+ quiet: QuietOpt = False,
854
+ verbose: VerboseOpt = False,
855
+ no_color: NoColorOpt = False,
856
+ debug: DebugOpt = False,
857
+ ) -> None:
858
+ """Write a local HTML page showing what sanitize would change, with run metrics.
859
+
860
+ Writes no sanitized file. The page holds the would-be output; with --include-input it also
861
+ holds the raw input. The exit code is the plan's (1 blocked/review, 6 key, 7 rescan).
862
+ """
863
+ _configure_logging(verbose, debug)
864
+
865
+ def body() -> int:
866
+ if is_stdio(input_path) or is_stdio(output):
867
+ raise UsageError(
868
+ "review needs a file INPUT and an --output file", code="stdio_unsupported"
869
+ )
870
+ if _is_directory_input(input_path):
871
+ raise UsageError("review takes a single file, not a directory", code="review_directory")
872
+ runtime = _runtime(
873
+ max_bytes, encoding, path_display, extension_formats=extension_format, fmt=fmt
874
+ )
875
+ options = ReviewOptions(
876
+ force=force,
877
+ include_input=include_input,
878
+ format=fmt,
879
+ runtime=runtime,
880
+ protected=(policy,),
881
+ )
882
+ data = review_file(
883
+ input_path,
884
+ output,
885
+ load_policy(policy),
886
+ _context(environment, target_environment, destination, use_case, profile),
887
+ options,
888
+ )
889
+ code = _emit(
890
+ data.report,
891
+ report_path=None,
892
+ report_format=None,
893
+ quiet=quiet,
894
+ verbose=verbose,
895
+ force=force,
896
+ )
897
+ held = "the would-be output" if data.has_output else "the plan only"
898
+ if include_input:
899
+ held += " and the RAW INPUT; handle it like the input file"
900
+ sys.stdout.flush()
901
+ sys.stderr.write(f"review page written: {escape_control(str(output))} ({held})\n")
902
+ return code
903
+
904
+ _guard(debug, body)
905
+
906
+
826
907
  @app.command("explain")
827
908
  def explain_command(
828
909
  policy: PolicyOpt,
@@ -11,6 +11,7 @@ from enum import StrEnum
11
11
  from typing import Any
12
12
 
13
13
  from ciphergate.constants import DISCLAIMER, REPORT_SCHEMA, ExitCode
14
+ from ciphergate.reporting.next_steps import next_step_items
14
15
 
15
16
 
16
17
  class RunStatus(StrEnum):
@@ -89,7 +90,7 @@ class RunReport:
89
90
  return self.exit_code == 0
90
91
 
91
92
  def to_dict(self) -> dict[str, Any]:
92
- return {
93
+ data: dict[str, Any] = {
93
94
  "schema": REPORT_SCHEMA,
94
95
  "command": self.command,
95
96
  "status": str(self.status),
@@ -111,5 +112,8 @@ class RunReport:
111
112
  "output": self.output,
112
113
  "rescan": self.rescan.to_dict() if self.rescan else None,
113
114
  "warnings": self.warnings,
114
- "disclaimer": DISCLAIMER,
115
115
  }
116
+ # Fixed hint text chosen by reason code: never contains input values.
117
+ data["next_steps"] = next_step_items(data)
118
+ data["disclaimer"] = DISCLAIMER
119
+ return data
@@ -52,6 +52,7 @@ from ciphergate.pipeline.sanitizer import (
52
52
  from ciphergate.policy.validator import Policy
53
53
  from ciphergate.readers import EXTENSION_FORMATS
54
54
  from ciphergate.reporting.audit import new_run_id, utc_now
55
+ from ciphergate.reporting.next_steps import merge_next_steps
55
56
 
56
57
  logger = logging.getLogger("ciphergate")
57
58
 
@@ -136,6 +137,7 @@ class BatchReport:
136
137
 
137
138
  def to_dict(self) -> dict[str, Any]:
138
139
  first = next((r.report for r in self.files if r.report is not None), None)
140
+ files = [r.to_dict() for r in self.files]
139
141
  return {
140
142
  "schema": BATCH_REPORT_SCHEMA,
141
143
  "command": self.command,
@@ -148,10 +150,11 @@ class BatchReport:
148
150
  "policy": first.policy if first is not None else None,
149
151
  "context": first.context if first is not None else None,
150
152
  "summary": self.summary(),
151
- "files": [r.to_dict() for r in self.files],
153
+ "files": files,
152
154
  "skipped": self.skipped,
153
155
  "output": self.output,
154
156
  "warnings": self.warnings,
157
+ "next_steps": merge_next_steps([f["report"] for f in files if "report" in f]),
155
158
  "disclaimer": DISCLAIMER,
156
159
  }
157
160
 
@@ -324,7 +327,8 @@ def coverage_warnings(
324
327
  def _require_files(discovery: Discovery) -> None:
325
328
  if not discovery.files:
326
329
  raise InputFormatError(
327
- "no supported input files found in the directory (see --format, --recursive)",
330
+ "no supported input files found in the directory (see --extension-format, --format, "
331
+ "--recursive)",
328
332
  code="batch_empty",
329
333
  )
330
334