secrethider 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. secrethider-0.1.0/.clang-format +3 -0
  2. secrethider-0.1.0/.editorconfig +9 -0
  3. secrethider-0.1.0/.gitignore +22 -0
  4. secrethider-0.1.0/.pre-commit-config.yaml +19 -0
  5. secrethider-0.1.0/.ubsan-ignorelist +4 -0
  6. secrethider-0.1.0/CMakeLists.txt +51 -0
  7. secrethider-0.1.0/CMakePresets.json +44 -0
  8. secrethider-0.1.0/LICENSE +21 -0
  9. secrethider-0.1.0/Makefile +148 -0
  10. secrethider-0.1.0/PKG-INFO +109 -0
  11. secrethider-0.1.0/README.md +96 -0
  12. secrethider-0.1.0/bindings/node/CMakeLists.txt +62 -0
  13. secrethider-0.1.0/bindings/python/CMakeLists.txt +6 -0
  14. secrethider-0.1.0/bindings/python/bench/bench_redact.py +89 -0
  15. secrethider-0.1.0/bindings/python/qa/leakcheck.py +195 -0
  16. secrethider-0.1.0/bindings/python/secrethider/__init__.py +57 -0
  17. secrethider-0.1.0/bindings/python/secrethider/__main__.py +3 -0
  18. secrethider-0.1.0/bindings/python/secrethider/_core.pyi +93 -0
  19. secrethider-0.1.0/bindings/python/secrethider/cli.py +163 -0
  20. secrethider-0.1.0/bindings/python/secrethider/fake.py +122 -0
  21. secrethider-0.1.0/bindings/python/secrethider/logging.py +39 -0
  22. secrethider-0.1.0/bindings/python/secrethider/py.typed +0 -0
  23. secrethider-0.1.0/bindings/python/secrethider/redact.py +216 -0
  24. secrethider-0.1.0/bindings/python/src/module.cpp +37 -0
  25. secrethider-0.1.0/bindings/python/src/redact.cpp +216 -0
  26. secrethider-0.1.0/bindings/python/tests/test_features.py +156 -0
  27. secrethider-0.1.0/bindings/python/tests/test_redact.py +163 -0
  28. secrethider-0.1.0/bindings/python/tests/test_secrethider.py +22 -0
  29. secrethider-0.1.0/cmake/Dependencies.cmake +70 -0
  30. secrethider-0.1.0/cmake/Warnings.cmake +7 -0
  31. secrethider-0.1.0/core/CMakeLists.txt +71 -0
  32. secrethider-0.1.0/core/bench/bench.cpp +29 -0
  33. secrethider-0.1.0/core/bench/bench_redact.cpp +391 -0
  34. secrethider-0.1.0/core/include/secrethider/json.hpp +19 -0
  35. secrethider-0.1.0/core/include/secrethider/redact.hpp +197 -0
  36. secrethider-0.1.0/core/include/secrethider/strings.hpp +25 -0
  37. secrethider-0.1.0/core/src/json.cpp +35 -0
  38. secrethider-0.1.0/core/src/redact/aho_corasick.cpp +217 -0
  39. secrethider-0.1.0/core/src/redact/aho_corasick.hpp +110 -0
  40. secrethider-0.1.0/core/src/redact/detectors.cpp +709 -0
  41. secrethider-0.1.0/core/src/redact/impl.hpp +121 -0
  42. secrethider-0.1.0/core/src/redact/json_redact.cpp +188 -0
  43. secrethider-0.1.0/core/src/redact/mask.cpp +202 -0
  44. secrethider-0.1.0/core/src/redact/redactor.cpp +488 -0
  45. secrethider-0.1.0/core/src/redact/regex_re2.cpp +68 -0
  46. secrethider-0.1.0/core/src/redact/sha256.hpp +155 -0
  47. secrethider-0.1.0/core/src/redact/stream.cpp +107 -0
  48. secrethider-0.1.0/core/src/strings.cpp +73 -0
  49. secrethider-0.1.0/core/tests/fuzz_redact.cpp +15 -0
  50. secrethider-0.1.0/core/tests/leakcheck.cpp +321 -0
  51. secrethider-0.1.0/core/tests/property.hpp +209 -0
  52. secrethider-0.1.0/core/tests/qa_repro.cpp +27 -0
  53. secrethider-0.1.0/core/tests/test_aho_corasick.cpp +149 -0
  54. secrethider-0.1.0/core/tests/test_features.cpp +175 -0
  55. secrethider-0.1.0/core/tests/test_json.cpp +25 -0
  56. secrethider-0.1.0/core/tests/test_json_redact.cpp +57 -0
  57. secrethider-0.1.0/core/tests/test_mask.cpp +62 -0
  58. secrethider-0.1.0/core/tests/test_property.cpp +157 -0
  59. secrethider-0.1.0/core/tests/test_redact.cpp +143 -0
  60. secrethider-0.1.0/core/tests/test_stream.cpp +118 -0
  61. secrethider-0.1.0/core/tests/test_strings.cpp +29 -0
  62. secrethider-0.1.0/docs/README.md +17 -0
  63. secrethider-0.1.0/docs/adr/0001-matching-engine.md +37 -0
  64. secrethider-0.1.0/docs/adr/0002-streaming-semantics.md +35 -0
  65. secrethider-0.1.0/docs/architecture/01-overview.md +55 -0
  66. secrethider-0.1.0/docs/architecture/02-interfaces.md +250 -0
  67. secrethider-0.1.0/docs/architecture/03-log-processing.md +112 -0
  68. secrethider-0.1.0/docs/architecture/04-rules-and-masking.md +71 -0
  69. secrethider-0.1.0/docs/architecture/05-build-and-platforms.md +72 -0
  70. secrethider-0.1.0/docs/architecture/06-performance.md +84 -0
  71. secrethider-0.1.0/docs/architecture/07-security-model.md +38 -0
  72. secrethider-0.1.0/docs/architecture/08-testing-and-qa.md +68 -0
  73. secrethider-0.1.0/docs/guides/removing-secrets-from-logs.md +106 -0
  74. secrethider-0.1.0/pyproject.toml +62 -0
  75. secrethider-0.1.0/scripts/msvc-env.bat +22 -0
@@ -0,0 +1,3 @@
1
+ BasedOnStyle: Google
2
+ IndentWidth: 4
3
+ ColumnLimit: 100
@@ -0,0 +1,9 @@
1
+ root = true
2
+ [*]
3
+ indent_style = space
4
+ indent_size = 4
5
+ end_of_line = lf
6
+ insert_final_newline = true
7
+ trim_trailing_whitespace = true
8
+ [*.{yml,yaml,json,js}]
9
+ indent_size = 2
@@ -0,0 +1,22 @@
1
+ build/
2
+ out/
3
+ dist/
4
+ .venv/
5
+ __pycache__/
6
+ *.egg-info/
7
+ *.pyd
8
+ *.so
9
+ *.dylib
10
+ *.node
11
+ node_modules/
12
+ prebuilds/
13
+ .cache/
14
+ compile_commands.json
15
+ .vs/
16
+ .pytest_cache/
17
+ .ruff_cache/
18
+
19
+ # local secrets (registry tokens): never commit
20
+ .env
21
+ .env.*
22
+ !.env.example
@@ -0,0 +1,19 @@
1
+ repos:
2
+ - repo: https://github.com/pre-commit/pre-commit-hooks
3
+ rev: v5.0.0
4
+ hooks:
5
+ - id: end-of-file-fixer
6
+ - id: trailing-whitespace
7
+ - id: check-yaml
8
+ - id: check-json
9
+ - repo: https://github.com/astral-sh/ruff-pre-commit
10
+ rev: v0.8.0
11
+ hooks:
12
+ - id: ruff
13
+ args: [--fix]
14
+ - id: ruff-format
15
+ - repo: https://github.com/pre-commit/mirrors-clang-format
16
+ rev: v19.1.0
17
+ hooks:
18
+ - id: clang-format
19
+ types_or: [c++]
@@ -0,0 +1,4 @@
1
+ # UBSan ignore list for sanitizer builds: third-party sources are not ours to fix
2
+ # (for example RE2 constructs a zero-length array at a null pointer). ASan still instruments everything.
3
+ [undefined]
4
+ src:*_deps*
@@ -0,0 +1,51 @@
1
+ cmake_minimum_required(VERSION 3.25)
2
+ project(secrethider VERSION 0.1.0 LANGUAGES CXX)
3
+
4
+ set(CMAKE_CXX_STANDARD 20)
5
+ set(CMAKE_CXX_STANDARD_REQUIRED ON)
6
+ set(CMAKE_CXX_EXTENSIONS OFF)
7
+ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
8
+ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
9
+ set(CMAKE_CXX_VISIBILITY_PRESET hidden)
10
+ set(CMAKE_VISIBILITY_INLINES_HIDDEN ON)
11
+
12
+ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
13
+ set(CMAKE_BUILD_TYPE Release)
14
+ endif()
15
+
16
+ option(SECRETHIDER_BUILD_TESTS "Build C++ unit tests" ${PROJECT_IS_TOP_LEVEL})
17
+ option(SECRETHIDER_BUILD_BENCH "Build C++ benchmarks" OFF)
18
+ option(SECRETHIDER_BUILD_PYTHON "Build the Python extension" OFF)
19
+ option(SECRETHIDER_SANITIZE "Enable ASan+UBSan (gcc/clang)" OFF)
20
+ option(SECRETHIDER_BUILD_FUZZ "Build the libFuzzer target (clang)" OFF)
21
+ option(SECRETHIDER_WITH_RE2 "Custom regex rules via RE2 (pulls abseil)" ON)
22
+
23
+ if(MSVC)
24
+ add_compile_options(/EHsc /utf-8)
25
+ endif()
26
+
27
+ if(SECRETHIDER_BUILD_TESTS)
28
+ enable_testing()
29
+ endif()
30
+
31
+ # Sanitizers must reach every target (static libraries need them at the final link, and the
32
+ # dependencies should be instrumented too), so they are global rather than per target.
33
+ if(SECRETHIDER_SANITIZE AND NOT MSVC)
34
+ add_compile_options(-fsanitize=address,undefined -fno-omit-frame-pointer)
35
+ add_link_options(-fsanitize=address,undefined)
36
+ if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
37
+ # third-party sources are not ours to fix (RE2 constructs a zero-length array at a null pointer)
38
+ add_compile_options(-fsanitize-ignorelist=${CMAKE_CURRENT_SOURCE_DIR}/.ubsan-ignorelist)
39
+ endif()
40
+ endif()
41
+
42
+ list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
43
+ include(Warnings)
44
+ include(Dependencies)
45
+
46
+ add_subdirectory(core)
47
+
48
+ if(SECRETHIDER_BUILD_PYTHON)
49
+ add_subdirectory(bindings/python)
50
+ endif()
51
+ # The Node addon is built by cmake-js from bindings/node/CMakeLists.txt.
@@ -0,0 +1,44 @@
1
+ {
2
+ "version": 6,
3
+ "configurePresets": [
4
+ {
5
+ "name": "dev",
6
+ "displayName": "Dev (tests + bench, Release)",
7
+ "binaryDir": "${sourceDir}/build/dev",
8
+ "generator": "Ninja",
9
+ "cacheVariables": {
10
+ "CMAKE_BUILD_TYPE": "Release",
11
+ "SECRETHIDER_BUILD_TESTS": "ON",
12
+ "SECRETHIDER_BUILD_BENCH": "ON"
13
+ }
14
+ },
15
+ {
16
+ "name": "dev-msvc",
17
+ "displayName": "Dev with MSVC (Windows; run from a VS developer prompt)",
18
+ "inherits": "dev",
19
+ "binaryDir": "${sourceDir}/build/dev-msvc",
20
+ "cacheVariables": { "CMAKE_CXX_COMPILER": "cl", "CMAKE_C_COMPILER": "cl" },
21
+ "condition": { "type": "equals", "lhs": "${hostSystemName}", "rhs": "Windows" }
22
+ },
23
+ {
24
+ "name": "asan",
25
+ "displayName": "Debug + ASan/UBSan (Linux/macOS)",
26
+ "inherits": "dev",
27
+ "binaryDir": "${sourceDir}/build/asan",
28
+ "cacheVariables": {
29
+ "CMAKE_BUILD_TYPE": "Debug",
30
+ "SECRETHIDER_BUILD_BENCH": "OFF",
31
+ "SECRETHIDER_SANITIZE": "ON"
32
+ }
33
+ }
34
+ ],
35
+ "buildPresets": [
36
+ { "name": "dev", "configurePreset": "dev" },
37
+ { "name": "dev-msvc", "configurePreset": "dev-msvc" },
38
+ { "name": "asan", "configurePreset": "asan" }
39
+ ],
40
+ "testPresets": [
41
+ { "name": "dev", "configurePreset": "dev", "output": { "outputOnFailure": true } },
42
+ { "name": "dev-msvc", "configurePreset": "dev-msvc", "output": { "outputOnFailure": true } }
43
+ ]
44
+ }
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 NoobforAl
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,148 @@
1
+ # secrethider developer commands. Run `make help`.
2
+ #
3
+ # make fix format + auto-fix lint: C++ (clang-format), Python (ruff), JavaScript/TypeScript (prettier)
4
+ # make test build everything, run every test suite (C++, Python, Node), then the benchmarks
5
+ #
6
+ # Needs: cmake, ninja, a C++20 compiler, Python 3.9+ (uv is used when installed), Node 18+, clang-format.
7
+ # Windows: GNU make with Git's sh. Visual Studio's C++ tools are found automatically (scripts/msvc-env.bat).
8
+
9
+ .DEFAULT_GOAL := help
10
+
11
+ # ---- platform ----------------------------------------------------------------------------------
12
+ ifeq ($(OS),Windows_NT)
13
+ PRESET ?= dev-msvc
14
+ VENV_BIN := .venv/Scripts
15
+ EXE := .exe
16
+ ENVRUN := $(CURDIR)/scripts/msvc-env.bat
17
+ PYBUILD_ENV := CMAKE_GENERATOR=Ninja CC=cl CXX=cl
18
+ # the extensionless npm/npx are shell scripts that pick WSL's bash under Git's sh: use the .cmd shims
19
+ NPM := npm.cmd
20
+ NPX := npx.cmd
21
+ NODE_BUILD := npx.cmd cmake-js compile -G Ninja --CDCMAKE_C_COMPILER=cl --CDCMAKE_CXX_COMPILER=cl
22
+ else
23
+ PRESET ?= dev
24
+ VENV_BIN := .venv/bin
25
+ EXE :=
26
+ ENVRUN :=
27
+ PYBUILD_ENV := CMAKE_GENERATOR=Ninja
28
+ NPM := npm
29
+ NPX := npx
30
+ NODE_BUILD := npx cmake-js compile
31
+ endif
32
+
33
+ # ---- tools -------------------------------------------------------------------------------------
34
+ PYTHON ?= python
35
+ CLANG_FORMAT ?= clang-format
36
+ HAVE_UV := $(shell command -v uv 2>/dev/null)
37
+ HAVE_RUFF := $(shell command -v ruff 2>/dev/null)
38
+ PY := $(VENV_BIN)/python$(EXE)
39
+ export UV_LINK_MODE := copy
40
+ ifdef HAVE_UV
41
+ VENV_CREATE := uv venv .venv
42
+ PIP_INSTALL := uv pip install --python $(PY)
43
+ else
44
+ VENV_CREATE := $(PYTHON) -m venv .venv
45
+ PIP_INSTALL := $(PY) -m pip install
46
+ endif
47
+ ifdef HAVE_RUFF
48
+ RUFF ?= ruff
49
+ else ifdef HAVE_UV
50
+ RUFF ?= uv tool run ruff
51
+ else
52
+ RUFF ?= $(PYTHON) -m ruff
53
+ endif
54
+
55
+ # ---- sources (tracked or untracked-but-not-ignored, so it works before the first commit) -------
56
+ GIT_FILES = git ls-files --cached --others --exclude-standard
57
+ CPP_FILES := $(shell $(GIT_FILES) -- '*.cpp' '*.hpp' '*.h')
58
+ SRC_CORE := $(shell $(GIT_FILES) -- core cmake CMakeLists.txt)
59
+ SRC_PY := $(SRC_CORE) $(shell $(GIT_FILES) -- bindings/python pyproject.toml README.md)
60
+ SRC_NODE := $(SRC_CORE) $(shell $(GIT_FILES) -- bindings/node/src bindings/node/CMakeLists.txt bindings/node/package.json)
61
+
62
+ BUILD_DIR := build/$(PRESET)
63
+ BENCH_FILTER ?= -Pathological
64
+
65
+ NODE_DEPS := bindings/node/node_modules/.installed
66
+ NODE_ADDON := bindings/node/build/Release/secrethider.node
67
+ PY_STAMP := .venv/.installed
68
+
69
+ .PHONY: help fix fix-cpp fix-python fix-js lint test check bench bench-full build \
70
+ test-core test-python test-node bench-core bench-python bench-node qa clean
71
+
72
+ help: ## list the commands
73
+ @grep -E '^[a-zA-Z_-]+:.*## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*## "}; {printf " make %-13s %s\n", $$1, $$2}'
74
+
75
+ # ---- format / lint -----------------------------------------------------------------------------
76
+ fix: fix-cpp fix-python fix-js ## format and auto-fix lint in place (C++, Python, JS/TS)
77
+
78
+ fix-cpp:
79
+ $(CLANG_FORMAT) -i $(CPP_FILES)
80
+
81
+ fix-python:
82
+ $(RUFF) check --fix bindings/python
83
+ $(RUFF) format bindings/python
84
+
85
+ fix-js: $(NODE_DEPS)
86
+ cd bindings/node && $(NPX) prettier --write "lib/**/*.{js,ts}" "test/**/*.js" "qa/**/*.js" "bench/**/*.js"
87
+
88
+ lint: $(NODE_DEPS) ## check formatting and lint without changing files (what CI should run)
89
+ $(CLANG_FORMAT) --dry-run --Werror $(CPP_FILES)
90
+ $(RUFF) check bindings/python
91
+ $(RUFF) format --check bindings/python
92
+ cd bindings/node && $(NPX) prettier --check "lib/**/*.{js,ts}" "test/**/*.js" "qa/**/*.js" "bench/**/*.js"
93
+
94
+ # ---- build -------------------------------------------------------------------------------------
95
+ build: ## configure and compile the C++ core, tests and benchmarks
96
+ $(ENVRUN) cmake --preset $(PRESET)
97
+ $(ENVRUN) cmake --build --preset $(PRESET)
98
+
99
+ $(PY_STAMP): $(SRC_PY)
100
+ test -x $(PY) || $(VENV_CREATE)
101
+ $(PYBUILD_ENV) $(ENVRUN) $(PIP_INSTALL) --reinstall . pytest faker psutil
102
+ @touch $@
103
+
104
+ $(NODE_DEPS): bindings/node/package.json
105
+ cd bindings/node && $(NPM) install --no-audit --no-fund
106
+ @touch $@
107
+
108
+ $(NODE_ADDON): $(NODE_DEPS) $(SRC_NODE)
109
+ cd bindings/node && $(ENVRUN) $(NODE_BUILD)
110
+
111
+ # ---- tests -------------------------------------------------------------------------------------
112
+ test: test-core test-python test-node bench ## build, run all tests (C++, Python, Node), then the benchmarks
113
+
114
+ check: test-core test-python test-node ## all tests, no benchmarks
115
+
116
+ test-core: build
117
+ $(ENVRUN) ctest --preset $(PRESET)
118
+
119
+ test-python: $(PY_STAMP)
120
+ $(PY) -m pytest -q
121
+
122
+ test-node: $(NODE_ADDON)
123
+ cd bindings/node && $(NPM) test
124
+
125
+ # ---- benchmarks --------------------------------------------------------------------------------
126
+ bench: bench-core bench-python bench-node ## quick benchmarks for all three languages (BENCH_FILTER=. for everything)
127
+
128
+ bench-full: ## include the adversarial-input benchmarks
129
+ $(MAKE) bench BENCH_FILTER=.
130
+
131
+ bench-core: build
132
+ ./$(BUILD_DIR)/core/secrethider_bench$(EXE) --benchmark_min_time=0.2s --benchmark_filter='$(BENCH_FILTER)'
133
+
134
+ bench-python: $(PY_STAMP)
135
+ $(PY) bindings/python/bench/bench_redact.py
136
+
137
+ bench-node: $(NODE_ADDON)
138
+ cd bindings/node && node bench/bench.js
139
+
140
+ # ---- deeper checks -----------------------------------------------------------------------------
141
+ qa: build $(PY_STAMP) $(NODE_ADDON) ## leak checks (C++, Python, Node) and a long randomized property run
142
+ ./$(BUILD_DIR)/core/secrethider_leakcheck$(EXE) 3000
143
+ SECRETHIDER_PROP_ITERS=300000 ./$(BUILD_DIR)/core/secrethider_tests$(EXE) "property: random structured inputs keep all invariants"
144
+ $(PY) bindings/python/qa/leakcheck.py 3000
145
+ cd bindings/node && node --expose-gc qa/leakcheck.js 3000
146
+
147
+ clean: ## remove build outputs and the Python venv
148
+ rm -rf build bindings/node/build .venv bindings/node/node_modules
@@ -0,0 +1,109 @@
1
+ Metadata-Version: 2.1
2
+ Name: secrethider
3
+ Version: 0.1.0
4
+ Summary: Fast string and JSON processing backed by a C++ core (simdjson, simdutf)
5
+ License: MIT
6
+ Classifier: Programming Language :: C++
7
+ Classifier: Programming Language :: Python :: 3
8
+ Classifier: Typing :: Typed
9
+ Requires-Python: >=3.9
10
+ Provides-Extra: faker
11
+ Requires-Dist: faker>=24; extra == "faker"
12
+ Description-Content-Type: text/markdown
13
+
14
+ # secretHider
15
+
16
+ Fast **secret redaction**, log processing, string and JSON tools: one C++20 core, thin bindings for Python and Node.js.
17
+
18
+ - **Remove secrets from logs** in **plain text, logfmt and JSON lines**: AWS/GitHub/Slack/Stripe/Google keys, JWTs, PEM private keys, `Bearer` tokens, URL credentials, `password=...` fields, and optional PII (email, phone, card, SSN, IBAN, IPv4).
19
+ - **Tell it what is secret** with plain lists: known values, field names and formats (or one `secrets.txt` file), on top of the built-in patterns.
20
+ - **Choose what happens**: delete the whole field (`drop_field`), mask (full, typed, keep first/last N, preserve length, partial email/phone), stable salted hash, **realistic fake data via Faker** (Python `faker`, JS `@faker-js/faker`), or your own function. Partial masks never reveal more than 25% of a secret.
21
+ - **Stream it**: chunk-safe redaction for files, pipes and sockets (output equals one-shot output at every split point), a `secrethider-redact` command line, and in-place/`.gz` file helpers.
22
+ - **Fast**: about 600 MiB/s per core on clean logs, roughly 38x a hand-written Python `re` pipeline and comparable to V8's RegExp for a few simple patterns in Node ([numbers](docs/architecture/06-performance.md)). Built on [simdjson](https://github.com/simdjson/simdjson), [simdutf](https://github.com/simdutf/simdutf), [RE2](https://github.com/google/re2).
23
+ - Also: `json_get` (JSON Pointer), `json_minify`, `json_valid`, UTF-8 helpers.
24
+
25
+ ```sh
26
+ secrethider-redact --in-place --drop-fields --secrets-file secrets.txt app.log # a=1 password=x b=2 -> a=1 b=2
27
+ secrethider-redact --pii --mask fake app.jsonl --json-lines # fake emails/phones instead of [REDACTED]
28
+ ```
29
+
30
+ ```python
31
+ import secrethider
32
+ from secrethider import Mask, RedactorBuilder
33
+
34
+ r = (RedactorBuilder()
35
+ .enable("pii")
36
+ .secret_values(["my-known-secret"]) # also: secret_keys([...]), secret_formats([...])
37
+ .mask(Mask.keep_prefix(4)) # or Mask.drop_field(), secrethider.fake_mask(), ...
38
+ .build())
39
+
40
+ r.redact("login password=hunter2hunter2hunter2 key=AKIAIOSFODNN7EXAMPLE")
41
+ # 'login password=hunt**** key=AKIA****'
42
+ # (short secrets fall back to a full mask: 4 revealed characters of a 10-character password would exceed 25%)
43
+
44
+ # logging, files, streams
45
+ from secrethider.logging import RedactingFormatter
46
+ handler.setFormatter(RedactingFormatter(r, "%(levelname)s %(message)s"))
47
+ secrethider.redact_file(open("in.log"), open("out.log", "w"), r)
48
+ ```
49
+
50
+ ```js
51
+ const fk = require('secrethider');
52
+ const r = new fk.RedactorBuilder().mask(fk.Mask.keepPrefix(4)).build();
53
+ process.stdin.pipe(fk.createRedactStream(r)).pipe(process.stdout);
54
+ ```
55
+
56
+ ## Layout
57
+
58
+ ```
59
+ core/ C++20 library (no Python/Node headers), tests, benchmarks
60
+ bindings/python/ nanobind module + `secrethider` package (built via scikit-build-core, root pyproject.toml)
61
+ bindings/node/ Node-API addon (node-addon-api + cmake-js)
62
+ cmake/ shared CMake modules (dependencies, warnings)
63
+ docs/ architecture: interfaces, log processing, rules and masking, builds, performance, security
64
+ ```
65
+
66
+ ## Documentation
67
+
68
+ Start with the [guide to removing secrets from logs](docs/guides/removing-secrets-from-logs.md), or [docs/README.md](docs/README.md) for everything: [interfaces](docs/architecture/02-interfaces.md), [log processing pipeline](docs/architecture/03-log-processing.md),
69
+ [rules and masking](docs/architecture/04-rules-and-masking.md), [build and platforms](docs/architecture/05-build-and-platforms.md),
70
+ [performance](docs/architecture/06-performance.md), [security model](docs/architecture/07-security-model.md).
71
+
72
+ ## Develop
73
+
74
+ ```sh
75
+ make fix # format + auto-fix lint: C++ (clang-format), Python (ruff), JS/TS (prettier)
76
+ make test # build, run all tests (C++, Python, Node), then the benchmarks
77
+ make help # everything else: lint, check (tests only), bench, bench-full, qa (leak checks + long fuzz), clean
78
+ ```
79
+
80
+ `make` finds Visual Studio's C++ tools on Windows by itself. Without `make`, the underlying steps are:
81
+
82
+ ```sh
83
+ # C++ core: tests + benchmarks (Windows: use the dev-msvc preset from a VS developer prompt)
84
+ cmake --preset dev && cmake --build --preset dev && ctest --preset dev
85
+ ./build/dev/core/secrethider_bench # secrethider_bench.exe on Windows
86
+
87
+ # Python
88
+ python -m venv .venv && . .venv/bin/activate # Windows: .venv\Scripts\activate
89
+ pip install . pytest faker && pytest
90
+
91
+ # Node
92
+ cd bindings/node && npm install && npm run build && npm test
93
+ ```
94
+
95
+ `-DSECRETHIDER_WITH_RE2=OFF` skips RE2/abseil for a much faster build; custom `regex()` rules are then unavailable.
96
+
97
+ ## Cross-platform builds
98
+
99
+ - Python wheels: `cibuildwheel` (Linux manylinux/musllinux, macOS x86_64 + arm64, Windows), stable-ABI (`abi3`) wheels, via `.github/workflows/release.yml`.
100
+ - Node: Node-API is ABI-stable, so one binary per OS/arch; CI collects `prebuilds/<platform>-<arch>/secrethider.node`.
101
+ - SIMD is selected at run time by simdjson/simdutf, so no `-march=native` and one binary runs fast everywhere.
102
+
103
+ ## Limits
104
+
105
+ Redaction is defence in depth, not a guarantee: it finds known formats, sensitive key names and values you register. See the [security model](docs/architecture/07-security-model.md).
106
+
107
+ ## License
108
+
109
+ MIT, see [LICENSE](LICENSE).
@@ -0,0 +1,96 @@
1
+ # secretHider
2
+
3
+ Fast **secret redaction**, log processing, string and JSON tools: one C++20 core, thin bindings for Python and Node.js.
4
+
5
+ - **Remove secrets from logs** in **plain text, logfmt and JSON lines**: AWS/GitHub/Slack/Stripe/Google keys, JWTs, PEM private keys, `Bearer` tokens, URL credentials, `password=...` fields, and optional PII (email, phone, card, SSN, IBAN, IPv4).
6
+ - **Tell it what is secret** with plain lists: known values, field names and formats (or one `secrets.txt` file), on top of the built-in patterns.
7
+ - **Choose what happens**: delete the whole field (`drop_field`), mask (full, typed, keep first/last N, preserve length, partial email/phone), stable salted hash, **realistic fake data via Faker** (Python `faker`, JS `@faker-js/faker`), or your own function. Partial masks never reveal more than 25% of a secret.
8
+ - **Stream it**: chunk-safe redaction for files, pipes and sockets (output equals one-shot output at every split point), a `secrethider-redact` command line, and in-place/`.gz` file helpers.
9
+ - **Fast**: about 600 MiB/s per core on clean logs, roughly 38x a hand-written Python `re` pipeline and comparable to V8's RegExp for a few simple patterns in Node ([numbers](docs/architecture/06-performance.md)). Built on [simdjson](https://github.com/simdjson/simdjson), [simdutf](https://github.com/simdutf/simdutf), [RE2](https://github.com/google/re2).
10
+ - Also: `json_get` (JSON Pointer), `json_minify`, `json_valid`, UTF-8 helpers.
11
+
12
+ ```sh
13
+ secrethider-redact --in-place --drop-fields --secrets-file secrets.txt app.log # a=1 password=x b=2 -> a=1 b=2
14
+ secrethider-redact --pii --mask fake app.jsonl --json-lines # fake emails/phones instead of [REDACTED]
15
+ ```
16
+
17
+ ```python
18
+ import secrethider
19
+ from secrethider import Mask, RedactorBuilder
20
+
21
+ r = (RedactorBuilder()
22
+ .enable("pii")
23
+ .secret_values(["my-known-secret"]) # also: secret_keys([...]), secret_formats([...])
24
+ .mask(Mask.keep_prefix(4)) # or Mask.drop_field(), secrethider.fake_mask(), ...
25
+ .build())
26
+
27
+ r.redact("login password=hunter2hunter2hunter2 key=AKIAIOSFODNN7EXAMPLE")
28
+ # 'login password=hunt**** key=AKIA****'
29
+ # (short secrets fall back to a full mask: 4 revealed characters of a 10-character password would exceed 25%)
30
+
31
+ # logging, files, streams
32
+ from secrethider.logging import RedactingFormatter
33
+ handler.setFormatter(RedactingFormatter(r, "%(levelname)s %(message)s"))
34
+ secrethider.redact_file(open("in.log"), open("out.log", "w"), r)
35
+ ```
36
+
37
+ ```js
38
+ const fk = require('secrethider');
39
+ const r = new fk.RedactorBuilder().mask(fk.Mask.keepPrefix(4)).build();
40
+ process.stdin.pipe(fk.createRedactStream(r)).pipe(process.stdout);
41
+ ```
42
+
43
+ ## Layout
44
+
45
+ ```
46
+ core/ C++20 library (no Python/Node headers), tests, benchmarks
47
+ bindings/python/ nanobind module + `secrethider` package (built via scikit-build-core, root pyproject.toml)
48
+ bindings/node/ Node-API addon (node-addon-api + cmake-js)
49
+ cmake/ shared CMake modules (dependencies, warnings)
50
+ docs/ architecture: interfaces, log processing, rules and masking, builds, performance, security
51
+ ```
52
+
53
+ ## Documentation
54
+
55
+ Start with the [guide to removing secrets from logs](docs/guides/removing-secrets-from-logs.md), or [docs/README.md](docs/README.md) for everything: [interfaces](docs/architecture/02-interfaces.md), [log processing pipeline](docs/architecture/03-log-processing.md),
56
+ [rules and masking](docs/architecture/04-rules-and-masking.md), [build and platforms](docs/architecture/05-build-and-platforms.md),
57
+ [performance](docs/architecture/06-performance.md), [security model](docs/architecture/07-security-model.md).
58
+
59
+ ## Develop
60
+
61
+ ```sh
62
+ make fix # format + auto-fix lint: C++ (clang-format), Python (ruff), JS/TS (prettier)
63
+ make test # build, run all tests (C++, Python, Node), then the benchmarks
64
+ make help # everything else: lint, check (tests only), bench, bench-full, qa (leak checks + long fuzz), clean
65
+ ```
66
+
67
+ `make` finds Visual Studio's C++ tools on Windows by itself. Without `make`, the underlying steps are:
68
+
69
+ ```sh
70
+ # C++ core: tests + benchmarks (Windows: use the dev-msvc preset from a VS developer prompt)
71
+ cmake --preset dev && cmake --build --preset dev && ctest --preset dev
72
+ ./build/dev/core/secrethider_bench # secrethider_bench.exe on Windows
73
+
74
+ # Python
75
+ python -m venv .venv && . .venv/bin/activate # Windows: .venv\Scripts\activate
76
+ pip install . pytest faker && pytest
77
+
78
+ # Node
79
+ cd bindings/node && npm install && npm run build && npm test
80
+ ```
81
+
82
+ `-DSECRETHIDER_WITH_RE2=OFF` skips RE2/abseil for a much faster build; custom `regex()` rules are then unavailable.
83
+
84
+ ## Cross-platform builds
85
+
86
+ - Python wheels: `cibuildwheel` (Linux manylinux/musllinux, macOS x86_64 + arm64, Windows), stable-ABI (`abi3`) wheels, via `.github/workflows/release.yml`.
87
+ - Node: Node-API is ABI-stable, so one binary per OS/arch; CI collects `prebuilds/<platform>-<arch>/secrethider.node`.
88
+ - SIMD is selected at run time by simdjson/simdutf, so no `-march=native` and one binary runs fast everywhere.
89
+
90
+ ## Limits
91
+
92
+ Redaction is defence in depth, not a guarantee: it finds known formats, sensitive key names and values you register. See the [security model](docs/architecture/07-security-model.md).
93
+
94
+ ## License
95
+
96
+ MIT, see [LICENSE](LICENSE).
@@ -0,0 +1,62 @@
1
+ cmake_minimum_required(VERSION 3.25)
2
+ cmake_policy(SET CMP0091 NEW) # MSVC runtime selection (cmake-js)
3
+ cmake_policy(SET CMP0042 NEW) # macOS @rpath
4
+
5
+ # npm's `rc` package installs node_modules/.bin/rc, which shadows the Windows
6
+ # resource compiler (rc.exe) when cmake-js runs under npm. Drop .bin from PATH.
7
+ if(WIN32)
8
+ set(_fk_path "$ENV{PATH}")
9
+ list(FILTER _fk_path EXCLUDE REGEX "node_modules[/\\\\]\\.bin")
10
+ set(ENV{PATH} "${_fk_path}")
11
+ endif()
12
+
13
+ project(secrethider_node LANGUAGES C CXX)
14
+
15
+ set(CMAKE_CXX_STANDARD 20)
16
+ set(CMAKE_CXX_STANDARD_REQUIRED ON)
17
+ set(CMAKE_CXX_EXTENSIONS OFF)
18
+ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
19
+ if(MSVC)
20
+ add_compile_options(/EHsc /utf-8)
21
+ endif()
22
+ set(CMAKE_CXX_VISIBILITY_PRESET hidden)
23
+ set(CMAKE_VISIBILITY_INLINES_HIDDEN ON)
24
+ if(NOT CMAKE_BUILD_TYPE)
25
+ set(CMAKE_BUILD_TYPE Release)
26
+ endif()
27
+
28
+ # Reuse the shared C++ core (this tree is built by cmake-js, not the top-level project).
29
+ set(REPO_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..)
30
+ list(APPEND CMAKE_MODULE_PATH ${REPO_ROOT}/cmake)
31
+ option(SECRETHIDER_WITH_RE2 "Custom regex rules via RE2 (pulls abseil)" ON)
32
+ # cmake-js builds addons against the static MSVC runtime (/MT); abseil defaults to /MD.
33
+ set(ABSL_MSVC_STATIC_RUNTIME ON CACHE BOOL "" FORCE)
34
+ include(Warnings)
35
+ include(Dependencies)
36
+ add_subdirectory(${REPO_ROOT}/core ${CMAKE_BINARY_DIR}/core EXCLUDE_FROM_ALL)
37
+
38
+ # Node-API (node-addon-api headers from node_modules)
39
+ execute_process(COMMAND node -p "require('node-addon-api').include_dir"
40
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
41
+ OUTPUT_VARIABLE NODE_ADDON_API_DIR OUTPUT_STRIP_TRAILING_WHITESPACE)
42
+ string(REPLACE "\"" "" NODE_ADDON_API_DIR "${NODE_ADDON_API_DIR}")
43
+
44
+ add_library(secrethider_node SHARED src/addon.cpp src/redact.cpp ${CMAKE_JS_SRC})
45
+ set_target_properties(secrethider_node PROPERTIES
46
+ PREFIX "" SUFFIX ".node" OUTPUT_NAME "secrethider"
47
+ # same location for Ninja and Visual Studio generators: build/Release/secrethider.node
48
+ RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}/Release
49
+ LIBRARY_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}/Release)
50
+ target_include_directories(secrethider_node PRIVATE ${CMAKE_JS_INC} ${NODE_ADDON_API_DIR})
51
+ target_link_libraries(secrethider_node PRIVATE secrethider::core ${CMAKE_JS_LIB})
52
+ target_compile_definitions(secrethider_node PRIVATE NAPI_VERSION=8 NAPI_CPP_EXCEPTIONS)
53
+ secrethider_set_warnings(secrethider_node)
54
+
55
+ if(MSVC)
56
+ target_link_libraries(secrethider_node PRIVATE delayimp) # needed by cmake-js's win_delay_load_hook
57
+ endif()
58
+
59
+ if(MSVC AND CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET)
60
+ execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF}
61
+ /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS})
62
+ endif()
@@ -0,0 +1,6 @@
1
+ nanobind_add_module(_core STABLE_ABI NB_STATIC src/module.cpp src/redact.cpp)
2
+ target_link_libraries(_core PRIVATE secrethider::core)
3
+ secrethider_set_warnings(_core)
4
+
5
+ # Only the extension goes into the wheel (not simdjson/simdutf/re2/abseil headers or libs).
6
+ install(TARGETS _core LIBRARY DESTINATION secrethider COMPONENT python_module)