secrethider 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- secrethider-0.1.0/.clang-format +3 -0
- secrethider-0.1.0/.editorconfig +9 -0
- secrethider-0.1.0/.gitignore +22 -0
- secrethider-0.1.0/.pre-commit-config.yaml +19 -0
- secrethider-0.1.0/.ubsan-ignorelist +4 -0
- secrethider-0.1.0/CMakeLists.txt +51 -0
- secrethider-0.1.0/CMakePresets.json +44 -0
- secrethider-0.1.0/LICENSE +21 -0
- secrethider-0.1.0/Makefile +148 -0
- secrethider-0.1.0/PKG-INFO +109 -0
- secrethider-0.1.0/README.md +96 -0
- secrethider-0.1.0/bindings/node/CMakeLists.txt +62 -0
- secrethider-0.1.0/bindings/python/CMakeLists.txt +6 -0
- secrethider-0.1.0/bindings/python/bench/bench_redact.py +89 -0
- secrethider-0.1.0/bindings/python/qa/leakcheck.py +195 -0
- secrethider-0.1.0/bindings/python/secrethider/__init__.py +57 -0
- secrethider-0.1.0/bindings/python/secrethider/__main__.py +3 -0
- secrethider-0.1.0/bindings/python/secrethider/_core.pyi +93 -0
- secrethider-0.1.0/bindings/python/secrethider/cli.py +163 -0
- secrethider-0.1.0/bindings/python/secrethider/fake.py +122 -0
- secrethider-0.1.0/bindings/python/secrethider/logging.py +39 -0
- secrethider-0.1.0/bindings/python/secrethider/py.typed +0 -0
- secrethider-0.1.0/bindings/python/secrethider/redact.py +216 -0
- secrethider-0.1.0/bindings/python/src/module.cpp +37 -0
- secrethider-0.1.0/bindings/python/src/redact.cpp +216 -0
- secrethider-0.1.0/bindings/python/tests/test_features.py +156 -0
- secrethider-0.1.0/bindings/python/tests/test_redact.py +163 -0
- secrethider-0.1.0/bindings/python/tests/test_secrethider.py +22 -0
- secrethider-0.1.0/cmake/Dependencies.cmake +70 -0
- secrethider-0.1.0/cmake/Warnings.cmake +7 -0
- secrethider-0.1.0/core/CMakeLists.txt +71 -0
- secrethider-0.1.0/core/bench/bench.cpp +29 -0
- secrethider-0.1.0/core/bench/bench_redact.cpp +391 -0
- secrethider-0.1.0/core/include/secrethider/json.hpp +19 -0
- secrethider-0.1.0/core/include/secrethider/redact.hpp +197 -0
- secrethider-0.1.0/core/include/secrethider/strings.hpp +25 -0
- secrethider-0.1.0/core/src/json.cpp +35 -0
- secrethider-0.1.0/core/src/redact/aho_corasick.cpp +217 -0
- secrethider-0.1.0/core/src/redact/aho_corasick.hpp +110 -0
- secrethider-0.1.0/core/src/redact/detectors.cpp +709 -0
- secrethider-0.1.0/core/src/redact/impl.hpp +121 -0
- secrethider-0.1.0/core/src/redact/json_redact.cpp +188 -0
- secrethider-0.1.0/core/src/redact/mask.cpp +202 -0
- secrethider-0.1.0/core/src/redact/redactor.cpp +488 -0
- secrethider-0.1.0/core/src/redact/regex_re2.cpp +68 -0
- secrethider-0.1.0/core/src/redact/sha256.hpp +155 -0
- secrethider-0.1.0/core/src/redact/stream.cpp +107 -0
- secrethider-0.1.0/core/src/strings.cpp +73 -0
- secrethider-0.1.0/core/tests/fuzz_redact.cpp +15 -0
- secrethider-0.1.0/core/tests/leakcheck.cpp +321 -0
- secrethider-0.1.0/core/tests/property.hpp +209 -0
- secrethider-0.1.0/core/tests/qa_repro.cpp +27 -0
- secrethider-0.1.0/core/tests/test_aho_corasick.cpp +149 -0
- secrethider-0.1.0/core/tests/test_features.cpp +175 -0
- secrethider-0.1.0/core/tests/test_json.cpp +25 -0
- secrethider-0.1.0/core/tests/test_json_redact.cpp +57 -0
- secrethider-0.1.0/core/tests/test_mask.cpp +62 -0
- secrethider-0.1.0/core/tests/test_property.cpp +157 -0
- secrethider-0.1.0/core/tests/test_redact.cpp +143 -0
- secrethider-0.1.0/core/tests/test_stream.cpp +118 -0
- secrethider-0.1.0/core/tests/test_strings.cpp +29 -0
- secrethider-0.1.0/docs/README.md +17 -0
- secrethider-0.1.0/docs/adr/0001-matching-engine.md +37 -0
- secrethider-0.1.0/docs/adr/0002-streaming-semantics.md +35 -0
- secrethider-0.1.0/docs/architecture/01-overview.md +55 -0
- secrethider-0.1.0/docs/architecture/02-interfaces.md +250 -0
- secrethider-0.1.0/docs/architecture/03-log-processing.md +112 -0
- secrethider-0.1.0/docs/architecture/04-rules-and-masking.md +71 -0
- secrethider-0.1.0/docs/architecture/05-build-and-platforms.md +72 -0
- secrethider-0.1.0/docs/architecture/06-performance.md +84 -0
- secrethider-0.1.0/docs/architecture/07-security-model.md +38 -0
- secrethider-0.1.0/docs/architecture/08-testing-and-qa.md +68 -0
- secrethider-0.1.0/docs/guides/removing-secrets-from-logs.md +106 -0
- secrethider-0.1.0/pyproject.toml +62 -0
- secrethider-0.1.0/scripts/msvc-env.bat +22 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
build/
|
|
2
|
+
out/
|
|
3
|
+
dist/
|
|
4
|
+
.venv/
|
|
5
|
+
__pycache__/
|
|
6
|
+
*.egg-info/
|
|
7
|
+
*.pyd
|
|
8
|
+
*.so
|
|
9
|
+
*.dylib
|
|
10
|
+
*.node
|
|
11
|
+
node_modules/
|
|
12
|
+
prebuilds/
|
|
13
|
+
.cache/
|
|
14
|
+
compile_commands.json
|
|
15
|
+
.vs/
|
|
16
|
+
.pytest_cache/
|
|
17
|
+
.ruff_cache/
|
|
18
|
+
|
|
19
|
+
# local secrets (registry tokens): never commit
|
|
20
|
+
.env
|
|
21
|
+
.env.*
|
|
22
|
+
!.env.example
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
3
|
+
rev: v5.0.0
|
|
4
|
+
hooks:
|
|
5
|
+
- id: end-of-file-fixer
|
|
6
|
+
- id: trailing-whitespace
|
|
7
|
+
- id: check-yaml
|
|
8
|
+
- id: check-json
|
|
9
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
10
|
+
rev: v0.8.0
|
|
11
|
+
hooks:
|
|
12
|
+
- id: ruff
|
|
13
|
+
args: [--fix]
|
|
14
|
+
- id: ruff-format
|
|
15
|
+
- repo: https://github.com/pre-commit/mirrors-clang-format
|
|
16
|
+
rev: v19.1.0
|
|
17
|
+
hooks:
|
|
18
|
+
- id: clang-format
|
|
19
|
+
types_or: [c++]
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
cmake_minimum_required(VERSION 3.25)
|
|
2
|
+
project(secrethider VERSION 0.1.0 LANGUAGES CXX)
|
|
3
|
+
|
|
4
|
+
set(CMAKE_CXX_STANDARD 20)
|
|
5
|
+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
|
6
|
+
set(CMAKE_CXX_EXTENSIONS OFF)
|
|
7
|
+
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
|
8
|
+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
|
|
9
|
+
set(CMAKE_CXX_VISIBILITY_PRESET hidden)
|
|
10
|
+
set(CMAKE_VISIBILITY_INLINES_HIDDEN ON)
|
|
11
|
+
|
|
12
|
+
if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
|
|
13
|
+
set(CMAKE_BUILD_TYPE Release)
|
|
14
|
+
endif()
|
|
15
|
+
|
|
16
|
+
option(SECRETHIDER_BUILD_TESTS "Build C++ unit tests" ${PROJECT_IS_TOP_LEVEL})
|
|
17
|
+
option(SECRETHIDER_BUILD_BENCH "Build C++ benchmarks" OFF)
|
|
18
|
+
option(SECRETHIDER_BUILD_PYTHON "Build the Python extension" OFF)
|
|
19
|
+
option(SECRETHIDER_SANITIZE "Enable ASan+UBSan (gcc/clang)" OFF)
|
|
20
|
+
option(SECRETHIDER_BUILD_FUZZ "Build the libFuzzer target (clang)" OFF)
|
|
21
|
+
option(SECRETHIDER_WITH_RE2 "Custom regex rules via RE2 (pulls abseil)" ON)
|
|
22
|
+
|
|
23
|
+
if(MSVC)
|
|
24
|
+
add_compile_options(/EHsc /utf-8)
|
|
25
|
+
endif()
|
|
26
|
+
|
|
27
|
+
if(SECRETHIDER_BUILD_TESTS)
|
|
28
|
+
enable_testing()
|
|
29
|
+
endif()
|
|
30
|
+
|
|
31
|
+
# Sanitizers must reach every target (static libraries need them at the final link, and the
|
|
32
|
+
# dependencies should be instrumented too), so they are global rather than per target.
|
|
33
|
+
if(SECRETHIDER_SANITIZE AND NOT MSVC)
|
|
34
|
+
add_compile_options(-fsanitize=address,undefined -fno-omit-frame-pointer)
|
|
35
|
+
add_link_options(-fsanitize=address,undefined)
|
|
36
|
+
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
|
|
37
|
+
# third-party sources are not ours to fix (RE2 constructs a zero-length array at a null pointer)
|
|
38
|
+
add_compile_options(-fsanitize-ignorelist=${CMAKE_CURRENT_SOURCE_DIR}/.ubsan-ignorelist)
|
|
39
|
+
endif()
|
|
40
|
+
endif()
|
|
41
|
+
|
|
42
|
+
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
|
|
43
|
+
include(Warnings)
|
|
44
|
+
include(Dependencies)
|
|
45
|
+
|
|
46
|
+
add_subdirectory(core)
|
|
47
|
+
|
|
48
|
+
if(SECRETHIDER_BUILD_PYTHON)
|
|
49
|
+
add_subdirectory(bindings/python)
|
|
50
|
+
endif()
|
|
51
|
+
# The Node addon is built by cmake-js from bindings/node/CMakeLists.txt.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 6,
|
|
3
|
+
"configurePresets": [
|
|
4
|
+
{
|
|
5
|
+
"name": "dev",
|
|
6
|
+
"displayName": "Dev (tests + bench, Release)",
|
|
7
|
+
"binaryDir": "${sourceDir}/build/dev",
|
|
8
|
+
"generator": "Ninja",
|
|
9
|
+
"cacheVariables": {
|
|
10
|
+
"CMAKE_BUILD_TYPE": "Release",
|
|
11
|
+
"SECRETHIDER_BUILD_TESTS": "ON",
|
|
12
|
+
"SECRETHIDER_BUILD_BENCH": "ON"
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
"name": "dev-msvc",
|
|
17
|
+
"displayName": "Dev with MSVC (Windows; run from a VS developer prompt)",
|
|
18
|
+
"inherits": "dev",
|
|
19
|
+
"binaryDir": "${sourceDir}/build/dev-msvc",
|
|
20
|
+
"cacheVariables": { "CMAKE_CXX_COMPILER": "cl", "CMAKE_C_COMPILER": "cl" },
|
|
21
|
+
"condition": { "type": "equals", "lhs": "${hostSystemName}", "rhs": "Windows" }
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"name": "asan",
|
|
25
|
+
"displayName": "Debug + ASan/UBSan (Linux/macOS)",
|
|
26
|
+
"inherits": "dev",
|
|
27
|
+
"binaryDir": "${sourceDir}/build/asan",
|
|
28
|
+
"cacheVariables": {
|
|
29
|
+
"CMAKE_BUILD_TYPE": "Debug",
|
|
30
|
+
"SECRETHIDER_BUILD_BENCH": "OFF",
|
|
31
|
+
"SECRETHIDER_SANITIZE": "ON"
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
],
|
|
35
|
+
"buildPresets": [
|
|
36
|
+
{ "name": "dev", "configurePreset": "dev" },
|
|
37
|
+
{ "name": "dev-msvc", "configurePreset": "dev-msvc" },
|
|
38
|
+
{ "name": "asan", "configurePreset": "asan" }
|
|
39
|
+
],
|
|
40
|
+
"testPresets": [
|
|
41
|
+
{ "name": "dev", "configurePreset": "dev", "output": { "outputOnFailure": true } },
|
|
42
|
+
{ "name": "dev-msvc", "configurePreset": "dev-msvc", "output": { "outputOnFailure": true } }
|
|
43
|
+
]
|
|
44
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 NoobforAl
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
# secrethider developer commands. Run `make help`.
|
|
2
|
+
#
|
|
3
|
+
# make fix format + auto-fix lint: C++ (clang-format), Python (ruff), JavaScript/TypeScript (prettier)
|
|
4
|
+
# make test build everything, run every test suite (C++, Python, Node), then the benchmarks
|
|
5
|
+
#
|
|
6
|
+
# Needs: cmake, ninja, a C++20 compiler, Python 3.9+ (uv is used when installed), Node 18+, clang-format.
|
|
7
|
+
# Windows: GNU make with Git's sh. Visual Studio's C++ tools are found automatically (scripts/msvc-env.bat).
|
|
8
|
+
|
|
9
|
+
.DEFAULT_GOAL := help
|
|
10
|
+
|
|
11
|
+
# ---- platform ----------------------------------------------------------------------------------
|
|
12
|
+
ifeq ($(OS),Windows_NT)
|
|
13
|
+
PRESET ?= dev-msvc
|
|
14
|
+
VENV_BIN := .venv/Scripts
|
|
15
|
+
EXE := .exe
|
|
16
|
+
ENVRUN := $(CURDIR)/scripts/msvc-env.bat
|
|
17
|
+
PYBUILD_ENV := CMAKE_GENERATOR=Ninja CC=cl CXX=cl
|
|
18
|
+
# the extensionless npm/npx are shell scripts that pick WSL's bash under Git's sh: use the .cmd shims
|
|
19
|
+
NPM := npm.cmd
|
|
20
|
+
NPX := npx.cmd
|
|
21
|
+
NODE_BUILD := npx.cmd cmake-js compile -G Ninja --CDCMAKE_C_COMPILER=cl --CDCMAKE_CXX_COMPILER=cl
|
|
22
|
+
else
|
|
23
|
+
PRESET ?= dev
|
|
24
|
+
VENV_BIN := .venv/bin
|
|
25
|
+
EXE :=
|
|
26
|
+
ENVRUN :=
|
|
27
|
+
PYBUILD_ENV := CMAKE_GENERATOR=Ninja
|
|
28
|
+
NPM := npm
|
|
29
|
+
NPX := npx
|
|
30
|
+
NODE_BUILD := npx cmake-js compile
|
|
31
|
+
endif
|
|
32
|
+
|
|
33
|
+
# ---- tools -------------------------------------------------------------------------------------
|
|
34
|
+
PYTHON ?= python
|
|
35
|
+
CLANG_FORMAT ?= clang-format
|
|
36
|
+
HAVE_UV := $(shell command -v uv 2>/dev/null)
|
|
37
|
+
HAVE_RUFF := $(shell command -v ruff 2>/dev/null)
|
|
38
|
+
PY := $(VENV_BIN)/python$(EXE)
|
|
39
|
+
export UV_LINK_MODE := copy
|
|
40
|
+
ifdef HAVE_UV
|
|
41
|
+
VENV_CREATE := uv venv .venv
|
|
42
|
+
PIP_INSTALL := uv pip install --python $(PY)
|
|
43
|
+
else
|
|
44
|
+
VENV_CREATE := $(PYTHON) -m venv .venv
|
|
45
|
+
PIP_INSTALL := $(PY) -m pip install
|
|
46
|
+
endif
|
|
47
|
+
ifdef HAVE_RUFF
|
|
48
|
+
RUFF ?= ruff
|
|
49
|
+
else ifdef HAVE_UV
|
|
50
|
+
RUFF ?= uv tool run ruff
|
|
51
|
+
else
|
|
52
|
+
RUFF ?= $(PYTHON) -m ruff
|
|
53
|
+
endif
|
|
54
|
+
|
|
55
|
+
# ---- sources (tracked or untracked-but-not-ignored, so it works before the first commit) -------
|
|
56
|
+
GIT_FILES = git ls-files --cached --others --exclude-standard
|
|
57
|
+
CPP_FILES := $(shell $(GIT_FILES) -- '*.cpp' '*.hpp' '*.h')
|
|
58
|
+
SRC_CORE := $(shell $(GIT_FILES) -- core cmake CMakeLists.txt)
|
|
59
|
+
SRC_PY := $(SRC_CORE) $(shell $(GIT_FILES) -- bindings/python pyproject.toml README.md)
|
|
60
|
+
SRC_NODE := $(SRC_CORE) $(shell $(GIT_FILES) -- bindings/node/src bindings/node/CMakeLists.txt bindings/node/package.json)
|
|
61
|
+
|
|
62
|
+
BUILD_DIR := build/$(PRESET)
|
|
63
|
+
BENCH_FILTER ?= -Pathological
|
|
64
|
+
|
|
65
|
+
NODE_DEPS := bindings/node/node_modules/.installed
|
|
66
|
+
NODE_ADDON := bindings/node/build/Release/secrethider.node
|
|
67
|
+
PY_STAMP := .venv/.installed
|
|
68
|
+
|
|
69
|
+
.PHONY: help fix fix-cpp fix-python fix-js lint test check bench bench-full build \
|
|
70
|
+
test-core test-python test-node bench-core bench-python bench-node qa clean
|
|
71
|
+
|
|
72
|
+
help: ## list the commands
|
|
73
|
+
@grep -E '^[a-zA-Z_-]+:.*## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*## "}; {printf " make %-13s %s\n", $$1, $$2}'
|
|
74
|
+
|
|
75
|
+
# ---- format / lint -----------------------------------------------------------------------------
|
|
76
|
+
fix: fix-cpp fix-python fix-js ## format and auto-fix lint in place (C++, Python, JS/TS)
|
|
77
|
+
|
|
78
|
+
fix-cpp:
|
|
79
|
+
$(CLANG_FORMAT) -i $(CPP_FILES)
|
|
80
|
+
|
|
81
|
+
fix-python:
|
|
82
|
+
$(RUFF) check --fix bindings/python
|
|
83
|
+
$(RUFF) format bindings/python
|
|
84
|
+
|
|
85
|
+
fix-js: $(NODE_DEPS)
|
|
86
|
+
cd bindings/node && $(NPX) prettier --write "lib/**/*.{js,ts}" "test/**/*.js" "qa/**/*.js" "bench/**/*.js"
|
|
87
|
+
|
|
88
|
+
lint: $(NODE_DEPS) ## check formatting and lint without changing files (what CI should run)
|
|
89
|
+
$(CLANG_FORMAT) --dry-run --Werror $(CPP_FILES)
|
|
90
|
+
$(RUFF) check bindings/python
|
|
91
|
+
$(RUFF) format --check bindings/python
|
|
92
|
+
cd bindings/node && $(NPX) prettier --check "lib/**/*.{js,ts}" "test/**/*.js" "qa/**/*.js" "bench/**/*.js"
|
|
93
|
+
|
|
94
|
+
# ---- build -------------------------------------------------------------------------------------
|
|
95
|
+
build: ## configure and compile the C++ core, tests and benchmarks
|
|
96
|
+
$(ENVRUN) cmake --preset $(PRESET)
|
|
97
|
+
$(ENVRUN) cmake --build --preset $(PRESET)
|
|
98
|
+
|
|
99
|
+
$(PY_STAMP): $(SRC_PY)
|
|
100
|
+
test -x $(PY) || $(VENV_CREATE)
|
|
101
|
+
$(PYBUILD_ENV) $(ENVRUN) $(PIP_INSTALL) --reinstall . pytest faker psutil
|
|
102
|
+
@touch $@
|
|
103
|
+
|
|
104
|
+
$(NODE_DEPS): bindings/node/package.json
|
|
105
|
+
cd bindings/node && $(NPM) install --no-audit --no-fund
|
|
106
|
+
@touch $@
|
|
107
|
+
|
|
108
|
+
$(NODE_ADDON): $(NODE_DEPS) $(SRC_NODE)
|
|
109
|
+
cd bindings/node && $(ENVRUN) $(NODE_BUILD)
|
|
110
|
+
|
|
111
|
+
# ---- tests -------------------------------------------------------------------------------------
|
|
112
|
+
test: test-core test-python test-node bench ## build, run all tests (C++, Python, Node), then the benchmarks
|
|
113
|
+
|
|
114
|
+
check: test-core test-python test-node ## all tests, no benchmarks
|
|
115
|
+
|
|
116
|
+
test-core: build
|
|
117
|
+
$(ENVRUN) ctest --preset $(PRESET)
|
|
118
|
+
|
|
119
|
+
test-python: $(PY_STAMP)
|
|
120
|
+
$(PY) -m pytest -q
|
|
121
|
+
|
|
122
|
+
test-node: $(NODE_ADDON)
|
|
123
|
+
cd bindings/node && $(NPM) test
|
|
124
|
+
|
|
125
|
+
# ---- benchmarks --------------------------------------------------------------------------------
|
|
126
|
+
bench: bench-core bench-python bench-node ## quick benchmarks for all three languages (BENCH_FILTER=. for everything)
|
|
127
|
+
|
|
128
|
+
bench-full: ## include the adversarial-input benchmarks
|
|
129
|
+
$(MAKE) bench BENCH_FILTER=.
|
|
130
|
+
|
|
131
|
+
bench-core: build
|
|
132
|
+
./$(BUILD_DIR)/core/secrethider_bench$(EXE) --benchmark_min_time=0.2s --benchmark_filter='$(BENCH_FILTER)'
|
|
133
|
+
|
|
134
|
+
bench-python: $(PY_STAMP)
|
|
135
|
+
$(PY) bindings/python/bench/bench_redact.py
|
|
136
|
+
|
|
137
|
+
bench-node: $(NODE_ADDON)
|
|
138
|
+
cd bindings/node && node bench/bench.js
|
|
139
|
+
|
|
140
|
+
# ---- deeper checks -----------------------------------------------------------------------------
|
|
141
|
+
qa: build $(PY_STAMP) $(NODE_ADDON) ## leak checks (C++, Python, Node) and a long randomized property run
|
|
142
|
+
./$(BUILD_DIR)/core/secrethider_leakcheck$(EXE) 3000
|
|
143
|
+
SECRETHIDER_PROP_ITERS=300000 ./$(BUILD_DIR)/core/secrethider_tests$(EXE) "property: random structured inputs keep all invariants"
|
|
144
|
+
$(PY) bindings/python/qa/leakcheck.py 3000
|
|
145
|
+
cd bindings/node && node --expose-gc qa/leakcheck.js 3000
|
|
146
|
+
|
|
147
|
+
clean: ## remove build outputs and the Python venv
|
|
148
|
+
rm -rf build bindings/node/build .venv bindings/node/node_modules
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: secrethider
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Fast string and JSON processing backed by a C++ core (simdjson, simdutf)
|
|
5
|
+
License: MIT
|
|
6
|
+
Classifier: Programming Language :: C++
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: Typing :: Typed
|
|
9
|
+
Requires-Python: >=3.9
|
|
10
|
+
Provides-Extra: faker
|
|
11
|
+
Requires-Dist: faker>=24; extra == "faker"
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# secretHider
|
|
15
|
+
|
|
16
|
+
Fast **secret redaction**, log processing, string and JSON tools: one C++20 core, thin bindings for Python and Node.js.
|
|
17
|
+
|
|
18
|
+
- **Remove secrets from logs** in **plain text, logfmt and JSON lines**: AWS/GitHub/Slack/Stripe/Google keys, JWTs, PEM private keys, `Bearer` tokens, URL credentials, `password=...` fields, and optional PII (email, phone, card, SSN, IBAN, IPv4).
|
|
19
|
+
- **Tell it what is secret** with plain lists: known values, field names and formats (or one `secrets.txt` file), on top of the built-in patterns.
|
|
20
|
+
- **Choose what happens**: delete the whole field (`drop_field`), mask (full, typed, keep first/last N, preserve length, partial email/phone), stable salted hash, **realistic fake data via Faker** (Python `faker`, JS `@faker-js/faker`), or your own function. Partial masks never reveal more than 25% of a secret.
|
|
21
|
+
- **Stream it**: chunk-safe redaction for files, pipes and sockets (output equals one-shot output at every split point), a `secrethider-redact` command line, and in-place/`.gz` file helpers.
|
|
22
|
+
- **Fast**: about 600 MiB/s per core on clean logs, roughly 38x a hand-written Python `re` pipeline and comparable to V8's RegExp for a few simple patterns in Node ([numbers](docs/architecture/06-performance.md)). Built on [simdjson](https://github.com/simdjson/simdjson), [simdutf](https://github.com/simdutf/simdutf), [RE2](https://github.com/google/re2).
|
|
23
|
+
- Also: `json_get` (JSON Pointer), `json_minify`, `json_valid`, UTF-8 helpers.
|
|
24
|
+
|
|
25
|
+
```sh
|
|
26
|
+
secrethider-redact --in-place --drop-fields --secrets-file secrets.txt app.log # a=1 password=x b=2 -> a=1 b=2
|
|
27
|
+
secrethider-redact --pii --mask fake app.jsonl --json-lines # fake emails/phones instead of [REDACTED]
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
import secrethider
|
|
32
|
+
from secrethider import Mask, RedactorBuilder
|
|
33
|
+
|
|
34
|
+
r = (RedactorBuilder()
|
|
35
|
+
.enable("pii")
|
|
36
|
+
.secret_values(["my-known-secret"]) # also: secret_keys([...]), secret_formats([...])
|
|
37
|
+
.mask(Mask.keep_prefix(4)) # or Mask.drop_field(), secrethider.fake_mask(), ...
|
|
38
|
+
.build())
|
|
39
|
+
|
|
40
|
+
r.redact("login password=hunter2hunter2hunter2 key=AKIAIOSFODNN7EXAMPLE")
|
|
41
|
+
# 'login password=hunt**** key=AKIA****'
|
|
42
|
+
# (short secrets fall back to a full mask: 4 revealed characters of a 10-character password would exceed 25%)
|
|
43
|
+
|
|
44
|
+
# logging, files, streams
|
|
45
|
+
from secrethider.logging import RedactingFormatter
|
|
46
|
+
handler.setFormatter(RedactingFormatter(r, "%(levelname)s %(message)s"))
|
|
47
|
+
secrethider.redact_file(open("in.log"), open("out.log", "w"), r)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
```js
|
|
51
|
+
const fk = require('secrethider');
|
|
52
|
+
const r = new fk.RedactorBuilder().mask(fk.Mask.keepPrefix(4)).build();
|
|
53
|
+
process.stdin.pipe(fk.createRedactStream(r)).pipe(process.stdout);
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Layout
|
|
57
|
+
|
|
58
|
+
```
|
|
59
|
+
core/ C++20 library (no Python/Node headers), tests, benchmarks
|
|
60
|
+
bindings/python/ nanobind module + `secrethider` package (built via scikit-build-core, root pyproject.toml)
|
|
61
|
+
bindings/node/ Node-API addon (node-addon-api + cmake-js)
|
|
62
|
+
cmake/ shared CMake modules (dependencies, warnings)
|
|
63
|
+
docs/ architecture: interfaces, log processing, rules and masking, builds, performance, security
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Documentation
|
|
67
|
+
|
|
68
|
+
Start with the [guide to removing secrets from logs](docs/guides/removing-secrets-from-logs.md), or [docs/README.md](docs/README.md) for everything: [interfaces](docs/architecture/02-interfaces.md), [log processing pipeline](docs/architecture/03-log-processing.md),
|
|
69
|
+
[rules and masking](docs/architecture/04-rules-and-masking.md), [build and platforms](docs/architecture/05-build-and-platforms.md),
|
|
70
|
+
[performance](docs/architecture/06-performance.md), [security model](docs/architecture/07-security-model.md).
|
|
71
|
+
|
|
72
|
+
## Develop
|
|
73
|
+
|
|
74
|
+
```sh
|
|
75
|
+
make fix # format + auto-fix lint: C++ (clang-format), Python (ruff), JS/TS (prettier)
|
|
76
|
+
make test # build, run all tests (C++, Python, Node), then the benchmarks
|
|
77
|
+
make help # everything else: lint, check (tests only), bench, bench-full, qa (leak checks + long fuzz), clean
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
`make` finds Visual Studio's C++ tools on Windows by itself. Without `make`, the underlying steps are:
|
|
81
|
+
|
|
82
|
+
```sh
|
|
83
|
+
# C++ core: tests + benchmarks (Windows: use the dev-msvc preset from a VS developer prompt)
|
|
84
|
+
cmake --preset dev && cmake --build --preset dev && ctest --preset dev
|
|
85
|
+
./build/dev/core/secrethider_bench # secrethider_bench.exe on Windows
|
|
86
|
+
|
|
87
|
+
# Python
|
|
88
|
+
python -m venv .venv && . .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
89
|
+
pip install . pytest faker && pytest
|
|
90
|
+
|
|
91
|
+
# Node
|
|
92
|
+
cd bindings/node && npm install && npm run build && npm test
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
`-DSECRETHIDER_WITH_RE2=OFF` skips RE2/abseil for a much faster build; custom `regex()` rules are then unavailable.
|
|
96
|
+
|
|
97
|
+
## Cross-platform builds
|
|
98
|
+
|
|
99
|
+
- Python wheels: `cibuildwheel` (Linux manylinux/musllinux, macOS x86_64 + arm64, Windows), stable-ABI (`abi3`) wheels, via `.github/workflows/release.yml`.
|
|
100
|
+
- Node: Node-API is ABI-stable, so one binary per OS/arch; CI collects `prebuilds/<platform>-<arch>/secrethider.node`.
|
|
101
|
+
- SIMD is selected at run time by simdjson/simdutf, so no `-march=native` and one binary runs fast everywhere.
|
|
102
|
+
|
|
103
|
+
## Limits
|
|
104
|
+
|
|
105
|
+
Redaction is defence in depth, not a guarantee: it finds known formats, sensitive key names and values you register. See the [security model](docs/architecture/07-security-model.md).
|
|
106
|
+
|
|
107
|
+
## License
|
|
108
|
+
|
|
109
|
+
MIT, see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# secretHider
|
|
2
|
+
|
|
3
|
+
Fast **secret redaction**, log processing, string and JSON tools: one C++20 core, thin bindings for Python and Node.js.
|
|
4
|
+
|
|
5
|
+
- **Remove secrets from logs** in **plain text, logfmt and JSON lines**: AWS/GitHub/Slack/Stripe/Google keys, JWTs, PEM private keys, `Bearer` tokens, URL credentials, `password=...` fields, and optional PII (email, phone, card, SSN, IBAN, IPv4).
|
|
6
|
+
- **Tell it what is secret** with plain lists: known values, field names and formats (or one `secrets.txt` file), on top of the built-in patterns.
|
|
7
|
+
- **Choose what happens**: delete the whole field (`drop_field`), mask (full, typed, keep first/last N, preserve length, partial email/phone), stable salted hash, **realistic fake data via Faker** (Python `faker`, JS `@faker-js/faker`), or your own function. Partial masks never reveal more than 25% of a secret.
|
|
8
|
+
- **Stream it**: chunk-safe redaction for files, pipes and sockets (output equals one-shot output at every split point), a `secrethider-redact` command line, and in-place/`.gz` file helpers.
|
|
9
|
+
- **Fast**: about 600 MiB/s per core on clean logs, roughly 38x a hand-written Python `re` pipeline and comparable to V8's RegExp for a few simple patterns in Node ([numbers](docs/architecture/06-performance.md)). Built on [simdjson](https://github.com/simdjson/simdjson), [simdutf](https://github.com/simdutf/simdutf), [RE2](https://github.com/google/re2).
|
|
10
|
+
- Also: `json_get` (JSON Pointer), `json_minify`, `json_valid`, UTF-8 helpers.
|
|
11
|
+
|
|
12
|
+
```sh
|
|
13
|
+
secrethider-redact --in-place --drop-fields --secrets-file secrets.txt app.log # a=1 password=x b=2 -> a=1 b=2
|
|
14
|
+
secrethider-redact --pii --mask fake app.jsonl --json-lines # fake emails/phones instead of [REDACTED]
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
import secrethider
|
|
19
|
+
from secrethider import Mask, RedactorBuilder
|
|
20
|
+
|
|
21
|
+
r = (RedactorBuilder()
|
|
22
|
+
.enable("pii")
|
|
23
|
+
.secret_values(["my-known-secret"]) # also: secret_keys([...]), secret_formats([...])
|
|
24
|
+
.mask(Mask.keep_prefix(4)) # or Mask.drop_field(), secrethider.fake_mask(), ...
|
|
25
|
+
.build())
|
|
26
|
+
|
|
27
|
+
r.redact("login password=hunter2hunter2hunter2 key=AKIAIOSFODNN7EXAMPLE")
|
|
28
|
+
# 'login password=hunt**** key=AKIA****'
|
|
29
|
+
# (short secrets fall back to a full mask: 4 revealed characters of a 10-character password would exceed 25%)
|
|
30
|
+
|
|
31
|
+
# logging, files, streams
|
|
32
|
+
from secrethider.logging import RedactingFormatter
|
|
33
|
+
handler.setFormatter(RedactingFormatter(r, "%(levelname)s %(message)s"))
|
|
34
|
+
secrethider.redact_file(open("in.log"), open("out.log", "w"), r)
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
```js
|
|
38
|
+
const fk = require('secrethider');
|
|
39
|
+
const r = new fk.RedactorBuilder().mask(fk.Mask.keepPrefix(4)).build();
|
|
40
|
+
process.stdin.pipe(fk.createRedactStream(r)).pipe(process.stdout);
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Layout
|
|
44
|
+
|
|
45
|
+
```
|
|
46
|
+
core/ C++20 library (no Python/Node headers), tests, benchmarks
|
|
47
|
+
bindings/python/ nanobind module + `secrethider` package (built via scikit-build-core, root pyproject.toml)
|
|
48
|
+
bindings/node/ Node-API addon (node-addon-api + cmake-js)
|
|
49
|
+
cmake/ shared CMake modules (dependencies, warnings)
|
|
50
|
+
docs/ architecture: interfaces, log processing, rules and masking, builds, performance, security
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Documentation
|
|
54
|
+
|
|
55
|
+
Start with the [guide to removing secrets from logs](docs/guides/removing-secrets-from-logs.md), or [docs/README.md](docs/README.md) for everything: [interfaces](docs/architecture/02-interfaces.md), [log processing pipeline](docs/architecture/03-log-processing.md),
|
|
56
|
+
[rules and masking](docs/architecture/04-rules-and-masking.md), [build and platforms](docs/architecture/05-build-and-platforms.md),
|
|
57
|
+
[performance](docs/architecture/06-performance.md), [security model](docs/architecture/07-security-model.md).
|
|
58
|
+
|
|
59
|
+
## Develop
|
|
60
|
+
|
|
61
|
+
```sh
|
|
62
|
+
make fix # format + auto-fix lint: C++ (clang-format), Python (ruff), JS/TS (prettier)
|
|
63
|
+
make test # build, run all tests (C++, Python, Node), then the benchmarks
|
|
64
|
+
make help # everything else: lint, check (tests only), bench, bench-full, qa (leak checks + long fuzz), clean
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`make` finds Visual Studio's C++ tools on Windows by itself. Without `make`, the underlying steps are:
|
|
68
|
+
|
|
69
|
+
```sh
|
|
70
|
+
# C++ core: tests + benchmarks (Windows: use the dev-msvc preset from a VS developer prompt)
|
|
71
|
+
cmake --preset dev && cmake --build --preset dev && ctest --preset dev
|
|
72
|
+
./build/dev/core/secrethider_bench # secrethider_bench.exe on Windows
|
|
73
|
+
|
|
74
|
+
# Python
|
|
75
|
+
python -m venv .venv && . .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
76
|
+
pip install . pytest faker && pytest
|
|
77
|
+
|
|
78
|
+
# Node
|
|
79
|
+
cd bindings/node && npm install && npm run build && npm test
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
`-DSECRETHIDER_WITH_RE2=OFF` skips RE2/abseil for a much faster build; custom `regex()` rules are then unavailable.
|
|
83
|
+
|
|
84
|
+
## Cross-platform builds
|
|
85
|
+
|
|
86
|
+
- Python wheels: `cibuildwheel` (Linux manylinux/musllinux, macOS x86_64 + arm64, Windows), stable-ABI (`abi3`) wheels, via `.github/workflows/release.yml`.
|
|
87
|
+
- Node: Node-API is ABI-stable, so one binary per OS/arch; CI collects `prebuilds/<platform>-<arch>/secrethider.node`.
|
|
88
|
+
- SIMD is selected at run time by simdjson/simdutf, so no `-march=native` and one binary runs fast everywhere.
|
|
89
|
+
|
|
90
|
+
## Limits
|
|
91
|
+
|
|
92
|
+
Redaction is defence in depth, not a guarantee: it finds known formats, sensitive key names and values you register. See the [security model](docs/architecture/07-security-model.md).
|
|
93
|
+
|
|
94
|
+
## License
|
|
95
|
+
|
|
96
|
+
MIT, see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
cmake_minimum_required(VERSION 3.25)
|
|
2
|
+
cmake_policy(SET CMP0091 NEW) # MSVC runtime selection (cmake-js)
|
|
3
|
+
cmake_policy(SET CMP0042 NEW) # macOS @rpath
|
|
4
|
+
|
|
5
|
+
# npm's `rc` package installs node_modules/.bin/rc, which shadows the Windows
|
|
6
|
+
# resource compiler (rc.exe) when cmake-js runs under npm. Drop .bin from PATH.
|
|
7
|
+
if(WIN32)
|
|
8
|
+
set(_fk_path "$ENV{PATH}")
|
|
9
|
+
list(FILTER _fk_path EXCLUDE REGEX "node_modules[/\\\\]\\.bin")
|
|
10
|
+
set(ENV{PATH} "${_fk_path}")
|
|
11
|
+
endif()
|
|
12
|
+
|
|
13
|
+
project(secrethider_node LANGUAGES C CXX)
|
|
14
|
+
|
|
15
|
+
set(CMAKE_CXX_STANDARD 20)
|
|
16
|
+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
|
17
|
+
set(CMAKE_CXX_EXTENSIONS OFF)
|
|
18
|
+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
|
|
19
|
+
if(MSVC)
|
|
20
|
+
add_compile_options(/EHsc /utf-8)
|
|
21
|
+
endif()
|
|
22
|
+
set(CMAKE_CXX_VISIBILITY_PRESET hidden)
|
|
23
|
+
set(CMAKE_VISIBILITY_INLINES_HIDDEN ON)
|
|
24
|
+
if(NOT CMAKE_BUILD_TYPE)
|
|
25
|
+
set(CMAKE_BUILD_TYPE Release)
|
|
26
|
+
endif()
|
|
27
|
+
|
|
28
|
+
# Reuse the shared C++ core (this tree is built by cmake-js, not the top-level project).
|
|
29
|
+
set(REPO_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..)
|
|
30
|
+
list(APPEND CMAKE_MODULE_PATH ${REPO_ROOT}/cmake)
|
|
31
|
+
option(SECRETHIDER_WITH_RE2 "Custom regex rules via RE2 (pulls abseil)" ON)
|
|
32
|
+
# cmake-js builds addons against the static MSVC runtime (/MT); abseil defaults to /MD.
|
|
33
|
+
set(ABSL_MSVC_STATIC_RUNTIME ON CACHE BOOL "" FORCE)
|
|
34
|
+
include(Warnings)
|
|
35
|
+
include(Dependencies)
|
|
36
|
+
add_subdirectory(${REPO_ROOT}/core ${CMAKE_BINARY_DIR}/core EXCLUDE_FROM_ALL)
|
|
37
|
+
|
|
38
|
+
# Node-API (node-addon-api headers from node_modules)
|
|
39
|
+
execute_process(COMMAND node -p "require('node-addon-api').include_dir"
|
|
40
|
+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
|
41
|
+
OUTPUT_VARIABLE NODE_ADDON_API_DIR OUTPUT_STRIP_TRAILING_WHITESPACE)
|
|
42
|
+
string(REPLACE "\"" "" NODE_ADDON_API_DIR "${NODE_ADDON_API_DIR}")
|
|
43
|
+
|
|
44
|
+
add_library(secrethider_node SHARED src/addon.cpp src/redact.cpp ${CMAKE_JS_SRC})
|
|
45
|
+
set_target_properties(secrethider_node PROPERTIES
|
|
46
|
+
PREFIX "" SUFFIX ".node" OUTPUT_NAME "secrethider"
|
|
47
|
+
# same location for Ninja and Visual Studio generators: build/Release/secrethider.node
|
|
48
|
+
RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}/Release
|
|
49
|
+
LIBRARY_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}/Release)
|
|
50
|
+
target_include_directories(secrethider_node PRIVATE ${CMAKE_JS_INC} ${NODE_ADDON_API_DIR})
|
|
51
|
+
target_link_libraries(secrethider_node PRIVATE secrethider::core ${CMAKE_JS_LIB})
|
|
52
|
+
target_compile_definitions(secrethider_node PRIVATE NAPI_VERSION=8 NAPI_CPP_EXCEPTIONS)
|
|
53
|
+
secrethider_set_warnings(secrethider_node)
|
|
54
|
+
|
|
55
|
+
if(MSVC)
|
|
56
|
+
target_link_libraries(secrethider_node PRIVATE delayimp) # needed by cmake-js's win_delay_load_hook
|
|
57
|
+
endif()
|
|
58
|
+
|
|
59
|
+
if(MSVC AND CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET)
|
|
60
|
+
execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF}
|
|
61
|
+
/out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS})
|
|
62
|
+
endif()
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
nanobind_add_module(_core STABLE_ABI NB_STATIC src/module.cpp src/redact.cpp)
|
|
2
|
+
target_link_libraries(_core PRIVATE secrethider::core)
|
|
3
|
+
secrethider_set_warnings(_core)
|
|
4
|
+
|
|
5
|
+
# Only the extension goes into the wheel (not simdjson/simdutf/re2/abseil headers or libs).
|
|
6
|
+
install(TARGETS _core LIBRARY DESTINATION secrethider COMPONENT python_module)
|