configreach 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- configreach-0.9.0/LICENSE +21 -0
- configreach-0.9.0/PKG-INFO +333 -0
- configreach-0.9.0/README.md +307 -0
- configreach-0.9.0/pyproject.toml +47 -0
- configreach-0.9.0/setup.cfg +4 -0
- configreach-0.9.0/src/configreach/__init__.py +3 -0
- configreach-0.9.0/src/configreach/__main__.py +3 -0
- configreach-0.9.0/src/configreach/_trace_exec.py +116 -0
- configreach-0.9.0/src/configreach/baseline.py +50 -0
- configreach-0.9.0/src/configreach/cache.py +55 -0
- configreach-0.9.0/src/configreach/capabilities.py +80 -0
- configreach-0.9.0/src/configreach/cli.py +370 -0
- configreach-0.9.0/src/configreach/config.py +87 -0
- configreach-0.9.0/src/configreach/diffing.py +158 -0
- configreach-0.9.0/src/configreach/discover.py +725 -0
- configreach-0.9.0/src/configreach/engine.py +174 -0
- configreach-0.9.0/src/configreach/entrypoint.py +240 -0
- configreach-0.9.0/src/configreach/fixture_exporters.py +179 -0
- configreach-0.9.0/src/configreach/models.py +459 -0
- configreach-0.9.0/src/configreach/planner.py +312 -0
- configreach-0.9.0/src/configreach/plugins.py +105 -0
- configreach-0.9.0/src/configreach/release.py +274 -0
- configreach-0.9.0/src/configreach/reporters.py +185 -0
- configreach-0.9.0/src/configreach/reproducibility.py +72 -0
- configreach-0.9.0/src/configreach/schemas.py +187 -0
- configreach-0.9.0/src/configreach/semantic_adapters.py +400 -0
- configreach-0.9.0/src/configreach/tracer.py +42 -0
- configreach-0.9.0/src/configreach/validator_adapters.py +150 -0
- configreach-0.9.0/src/configreach/workspace.py +200 -0
- configreach-0.9.0/src/configreach.egg-info/PKG-INFO +333 -0
- configreach-0.9.0/src/configreach.egg-info/SOURCES.txt +46 -0
- configreach-0.9.0/src/configreach.egg-info/dependency_links.txt +1 -0
- configreach-0.9.0/src/configreach.egg-info/entry_points.txt +2 -0
- configreach-0.9.0/src/configreach.egg-info/requires.txt +7 -0
- configreach-0.9.0/src/configreach.egg-info/top_level.txt +1 -0
- configreach-0.9.0/tests/test_cli.py +17 -0
- configreach-0.9.0/tests/test_properties.py +33 -0
- configreach-0.9.0/tests/test_reporters.py +15 -0
- configreach-0.9.0/tests/test_scan.py +41 -0
- configreach-0.9.0/tests/test_v02_features.py +98 -0
- configreach-0.9.0/tests/test_v03_semantics.py +129 -0
- configreach-0.9.0/tests/test_v04_semantics.py +163 -0
- configreach-0.9.0/tests/test_v05_planning.py +74 -0
- configreach-0.9.0/tests/test_v06_scale_exporters.py +135 -0
- configreach-0.9.0/tests/test_v07_plugins_repro.py +88 -0
- configreach-0.9.0/tests/test_v08_stability.py +124 -0
- configreach-0.9.0/tests/test_v09_compatibility_corpus.py +50 -0
- configreach-0.9.0/tests/test_v09_release.py +107 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Saurav Singla
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: configreach
|
|
3
|
+
Version: 0.9.0
|
|
4
|
+
Summary: Deterministic configuration coverage for software repositories
|
|
5
|
+
Author: Saurav Singla
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Keywords: configuration,coverage,testing,ci,devtools,feature-flags,environment-variables
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Environment :: Console
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
15
|
+
Classifier: Topic :: Software Development :: Testing
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
21
|
+
Requires-Dist: pytest-cov>=5; extra == "dev"
|
|
22
|
+
Requires-Dist: build==1.6.1; extra == "dev"
|
|
23
|
+
Requires-Dist: wheel==0.48.0; extra == "dev"
|
|
24
|
+
Requires-Dist: setuptools==84.0.0; extra == "dev"
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# ConfigReach
|
|
28
|
+
|
|
29
|
+
> **Your tests have 94% code coverage. But only 31% configuration coverage. ConfigReach tells you the difference.**
|
|
30
|
+
|
|
31
|
+
**ConfigReach is a deterministic, CPU-only, offline configuration coverage analyzer that shows which runtime configuration inputs, values, branches and important combinations your tests actually exercise.** Think **Codecov for configuration space**.
|
|
32
|
+
|
|
33
|
+
[](https://github.com/sauravsingla/ConfigReach/actions/workflows/ci.yml)
|
|
34
|
+
[](https://github.com/sauravsingla/ConfigReach/actions/workflows/codeql.yml)
|
|
35
|
+
[](https://github.com/sauravsingla/ConfigReach/actions/workflows/reproducibility.yml)
|
|
36
|
+
[](https://github.com/sauravsingla/ConfigReach/actions/workflows/performance.yml)
|
|
37
|
+
[](LICENSE)
|
|
38
|
+
[](https://www.python.org/)
|
|
39
|
+
[](pyproject.toml)
|
|
40
|
+
|
|
41
|
+
ConfigReach needs **no GPU, no LLM, no API key, no hosted service, no telemetry and no paid dependency**. Static analysis does not execute the target repository. Machine-readable results are deterministic for the same repository state and configuration.
|
|
42
|
+
|
|
43
|
+

|
|
44
|
+
|
|
45
|
+
## Why configuration coverage?
|
|
46
|
+
|
|
47
|
+
Code coverage can tell you that a line executed. It cannot tell you whether the configuration states that change that line's behavior were exercised.
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
mode = os.getenv("PAYMENT_MODE", "sandbox")
|
|
51
|
+
if mode == "live":
|
|
52
|
+
charge_real_card()
|
|
53
|
+
else:
|
|
54
|
+
simulate_charge()
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
A suite can execute that block every time and still never test `PAYMENT_MODE=live`. ConfigReach inventories configuration reads and declarations, maps them to test evidence, tracks known values and branches, measures configuration combinations, and reports the gaps.
|
|
58
|
+
|
|
59
|
+
## Quick start
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
git clone https://github.com/sauravsingla/ConfigReach.git
|
|
63
|
+
cd ConfigReach
|
|
64
|
+
python -m pip install -e .
|
|
65
|
+
configreach scan .
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Useful examples:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
configreach scan examples/polyglot
|
|
72
|
+
configreach explain PAYMENT_MODE examples/polyglot
|
|
73
|
+
configreach matrix examples/polyglot
|
|
74
|
+
configreach scan examples/polyglot --format html --output configreach.html
|
|
75
|
+
configreach plan examples/combinations --format markdown
|
|
76
|
+
configreach plan examples/combinations --fixture pytest --output configreach_cases.py
|
|
77
|
+
configreach workspace . --format json --output configreach-workspaces.json
|
|
78
|
+
configreach adapters --format json
|
|
79
|
+
configreach reproduce examples/combinations --runs 3
|
|
80
|
+
configreach schema --format json
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Observable metrics — no opaque AI score
|
|
84
|
+
|
|
85
|
+
ConfigReach reports evidence-based metrics independently:
|
|
86
|
+
|
|
87
|
+
- **Key coverage** — discovered configuration inputs with detected test/runtime evidence.
|
|
88
|
+
- **Value coverage** — explicitly tested values / explicitly known values.
|
|
89
|
+
- **Boolean coverage** — tested `true`/`false` states where a boolean domain is known.
|
|
90
|
+
- **Enum coverage** — exercised known discrete values for non-boolean finite domains.
|
|
91
|
+
- **Branch-state coverage** — configuration-dependent branch states with explicit test-value evidence.
|
|
92
|
+
- **Pairwise key coverage** — interacting keys receiving joint test evidence.
|
|
93
|
+
- **Pairwise value-state coverage** — known value combinations observed together in the same detected test scenario.
|
|
94
|
+
- **Blast radius** — files and top-level modules reading a configuration key.
|
|
95
|
+
- **Workspace coverage** — weighted configuration coverage across independently cached monorepo workspaces.
|
|
96
|
+
|
|
97
|
+
Unknown domains remain unknown. ConfigReach never asks a model whether something is “probably covered.” See [docs/metrics.md](docs/metrics.md).
|
|
98
|
+
|
|
99
|
+
## Discovery coverage
|
|
100
|
+
|
|
101
|
+
### Runtime reads
|
|
102
|
+
|
|
103
|
+
| Ecosystem | Examples | Current analysis |
|
|
104
|
+
|---|---|---|
|
|
105
|
+
| Python | `os.getenv`, `os.environ[...]`, `os.environ.get`, `setdefault` | AST-backed |
|
|
106
|
+
| Pydantic Settings | `BaseSettings`, aliases, `Literal`, Enum, bool domains, Field validators | AST-backed |
|
|
107
|
+
| Python CLI | `argparse`, common Click/Typer option forms | AST-backed/conservative |
|
|
108
|
+
| Feature flags | `is_enabled`, `feature_enabled`, LaunchDarkly-style variation calls | AST + deterministic patterns |
|
|
109
|
+
| JavaScript / TypeScript | `process.env`, Deno/Bun environment access, function-scoped comparisons | deterministic semantic adapter |
|
|
110
|
+
| Zod | `z.enum`, `z.boolean`, `z.literal`, min/max/regex/url/email/nonempty | deterministic validator adapter |
|
|
111
|
+
| Go | `os.Getenv`, `os.LookupEnv`, `t.Setenv`, function-scoped comparisons | deterministic semantic adapter |
|
|
112
|
+
| Java / Spring | `System.getenv`, `System.getProperty`, `@Value`, `Environment.getProperty`, `@ConfigurationProperties` | deterministic semantic adapter |
|
|
113
|
+
| Java Bean Validation | `@Min`, `@Max`, `@Size`, `@Pattern`, null/blank/sign constraints on `@Value` fields | deterministic validator adapter |
|
|
114
|
+
| .NET / C# | environment variables, `IConfiguration`, `GetValue`, feature flags | deterministic semantic adapter |
|
|
115
|
+
| Rust | `env::var`, `env::var_os` | deterministic pattern adapter |
|
|
116
|
+
| Ruby | `ENV[...]`, `ENV.fetch(...)` | deterministic pattern adapter |
|
|
117
|
+
| PHP | `getenv(...)`, common `env(...)` | deterministic pattern adapter |
|
|
118
|
+
| Shell | `$VAR`, `${VAR}` | deterministic pattern adapter |
|
|
119
|
+
|
|
120
|
+
### Declarations, schemas and deployment sources
|
|
121
|
+
|
|
122
|
+
ConfigReach recognizes `.env.example`, `.env.template`, `.env*` templates, JSON, TOML, INI/CFG, Java properties, YAML environment declarations, Dockerfiles/Containerfiles, Docker Compose, Kubernetes-style environment declarations, Helm `values.yaml`, GitHub Actions `${{ vars.* }}` and `${{ secrets.* }}`, Terraform variables, Makefile variables, Pydantic settings, JSON Schema finite domains/validators, Zod schemas and CLI options.
|
|
123
|
+
|
|
124
|
+
## Commands
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
configreach scan [PATH]
|
|
128
|
+
configreach coverage [PATH]
|
|
129
|
+
configreach explain KEY [PATH]
|
|
130
|
+
configreach matrix [PATH]
|
|
131
|
+
configreach plan [PATH]
|
|
132
|
+
configreach plan [PATH] --strength 3 --max-cases 40
|
|
133
|
+
configreach plan [PATH] --fixture pytest|jest|go|shell|junit|xunit
|
|
134
|
+
configreach workspace [PATH]
|
|
135
|
+
configreach adapters
|
|
136
|
+
configreach reproduce [PATH] --runs 3
|
|
137
|
+
configreach schema
|
|
138
|
+
configreach schema --kind report --check configreach.json
|
|
139
|
+
configreach diff origin/main...HEAD [PATH]
|
|
140
|
+
configreach pr-comment origin/main...HEAD [PATH]
|
|
141
|
+
configreach doctor [PATH]
|
|
142
|
+
configreach export [PATH] --format json
|
|
143
|
+
configreach export [PATH] --format sarif
|
|
144
|
+
configreach export [PATH] --format html
|
|
145
|
+
configreach baseline create [PATH]
|
|
146
|
+
configreach cache clear [PATH]
|
|
147
|
+
configreach init [PATH]
|
|
148
|
+
configreach trace --path . -- pytest -q
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
## Deterministic test planning
|
|
152
|
+
|
|
153
|
+
`configreach plan` converts already-known finite configuration domains into bounded 1-wise, 2-wise or 3-wise suggestions. Existing test scenarios are subtracted first, sensitive-looking keys are excluded, and CPU-safety limits prevent Cartesian-product explosions.
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
configreach plan . --strength 2
|
|
157
|
+
configreach plan . --format json --output configreach-plan.json
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
The planner does not invent values, execute the application, synthesize assertions or call a model. See [docs/planning.md](docs/planning.md).
|
|
161
|
+
|
|
162
|
+
## Fixture exporters
|
|
163
|
+
|
|
164
|
+
Turn the deterministic plan into lightweight scaffolding for your own tests:
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
configreach plan . --fixture pytest --output test_configreach_cases.py
|
|
168
|
+
configreach plan . --fixture jest --output configreach.cases.ts
|
|
169
|
+
configreach plan . --fixture go --output configreach_cases_test.go
|
|
170
|
+
configreach plan . --fixture shell --output configreach_cases.sh
|
|
171
|
+
configreach plan . --fixture junit --output ConfigReachCases.java
|
|
172
|
+
configreach plan . --fixture xunit --output ConfigReachCases.cs
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Exporters provide configuration cases only; they deliberately do not invent expected business outcomes. See [docs/fixtures.md](docs/fixtures.md).
|
|
176
|
+
|
|
177
|
+
## Monorepos and workspace-local incremental caching
|
|
178
|
+
|
|
179
|
+
`configreach workspace` detects Python, Node, Go, Rust, Maven/Gradle and `.csproj` workspace roots. Each workspace receives an independent `.configreach/cache/` boundary, so changing one package does not invalidate unrelated warmed workspace caches. Parent workspaces ignore nested workspace directories to avoid double counting.
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
configreach workspace .
|
|
183
|
+
configreach workspace . --format markdown
|
|
184
|
+
configreach workspace . --format json --output workspaces.json
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
The normal `configreach scan .` remains the combined repository view. See [docs/workspaces.md](docs/workspaces.md).
|
|
188
|
+
|
|
189
|
+
## Adapter API and optional parser-backed plugins
|
|
190
|
+
|
|
191
|
+
The base package remains dependency-free, but external deterministic adapters can register through the `configreach.adapters` entry-point group. Adapter API v1 includes compatibility version, parser identity, determinism declaration and capability metadata.
|
|
192
|
+
|
|
193
|
+
```bash
|
|
194
|
+
configreach adapters
|
|
195
|
+
configreach adapters --format json
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
ConfigReach rejects incompatible or explicitly non-deterministic plugins without crashing the core scanner. A real optional tree-sitter JavaScript adapter example lives under [`examples/plugins/tree_sitter_js`](examples/plugins/tree_sitter_js/); installing it is separate from installing ConfigReach. See [docs/plugin-sdk.md](docs/plugin-sdk.md) and [docs/adapter-capabilities.md](docs/adapter-capabilities.md).
|
|
199
|
+
|
|
200
|
+
## Reproducibility verification
|
|
201
|
+
|
|
202
|
+
```bash
|
|
203
|
+
configreach reproduce .
|
|
204
|
+
configreach reproduce . --runs 5 --format json --output repro.json
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
Repeated scans are uncached and converted to canonical JSON before SHA-256 hashing. Absolute root, timing and cache metadata are excluded because they are execution-environment metadata, not analysis semantics. The repository's reproducibility workflow compares canonical digests produced on **Ubuntu, macOS and Windows** and fails if they differ. See [docs/reproducibility.md](docs/reproducibility.md).
|
|
208
|
+
|
|
209
|
+
## Schema compatibility
|
|
210
|
+
|
|
211
|
+
ConfigReach publishes explicit versions for scan reports, workspace reports, deterministic plans, reproducibility results and adapter capability inventories.
|
|
212
|
+
|
|
213
|
+
```bash
|
|
214
|
+
configreach schema
|
|
215
|
+
configreach schema --format json
|
|
216
|
+
configreach schema --kind report --check configreach.json
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
The validator rejects unsupported future schemas instead of guessing their meaning. Report schemas 3-5 are accepted for structural compatibility checks, while new scan output remains report schema v5. Golden compatibility fixtures live in the test suite. See [docs/schema-compatibility.md](docs/schema-compatibility.md).
|
|
220
|
+
|
|
221
|
+
## CI gating
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
configreach scan . --fail-under 70
|
|
225
|
+
configreach scan . --fail-on error
|
|
226
|
+
configreach scan . --fail-on uncovered
|
|
227
|
+
configreach scan . --fail-on untested-values
|
|
228
|
+
configreach scan . --fail-on default-only
|
|
229
|
+
configreach scan . --fail-on global-env-overwrite
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
`--fail-under` gates key coverage. `--fail-on` can gate finding aliases, severities or exact `CRxxx` rule IDs.
|
|
233
|
+
|
|
234
|
+
## Pull-request configuration diff
|
|
235
|
+
|
|
236
|
+
```bash
|
|
237
|
+
configreach diff origin/main...HEAD
|
|
238
|
+
configreach pr-comment origin/main...HEAD --output /tmp/configreach-comment.md
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
ConfigReach resolves the local Git merge base, scans the base snapshot and current tree, and reports new/removed keys, changed domains/defaults, newly introduced untested configuration, new values without test evidence, changed-line configuration impact and blast radius. No external service is required.
|
|
242
|
+
|
|
243
|
+
## Searchable static HTML report
|
|
244
|
+
|
|
245
|
+
```bash
|
|
246
|
+
configreach scan . --format html --output configreach.html
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
The report is a single self-contained HTML file with no CDN/network dependency and includes searchable key/value/branch/combination evidence plus source links.
|
|
250
|
+
|
|
251
|
+
## Baselines and cache
|
|
252
|
+
|
|
253
|
+
```bash
|
|
254
|
+
configreach baseline create .
|
|
255
|
+
configreach scan . --no-cache
|
|
256
|
+
configreach cache clear .
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
Baseline keys remain visible but are excluded from CI key-coverage gating. Cache/timing state is excluded from JSON/SARIF result semantics. See [docs/baselines.md](docs/baselines.md).
|
|
260
|
+
|
|
261
|
+
## Optional lightweight runtime tracing
|
|
262
|
+
|
|
263
|
+
```bash
|
|
264
|
+
configreach trace -- pytest -q
|
|
265
|
+
configreach scan .
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
Tracing is explicit opt-in. The Python tracer records key names plus short SHA-256-derived value fingerprints; it does not persist raw runtime values.
|
|
269
|
+
|
|
270
|
+
## Deterministic findings
|
|
271
|
+
|
|
272
|
+
| Rule | Meaning | Default severity |
|
|
273
|
+
|---|---|---|
|
|
274
|
+
| `CR001` | configuration has no detected test/runtime evidence | warning |
|
|
275
|
+
| `CR002` | application read without a recognized declaration | warning |
|
|
276
|
+
| `CR003` | declaration without a recognized application read | note |
|
|
277
|
+
| `CR004` | known values are not all exercised | warning |
|
|
278
|
+
| `CR005` | sensitive-looking configuration has a non-empty static default | error |
|
|
279
|
+
| `CR006` | likely inconsistent names normalize to the same identifier | warning |
|
|
280
|
+
| `CR007` | production-like known value is not exercised | warning |
|
|
281
|
+
| `CR008` | explicit test values only exercise defaults | warning |
|
|
282
|
+
| `CR009` | a test mutates the global environment in a potentially leaky way | warning |
|
|
283
|
+
|
|
284
|
+
Every finding retains source provenance.
|
|
285
|
+
|
|
286
|
+
## GitHub Actions
|
|
287
|
+
|
|
288
|
+
```yaml
|
|
289
|
+
name: Configuration coverage
|
|
290
|
+
on: [pull_request]
|
|
291
|
+
|
|
292
|
+
jobs:
|
|
293
|
+
configreach:
|
|
294
|
+
runs-on: ubuntu-latest
|
|
295
|
+
steps:
|
|
296
|
+
- uses: actions/checkout@v4
|
|
297
|
+
with:
|
|
298
|
+
fetch-depth: 0
|
|
299
|
+
- uses: sauravsingla/ConfigReach@main
|
|
300
|
+
with:
|
|
301
|
+
path: .
|
|
302
|
+
format: markdown
|
|
303
|
+
fail-under: "60"
|
|
304
|
+
fail-on: error
|
|
305
|
+
```
|
|
306
|
+
|
|
307
|
+
Markdown output can be appended to the job summary, SARIF can be uploaded to Code Scanning, and the repository includes an optional PR-comment workflow.
|
|
308
|
+
|
|
309
|
+
## Benchmark, performance budget and testing
|
|
310
|
+
|
|
311
|
+
```bash
|
|
312
|
+
python benchmarks/bench_scan.py 1000
|
|
313
|
+
python benchmarks/perf_budget.py --files 800 --min-files-per-second 150
|
|
314
|
+
python -m pip install -e ".[dev]"
|
|
315
|
+
pytest
|
|
316
|
+
python -m compileall -q src tests
|
|
317
|
+
configreach reproduce examples/combinations --runs 3
|
|
318
|
+
```
|
|
319
|
+
|
|
320
|
+
The dedicated performance workflow runs the full semantic engine over a synthetic Python/TypeScript/Go/Java/.NET repository and uses a deliberately conservative throughput floor to catch order-of-magnitude regressions without turning runner noise into flaky CI.
|
|
321
|
+
|
|
322
|
+
The suite covers language/config discovery, deployment sources, validators, Pydantic/feature flags, branch provenance, combination metrics, real Git PR comparison, baselines, cache behavior, workspace-local invalidation, planners, fixture exporters, plugin compatibility, schema compatibility, reproducibility, performance gating, HTML/SARIF/JSON/Markdown reporters and CLI policies.
|
|
323
|
+
|
|
324
|
+
## Design principles
|
|
325
|
+
|
|
326
|
+
- CPU-only and zero runtime dependencies.
|
|
327
|
+
- No network, telemetry, model inference or paid API in the core.
|
|
328
|
+
- Static scanning never executes target application code.
|
|
329
|
+
- Runtime tracing is explicit opt-in.
|
|
330
|
+
- Unknown semantics stay unknown rather than being guessed.
|
|
331
|
+
- Machine output is designed for deterministic CI use.
|
|
332
|
+
|
|
333
|
+
See [docs/architecture.md](docs/architecture.md), [docs/threat-model.md](docs/threat-model.md), [docs/schema-compatibility.md](docs/schema-compatibility.md), [docs/roadmap.md](docs/roadmap.md) and [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
# ConfigReach
|
|
2
|
+
|
|
3
|
+
> **Your tests have 94% code coverage. But only 31% configuration coverage. ConfigReach tells you the difference.**
|
|
4
|
+
|
|
5
|
+
**ConfigReach is a deterministic, CPU-only, offline configuration coverage analyzer that shows which runtime configuration inputs, values, branches and important combinations your tests actually exercise.** Think **Codecov for configuration space**.
|
|
6
|
+
|
|
7
|
+
[](https://github.com/sauravsingla/ConfigReach/actions/workflows/ci.yml)
|
|
8
|
+
[](https://github.com/sauravsingla/ConfigReach/actions/workflows/codeql.yml)
|
|
9
|
+
[](https://github.com/sauravsingla/ConfigReach/actions/workflows/reproducibility.yml)
|
|
10
|
+
[](https://github.com/sauravsingla/ConfigReach/actions/workflows/performance.yml)
|
|
11
|
+
[](LICENSE)
|
|
12
|
+
[](https://www.python.org/)
|
|
13
|
+
[](pyproject.toml)
|
|
14
|
+
|
|
15
|
+
ConfigReach needs **no GPU, no LLM, no API key, no hosted service, no telemetry and no paid dependency**. Static analysis does not execute the target repository. Machine-readable results are deterministic for the same repository state and configuration.
|
|
16
|
+
|
|
17
|
+

|
|
18
|
+
|
|
19
|
+
## Why configuration coverage?
|
|
20
|
+
|
|
21
|
+
Code coverage can tell you that a line executed. It cannot tell you whether the configuration states that change that line's behavior were exercised.
|
|
22
|
+
|
|
23
|
+
```python
|
|
24
|
+
mode = os.getenv("PAYMENT_MODE", "sandbox")
|
|
25
|
+
if mode == "live":
|
|
26
|
+
charge_real_card()
|
|
27
|
+
else:
|
|
28
|
+
simulate_charge()
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
A suite can execute that block every time and still never test `PAYMENT_MODE=live`. ConfigReach inventories configuration reads and declarations, maps them to test evidence, tracks known values and branches, measures configuration combinations, and reports the gaps.
|
|
32
|
+
|
|
33
|
+
## Quick start
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
git clone https://github.com/sauravsingla/ConfigReach.git
|
|
37
|
+
cd ConfigReach
|
|
38
|
+
python -m pip install -e .
|
|
39
|
+
configreach scan .
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Useful examples:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
configreach scan examples/polyglot
|
|
46
|
+
configreach explain PAYMENT_MODE examples/polyglot
|
|
47
|
+
configreach matrix examples/polyglot
|
|
48
|
+
configreach scan examples/polyglot --format html --output configreach.html
|
|
49
|
+
configreach plan examples/combinations --format markdown
|
|
50
|
+
configreach plan examples/combinations --fixture pytest --output configreach_cases.py
|
|
51
|
+
configreach workspace . --format json --output configreach-workspaces.json
|
|
52
|
+
configreach adapters --format json
|
|
53
|
+
configreach reproduce examples/combinations --runs 3
|
|
54
|
+
configreach schema --format json
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Observable metrics — no opaque AI score
|
|
58
|
+
|
|
59
|
+
ConfigReach reports evidence-based metrics independently:
|
|
60
|
+
|
|
61
|
+
- **Key coverage** — discovered configuration inputs with detected test/runtime evidence.
|
|
62
|
+
- **Value coverage** — explicitly tested values / explicitly known values.
|
|
63
|
+
- **Boolean coverage** — tested `true`/`false` states where a boolean domain is known.
|
|
64
|
+
- **Enum coverage** — exercised known discrete values for non-boolean finite domains.
|
|
65
|
+
- **Branch-state coverage** — configuration-dependent branch states with explicit test-value evidence.
|
|
66
|
+
- **Pairwise key coverage** — interacting keys receiving joint test evidence.
|
|
67
|
+
- **Pairwise value-state coverage** — known value combinations observed together in the same detected test scenario.
|
|
68
|
+
- **Blast radius** — files and top-level modules reading a configuration key.
|
|
69
|
+
- **Workspace coverage** — weighted configuration coverage across independently cached monorepo workspaces.
|
|
70
|
+
|
|
71
|
+
Unknown domains remain unknown. ConfigReach never asks a model whether something is “probably covered.” See [docs/metrics.md](docs/metrics.md).
|
|
72
|
+
|
|
73
|
+
## Discovery coverage
|
|
74
|
+
|
|
75
|
+
### Runtime reads
|
|
76
|
+
|
|
77
|
+
| Ecosystem | Examples | Current analysis |
|
|
78
|
+
|---|---|---|
|
|
79
|
+
| Python | `os.getenv`, `os.environ[...]`, `os.environ.get`, `setdefault` | AST-backed |
|
|
80
|
+
| Pydantic Settings | `BaseSettings`, aliases, `Literal`, Enum, bool domains, Field validators | AST-backed |
|
|
81
|
+
| Python CLI | `argparse`, common Click/Typer option forms | AST-backed/conservative |
|
|
82
|
+
| Feature flags | `is_enabled`, `feature_enabled`, LaunchDarkly-style variation calls | AST + deterministic patterns |
|
|
83
|
+
| JavaScript / TypeScript | `process.env`, Deno/Bun environment access, function-scoped comparisons | deterministic semantic adapter |
|
|
84
|
+
| Zod | `z.enum`, `z.boolean`, `z.literal`, min/max/regex/url/email/nonempty | deterministic validator adapter |
|
|
85
|
+
| Go | `os.Getenv`, `os.LookupEnv`, `t.Setenv`, function-scoped comparisons | deterministic semantic adapter |
|
|
86
|
+
| Java / Spring | `System.getenv`, `System.getProperty`, `@Value`, `Environment.getProperty`, `@ConfigurationProperties` | deterministic semantic adapter |
|
|
87
|
+
| Java Bean Validation | `@Min`, `@Max`, `@Size`, `@Pattern`, null/blank/sign constraints on `@Value` fields | deterministic validator adapter |
|
|
88
|
+
| .NET / C# | environment variables, `IConfiguration`, `GetValue`, feature flags | deterministic semantic adapter |
|
|
89
|
+
| Rust | `env::var`, `env::var_os` | deterministic pattern adapter |
|
|
90
|
+
| Ruby | `ENV[...]`, `ENV.fetch(...)` | deterministic pattern adapter |
|
|
91
|
+
| PHP | `getenv(...)`, common `env(...)` | deterministic pattern adapter |
|
|
92
|
+
| Shell | `$VAR`, `${VAR}` | deterministic pattern adapter |
|
|
93
|
+
|
|
94
|
+
### Declarations, schemas and deployment sources
|
|
95
|
+
|
|
96
|
+
ConfigReach recognizes `.env.example`, `.env.template`, `.env*` templates, JSON, TOML, INI/CFG, Java properties, YAML environment declarations, Dockerfiles/Containerfiles, Docker Compose, Kubernetes-style environment declarations, Helm `values.yaml`, GitHub Actions `${{ vars.* }}` and `${{ secrets.* }}`, Terraform variables, Makefile variables, Pydantic settings, JSON Schema finite domains/validators, Zod schemas and CLI options.
|
|
97
|
+
|
|
98
|
+
## Commands
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
configreach scan [PATH]
|
|
102
|
+
configreach coverage [PATH]
|
|
103
|
+
configreach explain KEY [PATH]
|
|
104
|
+
configreach matrix [PATH]
|
|
105
|
+
configreach plan [PATH]
|
|
106
|
+
configreach plan [PATH] --strength 3 --max-cases 40
|
|
107
|
+
configreach plan [PATH] --fixture pytest|jest|go|shell|junit|xunit
|
|
108
|
+
configreach workspace [PATH]
|
|
109
|
+
configreach adapters
|
|
110
|
+
configreach reproduce [PATH] --runs 3
|
|
111
|
+
configreach schema
|
|
112
|
+
configreach schema --kind report --check configreach.json
|
|
113
|
+
configreach diff origin/main...HEAD [PATH]
|
|
114
|
+
configreach pr-comment origin/main...HEAD [PATH]
|
|
115
|
+
configreach doctor [PATH]
|
|
116
|
+
configreach export [PATH] --format json
|
|
117
|
+
configreach export [PATH] --format sarif
|
|
118
|
+
configreach export [PATH] --format html
|
|
119
|
+
configreach baseline create [PATH]
|
|
120
|
+
configreach cache clear [PATH]
|
|
121
|
+
configreach init [PATH]
|
|
122
|
+
configreach trace --path . -- pytest -q
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## Deterministic test planning
|
|
126
|
+
|
|
127
|
+
`configreach plan` converts already-known finite configuration domains into bounded 1-wise, 2-wise or 3-wise suggestions. Existing test scenarios are subtracted first, sensitive-looking keys are excluded, and CPU-safety limits prevent Cartesian-product explosions.
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
configreach plan . --strength 2
|
|
131
|
+
configreach plan . --format json --output configreach-plan.json
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
The planner does not invent values, execute the application, synthesize assertions or call a model. See [docs/planning.md](docs/planning.md).
|
|
135
|
+
|
|
136
|
+
## Fixture exporters
|
|
137
|
+
|
|
138
|
+
Turn the deterministic plan into lightweight scaffolding for your own tests:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
configreach plan . --fixture pytest --output test_configreach_cases.py
|
|
142
|
+
configreach plan . --fixture jest --output configreach.cases.ts
|
|
143
|
+
configreach plan . --fixture go --output configreach_cases_test.go
|
|
144
|
+
configreach plan . --fixture shell --output configreach_cases.sh
|
|
145
|
+
configreach plan . --fixture junit --output ConfigReachCases.java
|
|
146
|
+
configreach plan . --fixture xunit --output ConfigReachCases.cs
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Exporters provide configuration cases only; they deliberately do not invent expected business outcomes. See [docs/fixtures.md](docs/fixtures.md).
|
|
150
|
+
|
|
151
|
+
## Monorepos and workspace-local incremental caching
|
|
152
|
+
|
|
153
|
+
`configreach workspace` detects Python, Node, Go, Rust, Maven/Gradle and `.csproj` workspace roots. Each workspace receives an independent `.configreach/cache/` boundary, so changing one package does not invalidate unrelated warmed workspace caches. Parent workspaces ignore nested workspace directories to avoid double counting.
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
configreach workspace .
|
|
157
|
+
configreach workspace . --format markdown
|
|
158
|
+
configreach workspace . --format json --output workspaces.json
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
The normal `configreach scan .` remains the combined repository view. See [docs/workspaces.md](docs/workspaces.md).
|
|
162
|
+
|
|
163
|
+
## Adapter API and optional parser-backed plugins
|
|
164
|
+
|
|
165
|
+
The base package remains dependency-free, but external deterministic adapters can register through the `configreach.adapters` entry-point group. Adapter API v1 includes compatibility version, parser identity, determinism declaration and capability metadata.
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
configreach adapters
|
|
169
|
+
configreach adapters --format json
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
ConfigReach rejects incompatible or explicitly non-deterministic plugins without crashing the core scanner. A real optional tree-sitter JavaScript adapter example lives under [`examples/plugins/tree_sitter_js`](examples/plugins/tree_sitter_js/); installing it is separate from installing ConfigReach. See [docs/plugin-sdk.md](docs/plugin-sdk.md) and [docs/adapter-capabilities.md](docs/adapter-capabilities.md).
|
|
173
|
+
|
|
174
|
+
## Reproducibility verification
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
configreach reproduce .
|
|
178
|
+
configreach reproduce . --runs 5 --format json --output repro.json
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Repeated scans are uncached and converted to canonical JSON before SHA-256 hashing. Absolute root, timing and cache metadata are excluded because they are execution-environment metadata, not analysis semantics. The repository's reproducibility workflow compares canonical digests produced on **Ubuntu, macOS and Windows** and fails if they differ. See [docs/reproducibility.md](docs/reproducibility.md).
|
|
182
|
+
|
|
183
|
+
## Schema compatibility
|
|
184
|
+
|
|
185
|
+
ConfigReach publishes explicit versions for scan reports, workspace reports, deterministic plans, reproducibility results and adapter capability inventories.
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
configreach schema
|
|
189
|
+
configreach schema --format json
|
|
190
|
+
configreach schema --kind report --check configreach.json
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
The validator rejects unsupported future schemas instead of guessing their meaning. Report schemas 3-5 are accepted for structural compatibility checks, while new scan output remains report schema v5. Golden compatibility fixtures live in the test suite. See [docs/schema-compatibility.md](docs/schema-compatibility.md).
|
|
194
|
+
|
|
195
|
+
## CI gating
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
configreach scan . --fail-under 70
|
|
199
|
+
configreach scan . --fail-on error
|
|
200
|
+
configreach scan . --fail-on uncovered
|
|
201
|
+
configreach scan . --fail-on untested-values
|
|
202
|
+
configreach scan . --fail-on default-only
|
|
203
|
+
configreach scan . --fail-on global-env-overwrite
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
`--fail-under` gates key coverage. `--fail-on` can gate finding aliases, severities or exact `CRxxx` rule IDs.
|
|
207
|
+
|
|
208
|
+
## Pull-request configuration diff
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
configreach diff origin/main...HEAD
|
|
212
|
+
configreach pr-comment origin/main...HEAD --output /tmp/configreach-comment.md
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
ConfigReach resolves the local Git merge base, scans the base snapshot and current tree, and reports new/removed keys, changed domains/defaults, newly introduced untested configuration, new values without test evidence, changed-line configuration impact and blast radius. No external service is required.
|
|
216
|
+
|
|
217
|
+
## Searchable static HTML report
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
configreach scan . --format html --output configreach.html
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
The report is a single self-contained HTML file with no CDN/network dependency and includes searchable key/value/branch/combination evidence plus source links.
|
|
224
|
+
|
|
225
|
+
## Baselines and cache
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
configreach baseline create .
|
|
229
|
+
configreach scan . --no-cache
|
|
230
|
+
configreach cache clear .
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
Baseline keys remain visible but are excluded from CI key-coverage gating. Cache/timing state is excluded from JSON/SARIF result semantics. See [docs/baselines.md](docs/baselines.md).
|
|
234
|
+
|
|
235
|
+
## Optional lightweight runtime tracing
|
|
236
|
+
|
|
237
|
+
```bash
|
|
238
|
+
configreach trace -- pytest -q
|
|
239
|
+
configreach scan .
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
Tracing is explicit opt-in. The Python tracer records key names plus short SHA-256-derived value fingerprints; it does not persist raw runtime values.
|
|
243
|
+
|
|
244
|
+
## Deterministic findings
|
|
245
|
+
|
|
246
|
+
| Rule | Meaning | Default severity |
|
|
247
|
+
|---|---|---|
|
|
248
|
+
| `CR001` | configuration has no detected test/runtime evidence | warning |
|
|
249
|
+
| `CR002` | application read without a recognized declaration | warning |
|
|
250
|
+
| `CR003` | declaration without a recognized application read | note |
|
|
251
|
+
| `CR004` | known values are not all exercised | warning |
|
|
252
|
+
| `CR005` | sensitive-looking configuration has a non-empty static default | error |
|
|
253
|
+
| `CR006` | likely inconsistent names normalize to the same identifier | warning |
|
|
254
|
+
| `CR007` | production-like known value is not exercised | warning |
|
|
255
|
+
| `CR008` | explicit test values only exercise defaults | warning |
|
|
256
|
+
| `CR009` | a test mutates the global environment in a potentially leaky way | warning |
|
|
257
|
+
|
|
258
|
+
Every finding retains source provenance.
|
|
259
|
+
|
|
260
|
+
## GitHub Actions
|
|
261
|
+
|
|
262
|
+
```yaml
|
|
263
|
+
name: Configuration coverage
|
|
264
|
+
on: [pull_request]
|
|
265
|
+
|
|
266
|
+
jobs:
|
|
267
|
+
configreach:
|
|
268
|
+
runs-on: ubuntu-latest
|
|
269
|
+
steps:
|
|
270
|
+
- uses: actions/checkout@v4
|
|
271
|
+
with:
|
|
272
|
+
fetch-depth: 0
|
|
273
|
+
- uses: sauravsingla/ConfigReach@main
|
|
274
|
+
with:
|
|
275
|
+
path: .
|
|
276
|
+
format: markdown
|
|
277
|
+
fail-under: "60"
|
|
278
|
+
fail-on: error
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
Markdown output can be appended to the job summary, SARIF can be uploaded to Code Scanning, and the repository includes an optional PR-comment workflow.
|
|
282
|
+
|
|
283
|
+
## Benchmark, performance budget and testing
|
|
284
|
+
|
|
285
|
+
```bash
|
|
286
|
+
python benchmarks/bench_scan.py 1000
|
|
287
|
+
python benchmarks/perf_budget.py --files 800 --min-files-per-second 150
|
|
288
|
+
python -m pip install -e ".[dev]"
|
|
289
|
+
pytest
|
|
290
|
+
python -m compileall -q src tests
|
|
291
|
+
configreach reproduce examples/combinations --runs 3
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
The dedicated performance workflow runs the full semantic engine over a synthetic Python/TypeScript/Go/Java/.NET repository and uses a deliberately conservative throughput floor to catch order-of-magnitude regressions without turning runner noise into flaky CI.
|
|
295
|
+
|
|
296
|
+
The suite covers language/config discovery, deployment sources, validators, Pydantic/feature flags, branch provenance, combination metrics, real Git PR comparison, baselines, cache behavior, workspace-local invalidation, planners, fixture exporters, plugin compatibility, schema compatibility, reproducibility, performance gating, HTML/SARIF/JSON/Markdown reporters and CLI policies.
|
|
297
|
+
|
|
298
|
+
## Design principles
|
|
299
|
+
|
|
300
|
+
- CPU-only and zero runtime dependencies.
|
|
301
|
+
- No network, telemetry, model inference or paid API in the core.
|
|
302
|
+
- Static scanning never executes target application code.
|
|
303
|
+
- Runtime tracing is explicit opt-in.
|
|
304
|
+
- Unknown semantics stay unknown rather than being guessed.
|
|
305
|
+
- Machine output is designed for deterministic CI use.
|
|
306
|
+
|
|
307
|
+
See [docs/architecture.md](docs/architecture.md), [docs/threat-model.md](docs/threat-model.md), [docs/schema-compatibility.md](docs/schema-compatibility.md), [docs/roadmap.md](docs/roadmap.md) and [CONTRIBUTING.md](CONTRIBUTING.md).
|