vulntrack 0.2.0b1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vulntrack-0.2.0b1/.env.example +20 -0
- vulntrack-0.2.0b1/.gitignore +22 -0
- vulntrack-0.2.0b1/LICENSE +21 -0
- vulntrack-0.2.0b1/PKG-INFO +187 -0
- vulntrack-0.2.0b1/WORKLOG.md +128 -0
- vulntrack-0.2.0b1/app.py +23 -0
- vulntrack-0.2.0b1/docs/HANDOFF.md +160 -0
- vulntrack-0.2.0b1/docs/index.md +574 -0
- vulntrack-0.2.0b1/docs/packaging.md +94 -0
- vulntrack-0.2.0b1/mkdocs.yml +35 -0
- vulntrack-0.2.0b1/packs.example/README.md +94 -0
- vulntrack-0.2.0b1/packs.example/deployments.tsv +4 -0
- vulntrack-0.2.0b1/packs.example/kev.tsv +3 -0
- vulntrack-0.2.0b1/packs.example/packs.toml +14 -0
- vulntrack-0.2.0b1/packs.example/project-tiers.tsv +3 -0
- vulntrack-0.2.0b1/packs.example/projects.tsv +4 -0
- vulntrack-0.2.0b1/plan.md +306 -0
- vulntrack-0.2.0b1/pyproject.toml +67 -0
- vulntrack-0.2.0b1/readme.md +168 -0
- vulntrack-0.2.0b1/store/rules.ndjson +4 -0
- vulntrack-0.2.0b1/tests/conftest.py +22 -0
- vulntrack-0.2.0b1/tests/test_cli_packs.py +106 -0
- vulntrack-0.2.0b1/tests/test_collect.py +362 -0
- vulntrack-0.2.0b1/tests/test_engine.py +207 -0
- vulntrack-0.2.0b1/tests/test_packs.py +290 -0
- vulntrack-0.2.0b1/tests/test_settings.py +46 -0
- vulntrack-0.2.0b1/tests/test_webpath.py +38 -0
- vulntrack-0.2.0b1/vulntrack/__init__.py +7 -0
- vulntrack-0.2.0b1/vulntrack/cli.py +702 -0
- vulntrack-0.2.0b1/vulntrack/engine/__init__.py +33 -0
- vulntrack-0.2.0b1/vulntrack/engine/frame.py +188 -0
- vulntrack-0.2.0b1/vulntrack/engine/packs.py +384 -0
- vulntrack-0.2.0b1/vulntrack/engine/rating.py +185 -0
- vulntrack-0.2.0b1/vulntrack/engine/rules.py +193 -0
- vulntrack-0.2.0b1/vulntrack/harbor.py +402 -0
- vulntrack-0.2.0b1/vulntrack/settings.py +145 -0
- vulntrack-0.2.0b1/vulntrack/store.py +326 -0
- vulntrack-0.2.0b1/vulntrack/task.md +435 -0
- vulntrack-0.2.0b1/vulntrack/tui.py +339 -0
- vulntrack-0.2.0b1/vulntrack/ui/__init__.py +1 -0
- vulntrack-0.2.0b1/vulntrack/ui/data.py +66 -0
- vulntrack-0.2.0b1/vulntrack/ui/pages_assess.py +294 -0
- vulntrack-0.2.0b1/vulntrack/ui/pages_images.py +83 -0
- vulntrack-0.2.0b1/vulntrack/ui/pages_rules.py +75 -0
- vulntrack-0.2.0b1/vulntrack/ui/pages_scans.py +56 -0
- vulntrack-0.2.0b1/vulntrack/webpath.py +159 -0
- vulntrack-0.2.0b1/vulntrack.toml.example +56 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Secrets only. Everything else is in vulntrack.toml.
|
|
2
|
+
# Copy to .env (gitignored) — or export these directly; a real environment
|
|
3
|
+
# variable always wins over the file.
|
|
4
|
+
#
|
|
5
|
+
# One password per Harbor project, project name uppercased:
|
|
6
|
+
# [projects.domino] in vulntrack.toml -> HARBOR_DOMINO_PASSWORD
|
|
7
|
+
#
|
|
8
|
+
# Anything that is not a letter or digit becomes an underscore, so the name stays
|
|
9
|
+
# exportable from a shell (a dash is a syntax error in `export`):
|
|
10
|
+
# [projects.my-project] -> HARBOR_MY_PROJECT_PASSWORD
|
|
11
|
+
# For a name of your own choosing, set password_env on the project in the TOML.
|
|
12
|
+
HARBOR_DOMINO_PASSWORD=
|
|
13
|
+
HARBOR_LIBRARY_PASSWORD=
|
|
14
|
+
|
|
15
|
+
# Azure blob store (only when `store` is az://…); the SDK reads these itself.
|
|
16
|
+
# AZURE_STORAGE_CONNECTION_STRING=
|
|
17
|
+
|
|
18
|
+
# Any vulntrack.toml key can also be overridden here, VULNTRACK_ prefixed:
|
|
19
|
+
# VULNTRACK_STORE=/tmp/vt-store
|
|
20
|
+
# VULNTRACK_CONCURRENCY=8
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
.venv/
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.pyc
|
|
4
|
+
.pytest_cache/
|
|
5
|
+
dist/
|
|
6
|
+
*.egg-info/
|
|
7
|
+
site/
|
|
8
|
+
.claude/
|
|
9
|
+
|
|
10
|
+
# the two configuration files; .example templates are committed
|
|
11
|
+
vulntrack.toml
|
|
12
|
+
.env
|
|
13
|
+
|
|
14
|
+
# runtime stores
|
|
15
|
+
vt-store/
|
|
16
|
+
vt-data/
|
|
17
|
+
|
|
18
|
+
# mapping packs are yours; packs.example/ is the template
|
|
19
|
+
packs/
|
|
20
|
+
|
|
21
|
+
# legacy: superseded by .env
|
|
22
|
+
harbor-credentials.txt
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 brunnelu
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: vulntrack
|
|
3
|
+
Version: 0.2.0b1
|
|
4
|
+
Summary: Assess Harbor/Trivy findings with stacked, auditable CVSS environmental rules (git NDJSON store)
|
|
5
|
+
Author-email: brunnelu <6707792+brunnelu@users.noreply.github.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Requires-Dist: click>=8.1
|
|
10
|
+
Requires-Dist: cvss>=3.6
|
|
11
|
+
Requires-Dist: httpx>=0.27
|
|
12
|
+
Requires-Dist: polars>=1.17
|
|
13
|
+
Requires-Dist: pydantic-settings>=2.14.2
|
|
14
|
+
Requires-Dist: pydantic>=2.7
|
|
15
|
+
Requires-Dist: streamlit>=1.45
|
|
16
|
+
Requires-Dist: tenacity>=9.1.4
|
|
17
|
+
Requires-Dist: textual>=8.2.8
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# vulntrack
|
|
21
|
+
|
|
22
|
+
Assess Harbor/Trivy findings with a **stacked, auditable rules engine**.
|
|
23
|
+
Persistence is **append-only parquet in blob storage** (Azure container, or a local
|
|
24
|
+
path). Collect spec: **[vulntrack/task.md](vulntrack/task.md)**; architecture:
|
|
25
|
+
**[docs/index.md](docs/index.md)** (its section 7 on storage is superseded by task.md).
|
|
26
|
+
What changed and what is still open: **[WORKLOG.md](WORKLOG.md)**.
|
|
27
|
+
|
|
28
|
+
## Quickstart
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
uv sync
|
|
32
|
+
cp vulntrack.toml.example vulntrack.toml # settings
|
|
33
|
+
cp .env.example .env # HARBOR_<PROJECT>_PASSWORD
|
|
34
|
+
|
|
35
|
+
uv run vulntrack collect --no-submit # progress bar on a terminal; --no-progress off
|
|
36
|
+
|
|
37
|
+
# -v logs every HTTP request (httpx's own line)
|
|
38
|
+
uv run vulntrack -v collect --no-submit
|
|
39
|
+
|
|
40
|
+
# map extra columns onto the frame — deployments, KEV, owners, anything
|
|
41
|
+
cp -r packs.example packs
|
|
42
|
+
uv run vulntrack packs list # resolution order and the dependency graph
|
|
43
|
+
uv run vulntrack packs check # duplicates, collisions, coverage, dead rules
|
|
44
|
+
|
|
45
|
+
# Textual TUI — walk residue by finding PK, apply rules (recommended)
|
|
46
|
+
uv run vulntrack ui
|
|
47
|
+
|
|
48
|
+
# same flow without TUI
|
|
49
|
+
uv run vulntrack assess
|
|
50
|
+
uv run vulntrack assess CVE-2024-0001 --set MA=N --because "…"
|
|
51
|
+
|
|
52
|
+
# optional browser UI
|
|
53
|
+
uv run vulntrack ui --web
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Data model: PK, joins, assessments
|
|
57
|
+
|
|
58
|
+
### Occurrence primary key (source row)
|
|
59
|
+
|
|
60
|
+
Every finding from Harbor is one row with this identity:
|
|
61
|
+
|
|
62
|
+
```text
|
|
63
|
+
project
|
|
64
|
+
project_version # if you maintain one; else empty
|
|
65
|
+
cve
|
|
66
|
+
image # repository without tag
|
|
67
|
+
image_tag # part of PK as you see it in the world
|
|
68
|
+
digest # part of PK (immutable content); usually hidden in UI
|
|
69
|
+
package
|
|
70
|
+
package_version
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
That whole key is the **occurrence**. Digests are long — keep them in the
|
|
74
|
+
data, **do not put them in the default table columns** (optional short
|
|
75
|
+
prefix on drill-down / copy).
|
|
76
|
+
|
|
77
|
+
Today’s names: `image_repository` ≈ `project/image`, `installed_version` ≈
|
|
78
|
+
`package_version`, `tags` ≈ `image_tag`.
|
|
79
|
+
|
|
80
|
+
### Augmented columns (joins — not PK)
|
|
81
|
+
|
|
82
|
+
Anything else is **joined on**, not part of the source PK. Joins come from
|
|
83
|
+
**packs**: one TSV per mapping in `packs/`, keyed on a column the frame already
|
|
84
|
+
has — including a column an earlier pack produced. See `packs.example/`.
|
|
85
|
+
|
|
86
|
+
```text
|
|
87
|
+
image ──join──► packs/deployments.tsv ──► image_kind, in_use, exposure, …
|
|
88
|
+
cve ──join──► packs/kev.tsv ──► kev, …
|
|
89
|
+
image ──join──► packs/projects.tsv ──► project_id
|
|
90
|
+
└─join──► packs/project-tiers.tsv ──► tier, data_class
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
The order is derived from the key columns, never the filenames; a key nothing
|
|
94
|
+
produces, a cycle, or two rows with the same key is an error that names the pack.
|
|
95
|
+
A pack never adds a row — a mapping row matching nothing is reported by
|
|
96
|
+
`vulntrack packs check`, not merged in. Every value a pack writes lands in
|
|
97
|
+
`mapping_trail` beside the `reasoning`, and `vulntrack explain <digest> <cve>`
|
|
98
|
+
walks one value back through the chain to the image it came from.
|
|
99
|
+
|
|
100
|
+
`image_kind` is **not** a Harbor field and **not** part of the PK. It is
|
|
101
|
+
augmented data: you maintain `image → image_kind` (and friends); we left-join
|
|
102
|
+
it onto occurrences before rules run. Missing join → null → class rules do
|
|
103
|
+
not match until the register is filled.
|
|
104
|
+
|
|
105
|
+
Same pattern later for KEV, distro status, etc.: new pack, join key, new
|
|
106
|
+
column, null when unknown.
|
|
107
|
+
|
|
108
|
+
### Assessment (computed, not stored)
|
|
109
|
+
|
|
110
|
+
```text
|
|
111
|
+
occurrence (+ joined columns) + ordered rules → env vector, severity, reasons
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Recomputed every time. No assessment table in git — only rules + inputs.
|
|
115
|
+
|
|
116
|
+
### How people actually assess (the product loop)
|
|
117
|
+
|
|
118
|
+
In practice almost all **new** work is **per CVE**:
|
|
119
|
+
|
|
120
|
+
1. Open a CVE still in residue (Critical/High after the stack).
|
|
121
|
+
2. See which occurrence rows / joined context it covers (images, tags,
|
|
122
|
+
packages, versions — digest hidden).
|
|
123
|
+
3. Write **one rule** with:
|
|
124
|
+
- `when`: at least `cve = …`, plus whatever else is true
|
|
125
|
+
(e.g. only workspace via joined `image_kind`, only one package, …)
|
|
126
|
+
- `set`: environmental metrics (and/or terminal severity)
|
|
127
|
+
- `because`: **custom reasoning text** (the audit trail)
|
|
128
|
+
4. Save → recompute → matching rows leave residue; continue.
|
|
129
|
+
|
|
130
|
+
You are not maintaining a spreadsheet of assessments. You add **rules with
|
|
131
|
+
reasons**; the PK table only shows *where* that rule applies after the join
|
|
132
|
+
and stack.
|
|
133
|
+
|
|
134
|
+
Broad posture rules (`project.*`, `workspace.posture`) stay few and shared.
|
|
135
|
+
Residue work is mostly `cve.…` rules with a human `because`.
|
|
136
|
+
|
|
137
|
+
### What the UI should show
|
|
138
|
+
|
|
139
|
+
```text
|
|
140
|
+
Queue: residue grouped or listed by CVE (work list)
|
|
141
|
+
Detail: description + joined facts + occurrence rows for that CVE
|
|
142
|
+
(PK columns; digest hidden)
|
|
143
|
+
Applied: which rules already fired / current env result
|
|
144
|
+
Write: scope (CVE ± package ± image ± image_kind) + metrics + because
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
Showing every PK row is fine at residue scale (~hundreds). Collapsing
|
|
148
|
+
identical env outcomes is optional sugar — not required to understand the
|
|
149
|
+
model.
|
|
150
|
+
|
|
151
|
+
### Rules vs PK (important)
|
|
152
|
+
|
|
153
|
+
| Prefer in `when` | Avoid in `when` |
|
|
154
|
+
|---|---|
|
|
155
|
+
| `cve`, `package`, `image`, joined `image_kind` | `digest` (breaks on rebuild) |
|
|
156
|
+
| other joined facts you trust | bare `image_tag` unless the claim is really tag-specific |
|
|
157
|
+
|
|
158
|
+
Tag and package_version **are** part of the occurrence PK (inventory truth).
|
|
159
|
+
Rules still usually **omit** them so one decision covers rebuilds; only add
|
|
160
|
+
them when the claim is truly version- or tag-specific.
|
|
161
|
+
|
|
162
|
+
### Mapping today
|
|
163
|
+
|
|
164
|
+
| surface | role |
|
|
165
|
+
|---|---|
|
|
166
|
+
| `vulntrack ui` | walk residue, write per-CVE rules |
|
|
167
|
+
| `vulntrack assess` | same in pure CLI |
|
|
168
|
+
| `vulntrack rate` | dump rated occurrences (includes digest) |
|
|
169
|
+
| `vulntrack packs list` | pack resolution order and the dependency graph |
|
|
170
|
+
| `vulntrack packs check` | lint the packs — non-zero on error, for CI |
|
|
171
|
+
| `vulntrack explain <digest> <cve>` | rules fired, their inputs, and the mapping chain |
|
|
172
|
+
|
|
173
|
+
## Layout
|
|
174
|
+
|
|
175
|
+
| path | role |
|
|
176
|
+
|---|---|
|
|
177
|
+
| `vulntrack/engine/` | frame + packs, stacking rules, CVSS env score |
|
|
178
|
+
| `vulntrack.toml` | every setting (gitignored; template `.example`) |
|
|
179
|
+
| `packs/*.tsv` | mapping packs joined onto the frame (template `packs.example/`) |
|
|
180
|
+
| `.env` | every secret — `HARBOR_<PROJECT>_PASSWORD` |
|
|
181
|
+
| `vulntrack/settings.py` | env → `.env` → `vulntrack.toml` → defaults |
|
|
182
|
+
| `vulntrack/store.py` | append-only parquet store, SCD2 views |
|
|
183
|
+
| `vulntrack/harbor.py` | Harbor client + collect job |
|
|
184
|
+
| `vulntrack/task.md` | collect-layer spec — read this before changing either |
|
|
185
|
+
| `vulntrack/tui.py` | Textual assess UI |
|
|
186
|
+
| `vulntrack/ui/` | optional Streamlit |
|
|
187
|
+
| `vt-store/` | runtime data (gitignored) |
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# Worklog
|
|
2
|
+
|
|
3
|
+
Running record of what changed and what is still open, so the work can be followed
|
|
4
|
+
without reading every diff. Newest entry first. Design rationale lives in
|
|
5
|
+
[plan.md](plan.md); the collect-layer spec lives in [vulntrack/task.md](vulntrack/task.md).
|
|
6
|
+
|
|
7
|
+
## Where the work lives
|
|
8
|
+
|
|
9
|
+
| branch | tip | state |
|
|
10
|
+
|---|---|---|
|
|
11
|
+
| `worktree-packs-mapping-chain` | `92c1ebf` (2026-08-31) | **current line of work.** Chained mapping packs. 77 tests pass. |
|
|
12
|
+
| `ingest-blob-collect-layer` | `b1516d9` (2026-07-29) | superseded — its uncommitted working tree is committed here as `43aa273` |
|
|
13
|
+
| `worktree-purpose-docs` | `c527bb8` (2026-07-25) | 8 documentation commits, **never merged anywhere** — see open work |
|
|
14
|
+
| `implement-stacking-engine` | `d2eb6ee` (2026-07-25) | historical |
|
|
15
|
+
| `main` | `c1c5021` (2026-07-23) | last release line: Streamlit UI + Harbor scan CSRF fix |
|
|
16
|
+
|
|
17
|
+
The main checkout at the repo root still sits on `ingest-blob-collect-layer` with a
|
|
18
|
+
large uncommitted diff. That diff is **not lost** — it is commit `43aa273` on this
|
|
19
|
+
branch, verified identical. Continue here, not there.
|
|
20
|
+
|
|
21
|
+
## Open work
|
|
22
|
+
|
|
23
|
+
Ordered by what blocks what.
|
|
24
|
+
|
|
25
|
+
0. **Make the installed package self-contained** (see `docs/packaging.md`). Under a
|
|
26
|
+
wheel, `cli._project_root()` is `site-packages/`, so `app.py` and
|
|
27
|
+
`store/rules.ndjson` resolve to nothing: `vulntrack ui --web` fails and there is no
|
|
28
|
+
default ruleset. Needs a decision, not a patch — move `app.py` into the package and
|
|
29
|
+
find it via `importlib.resources`; source the ruleset from the config/state dir.
|
|
30
|
+
Blocks any non-beta release.
|
|
31
|
+
1. **Decide rules versioning** (plan.md §5.5, §8.4). Rules are the last un-audited
|
|
32
|
+
input: `rules.ndjson` is rewritten in place by the UI, so an as-of replay would
|
|
33
|
+
replay old packs against *today's* rules. Either record a git commit per run, or
|
|
34
|
+
make rules a store table (recommended — same shape as packs). **This is a decision,
|
|
35
|
+
not code, and it blocks 2 being worth building.**
|
|
36
|
+
2. **Packs as an SCD2 store table** (plan.md §5.4, step 6). Gives change history,
|
|
37
|
+
tombstones and as-of replay for mapping tables.
|
|
38
|
+
3. **Run fingerprint** (plan.md §5.5, step 7). `run_id` + sha256 of each pack file and
|
|
39
|
+
of the ruleset, stored with every rated output, so an exported spreadsheet ties back
|
|
40
|
+
to the inputs that produced it.
|
|
41
|
+
4. **Answer plan.md §8 open questions 1, 5** — outer-join intent, and whether a pack is
|
|
42
|
+
always a file or may eventually be a SQL/API query fetched at collect time. Q2, Q3
|
|
43
|
+
and Q4 were settled by the implementation (manifest at `packs/packs.toml`; duplicate
|
|
44
|
+
keys are a hard error; rules versioning still open as item 1 above).
|
|
45
|
+
5. **Decide the fate of `worktree-purpose-docs`.** Eight commits rewriting the
|
|
46
|
+
architecture docs around the data rather than the code — including the rating model
|
|
47
|
+
correction and the asset-class CVSS work. Unmerged and drifting; either fold into
|
|
48
|
+
this branch or retire it deliberately.
|
|
49
|
+
6. **Merge path back to `main`.** `main` is six weeks behind and predates the rating
|
|
50
|
+
rewrite, the blob store and the chain.
|
|
51
|
+
|
|
52
|
+
## Log
|
|
53
|
+
|
|
54
|
+
### 2026-09-07 — beta release 0.2.0b1, to exercise the PyPI workflow
|
|
55
|
+
|
|
56
|
+
Version bumped in `pyproject.toml` and `vulntrack/__init__.py`. `uv build` +
|
|
57
|
+
`uvx twine check` pass locally on both artifacts; 77 tests pass.
|
|
58
|
+
|
|
59
|
+
Two packaging defects surfaced while checking what would actually be published:
|
|
60
|
+
|
|
61
|
+
- **The sdist was shipping gitignored `packs/`.** Hatchling's default file selection
|
|
62
|
+
swept in untracked working-directory files, and `packs/` holds the real
|
|
63
|
+
`image_repository → project_id → tier` mappings. A local `uv build` would have made
|
|
64
|
+
them permanently public. Fixed with an explicit
|
|
65
|
+
`[tool.hatch.build.targets.sdist] include`. CI builds from a clean checkout and never
|
|
66
|
+
saw this. The wheel was never affected.
|
|
67
|
+
- **The installed package is not self-contained** — `cli._project_root()` is
|
|
68
|
+
`Path(__file__).parent.parent`, which is `site-packages/` under a wheel, so `app.py`
|
|
69
|
+
and `store/rules.ndjson` both resolve to paths that do not exist there. Not fixed;
|
|
70
|
+
see open work item 0. This is why the release is a beta.
|
|
71
|
+
|
|
72
|
+
Wrote `docs/packaging.md` — the doc `publish.yml` has always pointed at and which had
|
|
73
|
+
never existed, so the one-time Trusted Publishing setup was undocumented.
|
|
74
|
+
|
|
75
|
+
**Workflow result — [run 34118819161](https://github.com/brunnelu/vulntrack/actions/runs/34118819161):
|
|
76
|
+
`build` passed, `publish` failed on `invalid-publisher`.** So the answer is that the
|
|
77
|
+
PyPI publisher was never registered — the credentials were not added. Everything on the
|
|
78
|
+
GitHub side is correct; the OIDC claims the run presented are exactly the values
|
|
79
|
+
`docs/packaging.md` says to register:
|
|
80
|
+
|
|
81
|
+
repository brunnelu/vulntrack
|
|
82
|
+
workflow_ref .github/workflows/publish.yml
|
|
83
|
+
environment pypi
|
|
84
|
+
|
|
85
|
+
The `pypi` GitHub environment did not exist beforehand and was created by this run, as
|
|
86
|
+
expected. Nothing was uploaded, and `0.2.0b1` is therefore still an unused version
|
|
87
|
+
number on PyPI — re-dispatching the same workflow after the registration will publish
|
|
88
|
+
it. **Blocked on the pypi.org registration, which needs the maintainer's account.**
|
|
89
|
+
|
|
90
|
+
### 2026-09-07 — surveyed the state, started this log
|
|
91
|
+
|
|
92
|
+
No code changed. Established where the work actually is:
|
|
93
|
+
|
|
94
|
+
- The newest branch is `worktree-packs-mapping-chain`; adopted it as the line of work.
|
|
95
|
+
- Verified the repo root's uncommitted diff is captured byte-for-byte by `43aa273`
|
|
96
|
+
here, so nothing is stranded in the main checkout.
|
|
97
|
+
- Confirmed the chain resolves from data, not filenames: `packs list` against
|
|
98
|
+
`packs.example/` orders `project-tiers` after `projects` because it keys on
|
|
99
|
+
`project_id`, a column `projects` produces.
|
|
100
|
+
- Test suite: 77 passed.
|
|
101
|
+
|
|
102
|
+
### 2026-08-31 — chained mapping packs (`92c1ebf`)
|
|
103
|
+
|
|
104
|
+
Implements plan.md steps 1–5 and 9. A pack may key on a column another pack produced
|
|
105
|
+
(`image_repository → project_id → tier`), and the order is derived from the key columns
|
|
106
|
+
by fixpoint, so renaming a file cannot change the result.
|
|
107
|
+
|
|
108
|
+
The join is the small part; the rest is making failure loud, because a broken chain
|
|
109
|
+
otherwise re-rates findings with no diff, no error and a plausible-looking reasoning
|
|
110
|
+
string:
|
|
111
|
+
|
|
112
|
+
- left join only — a mapping row matching nothing is a lint result, never a frame row
|
|
113
|
+
- declared keys in `packs/packs.toml`; canonical-name inference stays as the fallback,
|
|
114
|
+
so an un-chained folder needs no manifest
|
|
115
|
+
- duplicate key rows are an error listing the offending values, not `keep="last"`
|
|
116
|
+
- `mapping_trail` beside `reasoning`: which pack wrote each value and on which key,
|
|
117
|
+
with an explicit `[default]` entry where the heuristic guessed — silence would not
|
|
118
|
+
distinguish a guess from a fact
|
|
119
|
+
- `vulntrack packs list` / `packs check` (non-zero exit, for CI) / `vulntrack explain`
|
|
120
|
+
|
|
121
|
+
New: `vulntrack/engine/packs.py` (384 lines), `tests/test_packs.py`,
|
|
122
|
+
`tests/test_cli_packs.py`.
|
|
123
|
+
|
|
124
|
+
### 2026-08-31 — snapshot of the rating rewrite and blob collect layer (`43aa273`)
|
|
125
|
+
|
|
126
|
+
Committed what had been sitting uncommitted in the main checkout: `engine/cvss_env.py`
|
|
127
|
+
and `engine/pipeline.py` replaced by `engine/rating.py`; the blob-backed append-only
|
|
128
|
+
store; the Harbor collect layer; `packs.example/`.
|
vulntrack-0.2.0b1/app.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""vulntrack UI — st.navigation.
|
|
2
|
+
|
|
3
|
+
uv run vulntrack ui
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import streamlit as st
|
|
7
|
+
|
|
8
|
+
st.set_page_config(page_title="vulntrack", layout="wide", initial_sidebar_state="expanded")
|
|
9
|
+
|
|
10
|
+
from vulntrack.ui import pages_assess, pages_images, pages_rules, pages_scans # noqa: E402
|
|
11
|
+
|
|
12
|
+
st.navigation(
|
|
13
|
+
{
|
|
14
|
+
"Assess": [
|
|
15
|
+
st.Page(pages_assess.page, title="Assess", url_path="assess", default=True),
|
|
16
|
+
st.Page(pages_rules.page, title="Rules", url_path="rules"),
|
|
17
|
+
],
|
|
18
|
+
"Estate": [
|
|
19
|
+
st.Page(pages_images.page, title="Images", url_path="images"),
|
|
20
|
+
st.Page(pages_scans.page, title="Scans", url_path="scans"),
|
|
21
|
+
],
|
|
22
|
+
}
|
|
23
|
+
).run()
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# Handoff — continue here next session
|
|
2
|
+
|
|
3
|
+
**Date:** 2026-07-25
|
|
4
|
+
**Branch:** `implement-stacking-engine` (tracks `origin/implement-stacking-engine`)
|
|
5
|
+
**PR:** https://github.com/brunnelu/vulntrack/pull/2 → `main`
|
|
6
|
+
**HEAD:** `5b73804` — UI: st.navigation, Assess input, keep all Harbor CVSS vectors
|
|
7
|
+
**Tests:** `uv run pytest` → **43 passed**
|
|
8
|
+
|
|
9
|
+
**Directive for the next agent/session:**
|
|
10
|
+
More code is not the goal. Prefer fewer, clearer mechanisms. The user has already pushed back on bloat, on proxy rabbit-holes, and on TSV-for-rules. **Do not add features to “look busy.”** Decide, simplify, or document — then implement the minimum.
|
|
11
|
+
|
|
12
|
+
**Product direction (KISS):**
|
|
13
|
+
- Rule = `when` + `set{metrics}` + `because` (+ optional terminal severity).
|
|
14
|
+
- **Primary UI: Textual TUI** (`uv run vulntrack ui`) — select → apply rule → residue shrinks.
|
|
15
|
+
- **Identity (see readme):** occurrence PK includes tag + package_version + digest (digest hidden in UI). `image_kind` etc. are **joined** packs, not source PK. Assess = mostly per-CVE rules + custom `because`; ratings computed, not stored.
|
|
16
|
+
- CLI `assess` same loop; Streamlit only via `ui --web`.
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
## What this project is
|
|
21
|
+
|
|
22
|
+
**vulntrack** turns Harbor/Trivy scan output into **environmental severities** with an **auditable reason chain**, using a **stack of rules**. Ratings are **not stored**; they are recomputed from versioned inputs.
|
|
23
|
+
|
|
24
|
+
Product name is **vulntrack** only (not streamlit-editor).
|
|
25
|
+
|
|
26
|
+
Target design: `docs/index.md` (architecture). Implementation is a partial realization of that doc.
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## Current shape (lean inventory)
|
|
31
|
+
|
|
32
|
+
| Piece | Role | Notes |
|
|
33
|
+
|---|---|---|
|
|
34
|
+
| `vulntrack/engine/` | Pure rules + CVSS env score | Core value. Keep small. |
|
|
35
|
+
| `vulntrack/gitstore.py` | NDJSON under `vt-data/` | Works; **user may prefer TSV** (not done). |
|
|
36
|
+
| `vulntrack/collect.py` | Stateless Harbor collect + scan ledger | OK. |
|
|
37
|
+
| `vulntrack/harbor.py` | Client + report parse | Multi-vector parse added; **recollect needed** for store. |
|
|
38
|
+
| `vulntrack/webpath.py` | code-server `proxy` / `absproxy` | Painful history; **UI access works for the user** — do not reopen unless broken. |
|
|
39
|
+
| `vulntrack/ui/` + `app.py` | Streamlit multipage | `st.navigation` restored; Assess still **not liked**. |
|
|
40
|
+
| `store/rules.ndjson` | Seed rules | Shipped defaults. |
|
|
41
|
+
| `vt-data/` | Runtime store (gitignored) | NDJSON; not committed. |
|
|
42
|
+
| Old Delta `data/` | **Deleted** | No migration path; reassess from Harbor. |
|
|
43
|
+
|
|
44
|
+
**Removed on purpose:** Delta Lake, SCD2, hierarchical `resolve`, half-built `rating_rules`, mock-first product path, legacy Excel assessments pipeline.
|
|
45
|
+
|
|
46
|
+
---
|
|
47
|
+
|
|
48
|
+
## How to run (today)
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
cd /home/coder/projects/personal/streamlit-editor
|
|
52
|
+
uv sync
|
|
53
|
+
cp vulntrack.toml.example vulntrack.toml # settings
|
|
54
|
+
cp .env.example .env # HARBOR_<PROJECT>_PASSWORD
|
|
55
|
+
|
|
56
|
+
uv run vulntrack collect --no-submit # harvest into vt-data/
|
|
57
|
+
uv run vulntrack rate --residue-only # CLI summary
|
|
58
|
+
uv run vulntrack ui # Streamlit
|
|
59
|
+
uv run pytest
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
**UI / code-server:**
|
|
63
|
+
`VSCODE_PROXY_URI=https://code-server.bru.lu/proxy/{{port}}/`.
|
|
64
|
+
CLI defaults to **absproxy** + `baseUrlPath=absproxy/<port>` (port default **4000**).
|
|
65
|
+
Open the URL the CLI prints (e.g. `https://code-server.bru.lu/absproxy/4000/`).
|
|
66
|
+
Nested paths like `…/proxy/N/proxy/N/` = old wrong `baseUrlPath` + cached 301s — private window or new port.
|
|
67
|
+
|
|
68
|
+
User stated they **can access the UI**; stop spending cycles on proxy debugging unless something is actually broken for them.
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## What works
|
|
73
|
+
|
|
74
|
+
- Flat in-memory rate pipeline: findings + images + deployments + ordered rules → env severity + reason chain.
|
|
75
|
+
- Collect from real Harbor; gitstore write; scan ledger append.
|
|
76
|
+
- Seed stack: project MAV, workspace MA/MC/MPR/MS, not-in-use, DoS signal, KEV last.
|
|
77
|
+
- Measured effect (order of magnitude): ~22k findings → ~50–50-ish Critical/High CVEs after stack (recompute after recollect).
|
|
78
|
+
- Multi-vector **parse** in code: all Harbor CVSS sources → `cvss_vectors` JSON (+ convenience columns). **Existing vt-data rows may still lack `cvss_vectors` until recollect.**
|
|
79
|
+
- Streamlit `st.navigation` pages: Assess (default), Rules, Overview, Images, Scans.
|
|
80
|
+
- Config: one `vulntrack.toml`; one robot password per project in `.env` (or the environment).
|
|
81
|
+
|
|
82
|
+
---
|
|
83
|
+
|
|
84
|
+
## Open product decisions (user-driven — do these first, not more glue)
|
|
85
|
+
|
|
86
|
+
### 1. Storage format
|
|
87
|
+
|
|
88
|
+
- **Now:** sorted **NDJSON** under `vt-data/`.
|
|
89
|
+
- **User:** “maybe better storage; **TSV might be an option**.”
|
|
90
|
+
- **Engineering preference:** one format, human-diffable, spreadsheet-friendly if TSV; avoid dual NDJSON+TSV forever.
|
|
91
|
+
- **Not done.** Next session should **choose** TSV vs NDJSON (or TSV for rules/deployments only) and migrate once — not add a third backend.
|
|
92
|
+
|
|
93
|
+
### 2. Rule model / authoring
|
|
94
|
+
|
|
95
|
+
- **Now:** JSON-ish rules (`when` predicates + `then` effects + `because`), CLI + Assess form that builds them.
|
|
96
|
+
- **User:** “**I do not like the way you propose rules.**”
|
|
97
|
+
- **Also disliked:** radio-button page nav (fixed to `st.navigation`); earlier “propose rules” / over-automated downrank framing.
|
|
98
|
+
- **Not done.** Next session should redesign rule **representation and UX** with the user *before* coding more form widgets. Possible lean directions (examples only — agree first):
|
|
99
|
+
- Rules as a **TSV/table** the user edits (columns: id, when_*, metric, value, because, enabled).
|
|
100
|
+
- Closer to original Assess flow (severity + reason + simple scope) that **emits** one clear rule row, not a DSL form.
|
|
101
|
+
- Fewer effects; metric-only rules as default.
|
|
102
|
+
- **Do not** add rule mining, AI proposals, or more rule kinds until the core format is agreed.
|
|
103
|
+
|
|
104
|
+
### 3. Data model hygiene
|
|
105
|
+
|
|
106
|
+
- `cvss_vectors` on findings after recollect; normalize CVE text in `cves.*`.
|
|
107
|
+
- Deployments still often **inferred** from `*-environment` names if register empty — fine as bootstrap, not as silent policy.
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
|
|
111
|
+
## Explicit non-goals for the next session
|
|
112
|
+
|
|
113
|
+
- No more proxy/absproxy archaeology unless the user cannot open the UI.
|
|
114
|
+
- No Delta/SCD2/“restore old stack.”
|
|
115
|
+
- No second web framework (FastAPI UI was tried and dropped for Streamlit).
|
|
116
|
+
- No CI/packaging/linter sprawl.
|
|
117
|
+
- No expanding the engine with EPSS/KEV/distro packs until storage + rules feel right.
|
|
118
|
+
- No large UI redesigns without agreeing the rule model.
|
|
119
|
+
|
|
120
|
+
---
|
|
121
|
+
|
|
122
|
+
## Hard-won constraints (do not re-litigate)
|
|
123
|
+
|
|
124
|
+
1. **Assessment is the product** — environmental score + reasons; not “dismiss by no-fix.”
|
|
125
|
+
2. **Rules stack**; score **once** at the end from accumulated metrics.
|
|
126
|
+
3. Prefer **CVSS environmental metrics** over blanket severity overrides.
|
|
127
|
+
4. **No blanket MUI/MAC** as free levers; residue is often AV:N/AC:L/UI:N by selection.
|
|
128
|
+
5. **Image identity without tag** for register/rules so new digests inherit.
|
|
129
|
+
6. **Git-only persistence** for the new design; ratings not stored as authority.
|
|
130
|
+
7. Workspace posture (example): MA:N, MC:L, MPR:L, MS:U — domain-owned by the user, not the model.
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
134
|
+
## Suggested start next week (minimal)
|
|
135
|
+
|
|
136
|
+
1. Read this file + `docs/index.md` §1–2 and §6 (rating).
|
|
137
|
+
2. Skim `vulntrack/engine/` and `gitstore.py` only — not the whole UI.
|
|
138
|
+
3. **With the user, decide:**
|
|
139
|
+
- Store: NDJSON → **TSV** (or hybrid)?
|
|
140
|
+
- Rules: what does a rule look like on disk and in the Assess UI?
|
|
141
|
+
4. Implement **only** that decision (one storage format + one rule shape), delete what becomes redundant.
|
|
142
|
+
5. Recollect from Harbor; verify multi-vector rows and residue.
|
|
143
|
+
|
|
144
|
+
---
|
|
145
|
+
|
|
146
|
+
## Credentials / secrets (do not commit)
|
|
147
|
+
|
|
148
|
+
- `.env` — one `HARBOR_<PROJECT>_PASSWORD` per project, and nothing else; gitignored.
|
|
149
|
+
Exported environment variables override it, so CI needs no file.
|
|
150
|
+
- `vulntrack.toml` — every non-secret setting; gitignored, template committed as
|
|
151
|
+
`vulntrack.toml.example`. Points at Harbor `https://harbor.bru.lu`, projects
|
|
152
|
+
`domino` and `library`, store `vt-store`.
|
|
153
|
+
- Superseded and removed: `config.toml`, `harbor/credentials.txt`,
|
|
154
|
+
`harbor-credentials.txt`, `.streamlit/config.toml`.
|
|
155
|
+
|
|
156
|
+
---
|
|
157
|
+
|
|
158
|
+
## One-line summary
|
|
159
|
+
|
|
160
|
+
**Independent vulntrack on git NDJSON + stacking engine + Streamlit is on PR #2; stop bloating — next work is storage format (likely TSV) and a rule model the user actually accepts.**
|