vulntrack 0.2.0b1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. vulntrack-0.2.0b1/.env.example +20 -0
  2. vulntrack-0.2.0b1/.gitignore +22 -0
  3. vulntrack-0.2.0b1/LICENSE +21 -0
  4. vulntrack-0.2.0b1/PKG-INFO +187 -0
  5. vulntrack-0.2.0b1/WORKLOG.md +128 -0
  6. vulntrack-0.2.0b1/app.py +23 -0
  7. vulntrack-0.2.0b1/docs/HANDOFF.md +160 -0
  8. vulntrack-0.2.0b1/docs/index.md +574 -0
  9. vulntrack-0.2.0b1/docs/packaging.md +94 -0
  10. vulntrack-0.2.0b1/mkdocs.yml +35 -0
  11. vulntrack-0.2.0b1/packs.example/README.md +94 -0
  12. vulntrack-0.2.0b1/packs.example/deployments.tsv +4 -0
  13. vulntrack-0.2.0b1/packs.example/kev.tsv +3 -0
  14. vulntrack-0.2.0b1/packs.example/packs.toml +14 -0
  15. vulntrack-0.2.0b1/packs.example/project-tiers.tsv +3 -0
  16. vulntrack-0.2.0b1/packs.example/projects.tsv +4 -0
  17. vulntrack-0.2.0b1/plan.md +306 -0
  18. vulntrack-0.2.0b1/pyproject.toml +67 -0
  19. vulntrack-0.2.0b1/readme.md +168 -0
  20. vulntrack-0.2.0b1/store/rules.ndjson +4 -0
  21. vulntrack-0.2.0b1/tests/conftest.py +22 -0
  22. vulntrack-0.2.0b1/tests/test_cli_packs.py +106 -0
  23. vulntrack-0.2.0b1/tests/test_collect.py +362 -0
  24. vulntrack-0.2.0b1/tests/test_engine.py +207 -0
  25. vulntrack-0.2.0b1/tests/test_packs.py +290 -0
  26. vulntrack-0.2.0b1/tests/test_settings.py +46 -0
  27. vulntrack-0.2.0b1/tests/test_webpath.py +38 -0
  28. vulntrack-0.2.0b1/vulntrack/__init__.py +7 -0
  29. vulntrack-0.2.0b1/vulntrack/cli.py +702 -0
  30. vulntrack-0.2.0b1/vulntrack/engine/__init__.py +33 -0
  31. vulntrack-0.2.0b1/vulntrack/engine/frame.py +188 -0
  32. vulntrack-0.2.0b1/vulntrack/engine/packs.py +384 -0
  33. vulntrack-0.2.0b1/vulntrack/engine/rating.py +185 -0
  34. vulntrack-0.2.0b1/vulntrack/engine/rules.py +193 -0
  35. vulntrack-0.2.0b1/vulntrack/harbor.py +402 -0
  36. vulntrack-0.2.0b1/vulntrack/settings.py +145 -0
  37. vulntrack-0.2.0b1/vulntrack/store.py +326 -0
  38. vulntrack-0.2.0b1/vulntrack/task.md +435 -0
  39. vulntrack-0.2.0b1/vulntrack/tui.py +339 -0
  40. vulntrack-0.2.0b1/vulntrack/ui/__init__.py +1 -0
  41. vulntrack-0.2.0b1/vulntrack/ui/data.py +66 -0
  42. vulntrack-0.2.0b1/vulntrack/ui/pages_assess.py +294 -0
  43. vulntrack-0.2.0b1/vulntrack/ui/pages_images.py +83 -0
  44. vulntrack-0.2.0b1/vulntrack/ui/pages_rules.py +75 -0
  45. vulntrack-0.2.0b1/vulntrack/ui/pages_scans.py +56 -0
  46. vulntrack-0.2.0b1/vulntrack/webpath.py +159 -0
  47. vulntrack-0.2.0b1/vulntrack.toml.example +56 -0
@@ -0,0 +1,20 @@
1
+ # Secrets only. Everything else is in vulntrack.toml.
2
+ # Copy to .env (gitignored) — or export these directly; a real environment
3
+ # variable always wins over the file.
4
+ #
5
+ # One password per Harbor project, project name uppercased:
6
+ # [projects.domino] in vulntrack.toml -> HARBOR_DOMINO_PASSWORD
7
+ #
8
+ # Anything that is not a letter or digit becomes an underscore, so the name stays
9
+ # exportable from a shell (a dash is a syntax error in `export`):
10
+ # [projects.my-project] -> HARBOR_MY_PROJECT_PASSWORD
11
+ # For a name of your own choosing, set password_env on the project in the TOML.
12
+ HARBOR_DOMINO_PASSWORD=
13
+ HARBOR_LIBRARY_PASSWORD=
14
+
15
+ # Azure blob store (only when `store` is az://…); the SDK reads these itself.
16
+ # AZURE_STORAGE_CONNECTION_STRING=
17
+
18
+ # Any vulntrack.toml key can also be overridden here, VULNTRACK_ prefixed:
19
+ # VULNTRACK_STORE=/tmp/vt-store
20
+ # VULNTRACK_CONCURRENCY=8
@@ -0,0 +1,22 @@
1
+ .venv/
2
+ __pycache__/
3
+ *.pyc
4
+ .pytest_cache/
5
+ dist/
6
+ *.egg-info/
7
+ site/
8
+ .claude/
9
+
10
+ # the two configuration files; .example templates are committed
11
+ vulntrack.toml
12
+ .env
13
+
14
+ # runtime stores
15
+ vt-store/
16
+ vt-data/
17
+
18
+ # mapping packs are yours; packs.example/ is the template
19
+ packs/
20
+
21
+ # legacy: superseded by .env
22
+ harbor-credentials.txt
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 brunnelu
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,187 @@
1
+ Metadata-Version: 2.5
2
+ Name: vulntrack
3
+ Version: 0.2.0b1
4
+ Summary: Assess Harbor/Trivy findings with stacked, auditable CVSS environmental rules (git NDJSON store)
5
+ Author-email: brunnelu <6707792+brunnelu@users.noreply.github.com>
6
+ License-Expression: MIT
7
+ License-File: LICENSE
8
+ Requires-Python: >=3.11
9
+ Requires-Dist: click>=8.1
10
+ Requires-Dist: cvss>=3.6
11
+ Requires-Dist: httpx>=0.27
12
+ Requires-Dist: polars>=1.17
13
+ Requires-Dist: pydantic-settings>=2.14.2
14
+ Requires-Dist: pydantic>=2.7
15
+ Requires-Dist: streamlit>=1.45
16
+ Requires-Dist: tenacity>=9.1.4
17
+ Requires-Dist: textual>=8.2.8
18
+ Description-Content-Type: text/markdown
19
+
20
+ # vulntrack
21
+
22
+ Assess Harbor/Trivy findings with a **stacked, auditable rules engine**.
23
+ Persistence is **append-only parquet in blob storage** (Azure container, or a local
24
+ path). Collect spec: **[vulntrack/task.md](vulntrack/task.md)**; architecture:
25
+ **[docs/index.md](docs/index.md)** (its section 7 on storage is superseded by task.md).
26
+ What changed and what is still open: **[WORKLOG.md](WORKLOG.md)**.
27
+
28
+ ## Quickstart
29
+
30
+ ```bash
31
+ uv sync
32
+ cp vulntrack.toml.example vulntrack.toml # settings
33
+ cp .env.example .env # HARBOR_<PROJECT>_PASSWORD
34
+
35
+ uv run vulntrack collect --no-submit # progress bar on a terminal; --no-progress off
36
+
37
+ # -v logs every HTTP request (httpx's own line)
38
+ uv run vulntrack -v collect --no-submit
39
+
40
+ # map extra columns onto the frame — deployments, KEV, owners, anything
41
+ cp -r packs.example packs
42
+ uv run vulntrack packs list # resolution order and the dependency graph
43
+ uv run vulntrack packs check # duplicates, collisions, coverage, dead rules
44
+
45
+ # Textual TUI — walk residue by finding PK, apply rules (recommended)
46
+ uv run vulntrack ui
47
+
48
+ # same flow without TUI
49
+ uv run vulntrack assess
50
+ uv run vulntrack assess CVE-2024-0001 --set MA=N --because "…"
51
+
52
+ # optional browser UI
53
+ uv run vulntrack ui --web
54
+ ```
55
+
56
+ ## Data model: PK, joins, assessments
57
+
58
+ ### Occurrence primary key (source row)
59
+
60
+ Every finding from Harbor is one row with this identity:
61
+
62
+ ```text
63
+ project
64
+ project_version # if you maintain one; else empty
65
+ cve
66
+ image # repository without tag
67
+ image_tag # part of PK as you see it in the world
68
+ digest # part of PK (immutable content); usually hidden in UI
69
+ package
70
+ package_version
71
+ ```
72
+
73
+ That whole key is the **occurrence**. Digests are long — keep them in the
74
+ data, **do not put them in the default table columns** (optional short
75
+ prefix on drill-down / copy).
76
+
77
+ Today’s names: `image_repository` ≈ `project/image`, `installed_version` ≈
78
+ `package_version`, `tags` ≈ `image_tag`.
79
+
80
+ ### Augmented columns (joins — not PK)
81
+
82
+ Anything else is **joined on**, not part of the source PK. Joins come from
83
+ **packs**: one TSV per mapping in `packs/`, keyed on a column the frame already
84
+ has — including a column an earlier pack produced. See `packs.example/`.
85
+
86
+ ```text
87
+ image ──join──► packs/deployments.tsv ──► image_kind, in_use, exposure, …
88
+ cve ──join──► packs/kev.tsv ──► kev, …
89
+ image ──join──► packs/projects.tsv ──► project_id
90
+ └─join──► packs/project-tiers.tsv ──► tier, data_class
91
+ ```
92
+
93
+ The order is derived from the key columns, never the filenames; a key nothing
94
+ produces, a cycle, or two rows with the same key is an error that names the pack.
95
+ A pack never adds a row — a mapping row matching nothing is reported by
96
+ `vulntrack packs check`, not merged in. Every value a pack writes lands in
97
+ `mapping_trail` beside the `reasoning`, and `vulntrack explain <digest> <cve>`
98
+ walks one value back through the chain to the image it came from.
99
+
100
+ `image_kind` is **not** a Harbor field and **not** part of the PK. It is
101
+ augmented data: you maintain `image → image_kind` (and friends); we left-join
102
+ it onto occurrences before rules run. Missing join → null → class rules do
103
+ not match until the register is filled.
104
+
105
+ Same pattern later for KEV, distro status, etc.: new pack, join key, new
106
+ column, null when unknown.
107
+
108
+ ### Assessment (computed, not stored)
109
+
110
+ ```text
111
+ occurrence (+ joined columns) + ordered rules → env vector, severity, reasons
112
+ ```
113
+
114
+ Recomputed every time. No assessment table in git — only rules + inputs.
115
+
116
+ ### How people actually assess (the product loop)
117
+
118
+ In practice almost all **new** work is **per CVE**:
119
+
120
+ 1. Open a CVE still in residue (Critical/High after the stack).
121
+ 2. See which occurrence rows / joined context it covers (images, tags,
122
+ packages, versions — digest hidden).
123
+ 3. Write **one rule** with:
124
+ - `when`: at least `cve = …`, plus whatever else is true
125
+ (e.g. only workspace via joined `image_kind`, only one package, …)
126
+ - `set`: environmental metrics (and/or terminal severity)
127
+ - `because`: **custom reasoning text** (the audit trail)
128
+ 4. Save → recompute → matching rows leave residue; continue.
129
+
130
+ You are not maintaining a spreadsheet of assessments. You add **rules with
131
+ reasons**; the PK table only shows *where* that rule applies after the join
132
+ and stack.
133
+
134
+ Broad posture rules (`project.*`, `workspace.posture`) stay few and shared.
135
+ Residue work is mostly `cve.…` rules with a human `because`.
136
+
137
+ ### What the UI should show
138
+
139
+ ```text
140
+ Queue: residue grouped or listed by CVE (work list)
141
+ Detail: description + joined facts + occurrence rows for that CVE
142
+ (PK columns; digest hidden)
143
+ Applied: which rules already fired / current env result
144
+ Write: scope (CVE ± package ± image ± image_kind) + metrics + because
145
+ ```
146
+
147
+ Showing every PK row is fine at residue scale (~hundreds). Collapsing
148
+ identical env outcomes is optional sugar — not required to understand the
149
+ model.
150
+
151
+ ### Rules vs PK (important)
152
+
153
+ | Prefer in `when` | Avoid in `when` |
154
+ |---|---|
155
+ | `cve`, `package`, `image`, joined `image_kind` | `digest` (breaks on rebuild) |
156
+ | other joined facts you trust | bare `image_tag` unless the claim is really tag-specific |
157
+
158
+ Tag and package_version **are** part of the occurrence PK (inventory truth).
159
+ Rules still usually **omit** them so one decision covers rebuilds; only add
160
+ them when the claim is truly version- or tag-specific.
161
+
162
+ ### Mapping today
163
+
164
+ | surface | role |
165
+ |---|---|
166
+ | `vulntrack ui` | walk residue, write per-CVE rules |
167
+ | `vulntrack assess` | same in pure CLI |
168
+ | `vulntrack rate` | dump rated occurrences (includes digest) |
169
+ | `vulntrack packs list` | pack resolution order and the dependency graph |
170
+ | `vulntrack packs check` | lint the packs — non-zero on error, for CI |
171
+ | `vulntrack explain <digest> <cve>` | rules fired, their inputs, and the mapping chain |
172
+
173
+ ## Layout
174
+
175
+ | path | role |
176
+ |---|---|
177
+ | `vulntrack/engine/` | frame + packs, stacking rules, CVSS env score |
178
+ | `vulntrack.toml` | every setting (gitignored; template `.example`) |
179
+ | `packs/*.tsv` | mapping packs joined onto the frame (template `packs.example/`) |
180
+ | `.env` | every secret — `HARBOR_<PROJECT>_PASSWORD` |
181
+ | `vulntrack/settings.py` | env → `.env` → `vulntrack.toml` → defaults |
182
+ | `vulntrack/store.py` | append-only parquet store, SCD2 views |
183
+ | `vulntrack/harbor.py` | Harbor client + collect job |
184
+ | `vulntrack/task.md` | collect-layer spec — read this before changing either |
185
+ | `vulntrack/tui.py` | Textual assess UI |
186
+ | `vulntrack/ui/` | optional Streamlit |
187
+ | `vt-store/` | runtime data (gitignored) |
@@ -0,0 +1,128 @@
1
+ # Worklog
2
+
3
+ Running record of what changed and what is still open, so the work can be followed
4
+ without reading every diff. Newest entry first. Design rationale lives in
5
+ [plan.md](plan.md); the collect-layer spec lives in [vulntrack/task.md](vulntrack/task.md).
6
+
7
+ ## Where the work lives
8
+
9
+ | branch | tip | state |
10
+ |---|---|---|
11
+ | `worktree-packs-mapping-chain` | `92c1ebf` (2026-08-31) | **current line of work.** Chained mapping packs. 77 tests pass. |
12
+ | `ingest-blob-collect-layer` | `b1516d9` (2026-07-29) | superseded — its uncommitted working tree is committed here as `43aa273` |
13
+ | `worktree-purpose-docs` | `c527bb8` (2026-07-25) | 8 documentation commits, **never merged anywhere** — see open work |
14
+ | `implement-stacking-engine` | `d2eb6ee` (2026-07-25) | historical |
15
+ | `main` | `c1c5021` (2026-07-23) | last release line: Streamlit UI + Harbor scan CSRF fix |
16
+
17
+ The main checkout at the repo root still sits on `ingest-blob-collect-layer` with a
18
+ large uncommitted diff. That diff is **not lost** — it is commit `43aa273` on this
19
+ branch, verified identical. Continue here, not there.
20
+
21
+ ## Open work
22
+
23
+ Ordered by what blocks what.
24
+
25
+ 0. **Make the installed package self-contained** (see `docs/packaging.md`). Under a
26
+ wheel, `cli._project_root()` is `site-packages/`, so `app.py` and
27
+ `store/rules.ndjson` resolve to nothing: `vulntrack ui --web` fails and there is no
28
+ default ruleset. Needs a decision, not a patch — move `app.py` into the package and
29
+ find it via `importlib.resources`; source the ruleset from the config/state dir.
30
+ Blocks any non-beta release.
31
+ 1. **Decide rules versioning** (plan.md §5.5, §8.4). Rules are the last un-audited
32
+ input: `rules.ndjson` is rewritten in place by the UI, so an as-of replay would
33
+ replay old packs against *today's* rules. Either record a git commit per run, or
34
+ make rules a store table (recommended — same shape as packs). **This is a decision,
35
+ not code, and it blocks 2 being worth building.**
36
+ 2. **Packs as an SCD2 store table** (plan.md §5.4, step 6). Gives change history,
37
+ tombstones and as-of replay for mapping tables.
38
+ 3. **Run fingerprint** (plan.md §5.5, step 7). `run_id` + sha256 of each pack file and
39
+ of the ruleset, stored with every rated output, so an exported spreadsheet ties back
40
+ to the inputs that produced it.
41
+ 4. **Answer plan.md §8 open questions 1, 5** — outer-join intent, and whether a pack is
42
+ always a file or may eventually be a SQL/API query fetched at collect time. Q2, Q3
43
+ and Q4 were settled by the implementation (manifest at `packs/packs.toml`; duplicate
44
+ keys are a hard error; rules versioning still open as item 1 above).
45
+ 5. **Decide the fate of `worktree-purpose-docs`.** Eight commits rewriting the
46
+ architecture docs around the data rather than the code — including the rating model
47
+ correction and the asset-class CVSS work. Unmerged and drifting; either fold into
48
+ this branch or retire it deliberately.
49
+ 6. **Merge path back to `main`.** `main` is six weeks behind and predates the rating
50
+ rewrite, the blob store and the chain.
51
+
52
+ ## Log
53
+
54
+ ### 2026-09-07 — beta release 0.2.0b1, to exercise the PyPI workflow
55
+
56
+ Version bumped in `pyproject.toml` and `vulntrack/__init__.py`. `uv build` +
57
+ `uvx twine check` pass locally on both artifacts; 77 tests pass.
58
+
59
+ Two packaging defects surfaced while checking what would actually be published:
60
+
61
+ - **The sdist was shipping gitignored `packs/`.** Hatchling's default file selection
62
+ swept in untracked working-directory files, and `packs/` holds the real
63
+ `image_repository → project_id → tier` mappings. A local `uv build` would have made
64
+ them permanently public. Fixed with an explicit
65
+ `[tool.hatch.build.targets.sdist] include`. CI builds from a clean checkout and never
66
+ saw this. The wheel was never affected.
67
+ - **The installed package is not self-contained** — `cli._project_root()` is
68
+ `Path(__file__).parent.parent`, which is `site-packages/` under a wheel, so `app.py`
69
+ and `store/rules.ndjson` both resolve to paths that do not exist there. Not fixed;
70
+ see open work item 0. This is why the release is a beta.
71
+
72
+ Wrote `docs/packaging.md` — the doc `publish.yml` has always pointed at and which had
73
+ never existed, so the one-time Trusted Publishing setup was undocumented.
74
+
75
+ **Workflow result — [run 34118819161](https://github.com/brunnelu/vulntrack/actions/runs/34118819161):
76
+ `build` passed, `publish` failed on `invalid-publisher`.** So the answer is that the
77
+ PyPI publisher was never registered — the credentials were not added. Everything on the
78
+ GitHub side is correct; the OIDC claims the run presented are exactly the values
79
+ `docs/packaging.md` says to register:
80
+
81
+ repository brunnelu/vulntrack
82
+ workflow_ref .github/workflows/publish.yml
83
+ environment pypi
84
+
85
+ The `pypi` GitHub environment did not exist beforehand and was created by this run, as
86
+ expected. Nothing was uploaded, and `0.2.0b1` is therefore still an unused version
87
+ number on PyPI — re-dispatching the same workflow after the registration will publish
88
+ it. **Blocked on the pypi.org registration, which needs the maintainer's account.**
89
+
90
+ ### 2026-09-07 — surveyed the state, started this log
91
+
92
+ No code changed. Established where the work actually is:
93
+
94
+ - The newest branch is `worktree-packs-mapping-chain`; adopted it as the line of work.
95
+ - Verified the repo root's uncommitted diff is captured byte-for-byte by `43aa273`
96
+ here, so nothing is stranded in the main checkout.
97
+ - Confirmed the chain resolves from data, not filenames: `packs list` against
98
+ `packs.example/` orders `project-tiers` after `projects` because it keys on
99
+ `project_id`, a column `projects` produces.
100
+ - Test suite: 77 passed.
101
+
102
+ ### 2026-08-31 — chained mapping packs (`92c1ebf`)
103
+
104
+ Implements plan.md steps 1–5 and 9. A pack may key on a column another pack produced
105
+ (`image_repository → project_id → tier`), and the order is derived from the key columns
106
+ by fixpoint, so renaming a file cannot change the result.
107
+
108
+ The join is the small part; the rest is making failure loud, because a broken chain
109
+ otherwise re-rates findings with no diff, no error and a plausible-looking reasoning
110
+ string:
111
+
112
+ - left join only — a mapping row matching nothing is a lint result, never a frame row
113
+ - declared keys in `packs/packs.toml`; canonical-name inference stays as the fallback,
114
+ so an un-chained folder needs no manifest
115
+ - duplicate key rows are an error listing the offending values, not `keep="last"`
116
+ - `mapping_trail` beside `reasoning`: which pack wrote each value and on which key,
117
+ with an explicit `[default]` entry where the heuristic guessed — silence would not
118
+ distinguish a guess from a fact
119
+ - `vulntrack packs list` / `packs check` (non-zero exit, for CI) / `vulntrack explain`
120
+
121
+ New: `vulntrack/engine/packs.py` (384 lines), `tests/test_packs.py`,
122
+ `tests/test_cli_packs.py`.
123
+
124
+ ### 2026-08-31 — snapshot of the rating rewrite and blob collect layer (`43aa273`)
125
+
126
+ Committed what had been sitting uncommitted in the main checkout: `engine/cvss_env.py`
127
+ and `engine/pipeline.py` replaced by `engine/rating.py`; the blob-backed append-only
128
+ store; the Harbor collect layer; `packs.example/`.
@@ -0,0 +1,23 @@
1
+ """vulntrack UI — st.navigation.
2
+
3
+ uv run vulntrack ui
4
+ """
5
+
6
+ import streamlit as st
7
+
8
+ st.set_page_config(page_title="vulntrack", layout="wide", initial_sidebar_state="expanded")
9
+
10
+ from vulntrack.ui import pages_assess, pages_images, pages_rules, pages_scans # noqa: E402
11
+
12
+ st.navigation(
13
+ {
14
+ "Assess": [
15
+ st.Page(pages_assess.page, title="Assess", url_path="assess", default=True),
16
+ st.Page(pages_rules.page, title="Rules", url_path="rules"),
17
+ ],
18
+ "Estate": [
19
+ st.Page(pages_images.page, title="Images", url_path="images"),
20
+ st.Page(pages_scans.page, title="Scans", url_path="scans"),
21
+ ],
22
+ }
23
+ ).run()
@@ -0,0 +1,160 @@
1
+ # Handoff — continue here next session
2
+
3
+ **Date:** 2026-07-25
4
+ **Branch:** `implement-stacking-engine` (tracks `origin/implement-stacking-engine`)
5
+ **PR:** https://github.com/brunnelu/vulntrack/pull/2 → `main`
6
+ **HEAD:** `5b73804` — UI: st.navigation, Assess input, keep all Harbor CVSS vectors
7
+ **Tests:** `uv run pytest` → **43 passed**
8
+
9
+ **Directive for the next agent/session:**
10
+ More code is not the goal. Prefer fewer, clearer mechanisms. The user has already pushed back on bloat, on proxy rabbit-holes, and on TSV-for-rules. **Do not add features to “look busy.”** Decide, simplify, or document — then implement the minimum.
11
+
12
+ **Product direction (KISS):**
13
+ - Rule = `when` + `set{metrics}` + `because` (+ optional terminal severity).
14
+ - **Primary UI: Textual TUI** (`uv run vulntrack ui`) — select → apply rule → residue shrinks.
15
+ - **Identity (see readme):** occurrence PK includes tag + package_version + digest (digest hidden in UI). `image_kind` etc. are **joined** packs, not source PK. Assess = mostly per-CVE rules + custom `because`; ratings computed, not stored.
16
+ - CLI `assess` same loop; Streamlit only via `ui --web`.
17
+
18
+ ---
19
+
20
+ ## What this project is
21
+
22
+ **vulntrack** turns Harbor/Trivy scan output into **environmental severities** with an **auditable reason chain**, using a **stack of rules**. Ratings are **not stored**; they are recomputed from versioned inputs.
23
+
24
+ Product name is **vulntrack** only (not streamlit-editor).
25
+
26
+ Target design: `docs/index.md` (architecture). Implementation is a partial realization of that doc.
27
+
28
+ ---
29
+
30
+ ## Current shape (lean inventory)
31
+
32
+ | Piece | Role | Notes |
33
+ |---|---|---|
34
+ | `vulntrack/engine/` | Pure rules + CVSS env score | Core value. Keep small. |
35
+ | `vulntrack/gitstore.py` | NDJSON under `vt-data/` | Works; **user may prefer TSV** (not done). |
36
+ | `vulntrack/collect.py` | Stateless Harbor collect + scan ledger | OK. |
37
+ | `vulntrack/harbor.py` | Client + report parse | Multi-vector parse added; **recollect needed** for store. |
38
+ | `vulntrack/webpath.py` | code-server `proxy` / `absproxy` | Painful history; **UI access works for the user** — do not reopen unless broken. |
39
+ | `vulntrack/ui/` + `app.py` | Streamlit multipage | `st.navigation` restored; Assess still **not liked**. |
40
+ | `store/rules.ndjson` | Seed rules | Shipped defaults. |
41
+ | `vt-data/` | Runtime store (gitignored) | NDJSON; not committed. |
42
+ | Old Delta `data/` | **Deleted** | No migration path; reassess from Harbor. |
43
+
44
+ **Removed on purpose:** Delta Lake, SCD2, hierarchical `resolve`, half-built `rating_rules`, mock-first product path, legacy Excel assessments pipeline.
45
+
46
+ ---
47
+
48
+ ## How to run (today)
49
+
50
+ ```bash
51
+ cd /home/coder/projects/personal/streamlit-editor
52
+ uv sync
53
+ cp vulntrack.toml.example vulntrack.toml # settings
54
+ cp .env.example .env # HARBOR_<PROJECT>_PASSWORD
55
+
56
+ uv run vulntrack collect --no-submit # harvest into vt-data/
57
+ uv run vulntrack rate --residue-only # CLI summary
58
+ uv run vulntrack ui # Streamlit
59
+ uv run pytest
60
+ ```
61
+
62
+ **UI / code-server:**
63
+ `VSCODE_PROXY_URI=https://code-server.bru.lu/proxy/{{port}}/`.
64
+ CLI defaults to **absproxy** + `baseUrlPath=absproxy/<port>` (port default **4000**).
65
+ Open the URL the CLI prints (e.g. `https://code-server.bru.lu/absproxy/4000/`).
66
+ Nested paths like `…/proxy/N/proxy/N/` = old wrong `baseUrlPath` + cached 301s — private window or new port.
67
+
68
+ User stated they **can access the UI**; stop spending cycles on proxy debugging unless something is actually broken for them.
69
+
70
+ ---
71
+
72
+ ## What works
73
+
74
+ - Flat in-memory rate pipeline: findings + images + deployments + ordered rules → env severity + reason chain.
75
+ - Collect from real Harbor; gitstore write; scan ledger append.
76
+ - Seed stack: project MAV, workspace MA/MC/MPR/MS, not-in-use, DoS signal, KEV last.
77
+ - Measured effect (order of magnitude): ~22k findings → ~50–50-ish Critical/High CVEs after stack (recompute after recollect).
78
+ - Multi-vector **parse** in code: all Harbor CVSS sources → `cvss_vectors` JSON (+ convenience columns). **Existing vt-data rows may still lack `cvss_vectors` until recollect.**
79
+ - Streamlit `st.navigation` pages: Assess (default), Rules, Overview, Images, Scans.
80
+ - Config: one `vulntrack.toml`; one robot password per project in `.env` (or the environment).
81
+
82
+ ---
83
+
84
+ ## Open product decisions (user-driven — do these first, not more glue)
85
+
86
+ ### 1. Storage format
87
+
88
+ - **Now:** sorted **NDJSON** under `vt-data/`.
89
+ - **User:** “maybe better storage; **TSV might be an option**.”
90
+ - **Engineering preference:** one format, human-diffable, spreadsheet-friendly if TSV; avoid dual NDJSON+TSV forever.
91
+ - **Not done.** Next session should **choose** TSV vs NDJSON (or TSV for rules/deployments only) and migrate once — not add a third backend.
92
+
93
+ ### 2. Rule model / authoring
94
+
95
+ - **Now:** JSON-ish rules (`when` predicates + `then` effects + `because`), CLI + Assess form that builds them.
96
+ - **User:** “**I do not like the way you propose rules.**”
97
+ - **Also disliked:** radio-button page nav (fixed to `st.navigation`); earlier “propose rules” / over-automated downrank framing.
98
+ - **Not done.** Next session should redesign rule **representation and UX** with the user *before* coding more form widgets. Possible lean directions (examples only — agree first):
99
+ - Rules as a **TSV/table** the user edits (columns: id, when_*, metric, value, because, enabled).
100
+ - Closer to original Assess flow (severity + reason + simple scope) that **emits** one clear rule row, not a DSL form.
101
+ - Fewer effects; metric-only rules as default.
102
+ - **Do not** add rule mining, AI proposals, or more rule kinds until the core format is agreed.
103
+
104
+ ### 3. Data model hygiene
105
+
106
+ - `cvss_vectors` on findings after recollect; normalize CVE text in `cves.*`.
107
+ - Deployments still often **inferred** from `*-environment` names if register empty — fine as bootstrap, not as silent policy.
108
+
109
+ ---
110
+
111
+ ## Explicit non-goals for the next session
112
+
113
+ - No more proxy/absproxy archaeology unless the user cannot open the UI.
114
+ - No Delta/SCD2/“restore old stack.”
115
+ - No second web framework (FastAPI UI was tried and dropped for Streamlit).
116
+ - No CI/packaging/linter sprawl.
117
+ - No expanding the engine with EPSS/KEV/distro packs until storage + rules feel right.
118
+ - No large UI redesigns without agreeing the rule model.
119
+
120
+ ---
121
+
122
+ ## Hard-won constraints (do not re-litigate)
123
+
124
+ 1. **Assessment is the product** — environmental score + reasons; not “dismiss by no-fix.”
125
+ 2. **Rules stack**; score **once** at the end from accumulated metrics.
126
+ 3. Prefer **CVSS environmental metrics** over blanket severity overrides.
127
+ 4. **No blanket MUI/MAC** as free levers; residue is often AV:N/AC:L/UI:N by selection.
128
+ 5. **Image identity without tag** for register/rules so new digests inherit.
129
+ 6. **Git-only persistence** for the new design; ratings not stored as authority.
130
+ 7. Workspace posture (example): MA:N, MC:L, MPR:L, MS:U — domain-owned by the user, not the model.
131
+
132
+ ---
133
+
134
+ ## Suggested start next week (minimal)
135
+
136
+ 1. Read this file + `docs/index.md` §1–2 and §6 (rating).
137
+ 2. Skim `vulntrack/engine/` and `gitstore.py` only — not the whole UI.
138
+ 3. **With the user, decide:**
139
+ - Store: NDJSON → **TSV** (or hybrid)?
140
+ - Rules: what does a rule look like on disk and in the Assess UI?
141
+ 4. Implement **only** that decision (one storage format + one rule shape), delete what becomes redundant.
142
+ 5. Recollect from Harbor; verify multi-vector rows and residue.
143
+
144
+ ---
145
+
146
+ ## Credentials / secrets (do not commit)
147
+
148
+ - `.env` — one `HARBOR_<PROJECT>_PASSWORD` per project, and nothing else; gitignored.
149
+ Exported environment variables override it, so CI needs no file.
150
+ - `vulntrack.toml` — every non-secret setting; gitignored, template committed as
151
+ `vulntrack.toml.example`. Points at Harbor `https://harbor.bru.lu`, projects
152
+ `domino` and `library`, store `vt-store`.
153
+ - Superseded and removed: `config.toml`, `harbor/credentials.txt`,
154
+ `harbor-credentials.txt`, `.streamlit/config.toml`.
155
+
156
+ ---
157
+
158
+ ## One-line summary
159
+
160
+ **Independent vulntrack on git NDJSON + stacking engine + Streamlit is on PR #2; stop bloating — next work is storage format (likely TSV) and a rule model the user actually accepts.**