runledger-cli 0.1.6__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runledger_cli-0.2.1/PKG-INFO +224 -0
- runledger_cli-0.2.1/README.md +196 -0
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/cli/guided.py +25 -0
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/cli/ledger.py +25 -0
- runledger_cli-0.2.1/cli/tests/conftest.py +21 -0
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/cli/tests/test_integration.py +48 -9
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/cli/tests/test_ledger.py +5 -25
- runledger_cli-0.2.1/pyproject.toml +44 -0
- runledger_cli-0.2.1/runledger_cli.egg-info/PKG-INFO +224 -0
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/runledger_cli.egg-info/SOURCES.txt +1 -0
- runledger_cli-0.1.6/PKG-INFO +0 -126
- runledger_cli-0.1.6/README.md +0 -114
- runledger_cli-0.1.6/pyproject.toml +0 -26
- runledger_cli-0.1.6/runledger_cli.egg-info/PKG-INFO +0 -126
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/LICENSE +0 -0
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/cli/__init__.py +0 -0
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/cli/docker-compose.yml +0 -0
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/cli/tests/__init__.py +0 -0
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/cli/tests/test_data_features.py +0 -0
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/cli/tests/test_real_results.py +0 -0
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/runledger_cli.egg-info/dependency_links.txt +0 -0
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/runledger_cli.egg-info/entry_points.txt +0 -0
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/runledger_cli.egg-info/requires.txt +0 -0
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/runledger_cli.egg-info/top_level.txt +0 -0
- {runledger_cli-0.1.6 → runledger_cli-0.2.1}/setup.cfg +0 -0
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: runledger-cli
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: Self-hosted SQL-style query engine over your team's experiment JSON
|
|
5
|
+
Author: Ahmed Rafat
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Repository, https://github.com/ahmedrafatcse/run-ledger
|
|
8
|
+
Project-URL: Documentation, https://github.com/ahmedrafatcse/run-ledger/tree/master/docs
|
|
9
|
+
Project-URL: Issues, https://github.com/ahmedrafatcse/run-ledger/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/ahmedrafatcse/run-ledger/releases
|
|
11
|
+
Keywords: experiments,json,postgresql,sql,ml,research,cli
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Java
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering
|
|
21
|
+
Classifier: Topic :: Database
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: requests
|
|
26
|
+
Requires-Dist: rich
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# RunLedger
|
|
31
|
+
|
|
32
|
+
[](https://pypi.org/project/runledger-cli/)
|
|
33
|
+
[](LICENSE)
|
|
34
|
+
|
|
35
|
+
**A self-hosted, SQL-style query engine over your team's experiment JSON.**
|
|
36
|
+
|
|
37
|
+
Point it at the JSON your runs produce. Query nested fields, aggregate across runs, and pull back the exact array element that matched. Every submission is a version — never an overwrite — so you can see who submitted it, when, and whether the file has changed since.
|
|
38
|
+
|
|
39
|
+
> *How do you query JSON with no schema?*
|
|
40
|
+
>
|
|
41
|
+
> *Who on your team ran this? Has it changed since?*
|
|
42
|
+
|
|
43
|
+
*Built for labs, thesis groups, and small research teams.*
|
|
44
|
+
|
|
45
|
+
<!-- Add a 15-second GIF here: search nested JSON → inspect matching array element → aggregate → compare versions. -->
|
|
46
|
+
|
|
47
|
+
---
|
|
48
|
+
|
|
49
|
+
## Query your JSON like a database
|
|
50
|
+
|
|
51
|
+
Experiment output nests metrics inside configs, arrays, and per-sample results. RunLedger provides a purpose-built query interface over those fields — structured comparisons, full-text with stemming, fuzzy matching, compound conditions, aggregation — without a predefined schema or prior metric registration.
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
# Comparison operators over any dot-path
|
|
55
|
+
runledger search --metric accuracy --op gt --value 0.9 --batch sweep-42
|
|
56
|
+
|
|
57
|
+
# Reach inside an array — and get back the exact matching element
|
|
58
|
+
runledger search --metric results[].sst2_cacc --op lt --value 0.7 --batch sweep-42
|
|
59
|
+
|
|
60
|
+
# Compound AND/OR across nested paths
|
|
61
|
+
runledger search \
|
|
62
|
+
--metric accuracy --op gt --value 0.9 \
|
|
63
|
+
--metric loss --op lt --value 0.2 \
|
|
64
|
+
--combine and --batch sweep-42
|
|
65
|
+
|
|
66
|
+
# Aggregate and group, including over arrays
|
|
67
|
+
runledger aggregate --metric accuracy --agg AVG --group-by experiment --batch sweep-42
|
|
68
|
+
runledger aggregate --metric results[].accuracy --agg AVG --group-by experiment --batch sweep-42
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
**Exact array-element pointers.** When an array query matches, RunLedger returns *which field matched and where* — the specific element, not just "this run matched":
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
sweep-42/run-7.json
|
|
75
|
+
results[3].sst2_cacc = 0.62
|
|
76
|
+
results[9].sst2_cacc = 0.68
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
That's the difference between "some array element passed your filter" and "element 3 and element 9 passed, and here are their values."
|
|
80
|
+
|
|
81
|
+
The same query capabilities are available through the web UI — rendered as tables, with the same inline pointers.
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## What the shared record gives you
|
|
86
|
+
|
|
87
|
+
**Version history.** Resubmitting the same `_source.file` and `_source.index` creates a new version — never an overwrite. The old version stays. The run detail page links to `/runs/{id}/history` (every version of a run identity) and `/runs/compare?id1=X&id2=Y` (field-by-field diff between two versions).
|
|
88
|
+
|
|
89
|
+
**Team isolation.** Only your team sees your team's runs. A foreign team's rows are invisible, not forbidden — `SELECT` returns nothing, `/runs/{id}` returns 404. This is enforced at the database layer, so it holds even if a service method forgets to filter.
|
|
90
|
+
|
|
91
|
+
**Change detection.** Every submission stores a fingerprint of its payload. Open a run and RunLedger recomputes and compares — if the payload was modified after upload, the badge goes red. States: **Verified** / **Tampered** / **Stale** / **Cannot verify**.
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
## Stack
|
|
96
|
+
|
|
97
|
+
**PostgreSQL 16** with JSONB and GIN indexes. **Spring Boot 3.4** for the REST API and the server-rendered Thymeleaf UI. **Python 3.10+** for the CLI, which manages the Docker stack for you.
|
|
98
|
+
|
|
99
|
+
Team isolation is enforced by Row-Level Security in the database. The application connects as a non-owner role; every request runs under a session identity Postgres validates. Access rules live in RLS policies, not in application code.
|
|
100
|
+
|
|
101
|
+
---
|
|
102
|
+
|
|
103
|
+
## Install
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
pip install runledger-cli
|
|
107
|
+
runledger guided
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
**You need Docker** — the only required dependency beyond Python. The backend (Postgres + the Java app) runs entirely inside containers. The CLI starts and stops them automatically. If Docker isn't installed, the CLI detects that on first run and links you to the installer.
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## Quick start
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
runledger scan ~/experiments --batch my-sweep
|
|
118
|
+
runledger search --metric accuracy --op gt --value 0.9 --batch my-sweep
|
|
119
|
+
runledger aggregate --metric accuracy --agg AVG --group-by experiment --batch my-sweep
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
New here? Run `runledger` with no arguments — the guided wizard walks you through your first scan and search.
|
|
123
|
+
|
|
124
|
+
> **Tip:** `python -m cli.ledger` works interchangeably with `runledger` on any platform.
|
|
125
|
+
|
|
126
|
+
### Web UI
|
|
127
|
+
|
|
128
|
+
With the stack running, open `http://localhost:8080` and log in with a development user ID (see [Development identity](#development-identity)).
|
|
129
|
+
|
|
130
|
+
| Page | What it does |
|
|
131
|
+
|---|---|
|
|
132
|
+
| **Runs** (`/`) | Everything you can see. Click a file for its payload, integrity badge, and version history. |
|
|
133
|
+
| **Compare** (`/query/compare`) | N conditions with AND/OR. Match pointers show which array element hit. |
|
|
134
|
+
| **Phrase** (`/query/phrase`) | Full-text search with stemming; optional fuzzy matching for typos. |
|
|
135
|
+
| **Aggregate** (`/query/aggregate`) | `AVG` / `MAX` / `MIN` / `SUM` / `COUNT`, with optional group-by. Supervisors get a team selector. |
|
|
136
|
+
| **Batches** (`/batches`) | Every batch you can see, grouped by team. Click through to browse its metric keys. |
|
|
137
|
+
| **Saved** (`/saved`) | Named queries. Re-run opens the query page with params prefilled. Personal — no one else can see or delete yours. |
|
|
138
|
+
| **Submit** (`/submit`) | Paste JSON or upload a file. Researchers only. Requires `_source.file` in the payload. |
|
|
139
|
+
|
|
140
|
+
Server-rendered throughout. No SPA, no build step.
|
|
141
|
+
|
|
142
|
+
---
|
|
143
|
+
|
|
144
|
+
## Installing from source
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
git clone https://github.com/ahmedrafatcse/run-ledger.git
|
|
148
|
+
cd run-ledger
|
|
149
|
+
docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
|
|
150
|
+
pip install -e .
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
The dev override is required. It publishes port `8080`, which is what the browser UI and the Python tests expect. Starting with only `docker-compose.yml` doesn't expose `8080` and the UI will not be reachable.
|
|
154
|
+
|
|
155
|
+
### Troubleshooting
|
|
156
|
+
|
|
157
|
+
<details>
|
|
158
|
+
<summary>Windows – <code>runledger</code> not recognized</summary>
|
|
159
|
+
|
|
160
|
+
On some Windows systems the Python Scripts folder is not automatically added to PATH. Two options:
|
|
161
|
+
|
|
162
|
+
1. **Add the Scripts folder to PATH permanently** (recommended):
|
|
163
|
+
- Find the install location: `pip show runledger-cli | findstr Location`
|
|
164
|
+
- Navigate to that folder, then up one level to the `Scripts` directory.
|
|
165
|
+
- Add that `Scripts` path to your system PATH via **System Properties > Environment Variables**.
|
|
166
|
+
- Restart your terminal.
|
|
167
|
+
|
|
168
|
+
2. **Use the Python module fallback** (works immediately, no PATH changes):
|
|
169
|
+
```bash
|
|
170
|
+
python -m cli.ledger
|
|
171
|
+
```
|
|
172
|
+
All subcommands and the guided wizard work exactly the same way.
|
|
173
|
+
</details>
|
|
174
|
+
|
|
175
|
+
<details>
|
|
176
|
+
<summary>Fedora / RHEL – Backend can't reach PostgreSQL (UnknownHostException)</summary>
|
|
177
|
+
|
|
178
|
+
Fedora's `firewalld` can block Docker's internal bridge networking, causing the app container to fail DNS resolution for `postgres`. Fix:
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
sudo firewall-cmd --zone=trusted --add-interface=docker0 --permanent
|
|
182
|
+
sudo firewall-cmd --reload
|
|
183
|
+
sudo systemctl restart docker
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
If you still see a port conflict (`port is already allocated`), check whether another process is using 5432 (`sudo ss -tulpn | grep 5432`) and either stop it or change the host port mapping.
|
|
187
|
+
</details>
|
|
188
|
+
|
|
189
|
+
---
|
|
190
|
+
|
|
191
|
+
## Metric paths
|
|
192
|
+
|
|
193
|
+
Address any value with a dot-path — `accuracy`, `metrics.loss`, `results[].sst2_cacc`. Shorthand (`cacc`) resolves automatically when unambiguous. Run `runledger metrics --batch <batch>` to see everything discovered in a scan, or add `--verbose` for full paths.
|
|
194
|
+
|
|
195
|
+
---
|
|
196
|
+
|
|
197
|
+
## Development identity
|
|
198
|
+
|
|
199
|
+
The browser login is a **development identity stub, not authentication**. There is no password, no OAuth, no SSO. You enter a user ID; the session is populated from `app_users`.
|
|
200
|
+
|
|
201
|
+
**Do not expose this configuration to untrusted users or networks.** A network boundary does not turn a stub into authentication — every request that reaches the app carries whatever user ID the caller declares. A real deployment needs an authentication mechanism in front of this before relying on the identity model for access control.
|
|
202
|
+
|
|
203
|
+
Demo users from `demo/seed.sql`:
|
|
204
|
+
|
|
205
|
+
| ID | Role | Team |
|
|
206
|
+
|---|---|---|
|
|
207
|
+
| `aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa` | researcher | Team A |
|
|
208
|
+
| `bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb` | researcher | Team B |
|
|
209
|
+
| `cccccccc-cccc-cccc-cccc-cccccccccccc` | supervisor | assigned to both teams |
|
|
210
|
+
|
|
211
|
+
Visit `http://localhost:8080/login` and paste an ID. For API access, pass the same ID in an `X-User-Id` header. Role and team are looked up server-side — the client cannot declare which team it belongs to, but it *can* declare which user it is, which is exactly why this stub isn't authentication
|
|
212
|
+
|
|
213
|
+
---
|
|
214
|
+
|
|
215
|
+
## Docs
|
|
216
|
+
|
|
217
|
+
- [API reference](https://github.com/ahmedrafatcse/run-ledger/blob/master/docs/api.md)
|
|
218
|
+
- [Usage examples](https://github.com/ahmedrafatcse/run-ledger/blob/master/docs/examples.md)
|
|
219
|
+
|
|
220
|
+
---
|
|
221
|
+
|
|
222
|
+
## License
|
|
223
|
+
|
|
224
|
+
MIT
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
|
|
2
|
+
# RunLedger
|
|
3
|
+
|
|
4
|
+
[](https://pypi.org/project/runledger-cli/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
**A self-hosted, SQL-style query engine over your team's experiment JSON.**
|
|
8
|
+
|
|
9
|
+
Point it at the JSON your runs produce. Query nested fields, aggregate across runs, and pull back the exact array element that matched. Every submission is a version — never an overwrite — so you can see who submitted it, when, and whether the file has changed since.
|
|
10
|
+
|
|
11
|
+
> *How do you query JSON with no schema?*
|
|
12
|
+
>
|
|
13
|
+
> *Who on your team ran this? Has it changed since?*
|
|
14
|
+
|
|
15
|
+
*Built for labs, thesis groups, and small research teams.*
|
|
16
|
+
|
|
17
|
+
<!-- Add a 15-second GIF here: search nested JSON → inspect matching array element → aggregate → compare versions. -->
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
## Query your JSON like a database
|
|
22
|
+
|
|
23
|
+
Experiment output nests metrics inside configs, arrays, and per-sample results. RunLedger provides a purpose-built query interface over those fields — structured comparisons, full-text with stemming, fuzzy matching, compound conditions, aggregation — without a predefined schema or prior metric registration.
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
# Comparison operators over any dot-path
|
|
27
|
+
runledger search --metric accuracy --op gt --value 0.9 --batch sweep-42
|
|
28
|
+
|
|
29
|
+
# Reach inside an array — and get back the exact matching element
|
|
30
|
+
runledger search --metric results[].sst2_cacc --op lt --value 0.7 --batch sweep-42
|
|
31
|
+
|
|
32
|
+
# Compound AND/OR across nested paths
|
|
33
|
+
runledger search \
|
|
34
|
+
--metric accuracy --op gt --value 0.9 \
|
|
35
|
+
--metric loss --op lt --value 0.2 \
|
|
36
|
+
--combine and --batch sweep-42
|
|
37
|
+
|
|
38
|
+
# Aggregate and group, including over arrays
|
|
39
|
+
runledger aggregate --metric accuracy --agg AVG --group-by experiment --batch sweep-42
|
|
40
|
+
runledger aggregate --metric results[].accuracy --agg AVG --group-by experiment --batch sweep-42
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
**Exact array-element pointers.** When an array query matches, RunLedger returns *which field matched and where* — the specific element, not just "this run matched":
|
|
44
|
+
|
|
45
|
+
```
|
|
46
|
+
sweep-42/run-7.json
|
|
47
|
+
results[3].sst2_cacc = 0.62
|
|
48
|
+
results[9].sst2_cacc = 0.68
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
That's the difference between "some array element passed your filter" and "element 3 and element 9 passed, and here are their values."
|
|
52
|
+
|
|
53
|
+
The same query capabilities are available through the web UI — rendered as tables, with the same inline pointers.
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## What the shared record gives you
|
|
58
|
+
|
|
59
|
+
**Version history.** Resubmitting the same `_source.file` and `_source.index` creates a new version — never an overwrite. The old version stays. The run detail page links to `/runs/{id}/history` (every version of a run identity) and `/runs/compare?id1=X&id2=Y` (field-by-field diff between two versions).
|
|
60
|
+
|
|
61
|
+
**Team isolation.** Only your team sees your team's runs. A foreign team's rows are invisible, not forbidden — `SELECT` returns nothing, `/runs/{id}` returns 404. This is enforced at the database layer, so it holds even if a service method forgets to filter.
|
|
62
|
+
|
|
63
|
+
**Change detection.** Every submission stores a fingerprint of its payload. Open a run and RunLedger recomputes and compares — if the payload was modified after upload, the badge goes red. States: **Verified** / **Tampered** / **Stale** / **Cannot verify**.
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## Stack
|
|
68
|
+
|
|
69
|
+
**PostgreSQL 16** with JSONB and GIN indexes. **Spring Boot 3.4** for the REST API and the server-rendered Thymeleaf UI. **Python 3.10+** for the CLI, which manages the Docker stack for you.
|
|
70
|
+
|
|
71
|
+
Team isolation is enforced by Row-Level Security in the database. The application connects as a non-owner role; every request runs under a session identity Postgres validates. Access rules live in RLS policies, not in application code.
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
## Install
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
pip install runledger-cli
|
|
79
|
+
runledger guided
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
**You need Docker** — the only required dependency beyond Python. The backend (Postgres + the Java app) runs entirely inside containers. The CLI starts and stops them automatically. If Docker isn't installed, the CLI detects that on first run and links you to the installer.
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## Quick start
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
runledger scan ~/experiments --batch my-sweep
|
|
90
|
+
runledger search --metric accuracy --op gt --value 0.9 --batch my-sweep
|
|
91
|
+
runledger aggregate --metric accuracy --agg AVG --group-by experiment --batch my-sweep
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
New here? Run `runledger` with no arguments — the guided wizard walks you through your first scan and search.
|
|
95
|
+
|
|
96
|
+
> **Tip:** `python -m cli.ledger` works interchangeably with `runledger` on any platform.
|
|
97
|
+
|
|
98
|
+
### Web UI
|
|
99
|
+
|
|
100
|
+
With the stack running, open `http://localhost:8080` and log in with a development user ID (see [Development identity](#development-identity)).
|
|
101
|
+
|
|
102
|
+
| Page | What it does |
|
|
103
|
+
|---|---|
|
|
104
|
+
| **Runs** (`/`) | Everything you can see. Click a file for its payload, integrity badge, and version history. |
|
|
105
|
+
| **Compare** (`/query/compare`) | N conditions with AND/OR. Match pointers show which array element hit. |
|
|
106
|
+
| **Phrase** (`/query/phrase`) | Full-text search with stemming; optional fuzzy matching for typos. |
|
|
107
|
+
| **Aggregate** (`/query/aggregate`) | `AVG` / `MAX` / `MIN` / `SUM` / `COUNT`, with optional group-by. Supervisors get a team selector. |
|
|
108
|
+
| **Batches** (`/batches`) | Every batch you can see, grouped by team. Click through to browse its metric keys. |
|
|
109
|
+
| **Saved** (`/saved`) | Named queries. Re-run opens the query page with params prefilled. Personal — no one else can see or delete yours. |
|
|
110
|
+
| **Submit** (`/submit`) | Paste JSON or upload a file. Researchers only. Requires `_source.file` in the payload. |
|
|
111
|
+
|
|
112
|
+
Server-rendered throughout. No SPA, no build step.
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## Installing from source
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
git clone https://github.com/ahmedrafatcse/run-ledger.git
|
|
120
|
+
cd run-ledger
|
|
121
|
+
docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
|
|
122
|
+
pip install -e .
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
The dev override is required. It publishes port `8080`, which is what the browser UI and the Python tests expect. Starting with only `docker-compose.yml` doesn't expose `8080` and the UI will not be reachable.
|
|
126
|
+
|
|
127
|
+
### Troubleshooting
|
|
128
|
+
|
|
129
|
+
<details>
|
|
130
|
+
<summary>Windows – <code>runledger</code> not recognized</summary>
|
|
131
|
+
|
|
132
|
+
On some Windows systems the Python Scripts folder is not automatically added to PATH. Two options:
|
|
133
|
+
|
|
134
|
+
1. **Add the Scripts folder to PATH permanently** (recommended):
|
|
135
|
+
- Find the install location: `pip show runledger-cli | findstr Location`
|
|
136
|
+
- Navigate to that folder, then up one level to the `Scripts` directory.
|
|
137
|
+
- Add that `Scripts` path to your system PATH via **System Properties > Environment Variables**.
|
|
138
|
+
- Restart your terminal.
|
|
139
|
+
|
|
140
|
+
2. **Use the Python module fallback** (works immediately, no PATH changes):
|
|
141
|
+
```bash
|
|
142
|
+
python -m cli.ledger
|
|
143
|
+
```
|
|
144
|
+
All subcommands and the guided wizard work exactly the same way.
|
|
145
|
+
</details>
|
|
146
|
+
|
|
147
|
+
<details>
|
|
148
|
+
<summary>Fedora / RHEL – Backend can't reach PostgreSQL (UnknownHostException)</summary>
|
|
149
|
+
|
|
150
|
+
Fedora's `firewalld` can block Docker's internal bridge networking, causing the app container to fail DNS resolution for `postgres`. Fix:
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
sudo firewall-cmd --zone=trusted --add-interface=docker0 --permanent
|
|
154
|
+
sudo firewall-cmd --reload
|
|
155
|
+
sudo systemctl restart docker
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
If you still see a port conflict (`port is already allocated`), check whether another process is using 5432 (`sudo ss -tulpn | grep 5432`) and either stop it or change the host port mapping.
|
|
159
|
+
</details>
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
## Metric paths
|
|
164
|
+
|
|
165
|
+
Address any value with a dot-path — `accuracy`, `metrics.loss`, `results[].sst2_cacc`. Shorthand (`cacc`) resolves automatically when unambiguous. Run `runledger metrics --batch <batch>` to see everything discovered in a scan, or add `--verbose` for full paths.
|
|
166
|
+
|
|
167
|
+
---
|
|
168
|
+
|
|
169
|
+
## Development identity
|
|
170
|
+
|
|
171
|
+
The browser login is a **development identity stub, not authentication**. There is no password, no OAuth, no SSO. You enter a user ID; the session is populated from `app_users`.
|
|
172
|
+
|
|
173
|
+
**Do not expose this configuration to untrusted users or networks.** A network boundary does not turn a stub into authentication — every request that reaches the app carries whatever user ID the caller declares. A real deployment needs an authentication mechanism in front of this before relying on the identity model for access control.
|
|
174
|
+
|
|
175
|
+
Demo users from `demo/seed.sql`:
|
|
176
|
+
|
|
177
|
+
| ID | Role | Team |
|
|
178
|
+
|---|---|---|
|
|
179
|
+
| `aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa` | researcher | Team A |
|
|
180
|
+
| `bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb` | researcher | Team B |
|
|
181
|
+
| `cccccccc-cccc-cccc-cccc-cccccccccccc` | supervisor | assigned to both teams |
|
|
182
|
+
|
|
183
|
+
Visit `http://localhost:8080/login` and paste an ID. For API access, pass the same ID in an `X-User-Id` header. Role and team are looked up server-side — the client cannot declare which team it belongs to, but it *can* declare which user it is, which is exactly why this stub isn't authentication
|
|
184
|
+
|
|
185
|
+
---
|
|
186
|
+
|
|
187
|
+
## Docs
|
|
188
|
+
|
|
189
|
+
- [API reference](https://github.com/ahmedrafatcse/run-ledger/blob/master/docs/api.md)
|
|
190
|
+
- [Usage examples](https://github.com/ahmedrafatcse/run-ledger/blob/master/docs/examples.md)
|
|
191
|
+
|
|
192
|
+
---
|
|
193
|
+
|
|
194
|
+
## License
|
|
195
|
+
|
|
196
|
+
MIT
|
|
@@ -13,6 +13,31 @@ from rich.progress import Progress
|
|
|
13
13
|
from rich.prompt import Prompt, Confirm
|
|
14
14
|
from rich.table import Table
|
|
15
15
|
|
|
16
|
+
# ---------------------------------------------------------------------------
|
|
17
|
+
# Identity header — applied to every requests call in this file.
|
|
18
|
+
#
|
|
19
|
+
# The backend requires an X-User-Id header on every request (Slice 4). The
|
|
20
|
+
# filter rejects requests without it. Until real authentication exists, the
|
|
21
|
+
# CLI sends a development identity, overridable via RUNLEDGER_USER_ID.
|
|
22
|
+
# ---------------------------------------------------------------------------
|
|
23
|
+
import os as _os
|
|
24
|
+
import requests as _requests
|
|
25
|
+
|
|
26
|
+
_DEV_USER_ID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" # Alice, from dev seed
|
|
27
|
+
|
|
28
|
+
def _install_identity_header():
|
|
29
|
+
_original_request = _requests.Session.request
|
|
30
|
+
def _request_with_identity(self, method, url, **kwargs):
|
|
31
|
+
headers = dict(kwargs.pop("headers", {}) or {})
|
|
32
|
+
headers.setdefault(
|
|
33
|
+
"X-User-Id",
|
|
34
|
+
_os.environ.get("RUNLEDGER_USER_ID", _DEV_USER_ID),
|
|
35
|
+
)
|
|
36
|
+
return _original_request(self, method, url, headers=headers, **kwargs)
|
|
37
|
+
_requests.Session.request = _request_with_identity
|
|
38
|
+
|
|
39
|
+
_install_identity_header()
|
|
40
|
+
|
|
16
41
|
# ------------------------------------------------------------
|
|
17
42
|
# Configuration – must match backend settings
|
|
18
43
|
# ------------------------------------------------------------
|
|
@@ -36,6 +36,31 @@ from rich.progress import Progress
|
|
|
36
36
|
from rich.prompt import Prompt, Confirm
|
|
37
37
|
from rich.table import Table
|
|
38
38
|
|
|
39
|
+
# ---------------------------------------------------------------------------
|
|
40
|
+
# Identity header — applied to every requests call in this file.
|
|
41
|
+
#
|
|
42
|
+
# The backend requires an X-User-Id header on every request (Slice 4). The
|
|
43
|
+
# filter rejects requests without it. Until real authentication exists, the
|
|
44
|
+
# CLI sends a development identity, overridable via RUNLEDGER_USER_ID.
|
|
45
|
+
# ---------------------------------------------------------------------------
|
|
46
|
+
import os as _os
|
|
47
|
+
import requests as _requests
|
|
48
|
+
|
|
49
|
+
_DEV_USER_ID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" # Alice, from dev seed
|
|
50
|
+
|
|
51
|
+
def _install_identity_header():
|
|
52
|
+
_original_request = _requests.Session.request
|
|
53
|
+
def _request_with_identity(self, method, url, **kwargs):
|
|
54
|
+
headers = dict(kwargs.pop("headers", {}) or {})
|
|
55
|
+
headers.setdefault(
|
|
56
|
+
"X-User-Id",
|
|
57
|
+
_os.environ.get("RUNLEDGER_USER_ID", _DEV_USER_ID),
|
|
58
|
+
)
|
|
59
|
+
return _original_request(self, method, url, headers=headers, **kwargs)
|
|
60
|
+
_requests.Session.request = _request_with_identity
|
|
61
|
+
|
|
62
|
+
_install_identity_header()
|
|
63
|
+
|
|
39
64
|
# ------------------------------------------------------------
|
|
40
65
|
# Force UTF‑8 output so Rich can render box‑drawing characters
|
|
41
66
|
# ------------------------------------------------------------
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Test session configuration.
|
|
3
|
+
|
|
4
|
+
The CLI reads RUNLEDGER_BASE_URL at import time to compute API_BASE and
|
|
5
|
+
HEALTH_URL. Setting it here — before pytest imports any test module — makes
|
|
6
|
+
the tests deterministic regardless of the developer's shell environment.
|
|
7
|
+
|
|
8
|
+
Without this, running the tests in a shell where RUNLEDGER_BASE_URL is set
|
|
9
|
+
(e.g. pointing at a dev stack on :8081) causes tests that assert on
|
|
10
|
+
:8080 to fail for reasons unrelated to the code under test.
|
|
11
|
+
"""
|
|
12
|
+
import os
|
|
13
|
+
|
|
14
|
+
# Force the CLI module to compute against the default port when the test
|
|
15
|
+
# suite imports it. This must run before any test module is collected.
|
|
16
|
+
os.environ["RUNLEDGER_BASE_URL"] = "http://localhost:8080"
|
|
17
|
+
|
|
18
|
+
# RUNLEDGER_NO_DOCKER is honored by ensure_backend() but not by the CLI's
|
|
19
|
+
# top-level main() — leaving it set would produce different behavior on
|
|
20
|
+
# a developer's machine than in CI.
|
|
21
|
+
os.environ.pop("RUNLEDGER_NO_DOCKER", None)
|
|
@@ -4,16 +4,20 @@ import tempfile
|
|
|
4
4
|
import time
|
|
5
5
|
import uuid
|
|
6
6
|
import os
|
|
7
|
+
import sys
|
|
7
8
|
from pathlib import Path
|
|
8
9
|
|
|
9
10
|
import pytest
|
|
10
11
|
import requests
|
|
11
12
|
|
|
13
|
+
|
|
12
14
|
# ------------------------------------------------------------
|
|
13
15
|
# Global settings – will be overridden dynamically by the fixture
|
|
14
16
|
# ------------------------------------------------------------
|
|
15
17
|
PROJECT_ROOT = Path(__file__).parent.parent.parent
|
|
16
18
|
COMPOSE_FILE = PROJECT_ROOT / "docker-compose.test.yml"
|
|
19
|
+
COMPOUND_DIR = PROJECT_ROOT / "test-data" / "sample-compound-runs"
|
|
20
|
+
LEDGER_SCRIPT = PROJECT_ROOT / "cli" / "ledger.py"
|
|
17
21
|
|
|
18
22
|
# Prevent CLI from ever auto‑starting Docker
|
|
19
23
|
os.environ["RUNLEDGER_NO_DOCKER"] = "1"
|
|
@@ -71,18 +75,50 @@ def docker_stack():
|
|
|
71
75
|
else:
|
|
72
76
|
pytest.fail("Backend did not become healthy")
|
|
73
77
|
|
|
74
|
-
|
|
78
|
+
# Seed dev users so the identity filter accepts CLI requests.
|
|
79
|
+
SEED_SQL = """
|
|
80
|
+
INSERT INTO teams (id, name) VALUES
|
|
81
|
+
('11111111-1111-1111-1111-111111111111', 'Team A'),
|
|
82
|
+
('22222222-2222-2222-2222-222222222222', 'Team B')
|
|
83
|
+
ON CONFLICT (id) DO NOTHING;
|
|
84
|
+
INSERT INTO app_users (id, email, display_name, app_role, team_id) VALUES
|
|
85
|
+
('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'alice@example.com', 'Alice', 'researcher', '11111111-1111-1111-1111-111111111111'),
|
|
86
|
+
('bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', 'bob@example.com', 'Bob', 'researcher', '22222222-2222-2222-2222-222222222222'),
|
|
87
|
+
('cccccccc-cccc-cccc-cccc-cccccccccccc', 'sup@example.com', 'Sup', 'supervisor', NULL)
|
|
88
|
+
ON CONFLICT (id) DO NOTHING;
|
|
89
|
+
INSERT INTO supervisor_team_assignments (supervisor_id, team_id) VALUES
|
|
90
|
+
('cccccccc-cccc-cccc-cccc-cccccccccccc', '11111111-1111-1111-1111-111111111111'),
|
|
91
|
+
('cccccccc-cccc-cccc-cccc-cccccccccccc', '22222222-2222-2222-2222-222222222222')
|
|
92
|
+
ON CONFLICT DO NOTHING;
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
pg_container = subprocess.run(
|
|
96
|
+
["docker", "compose", "-f", str(COMPOSE_FILE), "-p", project,
|
|
97
|
+
"ps", "-q", "postgres"],
|
|
98
|
+
capture_output=True, text=True, cwd=str(PROJECT_ROOT)
|
|
99
|
+
).stdout.strip()
|
|
100
|
+
|
|
101
|
+
subprocess.run(
|
|
102
|
+
["docker", "exec", "-i", pg_container,
|
|
103
|
+
"psql", "-U", "runledger", "-d", "runledger", "-c", SEED_SQL],
|
|
104
|
+
check=True, cwd=str(PROJECT_ROOT)
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
yield api_base
|
|
75
108
|
|
|
76
109
|
# Tear everything down
|
|
77
110
|
subprocess.run(
|
|
78
111
|
["docker", "compose", "-f", str(COMPOSE_FILE), "-p", project, "down", "-v"],
|
|
79
112
|
cwd=str(PROJECT_ROOT), capture_output=True
|
|
80
113
|
)
|
|
114
|
+
# Restore env so subsequent test files don't inherit the dead port.
|
|
115
|
+
os.environ.pop("RUNLEDGER_BASE_URL", None)
|
|
116
|
+
os.environ.pop("RUNLEDGER_PORT", None)
|
|
81
117
|
|
|
82
118
|
|
|
83
119
|
def run_cli(*args):
|
|
84
120
|
"""Run the CLI in a subprocess, inheriting the environment (port, no-docker)."""
|
|
85
|
-
cmd = [
|
|
121
|
+
cmd = [sys.executable, str(LEDGER_SCRIPT), *args]
|
|
86
122
|
return subprocess.run(
|
|
87
123
|
cmd,
|
|
88
124
|
capture_output=True,
|
|
@@ -152,7 +188,8 @@ def test_full_golden_path(docker_stack, sample_folder):
|
|
|
152
188
|
assert "running" in result.stdout.lower()
|
|
153
189
|
|
|
154
190
|
# 6. deep API verification (uses dynamic API_BASE)
|
|
155
|
-
resp = requests.get(API_BASE, params={"batch": batch}
|
|
191
|
+
resp = requests.get(API_BASE, params={"batch": batch},
|
|
192
|
+
headers={"X-User-Id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"})
|
|
156
193
|
assert resp.status_code == 200
|
|
157
194
|
runs = resp.json()["content"]
|
|
158
195
|
assert len(runs) == 4
|
|
@@ -196,19 +233,21 @@ def test_search_no_results(docker_stack, sample_folder):
|
|
|
196
233
|
|
|
197
234
|
|
|
198
235
|
def test_multi_filter_and_returns_results():
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
pytest.skip(f"Folder {compound_dir} not found")
|
|
236
|
+
if not COMPOUND_DIR.is_dir():
|
|
237
|
+
pytest.skip(f"Folder {COMPOUND_DIR} not found")
|
|
202
238
|
|
|
203
239
|
scan_result = subprocess.run(
|
|
204
|
-
[
|
|
240
|
+
[sys.executable, str(LEDGER_SCRIPT), "scan", str(COMPOUND_DIR),
|
|
241
|
+
"--batch", "compound-int"],
|
|
205
242
|
capture_output=True, text=True, encoding="utf-8", errors="replace"
|
|
206
243
|
)
|
|
207
244
|
assert scan_result.returncode == 0, f"Scan failed: {scan_result.stderr}"
|
|
208
245
|
|
|
209
246
|
result = subprocess.run(
|
|
210
|
-
[
|
|
211
|
-
"--metric", "
|
|
247
|
+
[sys.executable, str(LEDGER_SCRIPT), "search",
|
|
248
|
+
"--metric", "accuracy", "--op", "gt", "--value", "0.9",
|
|
249
|
+
"--metric", "loss", "--op", "lt", "--value", "0.2",
|
|
250
|
+
"--combine", "and", "--batch", "compound-int"],
|
|
212
251
|
capture_output=True, text=True, encoding="utf-8", errors="replace"
|
|
213
252
|
)
|
|
214
253
|
assert result.returncode == 0, f"Search failed: {result.stderr}"
|