jev-col 0.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. jev_col-0.5.1/.github/workflows/publish.yml +76 -0
  2. jev_col-0.5.1/.gitignore +30 -0
  3. jev_col-0.5.1/CHANGELOG.md +30 -0
  4. jev_col-0.5.1/LICENSE +21 -0
  5. jev_col-0.5.1/Makefile +10 -0
  6. jev_col-0.5.1/PKG-INFO +381 -0
  7. jev_col-0.5.1/README.md +363 -0
  8. jev_col-0.5.1/SECURITY.md +18 -0
  9. jev_col-0.5.1/benchmarks/README.md +205 -0
  10. jev_col-0.5.1/benchmarks/local_models.py +391 -0
  11. jev_col-0.5.1/benchmarks/product_agreement.py +67 -0
  12. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/RESULTS-NOTES.md +40 -0
  13. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/diffusiongemma-product100/answers.json +2102 -0
  14. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/diffusiongemma-product100/manifest.json +133 -0
  15. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/diffusiongemma-product100/predictions.csv +101 -0
  16. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/diffusiongemma-product100/report.json +414 -0
  17. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/diffusiongemma-product1000/answers.json +21002 -0
  18. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/diffusiongemma-product1000/manifest.json +1033 -0
  19. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/diffusiongemma-product1000/predictions.csv +1001 -0
  20. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/diffusiongemma-product1000/report.json +1141 -0
  21. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/laya-product100/answers.json +1754 -0
  22. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/laya-product100/manifest.json +136 -0
  23. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/laya-product100/predictions.csv +101 -0
  24. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/laya-product100/report.json +760 -0
  25. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/laya-product1000/answers.json +19154 -0
  26. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/laya-product1000/manifest.json +1036 -0
  27. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/laya-product1000/predictions.csv +1001 -0
  28. jev_col-0.5.1/benchmarks/results/local-models-2026-09-22/laya-product1000/report.json +3912 -0
  29. jev_col-0.5.1/benchmarks/results/product-agreement-2026-09-21/answers.json +2102 -0
  30. jev_col-0.5.1/benchmarks/results/product-agreement-2026-09-21/manifest.json +114 -0
  31. jev_col-0.5.1/benchmarks/results/product-agreement-2026-09-21/predictions.csv +101 -0
  32. jev_col-0.5.1/benchmarks/results/product-agreement-2026-09-21/report.json +389 -0
  33. jev_col-0.5.1/data/SAMPLE.json +50 -0
  34. jev_col-0.5.1/data/complaints-5k.parquet +0 -0
  35. jev_col-0.5.1/docs/benchmarks/local-models-2026-09-22.json +398 -0
  36. jev_col-0.5.1/examples/complaints-codebook.json +24 -0
  37. jev_col-0.5.1/examples/multifield-codebook.json +23 -0
  38. jev_col-0.5.1/pyproject.toml +39 -0
  39. jev_col-0.5.1/src/jcol/__init__.py +9 -0
  40. jev_col-0.5.1/src/jcol/__main__.py +3 -0
  41. jev_col-0.5.1/src/jcol/app.py +231 -0
  42. jev_col-0.5.1/src/jcol/batch.py +122 -0
  43. jev_col-0.5.1/src/jcol/cli.py +413 -0
  44. jev_col-0.5.1/src/jcol/codebook.py +97 -0
  45. jev_col-0.5.1/src/jcol/core.py +5 -0
  46. jev_col-0.5.1/src/jcol/engine.py +293 -0
  47. jev_col-0.5.1/src/jcol/evaluation.py +89 -0
  48. jev_col-0.5.1/src/jcol/pool.py +177 -0
  49. jev_col-0.5.1/src/jcol/project.py +122 -0
  50. jev_col-0.5.1/src/jcol/spec.py +72 -0
  51. jev_col-0.5.1/src/jcol/static/index.html +329 -0
  52. jev_col-0.5.1/src/jcol/tables.py +129 -0
  53. jev_col-0.5.1/tests/conftest.py +27 -0
  54. jev_col-0.5.1/tests/fakes.py +80 -0
  55. jev_col-0.5.1/tests/test_app.py +259 -0
  56. jev_col-0.5.1/tests/test_batch.py +289 -0
  57. jev_col-0.5.1/tests/test_cli.py +277 -0
  58. jev_col-0.5.1/tests/test_core.py +236 -0
  59. jev_col-0.5.1/tests/test_engine.py +274 -0
  60. jev_col-0.5.1/tests/test_evaluation.py +52 -0
  61. jev_col-0.5.1/tests/test_process.py +170 -0
  62. jev_col-0.5.1/tests/test_spec.py +61 -0
  63. jev_col-0.5.1/uv.lock +649 -0
@@ -0,0 +1,76 @@
1
+ name: Test and publish
2
+ # Every push and pull request runs the tests. A pushed tag like v0.1.0 also publishes to PyPI once the
3
+ # tests pass. PyPI trusts this workflow file by name (Trusted Publishing), so there is no API token to
4
+ # store, leak or rotate; keep the filename.
5
+
6
+ on:
7
+ push:
8
+ branches: [main]
9
+ tags: ["v*"]
10
+ pull_request:
11
+
12
+ permissions:
13
+ contents: read
14
+
15
+ defaults:
16
+ run:
17
+ working-directory: consumer
18
+
19
+ jobs:
20
+ test:
21
+ runs-on: ubuntu-latest
22
+ timeout-minutes: 10
23
+ strategy:
24
+ matrix:
25
+ python: ["3.10", "3.13"]
26
+ steps:
27
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
28
+ with:
29
+ path: consumer
30
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
31
+ with:
32
+ repository: keltokhy/jevkit-core
33
+ ref: v0.3.1
34
+ path: jevkit-core
35
+ - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
36
+ with:
37
+ version: "0.12.9"
38
+ python-version: ${{ matrix.python }}
39
+ - run: uv sync --locked
40
+ - run: uv run pytest -q
41
+ - run: uv build --no-sources
42
+ - run: uvx --from twine==7.0.0 twine check --strict dist/*
43
+ - name: Install and smoke-test the wheel outside the checkout
44
+ run: |
45
+ uv build --project ../jevkit-core --no-sources --out-dir /tmp/jevkit-wheels
46
+ uv venv /tmp/jcol-wheel-smoke
47
+ uv pip install --python /tmp/jcol-wheel-smoke/bin/python /tmp/jevkit-wheels/*.whl dist/*.whl
48
+ /tmp/jcol-wheel-smoke/bin/python tests/test_process.py /tmp/jcol-wheel-smoke/bin/jcol
49
+ cd /tmp
50
+ /tmp/jcol-wheel-smoke/bin/jcol --version
51
+ /tmp/jcol-wheel-smoke/bin/jcol run --help
52
+ /tmp/jcol-wheel-smoke/bin/jcol evaluate --help
53
+ /tmp/jcol-wheel-smoke/bin/python -c 'import jcol; from importlib.resources import files; assert callable(jcol.annotate); assert files("jcol").joinpath("static/index.html").is_file()'
54
+
55
+ pypi:
56
+ if: startsWith(github.ref, 'refs/tags/v')
57
+ needs: test
58
+ runs-on: ubuntu-latest
59
+ environment:
60
+ name: pypi
61
+ url: https://pypi.org/project/jev-col/
62
+ permissions:
63
+ id-token: write
64
+ contents: read
65
+ steps:
66
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
67
+ with:
68
+ path: consumer
69
+ - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
70
+ with:
71
+ version: "0.12.9"
72
+ python-version: "3.13"
73
+ - name: Verify the release tag matches the package version
74
+ run: test "$GITHUB_REF_NAME" = "v$(uv version --short)"
75
+ - run: uv build --no-sources
76
+ - run: uv publish --trusted-publishing always
@@ -0,0 +1,30 @@
1
+ .venv/
2
+ __pycache__/
3
+ .pytest_cache/
4
+ dist/
5
+ data/cfpb-has-text-*.parquet
6
+ data/*.csv
7
+ data/*.zip
8
+ data/shards.txt
9
+ spike/texts.json
10
+ .env
11
+ .env.*
12
+ *.key
13
+ *.jcol.sqlite*
14
+
15
+ # Local development, agent notes and generated run artifacts.
16
+ .DS_Store
17
+ .codex/
18
+ AGENTS.md
19
+ work/
20
+ outputs/
21
+ internal/
22
+ spike/
23
+ *.sqlite
24
+ *.sqlite-wal
25
+ *.sqlite-shm
26
+ *.sqlite.lock
27
+ *.sqlite.lock-journal
28
+ *.log
29
+ bench/
30
+ benchmarks/results/**/*.parquet
@@ -0,0 +1,30 @@
1
+ # Changelog
2
+
3
+ ## 0.5.1
4
+
5
+ - First release on PyPI. The distribution is named `jev-col`, matching `jev-grep`, `jev-sort` and
6
+ `jev-select`. The import (`jcol`) and the command (`jcol`) are unchanged.
7
+ - README links point to GitHub, so they also work on PyPI.
8
+ - Restore the recorded product check in `benchmarks/`, with runs on the local DiffusionGemma and Laya
9
+ servers beside Jev. It measures proxy agreement with an administrative label, not accuracy.
10
+
11
+ ## 0.5.0
12
+
13
+ - Add `--api diffusiongemma` and `--api laya` for System One servers running on your own machine,
14
+ through `jevkit-runtime` 0.3: chosen only by name, no key needed, and metered at zero API fees
15
+ unless `JEV_PRICE_PER_MTOK` is set.
16
+
17
+ ## 0.4.0
18
+
19
+ - Move provider selection, answer identity, the cache, validation and metering to the shared
20
+ `jevkit-runtime` 0.2. The tool now only names the providers it offers.
21
+
22
+ ## 0.3.0
23
+
24
+ - Command-line annotation workflow: apply a codebook to a table, resume interrupted runs, export
25
+ labels and evaluate them against a gold column.
26
+ - Local browser access is hardened, and API responses with invalid metering are rejected.
27
+
28
+ ## 0.1.0
29
+
30
+ - First version: a browser table whose columns Jev fills from plain-English headers.
jev_col-0.5.1/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Khaled Eltokhy
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
jev_col-0.5.1/Makefile ADDED
@@ -0,0 +1,10 @@
1
+ .PHONY: install-local test build
2
+
3
+ install-local:
4
+ uv tool install --force .
5
+
6
+ test:
7
+ uv run pytest -q
8
+
9
+ build:
10
+ uv build
jev_col-0.5.1/PKG-INFO ADDED
@@ -0,0 +1,381 @@
1
+ Metadata-Version: 2.5
2
+ Name: jev-col
3
+ Version: 0.5.1
4
+ Summary: Apply natural-language codebooks to tables, with resumable annotation and label evaluation.
5
+ Project-URL: Repository, https://github.com/keltokhy/jcol
6
+ Project-URL: Issues, https://github.com/keltokhy/jcol/issues
7
+ Author: Khaled Eltokhy
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Requires-Python: >=3.10
11
+ Requires-Dist: httpx[http2]>=0.27
12
+ Requires-Dist: jevkit-runtime[http2]<0.4.0,>=0.3.0
13
+ Requires-Dist: polars>=1.0
14
+ Requires-Dist: starlette>=0.37
15
+ Requires-Dist: uvicorn>=0.30
16
+ Requires-Dist: websockets>=12
17
+ Description-Content-Type: text/markdown
18
+
19
+ # jcol
20
+
21
+ Apply a natural-language codebook to a table from your terminal. Validate the inputs,
22
+ run with checkpoints, resume interrupted work, and pipe or export the results.
23
+ Each definition becomes a column of judgments from
24
+ [Jev](https://docs.typesafe.ai), TypeSafe's decision model.
25
+
26
+ ```bash
27
+ jcol run data/complaints-5k.parquet \
28
+ --codebook examples/complaints-codebook.json \
29
+ --output coded.parquet --report report.json
30
+ ```
31
+
32
+ The output retains every source column and adds a value and confidence for each variable.
33
+ The command checkpoints successful cells in `coded.parquet.jcol.sqlite`; rerunning the same
34
+ command resumes missing cells. The browser also saves its committed columns across restarts.
35
+
36
+ Version 0.3.1 includes a Python API and an optional browser interface. It has **not been
37
+ published to PyPI**; install from Git or a built wheel. Software tests use a local fake
38
+ API and do not establish model accuracy. See [Validation](#validation).
39
+
40
+ ## Install
41
+
42
+ Python 3.10 or later:
43
+
44
+ ```bash
45
+ uv tool install 'git+https://github.com/keltokhy/jcol.git'
46
+ jcol --help
47
+ jcol doctor
48
+ ```
49
+
50
+ This installs `jcol` on PATH in its own environment. If your shell cannot find it, run
51
+ `uv tool update-shell` and restart the shell. For local development, clone the repository
52
+ and run `make install-local` (or `uv tool install --force .`). To build a package:
53
+
54
+ ```bash
55
+ uv build
56
+ uv tool install --force dist/jev_col-0.5.1-py3-none-any.whl
57
+ ```
58
+
59
+ Set `TYPESAFE_API_KEY` or `OPENROUTER_API_KEY`, or put a key in
60
+ `~/.config/jev/typesafe.key` or `~/.config/jev/openrouter.key`. With both keys, TypeSafe wins.
61
+ Choose explicitly with `--api` or `JEV_API`. The key stays in the Python process.
62
+ Only `run` and `browse` need a key. `doctor` checks local configuration; `doctor --check`
63
+ also sends an unauthenticated HEAD request to test reachability, without doing inference.
64
+ Reachability does not verify authentication or model availability.
65
+
66
+ ### Local servers (experimental)
67
+
68
+ `--api diffusiongemma` and `--api laya` send the same questions to a System One server on your own
69
+ machine, an [OpenJev](https://github.com/razorback16/openjev) or
70
+ [laya-mlx](https://github.com/mizorewww/laya-mlx) process that you run separately. They are never
71
+ chosen automatically, need no key, and count as $0 against the budget unless `JEV_PRICE_PER_MTOK`
72
+ is set; `doctor` lists them. The runtime's [DiffusionGemma](https://github.com/keltokhy/jevkit-core/blob/main/docs/diffusiongemma.md)
73
+ and [Laya](https://github.com/keltokhy/jevkit-core/blob/main/docs/laya.md) guides explain the setup; keep
74
+ `--concurrency` low while a local model warms up.
75
+ See [How well does it work](#how-well-does-it-work) for a comparison with Jev.
76
+
77
+ ## Terminal workflow
78
+
79
+ ```bash
80
+ jcol inspect reviews.csv
81
+ jcol init --input review --name refund_request \
82
+ --definition 'The customer explicitly asks for a refund.' -o codebook.json
83
+ jcol validate reviews.csv --codebook codebook.json
84
+ jcol run reviews.csv --codebook codebook.json --dry-run
85
+ jcol run reviews.csv --codebook codebook.json -o coded.parquet --budget 0.25
86
+ jcol status coded.parquet.jcol.sqlite
87
+ jcol export coded.parquet.jcol.sqlite --source reviews.csv -o recovered.csv
88
+ ```
89
+
90
+ Repeat the same `run` command to resume. Successful cells are retained across interrupts;
91
+ `export` recovers them without credentials or API calls. Export requires the original table,
92
+ including its schema and row order. `status` and `export` can read an active project.
93
+
94
+ | Command | Purpose |
95
+ |---|---|
96
+ | `init` | Generate a valid, editable codebook from a definition |
97
+ | `inspect` | Discover field names, types, null counts and row count |
98
+ | `validate` | Check inputs, codebook, gold labels and output-name conflicts offline |
99
+ | `run --dry-run` | Describe work and check an existing checkpoint without writes or calls |
100
+ | `run` | Annotate, checkpoint, resume and write results |
101
+ | `status` | Read completion per saved column |
102
+ | `export` | Recover saved results, including partial results |
103
+ | `evaluate` | Compare predictions with supplied reviewed labels |
104
+ | `doctor` | Inspect configuration without revealing keys |
105
+ | `browse` | Open the optional local browser interface |
106
+
107
+ For a category, add `--kind category --option billing --option other` to `init`.
108
+ For a scale, use `--kind scale` and repeat `--option` from lowest to highest.
109
+ Edit the generated JSON to add more variables or detailed option definitions.
110
+ Existing codebooks are protected unless `init --force` is given.
111
+
112
+ ### Pipes and automation
113
+
114
+ ```bash
115
+ cat reviews.csv | jcol run - --input-format csv --codebook codebook.json \
116
+ --project study.sqlite -o - --output-format jsonl --report report.json \
117
+ > coded.jsonl
118
+ jcol --json status study.sqlite
119
+ ```
120
+
121
+ Input/output formats are CSV, TSV, Parquet and JSONL (`.ndjson` is also recognized).
122
+ Stdin requires `--input-format`; stdout defaults to JSONL. A streamed run requires
123
+ `--project PATH` or an explicit `--no-project`. Tables are held in memory; pipe support
124
+ does not make processing out-of-core. Named table outputs and reports are replaced
125
+ atomically after successful serialization. These two files are written separately.
126
+
127
+ Metadata commands emit JSON objects. `--json`, before or after the subcommand, wraps
128
+ reports in `{"schema_version":1,"command":"status","ok":true,"data":{...}}`.
129
+ Errors use `{"schema_version":1,"command":"status","ok":false,"error":{"code":"command_error","message":"..."}}`.
130
+ Argument errors use `usage_error`; interruptions use `interrupted`. For a partial run,
131
+ the report is valid (`ok: true`), `data.run.complete` is false, and exit status is 2.
132
+ Without `--json`, `run` and `evaluate` retain their original unwrapped report formats.
133
+
134
+ With `-o -`, stdout contains only the table (or the raw codebook for `init`). Run/export
135
+ reports go to `--report` or stderr; errors also go to stderr. Use `--quiet` to suppress
136
+ human summaries and terminal progress when parsing stderr as JSON. Help and version are
137
+ always text. `browse` does not support JSON output.
138
+
139
+ Exit codes: **0** success (including exporting partial results or discovering missing
140
+ credentials with `doctor`), **1** invalid input or execution failure, **2** partial annotation,
141
+ **130** interrupted, **141** closed output pipe. In scripts, check `doctor`'s `ready` field,
142
+ `status`'s `complete` field, and the run exit code. No-argument `jcol` prints help.
143
+ Use `python -m jcol` when invoking an installed package through a specific Python environment.
144
+
145
+ ## Codebooks
146
+
147
+ A codebook is a versioned JSON file with explicit input fields and named output variables:
148
+
149
+ ```json
150
+ {
151
+ "version": 1,
152
+ "inputs": ["narrative", "issue"],
153
+ "columns": [
154
+ {
155
+ "name": "unauthorized_activity",
156
+ "kind": "binary",
157
+ "definition": "The consumer alleges an account or transaction they did not authorize. Include identity theft. Exclude disputes about authorized fees or loan terms.",
158
+ "gold": "reviewer_label"
159
+ }
160
+ ]
161
+ }
162
+ ```
163
+
164
+ `definition` is the actual instruction sent to Jev. It can contain punctuation, inclusion
165
+ rules and exclusions without the browser header grammar. Only `inputs` are sent. With
166
+ multiple inputs each row is a JSON object with field names and values; a single input is
167
+ sent as plain text. A missing single-field input becomes empty text. Source nulls remain
168
+ null in the result. Multiple-field inputs retain JSON nulls.
169
+
170
+ | Kind | Required options | Output |
171
+ |---|---|---|
172
+ | `binary` | None | Probability from 0 to 1 |
173
+ | `category` | 2–255 distinct labels, as a list or an object mapping labels to definitions | Chosen label |
174
+ | `scale` | 2–10 distinct levels, listed from lowest to highest | Numeric position from 0 to number of levels minus one |
175
+
176
+ Every variable also adds `NAME__confidence`. For a binary variable this is
177
+ `max(p, 1-p)`; for other kinds it is the API's confidence when supplied, otherwise zero.
178
+ These are model scores, not guarantees of correctness or calibrated confidence intervals.
179
+ Cell values retain the browser's rounding: binary probabilities and confidence to three
180
+ decimals, scale values to two. Saved projects also retain the unrounded per-question answer.
181
+
182
+ `gold` optionally names a reference-label column for evaluation. Declared gold fields
183
+ cannot also be model inputs. Unknown codebook fields, invalid labels, duplicate names and
184
+ output names that would overwrite source columns are rejected before a batch run makes calls.
185
+ Check the remaining inputs yourself for indirect label leakage.
186
+
187
+ Examples:
188
+
189
+ - [Product codebook](https://github.com/keltokhy/jcol/blob/main/examples/complaints-codebook.json): narrative-only product classification with proxy labels.
190
+ - [Multiple-field codebook](https://github.com/keltokhy/jcol/blob/main/examples/multifield-codebook.json): narrative and issue used together, with binary and category definitions.
191
+
192
+ ## Batch and Python
193
+
194
+ ```bash
195
+ jcol run reviews.csv --codebook codebook.json --output coded.csv
196
+ jcol run reviews.tsv --codebook codebook.json --output coded.parquet \
197
+ --project study.jcol.sqlite --model typesafe/jev-1.13 --api openrouter \
198
+ --budget 0.25 --concurrency 4 --report report.json
199
+ jcol evaluate coded.parquet --codebook codebook.json --report validation.json
200
+ ```
201
+
202
+ Tables can be CSV, TSV, Parquet or JSONL. Batch mode sends full serialized rows by default;
203
+ `--max-chars N` explicitly truncates them and reports how many were affected. Export always
204
+ preserves the full source values. Parquet preserves source types; CSV/TSV use their usual
205
+ text representations.
206
+
207
+ `--project` defaults to `OUTPUT.jcol.sqlite`. Successful cells are durable as they arrive,
208
+ including if the process crashes. A project checks the full table contents, schema and order,
209
+ selected inputs, codebook, model ID, endpoint and truncation setting before resuming.
210
+ Changed inputs require a new project path. One process may own a project at a time.
211
+
212
+ Partial output and the JSON report are still written after budget exhaustion or API failures.
213
+ A report goes to stdout unless `--report` names a file or the table is streamed to stdout.
214
+ All requested columns share each row's call; repeated identical rows share answers.
215
+
216
+ ```python
217
+ import jcol
218
+
219
+ result = jcol.annotate(
220
+ "reviews.parquet", "codebook.json",
221
+ project="study.jcol.sqlite",
222
+ budget=0.25,
223
+ concurrency=4,
224
+ )
225
+ result.write("coded.parquet")
226
+ print(result.complete)
227
+ print(result.evaluation)
228
+ ```
229
+
230
+ `annotate` accepts a Polars DataFrame or a table path; the codebook can be a path, dictionary
231
+ or `jcol.Codebook`. The result contains a Polars `table`, `metadata`, and `evaluation`.
232
+ The Python API persists when `project` is provided; without it, results live in the returned
233
+ object. In a notebook or another running event loop, use `await jcol.annotate_async(...)`.
234
+ Call `jcol.evaluate(dataframe, codebook)` to evaluate existing predictions without model calls.
235
+
236
+ ## Validation
237
+
238
+ Evaluation reports labeled rows, evaluated rows, missing predictions and coverage alongside
239
+ metrics. Missing labels are excluded. Missing predictions reduce coverage and are excluded
240
+ from the metric denominator, so always read the two together. Invalid nonmissing labels or
241
+ predictions are errors.
242
+
243
+ - **Binary:** accuracy at a configurable threshold (default 0.5), Brier score, per-class
244
+ precision/recall/F1, macro-F1 and a confusion matrix. Gold accepts true/false, yes/no or 1/0.
245
+ - **Category:** exact-label accuracy, per-class metrics, macro-F1 and a confusion matrix.
246
+ Matching is case-sensitive. Macro-F1 averages every declared category, including absent
247
+ classes, whose F1 is zero.
248
+ - **Scale:** mean absolute error in scale positions; gold may be a numeric position or an
249
+ exact level name.
250
+ - **Disagreements:** zero-based row indices with the reference and predicted values.
251
+
252
+ Freeze your definitions, reserve representative reviewed rows, and examine disagreements
253
+ before using a derived variable.
254
+
255
+ ## How well does it work
256
+
257
+ One recorded check codes 100 public complaints with the
258
+ [product codebook](https://github.com/keltokhy/jcol/blob/main/examples/complaints-codebook.json), which picks the financial product at the
259
+ center of each narrative from ten labels, and compares the answers with `product_group`, a grouping
260
+ of each complaint's administrative product label. This is proxy agreement, not accuracy: a narrative
261
+ can discuss several products, and the codebook draws some boundaries differently from the reference.
262
+ [benchmarks/README.md](https://github.com/keltokhy/jcol/blob/main/benchmarks/README.md) has the full record and its caveats.
263
+
264
+ | 100 complaints | Jev 1.13 (OpenRouter) | DiffusionGemma (`openjev-0.1`, local) | Laya (`laya-421m`, local) |
265
+ |---|---:|---:|---:|
266
+ | Rows answered | 100 | 100 | 73 |
267
+ | Agreement with `product_group` (majority-label baseline: 55) | 75 | 78 | 50 |
268
+ | Wall time | 6.63 s | 134.6 s | 71.4 s |
269
+
270
+ DiffusionGemma answered every row and agreed with `product_group` as often as Jev; 100 rows are too
271
+ few to separate the two. On 1,000 rows it agreed on 830, against 596 for the majority label, at about
272
+ 1.4 rows a second on an Apple M3 Ultra. Laya took under a minute on the same 1,000 rows, but it reads
273
+ at most 512 tokens, question included, and refused the narratives that did not fit: 27 of 100 and 202
274
+ of 1,000. Jev remains the default; DiffusionGemma is a reasonable substitute when the text must stay
275
+ on your machine, and Laya is not one for narratives like these.
276
+
277
+ ## Browser
278
+
279
+ ```bash
280
+ jcol browse data/complaints-5k.parquet --text narrative
281
+ jcol browse tickets.csv --text subject body --max-chars 0 --no-open
282
+ ```
283
+
284
+ The legacy `jcol FILE --text ...` invocation also works. The browser opens on
285
+ `http://127.0.0.1:8765`. Type a header, inspect the preview, and press Enter:
286
+
287
+ ```text
288
+ alleges fraud?
289
+ product: mortgage, credit card, other
290
+ tone: calm < upset < furious
291
+ ```
292
+
293
+ Headers without a colon are binary descriptions. A colon followed by comma-separated options
294
+ creates a category column; levels separated by `<` create a scale. The typed description is
295
+ inserted into a short question template. Use a JSON codebook and batch mode for detailed rules.
296
+
297
+ - Click a column name to sort, or a cell to filter. Click a filter chip to remove it.
298
+ - `show fields` reveals source fields. Click a row to read all selected inputs and fields.
299
+ - Category columns offer `compare with…` for a quick comparison with a source field.
300
+ - `export csv` downloads the filtered, sorted rows with **full source values**, numeric scale
301
+ positions, and committed values/confidences. A generated name that conflicts with a source
302
+ field is prefixed with `jcol_COLUMNID__` until it is unique.
303
+ - Columns persist in `FILE.jcol.sqlite` by default. Use `--project PATH` to choose a location
304
+ or `--no-project` for a temporary session. Removing a column also removes it from the project.
305
+
306
+ | Browser option | Default / meaning |
307
+ |---|---|
308
+ | `--text COLUMN [COLUMN ...]` | Selected inputs; default is the longest string column on average |
309
+ | `--limit N` | First N rows |
310
+ | `--max-chars N` | 4,000 characters per serialized row; warns on truncation; 0 sends full rows |
311
+ | `--budget DOLLARS` | 2.00; a per-invocation spending threshold, not a hard cap |
312
+ | `--workers N` / `--per-worker N` | 16 processes, 64 background calls each; workers 0 runs in-process |
313
+ | `--api` / `--model` | Backend and model ID; `JEV_MODEL` is also accepted |
314
+ | `--no-cache` | Disable the separate answer cache, while retaining project persistence |
315
+ | `--port` / `--no-open` | Port 8765; optionally suppress opening the browser |
316
+
317
+ The browser prioritizes visible rows and may resend a slow visible-row call. Background rows
318
+ follow a fixed shuffle (seed 70). Its running estimates describe model judgments over the table;
319
+ the displayed margins cover sampling uncertainty only. They do not measure classification error.
320
+
321
+ ## Costs, persistence and limits
322
+
323
+ - Budget checks use cost already reported by completed requests. In-flight calls can exceed the
324
+ threshold. Batch mode bounds them with `--concurrency`; browser previews can spend outside
325
+ background budget checks. Budget and cost counters reset each invocation. A saved project
326
+ preserves cells, not a cumulative spending limit.
327
+ - Answers also cache in `~/.cache/jev/answers.sqlite`, shared with the other JevKit tools. Keys
328
+ include provider, endpoint, requested model, serialized input and question; entries written by
329
+ versions before 0.4 are not reused. `JEV_URL` overrides the endpoint. No API keys are stored in
330
+ projects or exports.
331
+ - Default model IDs are aliases. Pin a model for a study; a project cannot detect a server moving
332
+ an alias to new model weights. Cached and saved answers remain frozen under the requested ID.
333
+ - The full table and results are held in memory. The browser receives previews and source fields
334
+ for every row and sorts locally. This is intended for thousands of rows, not out-of-core datasets.
335
+ - Transient failures are retried; the scheduler makes up to two additional sweeps for missing cells.
336
+ Invalid answers remain missing. Fatal authentication or credit errors stop new background work.
337
+ - Every browser tab shares the same columns and preview. The server binds only to localhost,
338
+ rejects nonlocal Host headers and cross-origin HTTP/WebSocket requests, and has no user
339
+ authentication. Other local programs can access it. Data sent to the model goes to the selected API provider.
340
+ - Jev follows the definition provided; text in a row can influence its answer. Review coded data,
341
+ and do not treat a judgment column as a security boundary.
342
+
343
+ ## Data and development
344
+
345
+ `data/complaints-5k.parquet` contains 5,000 public CFPB complaints and their fields.
346
+ [data/SAMPLE.json](https://github.com/keltokhy/jcol/blob/main/data/SAMPLE.json) records the Hugging Face snapshot, original sampling method,
347
+ seed, dates and exact mapping used to derive `product_group`. Original source files and the
348
+ original 5,000-row sampling script are not bundled.
349
+
350
+ ```bash
351
+ uv sync
352
+ uv run pytest -q # offline fake API; no credentials or external connections
353
+ uv build # source distribution and wheel
354
+ ```
355
+
356
+ CI runs tests and installs the built wheel outside the checkout on Python 3.10 and 3.13,
357
+ checking the CLI, public imports and packaged browser assets.
358
+
359
+ `codebook.py`, `batch.py`, `evaluation.py` and `project.py` implement the reusable workflow.
360
+ `engine.py` is the shared scheduler; `core.py` and `pool.py` implement API calls and caching.
361
+ `app.py` and `static/index.html` implement the browser. MIT license.
362
+
363
+ ## Shared JevKit development
364
+
365
+ This tool uses [`jevkit-runtime`](https://github.com/keltokhy/jevkit-core), imported
366
+ as `jevkit_runtime`. Clone that repository beside this one as `../jevkit-core`, then
367
+ run `uv sync`. Core Python edits apply on the next invocation of this tool;
368
+ restart long-lived Python processes after editing.
369
+
370
+ The distribution name is `jevkit-runtime` because `jevkit-core` on PyPI belongs
371
+ to a different project. The runtime is [available on PyPI](https://pypi.org/project/jevkit-runtime/).
372
+ Use the sibling checkout for shared development, or `uv sync --no-sources` for a
373
+ standalone source checkout. Existing published versions of this tool are
374
+ unaffected by this source migration.
375
+
376
+ From the core checkout, `python scripts/dev.py setup`, `check`, and `wheel-check`
377
+ set up and validate all five consumers in separate environments.
378
+ CI checks out core tag `v0.3.1`. Prompts, question construction, and budget policies
379
+ remain in this repository; answer identity, the answer store, transport, and metering
380
+ are the runtime's. Runtime 0.2 keys and stores answers differently from 0.1, so a cache
381
+ written by an earlier version is re-asked once after upgrading.