jev-select 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. jev_select-0.1.0/.github/workflows/publish.yml +50 -0
  2. jev_select-0.1.0/.gitignore +13 -0
  3. jev_select-0.1.0/LICENSE +21 -0
  4. jev_select-0.1.0/Makefile +9 -0
  5. jev_select-0.1.0/PKG-INFO +231 -0
  6. jev_select-0.1.0/README.md +208 -0
  7. jev_select-0.1.0/bench/retrieval.py +176 -0
  8. jev_select-0.1.0/bench/scale.py +52 -0
  9. jev_select-0.1.0/bench/scenarios.py +134 -0
  10. jev_select-0.1.0/docs/BENCHMARKS.md +105 -0
  11. jev_select-0.1.0/docs/OUTPUT.md +93 -0
  12. jev_select-0.1.0/docs/PLAN.md +54 -0
  13. jev_select-0.1.0/docs/RELEASING.md +22 -0
  14. jev_select-0.1.0/docs/benchmarks/scale-million.json +97 -0
  15. jev_select-0.1.0/docs/benchmarks/scenarios.json +858 -0
  16. jev_select-0.1.0/docs/benchmarks/scifact-cold.json +1929 -0
  17. jev_select-0.1.0/docs/verification.json +33 -0
  18. jev_select-0.1.0/examples/conversations.jsonl +6 -0
  19. jev_select-0.1.0/pyproject.toml +46 -0
  20. jev_select-0.1.0/skills/jselect/SKILL.md +50 -0
  21. jev_select-0.1.0/src/jselect/__init__.py +9 -0
  22. jev_select-0.1.0/src/jselect/__main__.py +3 -0
  23. jev_select-0.1.0/src/jselect/cli.py +272 -0
  24. jev_select-0.1.0/src/jselect/index.py +255 -0
  25. jev_select-0.1.0/src/jselect/inputs.py +307 -0
  26. jev_select-0.1.0/src/jselect/judge.py +345 -0
  27. jev_select-0.1.0/src/jselect/select.py +385 -0
  28. jev_select-0.1.0/src/jselect/text.py +107 -0
  29. jev_select-0.1.0/src/jselect/types.py +77 -0
  30. jev_select-0.1.0/tests/conftest.py +17 -0
  31. jev_select-0.1.0/tests/test_judge_cli.py +174 -0
  32. jev_select-0.1.0/tests/test_selection.py +180 -0
  33. jev_select-0.1.0/tests/test_streaming_groups.py +190 -0
  34. jev_select-0.1.0/uv.lock +665 -0
@@ -0,0 +1,50 @@
1
+ name: Test and publish
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ tags: ["v*"]
7
+ pull_request:
8
+
9
+ permissions:
10
+ contents: read
11
+
12
+ jobs:
13
+ test:
14
+ runs-on: ubuntu-latest
15
+ strategy:
16
+ matrix:
17
+ python: ["3.10", "3.13"]
18
+ steps:
19
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
20
+ - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
21
+ with:
22
+ version: "0.12.9"
23
+ python-version: ${{ matrix.python }}
24
+ - run: uv sync --locked
25
+ - run: uv run ruff check src tests bench
26
+ - run: uv run ruff format --check src tests bench
27
+ - run: uv run pytest -q
28
+ - run: uv build --no-sources
29
+ - run: uvx --from twine==7.0.0 twine check --strict dist/*
30
+
31
+ pypi:
32
+ if: startsWith(github.ref, 'refs/tags/v')
33
+ needs: test
34
+ runs-on: ubuntu-latest
35
+ environment:
36
+ name: pypi
37
+ url: https://pypi.org/project/jev-select/
38
+ permissions:
39
+ id-token: write
40
+ contents: read
41
+ steps:
42
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
43
+ - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
44
+ with:
45
+ version: "0.12.9"
46
+ python-version: "3.13"
47
+ - name: Verify the release tag matches the package version
48
+ run: test "$GITHUB_REF_NAME" = "v$(uv version --short)"
49
+ - run: uv build --no-sources
50
+ - run: uv publish --trusted-publishing always
@@ -0,0 +1,13 @@
1
+ .venv/
2
+ __pycache__/
3
+ .pytest_cache/
4
+ .ruff_cache/
5
+ dist/
6
+ *.egg-info/
7
+ *.jselect
8
+ *.jselect-*
9
+ bench/data/
10
+ bench/out/
11
+ .env
12
+ *.key
13
+ .DS_Store
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Khaled Eltokhy
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,9 @@
1
+ .PHONY: install-local check
2
+ install-local:
3
+ uv tool install --editable .
4
+
5
+ check:
6
+ uv run ruff check src tests bench
7
+ uv run ruff format --check src tests bench
8
+ uv run pytest -q
9
+ uv build
@@ -0,0 +1,231 @@
1
+ Metadata-Version: 2.5
2
+ Name: jev-select
3
+ Version: 0.1.0
4
+ Summary: Select useful, source-linked evidence for an AI task within a token budget
5
+ Project-URL: Repository, https://github.com/keltokhy/jselect
6
+ Project-URL: Documentation, https://github.com/keltokhy/jselect#readme
7
+ Project-URL: Issues, https://github.com/keltokhy/jselect/issues
8
+ Author: Khaled Eltokhy
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: agents,cli,context,jev,retrieval,semantic search
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Environment :: Console
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Topic :: Text Processing
17
+ Classifier: Topic :: Utilities
18
+ Requires-Python: >=3.10
19
+ Requires-Dist: httpx>=0.27
20
+ Requires-Dist: pathspec>=0.12
21
+ Requires-Dist: tiktoken>=0.8
22
+ Description-Content-Type: text/markdown
23
+
24
+ # jselect
25
+
26
+ **Useful evidence for your AI, within a token budget.**
27
+
28
+ Give jselect a task and your data. It finds relevant passages, favors different information over
29
+ repetition, and assembles a source-linked context your agent can read directly.
30
+
31
+ ```bash
32
+ jselect "Why are people giving up during signup?" conversations.jsonl --tokens 8000
33
+ ```
34
+
35
+ ```python
36
+ from jselect import select
37
+
38
+ evidence = select(records, task="Why are people giving up during signup?", tokens=8000)
39
+ print(evidence.context) # source excerpts and citations, within the token budget
40
+ ```
41
+
42
+ No labels, predefined categories, vector database, or generative model required. Runs on files,
43
+ directories, piped exports, or Python records. This is a **separate tool and package** from jgrep.
44
+
45
+ ## Install
46
+
47
+ Python 3.10+:
48
+
49
+ ```bash
50
+ uv tool install jev-select # installs the jselect command on PATH
51
+ pip install jev-select # use jselect from your Python application
52
+ ```
53
+
54
+ The distribution is named `jev-select`; the command and Python import are both `jselect`.
55
+
56
+ ```bash
57
+ jselect doctor --json
58
+ echo 'The signup verification email never arrived.' | jselect "signup problems" --tokens 600
59
+ ```
60
+
61
+ For development, clone [the repository](https://github.com/keltokhy/jselect), run `uv sync`,
62
+ then `uv run jselect ...`. For an editable command: `uv tool install --editable .`.
63
+
64
+ If a TypeSafe or OpenRouter key is configured, the default uses **semantic relevance scoring** with Jev.
65
+ Set `TYPESAFE_API_KEY` or `OPENROUTER_API_KEY`, or use existing credentials in
66
+ `~/.config/jev/typesafe.key` or `~/.config/jev/openrouter.key`. Credentials never appear in output.
67
+ `JEV_API`, `JEV_MODEL`, and `JEV_URL` overrides are supported. Gateways use `JEV_GATEWAY_URL` and
68
+ `JEV_GATEWAY_API_KEY`, or `~/.config/jev/gateway.url` and `gateway.key`.
69
+
70
+ Without credentials, jselect uses local lexical retrieval and says so. Force that with `--local`.
71
+ Use `--mode semantic` to require semantic scoring and fail if credentials are missing.
72
+
73
+ ## Use it across your data
74
+
75
+ ```bash
76
+ # Customer conversations, with common text fields detected automatically
77
+ jselect "What prevents users from finishing signup?" conversations.jsonl --tokens 2000
78
+
79
+ # Related turns stored as separate, possibly interleaved rows
80
+ jselect "Where does the assistant contradict itself?" turns.jsonl --group-by conversation_id
81
+
82
+ # Source code and documentation, respecting .gitignore and .ignore
83
+ jselect "How are database connections released?" src/ docs/ --tokens 4000
84
+
85
+ # Papers or interview responses in a CSV
86
+ jselect "Evidence that challenges the proposed explanation" papers.csv --field abstract --tokens 3000
87
+
88
+ # Export from another tool, preserving original IDs and field locations
89
+ cat events.jsonl | jselect "Why are requests waiting?" --format jsonl --field message --json
90
+
91
+ # Fully offline, including token accounting: byte count conservatively bounds byte-BPE token use
92
+ jselect "retry timeout configuration" src/ --local --encoding bytes --tokens 4000
93
+ ```
94
+
95
+ Supported inputs: UTF-8 text and code, logs, JSONL/NDJSON, JSON objects or arrays, CSV/TSV, directories,
96
+ and stdin. Formats are inferred from extensions. Stdin defaults to lines; use `--format jsonl` for
97
+ structured input. `.log` files default to one record per line; `--format text` preserves neighboring
98
+ log lines in overlapping passages. PDF, Word, images, and audio need text extraction first.
99
+
100
+ Common text fields are tried in this order: `text`, `content`, `message`, `messages`, `conversation`,
101
+ `body`, `abstract`. If none exists, the entire object is serialized as JSON. `--field event.message`
102
+ selects an exact or dotted field. Only selected text is sent to the scorer, not the entire original row.
103
+ IDs come from `id`, `_id`, or input position. A group uses complete serialized rows unless you explicitly
104
+ select a field, preserving roles and other turn context. Grouping preserves input order, not timestamp order.
105
+
106
+ ## Fast repeated investigations
107
+
108
+ Build a local index once and ask many questions:
109
+
110
+ ```bash
111
+ jselect index conversations.jsonl --output conversations.jselect
112
+ jselect "Confusion about cancellation" conversations.jselect --json --output first.json
113
+ jselect "Confusion about cancellation" conversations.jselect --against first.json --json --output next.json
114
+ jselect inspect conversations.jselect --json
115
+ jselect show conversations.jselect PASSAGE_ID --json
116
+ ```
117
+
118
+ `--against` excludes exact excerpts already returned and penalizes similar text. It is useful when an
119
+ agent asks for more evidence or brings an existing reference set. It does not promise every follow-up
120
+ will introduce a new semantic idea. Saved indexes are snapshots: rebuild with `index ... --force` when
121
+ the source changes. Replacement is atomic, so a failed rebuild leaves the previous index intact.
122
+
123
+ For maximum retrieval breadth, use `--scan all`. This scores every unique passage that fits the output
124
+ budget and is not already supplied in `--against`, then keeps a bounded pool for final selection.
125
+ The full scan is preflighted against the estimated dollar budget before any calls are made.
126
+
127
+ ```bash
128
+ jselect "Signs the customer has lost trust" conversations.jselect --scan all --budget 0.25
129
+ ```
130
+
131
+ ## Python and agents
132
+
133
+ ```python
134
+ from jselect import Index, Record, aselect, select
135
+
136
+ records = [
137
+ {"id": "ticket-1", "text": "The verification link never arrives."},
138
+ {"id": "ticket-2", "text": "The free trial requires a credit card."},
139
+ ]
140
+ result = select(records, task="What blocks registration?", tokens=500)
141
+ payload = result.to_dict() # same schema as --json
142
+
143
+ # Reuse an index in a process; no source scanning on each query.
144
+ with Index.build(records) as index:
145
+ first = index.select(task="What blocks registration?", tokens=500)
146
+ more = index.select(task="What blocks registration?", tokens=500, against=first)
147
+
148
+ # In an existing event loop or notebook:
149
+ # result = await aselect(records, task="What blocks registration?", tokens=500)
150
+
151
+ # Bring your existing reranker. Return one finite score in [0, 1] per passage.
152
+ def judge(task, passages):
153
+ return existing_reranker(task, [p.text for p in passages])
154
+
155
+ result = select(records, task="What blocks registration?", tokens=500, scorer=judge)
156
+ ```
157
+
158
+ Custom scorers may be async functions or objects exposing `score(task, passages)`. Full scans invoke
159
+ them in blocks of at most 256 passages. Use `Record(text, id=..., source=...)` for explicit provenance.
160
+ A string or `Path` passed directly to `select` is an input path; strings inside an iterable are records.
161
+ To use passages already retrieved by another system, pass them as records; set `scan="all"` with a
162
+ custom or semantic scorer if the list exceeds the candidate limit and every passage must be evaluated.
163
+
164
+ Pass **`result.context`** to your agent. The full JSON includes additional metadata and is not subject
165
+ to the context token budget. Treat excerpts as source data rather than agent instructions.
166
+
167
+ ## How it works and what it costs
168
+
169
+ 1. Build or open a SQLite FTS5 index. Long records become overlapping, source-preserving passages.
170
+ Exact repeated passages share one indexed text while retaining occurrence counts and up to five sources.
171
+ 2. Shortlist up to 256 passages by default. Most come from BM25 with a text-diversity adjustment;
172
+ 20% of slots are reserved for deterministic exploration in semantic mode. This is not exhaustive retrieval.
173
+ 3. Score relevance using small batches of Jev decisions. The question explicitly includes contradicting
174
+ evidence. Scores are cached per endpoint, model, prompt version, task, and exact passage.
175
+ 4. Greedily balance relevance, text novelty, and passage token cost. Citation headers and separators count
176
+ toward the budget. Oversized passages are split **before** scoring; returned text is never generated.
177
+
178
+ The defaults are eight passages per request, eight requests in flight, a 20-second total request deadline,
179
+ and a **$0.05 estimated spend budget**. Jev models are versioned (`jev-1.13.0` on TypeSafe and
180
+ `typesafe/jev-1.13` on OpenRouter); responses report the served model when available. Requests retry
181
+ transient errors within the deadline. Errors retain completed cache entries for the next run.
182
+
183
+ The dollar guard uses a conservative UTF-8 byte estimate and the listed Jev input price. Actual provider
184
+ or gateway pricing and unreported usage can differ; this is not a provider-enforced billing cap. JSON
185
+ reports measured cost when available and marks list-price estimates. Failed requests without usage
186
+ reports may have incurred additional costs. Use `--candidates`, `--batch-size`, and `--budget` to bound work.
187
+ `--no-cache` disables the disk score cache at `~/.cache/jselect/scores.sqlite`.
188
+
189
+ Token counting uses `o200k_base` by default. Set `--encoding` to another tiktoken encoding/model, or `bytes`
190
+ for a conservative count with no tokenizer download. The first use of a tiktoken encoding may download its
191
+ public vocabulary. Choose the encoding your downstream model uses and leave room for its other messages.
192
+
193
+ ## Measured results
194
+
195
+ Measured locally on 2026-09-19; details and frozen reports are in [the benchmark report](https://github.com/keltokhy/jselect/blob/main/docs/BENCHMARKS.md).
196
+
197
+ | Check | Observed result |
198
+ |---|---|
199
+ | One million distinct generated log records | 13.64 s to index; 0.40 s median repeated **local** query; 143 MB peak process memory |
200
+ | 30 SciFact research queries, 2,000 tokens, 256 candidates, cache disabled | 86.7% mean labeled-source recall vs 78.3% for BM25 ranking; 1.86 s median; $0.149 total |
201
+ | Handwritten support, code, and contract fixtures | All 4 intended evidence types in 4 passages in each fixture; relevance-only variant covered 3, 3, and 1 |
202
+
203
+ These are scoped measurements, not guarantees for arbitrary data, agent answer quality, or future API
204
+ latency. A selected set cannot establish prevalence or causation. Diversity is a lexical heuristic;
205
+ it does not certify balanced viewpoints or find every contradiction. Semantic scores are model judgments,
206
+ not calibrated confidence in a final answer. The JSON reports how much of the collection was considered.
207
+
208
+ ## Output and errors
209
+
210
+ Default stdout is the exact context string. `--json` returns one object with `schema_version: 1`,
211
+ `task`, `context`, `items`, `tokens`, `token_budget`, `encoding`, `stats`, and `warnings`.
212
+ Each item contains original `text`, a stable content-hash `id`, `sources`, `occurrences`, `relevance`,
213
+ `novelty`, and the selection rule used. See [the output contract](https://github.com/keltokhy/jselect/blob/main/docs/OUTPUT.md).
214
+
215
+ Exit 0 means success, including empty evidence. Exit 2 means invalid input, bad setup, budget refusal,
216
+ or a provider error. Exit 130 means interruption. JSON errors have an `error` object and any available
217
+ usage `stats`; diagnostics never contaminate JSON stdout. `--stats` writes timings and cost to stderr.
218
+
219
+ ## Development
220
+
221
+ ```bash
222
+ uv sync
223
+ uv run pytest -q
224
+ uv run ruff check src tests bench
225
+ uv run ruff format --check src tests bench
226
+ uv build
227
+ ```
228
+
229
+ See [the validation plan](https://github.com/keltokhy/jselect/blob/main/docs/PLAN.md),
230
+ [benchmarks](https://github.com/keltokhy/jselect/blob/main/docs/BENCHMARKS.md), and the
231
+ [companion agent skill](https://github.com/keltokhy/jselect/blob/main/skills/jselect/SKILL.md). MIT licensed.
@@ -0,0 +1,208 @@
1
+ # jselect
2
+
3
+ **Useful evidence for your AI, within a token budget.**
4
+
5
+ Give jselect a task and your data. It finds relevant passages, favors different information over
6
+ repetition, and assembles a source-linked context your agent can read directly.
7
+
8
+ ```bash
9
+ jselect "Why are people giving up during signup?" conversations.jsonl --tokens 8000
10
+ ```
11
+
12
+ ```python
13
+ from jselect import select
14
+
15
+ evidence = select(records, task="Why are people giving up during signup?", tokens=8000)
16
+ print(evidence.context) # source excerpts and citations, within the token budget
17
+ ```
18
+
19
+ No labels, predefined categories, vector database, or generative model required. Runs on files,
20
+ directories, piped exports, or Python records. This is a **separate tool and package** from jgrep.
21
+
22
+ ## Install
23
+
24
+ Python 3.10+:
25
+
26
+ ```bash
27
+ uv tool install jev-select # installs the jselect command on PATH
28
+ pip install jev-select # use jselect from your Python application
29
+ ```
30
+
31
+ The distribution is named `jev-select`; the command and Python import are both `jselect`.
32
+
33
+ ```bash
34
+ jselect doctor --json
35
+ echo 'The signup verification email never arrived.' | jselect "signup problems" --tokens 600
36
+ ```
37
+
38
+ For development, clone [the repository](https://github.com/keltokhy/jselect), run `uv sync`,
39
+ then `uv run jselect ...`. For an editable command: `uv tool install --editable .`.
40
+
41
+ If a TypeSafe or OpenRouter key is configured, the default uses **semantic relevance scoring** with Jev.
42
+ Set `TYPESAFE_API_KEY` or `OPENROUTER_API_KEY`, or use existing credentials in
43
+ `~/.config/jev/typesafe.key` or `~/.config/jev/openrouter.key`. Credentials never appear in output.
44
+ `JEV_API`, `JEV_MODEL`, and `JEV_URL` overrides are supported. Gateways use `JEV_GATEWAY_URL` and
45
+ `JEV_GATEWAY_API_KEY`, or `~/.config/jev/gateway.url` and `gateway.key`.
46
+
47
+ Without credentials, jselect uses local lexical retrieval and says so. Force that with `--local`.
48
+ Use `--mode semantic` to require semantic scoring and fail if credentials are missing.
49
+
50
+ ## Use it across your data
51
+
52
+ ```bash
53
+ # Customer conversations, with common text fields detected automatically
54
+ jselect "What prevents users from finishing signup?" conversations.jsonl --tokens 2000
55
+
56
+ # Related turns stored as separate, possibly interleaved rows
57
+ jselect "Where does the assistant contradict itself?" turns.jsonl --group-by conversation_id
58
+
59
+ # Source code and documentation, respecting .gitignore and .ignore
60
+ jselect "How are database connections released?" src/ docs/ --tokens 4000
61
+
62
+ # Papers or interview responses in a CSV
63
+ jselect "Evidence that challenges the proposed explanation" papers.csv --field abstract --tokens 3000
64
+
65
+ # Export from another tool, preserving original IDs and field locations
66
+ cat events.jsonl | jselect "Why are requests waiting?" --format jsonl --field message --json
67
+
68
+ # Fully offline, including token accounting: byte count conservatively bounds byte-BPE token use
69
+ jselect "retry timeout configuration" src/ --local --encoding bytes --tokens 4000
70
+ ```
71
+
72
+ Supported inputs: UTF-8 text and code, logs, JSONL/NDJSON, JSON objects or arrays, CSV/TSV, directories,
73
+ and stdin. Formats are inferred from extensions. Stdin defaults to lines; use `--format jsonl` for
74
+ structured input. `.log` files default to one record per line; `--format text` preserves neighboring
75
+ log lines in overlapping passages. PDF, Word, images, and audio need text extraction first.
76
+
77
+ Common text fields are tried in this order: `text`, `content`, `message`, `messages`, `conversation`,
78
+ `body`, `abstract`. If none exists, the entire object is serialized as JSON. `--field event.message`
79
+ selects an exact or dotted field. Only selected text is sent to the scorer, not the entire original row.
80
+ IDs come from `id`, `_id`, or input position. A group uses complete serialized rows unless you explicitly
81
+ select a field, preserving roles and other turn context. Grouping preserves input order, not timestamp order.
82
+
83
+ ## Fast repeated investigations
84
+
85
+ Build a local index once and ask many questions:
86
+
87
+ ```bash
88
+ jselect index conversations.jsonl --output conversations.jselect
89
+ jselect "Confusion about cancellation" conversations.jselect --json --output first.json
90
+ jselect "Confusion about cancellation" conversations.jselect --against first.json --json --output next.json
91
+ jselect inspect conversations.jselect --json
92
+ jselect show conversations.jselect PASSAGE_ID --json
93
+ ```
94
+
95
+ `--against` excludes exact excerpts already returned and penalizes similar text. It is useful when an
96
+ agent asks for more evidence or brings an existing reference set. It does not promise every follow-up
97
+ will introduce a new semantic idea. Saved indexes are snapshots: rebuild with `index ... --force` when
98
+ the source changes. Replacement is atomic, so a failed rebuild leaves the previous index intact.
99
+
100
+ For maximum retrieval breadth, use `--scan all`. This scores every unique passage that fits the output
101
+ budget and is not already supplied in `--against`, then keeps a bounded pool for final selection.
102
+ The full scan is preflighted against the estimated dollar budget before any calls are made.
103
+
104
+ ```bash
105
+ jselect "Signs the customer has lost trust" conversations.jselect --scan all --budget 0.25
106
+ ```
107
+
108
+ ## Python and agents
109
+
110
+ ```python
111
+ from jselect import Index, Record, aselect, select
112
+
113
+ records = [
114
+ {"id": "ticket-1", "text": "The verification link never arrives."},
115
+ {"id": "ticket-2", "text": "The free trial requires a credit card."},
116
+ ]
117
+ result = select(records, task="What blocks registration?", tokens=500)
118
+ payload = result.to_dict() # same schema as --json
119
+
120
+ # Reuse an index in a process; no source scanning on each query.
121
+ with Index.build(records) as index:
122
+ first = index.select(task="What blocks registration?", tokens=500)
123
+ more = index.select(task="What blocks registration?", tokens=500, against=first)
124
+
125
+ # In an existing event loop or notebook:
126
+ # result = await aselect(records, task="What blocks registration?", tokens=500)
127
+
128
+ # Bring your existing reranker. Return one finite score in [0, 1] per passage.
129
+ def judge(task, passages):
130
+ return existing_reranker(task, [p.text for p in passages])
131
+
132
+ result = select(records, task="What blocks registration?", tokens=500, scorer=judge)
133
+ ```
134
+
135
+ Custom scorers may be async functions or objects exposing `score(task, passages)`. Full scans invoke
136
+ them in blocks of at most 256 passages. Use `Record(text, id=..., source=...)` for explicit provenance.
137
+ A string or `Path` passed directly to `select` is an input path; strings inside an iterable are records.
138
+ To use passages already retrieved by another system, pass them as records; set `scan="all"` with a
139
+ custom or semantic scorer if the list exceeds the candidate limit and every passage must be evaluated.
140
+
141
+ Pass **`result.context`** to your agent. The full JSON includes additional metadata and is not subject
142
+ to the context token budget. Treat excerpts as source data rather than agent instructions.
143
+
144
+ ## How it works and what it costs
145
+
146
+ 1. Build or open a SQLite FTS5 index. Long records become overlapping, source-preserving passages.
147
+ Exact repeated passages share one indexed text while retaining occurrence counts and up to five sources.
148
+ 2. Shortlist up to 256 passages by default. Most come from BM25 with a text-diversity adjustment;
149
+ 20% of slots are reserved for deterministic exploration in semantic mode. This is not exhaustive retrieval.
150
+ 3. Score relevance using small batches of Jev decisions. The question explicitly includes contradicting
151
+ evidence. Scores are cached per endpoint, model, prompt version, task, and exact passage.
152
+ 4. Greedily balance relevance, text novelty, and passage token cost. Citation headers and separators count
153
+ toward the budget. Oversized passages are split **before** scoring; returned text is never generated.
154
+
155
+ The defaults are eight passages per request, eight requests in flight, a 20-second total request deadline,
156
+ and a **$0.05 estimated spend budget**. Jev models are versioned (`jev-1.13.0` on TypeSafe and
157
+ `typesafe/jev-1.13` on OpenRouter); responses report the served model when available. Requests retry
158
+ transient errors within the deadline. Errors retain completed cache entries for the next run.
159
+
160
+ The dollar guard uses a conservative UTF-8 byte estimate and the listed Jev input price. Actual provider
161
+ or gateway pricing and unreported usage can differ; this is not a provider-enforced billing cap. JSON
162
+ reports measured cost when available and marks list-price estimates. Failed requests without usage
163
+ reports may have incurred additional costs. Use `--candidates`, `--batch-size`, and `--budget` to bound work.
164
+ `--no-cache` disables the disk score cache at `~/.cache/jselect/scores.sqlite`.
165
+
166
+ Token counting uses `o200k_base` by default. Set `--encoding` to another tiktoken encoding/model, or `bytes`
167
+ for a conservative count with no tokenizer download. The first use of a tiktoken encoding may download its
168
+ public vocabulary. Choose the encoding your downstream model uses and leave room for its other messages.
169
+
170
+ ## Measured results
171
+
172
+ Measured locally on 2026-09-19; details and frozen reports are in [the benchmark report](https://github.com/keltokhy/jselect/blob/main/docs/BENCHMARKS.md).
173
+
174
+ | Check | Observed result |
175
+ |---|---|
176
+ | One million distinct generated log records | 13.64 s to index; 0.40 s median repeated **local** query; 143 MB peak process memory |
177
+ | 30 SciFact research queries, 2,000 tokens, 256 candidates, cache disabled | 86.7% mean labeled-source recall vs 78.3% for BM25 ranking; 1.86 s median; $0.149 total |
178
+ | Handwritten support, code, and contract fixtures | All 4 intended evidence types in 4 passages in each fixture; relevance-only variant covered 3, 3, and 1 |
179
+
180
+ These are scoped measurements, not guarantees for arbitrary data, agent answer quality, or future API
181
+ latency. A selected set cannot establish prevalence or causation. Diversity is a lexical heuristic;
182
+ it does not certify balanced viewpoints or find every contradiction. Semantic scores are model judgments,
183
+ not calibrated confidence in a final answer. The JSON reports how much of the collection was considered.
184
+
185
+ ## Output and errors
186
+
187
+ Default stdout is the exact context string. `--json` returns one object with `schema_version: 1`,
188
+ `task`, `context`, `items`, `tokens`, `token_budget`, `encoding`, `stats`, and `warnings`.
189
+ Each item contains original `text`, a stable content-hash `id`, `sources`, `occurrences`, `relevance`,
190
+ `novelty`, and the selection rule used. See [the output contract](https://github.com/keltokhy/jselect/blob/main/docs/OUTPUT.md).
191
+
192
+ Exit 0 means success, including empty evidence. Exit 2 means invalid input, bad setup, budget refusal,
193
+ or a provider error. Exit 130 means interruption. JSON errors have an `error` object and any available
194
+ usage `stats`; diagnostics never contaminate JSON stdout. `--stats` writes timings and cost to stderr.
195
+
196
+ ## Development
197
+
198
+ ```bash
199
+ uv sync
200
+ uv run pytest -q
201
+ uv run ruff check src tests bench
202
+ uv run ruff format --check src tests bench
203
+ uv build
204
+ ```
205
+
206
+ See [the validation plan](https://github.com/keltokhy/jselect/blob/main/docs/PLAN.md),
207
+ [benchmarks](https://github.com/keltokhy/jselect/blob/main/docs/BENCHMARKS.md), and the
208
+ [companion agent skill](https://github.com/keltokhy/jselect/blob/main/skills/jselect/SKILL.md). MIT licensed.