jev-select 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jev_select-0.1.0.dist-info/METADATA +231 -0
- jev_select-0.1.0.dist-info/RECORD +14 -0
- jev_select-0.1.0.dist-info/WHEEL +4 -0
- jev_select-0.1.0.dist-info/entry_points.txt +2 -0
- jev_select-0.1.0.dist-info/licenses/LICENSE +21 -0
- jselect/__init__.py +9 -0
- jselect/__main__.py +3 -0
- jselect/cli.py +272 -0
- jselect/index.py +255 -0
- jselect/inputs.py +307 -0
- jselect/judge.py +345 -0
- jselect/select.py +385 -0
- jselect/text.py +107 -0
- jselect/types.py +77 -0
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: jev-select
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Select useful, source-linked evidence for an AI task within a token budget
|
|
5
|
+
Project-URL: Repository, https://github.com/keltokhy/jselect
|
|
6
|
+
Project-URL: Documentation, https://github.com/keltokhy/jselect#readme
|
|
7
|
+
Project-URL: Issues, https://github.com/keltokhy/jselect/issues
|
|
8
|
+
Author: Khaled Eltokhy
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: agents,cli,context,jev,retrieval,semantic search
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Text Processing
|
|
17
|
+
Classifier: Topic :: Utilities
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Requires-Dist: httpx>=0.27
|
|
20
|
+
Requires-Dist: pathspec>=0.12
|
|
21
|
+
Requires-Dist: tiktoken>=0.8
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# jselect
|
|
25
|
+
|
|
26
|
+
**Useful evidence for your AI, within a token budget.**
|
|
27
|
+
|
|
28
|
+
Give jselect a task and your data. It finds relevant passages, favors different information over
|
|
29
|
+
repetition, and assembles a source-linked context your agent can read directly.
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
jselect "Why are people giving up during signup?" conversations.jsonl --tokens 8000
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
from jselect import select
|
|
37
|
+
|
|
38
|
+
evidence = select(records, task="Why are people giving up during signup?", tokens=8000)
|
|
39
|
+
print(evidence.context) # source excerpts and citations, within the token budget
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
No labels, predefined categories, vector database, or generative model required. Runs on files,
|
|
43
|
+
directories, piped exports, or Python records. This is a **separate tool and package** from jgrep.
|
|
44
|
+
|
|
45
|
+
## Install
|
|
46
|
+
|
|
47
|
+
Python 3.10+:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
uv tool install jev-select # installs the jselect command on PATH
|
|
51
|
+
pip install jev-select # use jselect from your Python application
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
The distribution is named `jev-select`; the command and Python import are both `jselect`.
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
jselect doctor --json
|
|
58
|
+
echo 'The signup verification email never arrived.' | jselect "signup problems" --tokens 600
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
For development, clone [the repository](https://github.com/keltokhy/jselect), run `uv sync`,
|
|
62
|
+
then `uv run jselect ...`. For an editable command: `uv tool install --editable .`.
|
|
63
|
+
|
|
64
|
+
If a TypeSafe or OpenRouter key is configured, the default uses **semantic relevance scoring** with Jev.
|
|
65
|
+
Set `TYPESAFE_API_KEY` or `OPENROUTER_API_KEY`, or use existing credentials in
|
|
66
|
+
`~/.config/jev/typesafe.key` or `~/.config/jev/openrouter.key`. Credentials never appear in output.
|
|
67
|
+
`JEV_API`, `JEV_MODEL`, and `JEV_URL` overrides are supported. Gateways use `JEV_GATEWAY_URL` and
|
|
68
|
+
`JEV_GATEWAY_API_KEY`, or `~/.config/jev/gateway.url` and `gateway.key`.
|
|
69
|
+
|
|
70
|
+
Without credentials, jselect uses local lexical retrieval and says so. Force that with `--local`.
|
|
71
|
+
Use `--mode semantic` to require semantic scoring and fail if credentials are missing.
|
|
72
|
+
|
|
73
|
+
## Use it across your data
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
# Customer conversations, with common text fields detected automatically
|
|
77
|
+
jselect "What prevents users from finishing signup?" conversations.jsonl --tokens 2000
|
|
78
|
+
|
|
79
|
+
# Related turns stored as separate, possibly interleaved rows
|
|
80
|
+
jselect "Where does the assistant contradict itself?" turns.jsonl --group-by conversation_id
|
|
81
|
+
|
|
82
|
+
# Source code and documentation, respecting .gitignore and .ignore
|
|
83
|
+
jselect "How are database connections released?" src/ docs/ --tokens 4000
|
|
84
|
+
|
|
85
|
+
# Papers or interview responses in a CSV
|
|
86
|
+
jselect "Evidence that challenges the proposed explanation" papers.csv --field abstract --tokens 3000
|
|
87
|
+
|
|
88
|
+
# Export from another tool, preserving original IDs and field locations
|
|
89
|
+
cat events.jsonl | jselect "Why are requests waiting?" --format jsonl --field message --json
|
|
90
|
+
|
|
91
|
+
# Fully offline, including token accounting: byte count conservatively bounds byte-BPE token use
|
|
92
|
+
jselect "retry timeout configuration" src/ --local --encoding bytes --tokens 4000
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Supported inputs: UTF-8 text and code, logs, JSONL/NDJSON, JSON objects or arrays, CSV/TSV, directories,
|
|
96
|
+
and stdin. Formats are inferred from extensions. Stdin defaults to lines; use `--format jsonl` for
|
|
97
|
+
structured input. `.log` files default to one record per line; `--format text` preserves neighboring
|
|
98
|
+
log lines in overlapping passages. PDF, Word, images, and audio need text extraction first.
|
|
99
|
+
|
|
100
|
+
Common text fields are tried in this order: `text`, `content`, `message`, `messages`, `conversation`,
|
|
101
|
+
`body`, `abstract`. If none exists, the entire object is serialized as JSON. `--field event.message`
|
|
102
|
+
selects an exact or dotted field. Only selected text is sent to the scorer, not the entire original row.
|
|
103
|
+
IDs come from `id`, `_id`, or input position. A group uses complete serialized rows unless you explicitly
|
|
104
|
+
select a field, preserving roles and other turn context. Grouping preserves input order, not timestamp order.
|
|
105
|
+
|
|
106
|
+
## Fast repeated investigations
|
|
107
|
+
|
|
108
|
+
Build a local index once and ask many questions:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
jselect index conversations.jsonl --output conversations.jselect
|
|
112
|
+
jselect "Confusion about cancellation" conversations.jselect --json --output first.json
|
|
113
|
+
jselect "Confusion about cancellation" conversations.jselect --against first.json --json --output next.json
|
|
114
|
+
jselect inspect conversations.jselect --json
|
|
115
|
+
jselect show conversations.jselect PASSAGE_ID --json
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
`--against` excludes exact excerpts already returned and penalizes similar text. It is useful when an
|
|
119
|
+
agent asks for more evidence or brings an existing reference set. It does not promise every follow-up
|
|
120
|
+
will introduce a new semantic idea. Saved indexes are snapshots: rebuild with `index ... --force` when
|
|
121
|
+
the source changes. Replacement is atomic, so a failed rebuild leaves the previous index intact.
|
|
122
|
+
|
|
123
|
+
For maximum retrieval breadth, use `--scan all`. This scores every unique passage that fits the output
|
|
124
|
+
budget and is not already supplied in `--against`, then keeps a bounded pool for final selection.
|
|
125
|
+
The full scan is preflighted against the estimated dollar budget before any calls are made.
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
jselect "Signs the customer has lost trust" conversations.jselect --scan all --budget 0.25
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Python and agents
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
from jselect import Index, Record, aselect, select
|
|
135
|
+
|
|
136
|
+
records = [
|
|
137
|
+
{"id": "ticket-1", "text": "The verification link never arrives."},
|
|
138
|
+
{"id": "ticket-2", "text": "The free trial requires a credit card."},
|
|
139
|
+
]
|
|
140
|
+
result = select(records, task="What blocks registration?", tokens=500)
|
|
141
|
+
payload = result.to_dict() # same schema as --json
|
|
142
|
+
|
|
143
|
+
# Reuse an index in a process; no source scanning on each query.
|
|
144
|
+
with Index.build(records) as index:
|
|
145
|
+
first = index.select(task="What blocks registration?", tokens=500)
|
|
146
|
+
more = index.select(task="What blocks registration?", tokens=500, against=first)
|
|
147
|
+
|
|
148
|
+
# In an existing event loop or notebook:
|
|
149
|
+
# result = await aselect(records, task="What blocks registration?", tokens=500)
|
|
150
|
+
|
|
151
|
+
# Bring your existing reranker. Return one finite score in [0, 1] per passage.
|
|
152
|
+
def judge(task, passages):
|
|
153
|
+
return existing_reranker(task, [p.text for p in passages])
|
|
154
|
+
|
|
155
|
+
result = select(records, task="What blocks registration?", tokens=500, scorer=judge)
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Custom scorers may be async functions or objects exposing `score(task, passages)`. Full scans invoke
|
|
159
|
+
them in blocks of at most 256 passages. Use `Record(text, id=..., source=...)` for explicit provenance.
|
|
160
|
+
A string or `Path` passed directly to `select` is an input path; strings inside an iterable are records.
|
|
161
|
+
To use passages already retrieved by another system, pass them as records; set `scan="all"` with a
|
|
162
|
+
custom or semantic scorer if the list exceeds the candidate limit and every passage must be evaluated.
|
|
163
|
+
|
|
164
|
+
Pass **`result.context`** to your agent. The full JSON includes additional metadata and is not subject
|
|
165
|
+
to the context token budget. Treat excerpts as source data rather than agent instructions.
|
|
166
|
+
|
|
167
|
+
## How it works and what it costs
|
|
168
|
+
|
|
169
|
+
1. Build or open a SQLite FTS5 index. Long records become overlapping, source-preserving passages.
|
|
170
|
+
Exact repeated passages share one indexed text while retaining occurrence counts and up to five sources.
|
|
171
|
+
2. Shortlist up to 256 passages by default. Most come from BM25 with a text-diversity adjustment;
|
|
172
|
+
20% of slots are reserved for deterministic exploration in semantic mode. This is not exhaustive retrieval.
|
|
173
|
+
3. Score relevance using small batches of Jev decisions. The question explicitly includes contradicting
|
|
174
|
+
evidence. Scores are cached per endpoint, model, prompt version, task, and exact passage.
|
|
175
|
+
4. Greedily balance relevance, text novelty, and passage token cost. Citation headers and separators count
|
|
176
|
+
toward the budget. Oversized passages are split **before** scoring; returned text is never generated.
|
|
177
|
+
|
|
178
|
+
The defaults are eight passages per request, eight requests in flight, a 20-second total request deadline,
|
|
179
|
+
and a **$0.05 estimated spend budget**. Jev models are versioned (`jev-1.13.0` on TypeSafe and
|
|
180
|
+
`typesafe/jev-1.13` on OpenRouter); responses report the served model when available. Requests retry
|
|
181
|
+
transient errors within the deadline. Errors retain completed cache entries for the next run.
|
|
182
|
+
|
|
183
|
+
The dollar guard uses a conservative UTF-8 byte estimate and the listed Jev input price. Actual provider
|
|
184
|
+
or gateway pricing and unreported usage can differ; this is not a provider-enforced billing cap. JSON
|
|
185
|
+
reports measured cost when available and marks list-price estimates. Failed requests without usage
|
|
186
|
+
reports may have incurred additional costs. Use `--candidates`, `--batch-size`, and `--budget` to bound work.
|
|
187
|
+
`--no-cache` disables the disk score cache at `~/.cache/jselect/scores.sqlite`.
|
|
188
|
+
|
|
189
|
+
Token counting uses `o200k_base` by default. Set `--encoding` to another tiktoken encoding/model, or `bytes`
|
|
190
|
+
for a conservative count with no tokenizer download. The first use of a tiktoken encoding may download its
|
|
191
|
+
public vocabulary. Choose the encoding your downstream model uses and leave room for its other messages.
|
|
192
|
+
|
|
193
|
+
## Measured results
|
|
194
|
+
|
|
195
|
+
Measured locally on 2026-09-19; details and frozen reports are in [the benchmark report](https://github.com/keltokhy/jselect/blob/main/docs/BENCHMARKS.md).
|
|
196
|
+
|
|
197
|
+
| Check | Observed result |
|
|
198
|
+
|---|---|
|
|
199
|
+
| One million distinct generated log records | 13.64 s to index; 0.40 s median repeated **local** query; 143 MB peak process memory |
|
|
200
|
+
| 30 SciFact research queries, 2,000 tokens, 256 candidates, cache disabled | 86.7% mean labeled-source recall vs 78.3% for BM25 ranking; 1.86 s median; $0.149 total |
|
|
201
|
+
| Handwritten support, code, and contract fixtures | All 4 intended evidence types in 4 passages in each fixture; relevance-only variant covered 3, 3, and 1 |
|
|
202
|
+
|
|
203
|
+
These are scoped measurements, not guarantees for arbitrary data, agent answer quality, or future API
|
|
204
|
+
latency. A selected set cannot establish prevalence or causation. Diversity is a lexical heuristic;
|
|
205
|
+
it does not certify balanced viewpoints or find every contradiction. Semantic scores are model judgments,
|
|
206
|
+
not calibrated confidence in a final answer. The JSON reports how much of the collection was considered.
|
|
207
|
+
|
|
208
|
+
## Output and errors
|
|
209
|
+
|
|
210
|
+
Default stdout is the exact context string. `--json` returns one object with `schema_version: 1`,
|
|
211
|
+
`task`, `context`, `items`, `tokens`, `token_budget`, `encoding`, `stats`, and `warnings`.
|
|
212
|
+
Each item contains original `text`, a stable content-hash `id`, `sources`, `occurrences`, `relevance`,
|
|
213
|
+
`novelty`, and the selection rule used. See [the output contract](https://github.com/keltokhy/jselect/blob/main/docs/OUTPUT.md).
|
|
214
|
+
|
|
215
|
+
Exit 0 means success, including empty evidence. Exit 2 means invalid input, bad setup, budget refusal,
|
|
216
|
+
or a provider error. Exit 130 means interruption. JSON errors have an `error` object and any available
|
|
217
|
+
usage `stats`; diagnostics never contaminate JSON stdout. `--stats` writes timings and cost to stderr.
|
|
218
|
+
|
|
219
|
+
## Development
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
uv sync
|
|
223
|
+
uv run pytest -q
|
|
224
|
+
uv run ruff check src tests bench
|
|
225
|
+
uv run ruff format --check src tests bench
|
|
226
|
+
uv build
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
See [the validation plan](https://github.com/keltokhy/jselect/blob/main/docs/PLAN.md),
|
|
230
|
+
[benchmarks](https://github.com/keltokhy/jselect/blob/main/docs/BENCHMARKS.md), and the
|
|
231
|
+
[companion agent skill](https://github.com/keltokhy/jselect/blob/main/skills/jselect/SKILL.md). MIT licensed.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
jselect/__init__.py,sha256=5g2DNxjJVGDwQySR0upXJwhkUjViVYrdU0KEmn8Qqeo,320
|
|
2
|
+
jselect/__main__.py,sha256=Huz0dExiaH0XTMLhfe3skkFD9-VH7ZUiWMbDR_J8TIY,28
|
|
3
|
+
jselect/cli.py,sha256=mEGCOmDDfqOB-ErGJ6cmmFmneUhOMUsBHBKJqq7CeZY,10875
|
|
4
|
+
jselect/index.py,sha256=kGEO36ZSVSVd3fLe6uq8HT2TpaPNdi-EKKpvzP35Qvg,10926
|
|
5
|
+
jselect/inputs.py,sha256=4Fw8CGiGy9bulUkDK8Y4AzRpe5wf43gZyCJ5hm4xf7g,12182
|
|
6
|
+
jselect/judge.py,sha256=_zSHexjtOQKYuxLAYY1-eaJ2muG49gEdWvN9RavTrjQ,15013
|
|
7
|
+
jselect/select.py,sha256=gPBteVvnWdvZNfOZ9h2x833EfU1fW14pMfb0XTVWW-o,15354
|
|
8
|
+
jselect/text.py,sha256=p954cyTLyC6wcnnA-uWIVUq2POE45jnRE9-rzoJu1Hk,4108
|
|
9
|
+
jselect/types.py,sha256=_Q2XdoFlmGTdkA9BQ_9IY8qcN9jfGBxcIgSCcAVcXJ0,1938
|
|
10
|
+
jev_select-0.1.0.dist-info/METADATA,sha256=gQkLrG-UZQMAtGoBmT-yqODryBBl-KCqEm9b0i0xpcM,12063
|
|
11
|
+
jev_select-0.1.0.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
|
|
12
|
+
jev_select-0.1.0.dist-info/entry_points.txt,sha256=_R5BGb8cm9en7hGwcVgHrMPyxo-9IEtfKhEKjaQj5RA,44
|
|
13
|
+
jev_select-0.1.0.dist-info/licenses/LICENSE,sha256=unAu2Ii_6qZZfNfkD44Vj0MNv9C7H6CBRje23cMNx4g,1071
|
|
14
|
+
jev_select-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Khaled Eltokhy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
jselect/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""Useful evidence within a token budget."""
|
|
2
|
+
|
|
3
|
+
from .index import Index
|
|
4
|
+
from .select import aselect, select
|
|
5
|
+
from .text import count_tokens
|
|
6
|
+
from .types import Evidence, Passage, Record, Selection
|
|
7
|
+
|
|
8
|
+
__all__ = ["Evidence", "Index", "Passage", "Record", "Selection", "aselect", "count_tokens", "select"]
|
|
9
|
+
__version__ = "0.1.0"
|
jselect/__main__.py
ADDED
jselect/cli.py
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
"""The shell interface. stdout is evidence (or JSON); diagnostics go to stderr."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sqlite3
|
|
8
|
+
import sys
|
|
9
|
+
from dataclasses import asdict
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from . import __version__
|
|
13
|
+
from .index import Index
|
|
14
|
+
from .inputs import read_paths
|
|
15
|
+
from .judge import SemanticError, resolve_backend
|
|
16
|
+
from .select import select
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Parser(argparse.ArgumentParser):
|
|
20
|
+
def error(self, message):
|
|
21
|
+
raise ValueError(message)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def inputs(parser):
|
|
25
|
+
parser.add_argument(
|
|
26
|
+
"paths", nargs="*", help="files/directories; '-' reads stdin; a .jselect file reuses an index"
|
|
27
|
+
)
|
|
28
|
+
parser.add_argument("--field", help="text field or dotted path; auto-detected for common JSON/CSV fields")
|
|
29
|
+
parser.add_argument(
|
|
30
|
+
"--group-by", help="combine rows by this field, e.g. conversation_id (in input order)"
|
|
31
|
+
)
|
|
32
|
+
parser.add_argument(
|
|
33
|
+
"--format",
|
|
34
|
+
choices=["auto", "text", "lines", "jsonl", "json", "csv", "tsv"],
|
|
35
|
+
default="auto",
|
|
36
|
+
help="input format (default: infer from extension)",
|
|
37
|
+
)
|
|
38
|
+
parser.add_argument("--glob", help="only read matching files in directories, e.g. '*.py'")
|
|
39
|
+
parser.add_argument(
|
|
40
|
+
"--exclude", action="append", default=[], help="gitignore-style exclusion; repeatable"
|
|
41
|
+
)
|
|
42
|
+
parser.add_argument(
|
|
43
|
+
"--chunk-size", type=int, default=1800, help="passage size in characters (default: 1800)"
|
|
44
|
+
)
|
|
45
|
+
parser.add_argument("--overlap", type=int, default=240, help="overlap in characters (default: 240)")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def source(args):
|
|
49
|
+
return read_paths(
|
|
50
|
+
args.paths or ["-"],
|
|
51
|
+
field=args.field,
|
|
52
|
+
format=args.format,
|
|
53
|
+
glob=args.glob,
|
|
54
|
+
exclude=args.exclude,
|
|
55
|
+
group_by=args.group_by,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def parser_for(command):
|
|
60
|
+
p = Parser(
|
|
61
|
+
prog="jselect" + (f" {command}" if command else ""),
|
|
62
|
+
description="Select useful, source-linked evidence within a token budget.",
|
|
63
|
+
epilog="Other commands: index (save a searchable collection), inspect (index stats), "
|
|
64
|
+
"show (read a passage by ID), doctor (check setup).",
|
|
65
|
+
)
|
|
66
|
+
p.add_argument("--version", action="version", version=f"jselect {__version__}")
|
|
67
|
+
p.add_argument("--json", action="store_true", help="emit a versioned JSON object, including JSON errors")
|
|
68
|
+
if command == "doctor":
|
|
69
|
+
p.add_argument("--api", choices=["typesafe", "openrouter", "gateway"])
|
|
70
|
+
return p
|
|
71
|
+
if command in {"inspect", "show"}:
|
|
72
|
+
p.add_argument("path", help="saved .jselect index")
|
|
73
|
+
if command == "show":
|
|
74
|
+
p.add_argument("id", help="full passage ID from JSON selection output")
|
|
75
|
+
return p
|
|
76
|
+
if command == "index":
|
|
77
|
+
inputs(p)
|
|
78
|
+
p.add_argument("--output", "-o", required=True, help="path for the saved .jselect index")
|
|
79
|
+
p.add_argument("--force", action="store_true", help="replace an existing index atomically")
|
|
80
|
+
return p
|
|
81
|
+
p.add_argument("task", help="the task, question, or information you need")
|
|
82
|
+
inputs(p)
|
|
83
|
+
p.add_argument(
|
|
84
|
+
"--tokens", type=int, default=8000, help="maximum tokens in the evidence context (default: 8000)"
|
|
85
|
+
)
|
|
86
|
+
p.add_argument(
|
|
87
|
+
"--encoding", default="o200k_base", help="tiktoken encoding/model, or 'bytes' (default: o200k_base)"
|
|
88
|
+
)
|
|
89
|
+
p.add_argument(
|
|
90
|
+
"--mode",
|
|
91
|
+
choices=["auto", "local", "semantic"],
|
|
92
|
+
default="auto",
|
|
93
|
+
help="auto uses semantic scoring when a key is configured, else local retrieval",
|
|
94
|
+
)
|
|
95
|
+
p.add_argument(
|
|
96
|
+
"--local", action="store_const", dest="mode", const="local", help="use local retrieval; no API calls"
|
|
97
|
+
)
|
|
98
|
+
p.add_argument("--candidates", type=int, default=256, help="maximum passages shortlisted (default: 256)")
|
|
99
|
+
p.add_argument(
|
|
100
|
+
"--scan", choices=["shortlist", "all"], default="shortlist", help="all scores every unique passage"
|
|
101
|
+
)
|
|
102
|
+
p.add_argument("--against", help="previous --json result or records: seek additional evidence")
|
|
103
|
+
p.add_argument(
|
|
104
|
+
"--diversity", type=float, default=0.7, help="penalty for repetitive text, 0..1 (default: 0.7)"
|
|
105
|
+
)
|
|
106
|
+
p.add_argument("--threshold", type=float, help="minimum relevance score (semantic default: 0.25)")
|
|
107
|
+
p.add_argument("-n", "--max-items", type=int, help="also cap the number of selected passages")
|
|
108
|
+
p.add_argument("--api", choices=["typesafe", "openrouter", "gateway"])
|
|
109
|
+
p.add_argument("--model", help="override the pinned Jev model")
|
|
110
|
+
p.add_argument(
|
|
111
|
+
"--budget",
|
|
112
|
+
type=float,
|
|
113
|
+
default=0.05,
|
|
114
|
+
help="semantic dollar budget; preflight estimate (default: 0.05)",
|
|
115
|
+
)
|
|
116
|
+
p.add_argument(
|
|
117
|
+
"-j", "--concurrency", type=int, default=8, help="maximum simultaneous requests (default: 8)"
|
|
118
|
+
)
|
|
119
|
+
p.add_argument("--batch-size", type=int, default=8, help="passages per API request, 1..16 (default: 8)")
|
|
120
|
+
p.add_argument("--timeout", type=float, default=20, help="total seconds per request, including retries")
|
|
121
|
+
p.add_argument("--no-cache", action="store_true", help="disable persistent semantic score caching")
|
|
122
|
+
p.add_argument("--stats", action="store_true", help="print timing and cost to stderr")
|
|
123
|
+
p.add_argument("--output", "-o", help="write evidence or JSON to this file instead of stdout")
|
|
124
|
+
return p
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def main(argv=None, *, out=None, err=None, transport=None) -> int:
|
|
128
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
129
|
+
out, err = out or sys.stdout, err or sys.stderr
|
|
130
|
+
json_mode = "--json" in argv
|
|
131
|
+
command = next((arg for arg in argv if arg != "--json"), "")
|
|
132
|
+
command = command if command in {"index", "inspect", "show", "doctor"} else ""
|
|
133
|
+
if command:
|
|
134
|
+
argv.remove(command)
|
|
135
|
+
try:
|
|
136
|
+
args = parser_for(command).parse_intermixed_args(argv)
|
|
137
|
+
if command == "doctor":
|
|
138
|
+
issue = None
|
|
139
|
+
try:
|
|
140
|
+
backend = resolve_backend(args.api)
|
|
141
|
+
except (OSError, ValueError) as e:
|
|
142
|
+
backend, issue = None, str(e)
|
|
143
|
+
db = sqlite3.connect(":memory:")
|
|
144
|
+
try:
|
|
145
|
+
db.execute("CREATE VIRTUAL TABLE health USING fts5(text)")
|
|
146
|
+
finally:
|
|
147
|
+
db.close()
|
|
148
|
+
payload = {
|
|
149
|
+
"schema_version": 1,
|
|
150
|
+
"version": __version__,
|
|
151
|
+
"local_ready": True,
|
|
152
|
+
"semantic_configured": backend is not None,
|
|
153
|
+
"auth_verified": False,
|
|
154
|
+
"api": backend.name if backend else None,
|
|
155
|
+
"model": backend.model if backend else None,
|
|
156
|
+
"auth_source": backend.auth_source if backend else "missing",
|
|
157
|
+
"hint": issue
|
|
158
|
+
or (None if backend else "Set TYPESAFE_API_KEY or OPENROUTER_API_KEY for semantic scoring."),
|
|
159
|
+
}
|
|
160
|
+
print(
|
|
161
|
+
json.dumps(payload)
|
|
162
|
+
if args.json
|
|
163
|
+
else f"jselect {__version__}: local mode ready; "
|
|
164
|
+
f"semantic {'configured' if backend else 'not configured'}"
|
|
165
|
+
+ (f"\n{payload['hint']}" if payload["hint"] else ""),
|
|
166
|
+
file=out,
|
|
167
|
+
)
|
|
168
|
+
return 0
|
|
169
|
+
if command == "index":
|
|
170
|
+
with Index.build(
|
|
171
|
+
source(args),
|
|
172
|
+
path=args.output,
|
|
173
|
+
chunk_size=args.chunk_size,
|
|
174
|
+
overlap=args.overlap,
|
|
175
|
+
force=args.force,
|
|
176
|
+
) as index:
|
|
177
|
+
payload = {"schema_version": 1, "path": str(index.path), "stats": index.stats}
|
|
178
|
+
print(
|
|
179
|
+
json.dumps(payload)
|
|
180
|
+
if args.json
|
|
181
|
+
else f"Indexed {payload['stats']['records']:,} records → {payload['path']}",
|
|
182
|
+
file=out,
|
|
183
|
+
)
|
|
184
|
+
return 0
|
|
185
|
+
if command in {"inspect", "show"}:
|
|
186
|
+
with Index(args.path) as index:
|
|
187
|
+
payload = (
|
|
188
|
+
{"schema_version": 1, "path": str(index.path), "stats": index.stats}
|
|
189
|
+
if command == "inspect"
|
|
190
|
+
else {"schema_version": 1, "passage": asdict(index.get(args.id))}
|
|
191
|
+
)
|
|
192
|
+
print(
|
|
193
|
+
json.dumps(payload, ensure_ascii=False)
|
|
194
|
+
if args.json or command == "inspect"
|
|
195
|
+
else payload["passage"]["text"],
|
|
196
|
+
file=out,
|
|
197
|
+
)
|
|
198
|
+
return 0
|
|
199
|
+
index = None
|
|
200
|
+
if len(args.paths) == 1 and Path(args.paths[0]).suffix == ".jselect":
|
|
201
|
+
index = Index(args.paths[0])
|
|
202
|
+
try:
|
|
203
|
+
result = select(
|
|
204
|
+
index or source(args),
|
|
205
|
+
task=args.task,
|
|
206
|
+
tokens=args.tokens,
|
|
207
|
+
encoding=args.encoding,
|
|
208
|
+
mode=args.mode,
|
|
209
|
+
candidates=args.candidates,
|
|
210
|
+
scan=args.scan,
|
|
211
|
+
diversity=args.diversity,
|
|
212
|
+
threshold=args.threshold,
|
|
213
|
+
max_items=args.max_items,
|
|
214
|
+
against=args.against,
|
|
215
|
+
api=args.api,
|
|
216
|
+
model=args.model,
|
|
217
|
+
budget=args.budget,
|
|
218
|
+
concurrency=args.concurrency,
|
|
219
|
+
batch_size=args.batch_size,
|
|
220
|
+
timeout=args.timeout,
|
|
221
|
+
cache=not args.no_cache,
|
|
222
|
+
chunk_size=args.chunk_size,
|
|
223
|
+
overlap=args.overlap,
|
|
224
|
+
_transport=transport,
|
|
225
|
+
)
|
|
226
|
+
finally:
|
|
227
|
+
if index:
|
|
228
|
+
index.close()
|
|
229
|
+
output = json.dumps(result.to_dict(), ensure_ascii=False) + "\n" if args.json else result.context
|
|
230
|
+
if args.output:
|
|
231
|
+
Path(args.output).write_text(output, encoding="utf-8")
|
|
232
|
+
else:
|
|
233
|
+
out.write(output)
|
|
234
|
+
if args.stats:
|
|
235
|
+
stats = result.stats
|
|
236
|
+
print(
|
|
237
|
+
f"jselect: {len(result.items)} passages, {result.tokens}/{result.token_budget} tokens; "
|
|
238
|
+
f"{stats.get('mode')}; {stats.get('calls', 0)} calls; ${stats.get('cost', 0):.6f}; "
|
|
239
|
+
f"{stats.get('seconds', 0):.3f}s",
|
|
240
|
+
file=err,
|
|
241
|
+
)
|
|
242
|
+
if not args.json:
|
|
243
|
+
for warning in result.warnings:
|
|
244
|
+
print(f"jselect: {warning}", file=err)
|
|
245
|
+
return 0
|
|
246
|
+
except BrokenPipeError:
|
|
247
|
+
return 0
|
|
248
|
+
except (ValueError, OSError, sqlite3.Error, SemanticError) as e:
|
|
249
|
+
if json_mode:
|
|
250
|
+
print(
|
|
251
|
+
json.dumps(
|
|
252
|
+
{
|
|
253
|
+
"schema_version": 1,
|
|
254
|
+
"error": {"type": type(e).__name__, "message": str(e)},
|
|
255
|
+
"stats": getattr(e, "stats", None),
|
|
256
|
+
}
|
|
257
|
+
),
|
|
258
|
+
file=out,
|
|
259
|
+
)
|
|
260
|
+
else:
|
|
261
|
+
print(f"jselect: {e}", file=err)
|
|
262
|
+
return 2
|
|
263
|
+
except KeyboardInterrupt:
|
|
264
|
+
print("jselect: interrupted", file=err)
|
|
265
|
+
return 130
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def cli():
|
|
269
|
+
try:
|
|
270
|
+
raise SystemExit(main())
|
|
271
|
+
except BrokenPipeError:
|
|
272
|
+
raise SystemExit(0) from None
|