jev-relevance 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jev_relevance-0.1.0/LICENSE +21 -0
- jev_relevance-0.1.0/PKG-INFO +168 -0
- jev_relevance-0.1.0/README.md +143 -0
- jev_relevance-0.1.0/pyproject.toml +57 -0
- jev_relevance-0.1.0/setup.cfg +4 -0
- jev_relevance-0.1.0/src/jev_relevance/__init__.py +74 -0
- jev_relevance-0.1.0/src/jev_relevance/builder.py +184 -0
- jev_relevance-0.1.0/src/jev_relevance/config.py +185 -0
- jev_relevance-0.1.0/src/jev_relevance/constants.py +164 -0
- jev_relevance-0.1.0/src/jev_relevance/provider.py +129 -0
- jev_relevance-0.1.0/src/jev_relevance/py.typed +0 -0
- jev_relevance-0.1.0/src/jev_relevance/retriever.py +167 -0
- jev_relevance-0.1.0/src/jev_relevance/scoring.py +98 -0
- jev_relevance-0.1.0/src/jev_relevance/strategies.py +281 -0
- jev_relevance-0.1.0/src/jev_relevance.egg-info/PKG-INFO +168 -0
- jev_relevance-0.1.0/src/jev_relevance.egg-info/SOURCES.txt +21 -0
- jev_relevance-0.1.0/src/jev_relevance.egg-info/dependency_links.txt +1 -0
- jev_relevance-0.1.0/src/jev_relevance.egg-info/requires.txt +6 -0
- jev_relevance-0.1.0/src/jev_relevance.egg-info/top_level.txt +1 -0
- jev_relevance-0.1.0/tests/test_builder.py +102 -0
- jev_relevance-0.1.0/tests/test_config.py +81 -0
- jev_relevance-0.1.0/tests/test_provider.py +72 -0
- jev_relevance-0.1.0/tests/test_retriever.py +161 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jev Relevance Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: jev-relevance
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Drop-in relevance filter for LangChain RAG retrievers powered by TypeSafe Jev.
|
|
5
|
+
Author: Jev Relevance Contributors
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/saksham-malhotra-27/jev-relevance
|
|
8
|
+
Project-URL: Repository, https://github.com/saksham-malhotra-27/jev-relevance
|
|
9
|
+
Project-URL: Bug Tracker, https://github.com/saksham-malhotra-27/jev-relevance/issues
|
|
10
|
+
Keywords: langchain,rag,retriever,relevance,rerank,classifier,jev,typesafe
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: langchain-core<2,>=1.0
|
|
20
|
+
Requires-Dist: langchain-typesafe<1,>=0.0.1a3
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
23
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# jev-relevance
|
|
27
|
+
|
|
28
|
+
A drop-in relevance filter for LangChain RAG retrievers, powered by
|
|
29
|
+
[TypeSafe Jev](https://typesafe.ai). Wrap any `BaseRetriever`, and Jev decides
|
|
30
|
+
which retrieved chunks actually answer the query. No chain plumbing, no prompt
|
|
31
|
+
engineering, one (or a few) API calls.
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
from jev_relevance import JevRelevanceRetrieverBuilder
|
|
35
|
+
|
|
36
|
+
retriever = (
|
|
37
|
+
JevRelevanceRetrieverBuilder()
|
|
38
|
+
.with_base_retriever(my_vector_retriever) # any LangChain BaseRetriever
|
|
39
|
+
.with_top_k(5)
|
|
40
|
+
.build()
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
docs = retriever.invoke("What is the capital of France?")
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Because `JevRelevanceRetriever` **is a** `BaseRetriever`, you can swap it into an
|
|
47
|
+
existing chain without touching anything else. It returns only the chunks that
|
|
48
|
+
clear the relevance threshold, sorted by Jev score, with the score attached in
|
|
49
|
+
`Document.metadata["relevance"]`.
|
|
50
|
+
|
|
51
|
+
## Features
|
|
52
|
+
|
|
53
|
+
- **Drop-in**: subclass of `BaseRetriever`; `astream`, `with_retriever`, agent
|
|
54
|
+
and chain wiring all keep working.
|
|
55
|
+
- **Two scoring modes**:
|
|
56
|
+
- `per_chunk` (default): one Jev call per query-passage pair, bounded by
|
|
57
|
+
`max_concurrency`. Most accurate.
|
|
58
|
+
- `batched`: every chunk judged in a single request for the "sub-second"
|
|
59
|
+
latency case (max 32 chunks).
|
|
60
|
+
- **Two providers**: TypeSafe (`typesafe`) and OpenRouter (`openrouter`), plus a
|
|
61
|
+
factory that lets you register more without touching the core.
|
|
62
|
+
- **fail-open**: on scoring errors you get the unfiltered candidates instead of
|
|
63
|
+
empty context (configurable).
|
|
64
|
+
- **Zero magic numbers**: every default lives in `constants.py` and every
|
|
65
|
+
behavior is builder-parameterized.
|
|
66
|
+
|
|
67
|
+
## Installation
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
pip install jev-relevance
|
|
71
|
+
# or, from this repo:
|
|
72
|
+
python -m venv .venv && .venv\Scripts\activate
|
|
73
|
+
pip install -e ".[dev]"
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Python 3.10+. The library itself only needs `langchain-core` and
|
|
77
|
+
`langchain-typesafe`.
|
|
78
|
+
|
|
79
|
+
## Providers
|
|
80
|
+
|
|
81
|
+
| Provider | Default model | Base URL |
|
|
82
|
+
| --- | --- | --- |
|
|
83
|
+
| `typesafe` | `jev-latest` | `https://api.typesafe.ai` |
|
|
84
|
+
| `openrouter` | `~typesafe/jev-latest` | `https://openrouter.ai/api` |
|
|
85
|
+
|
|
86
|
+
Credentials are **always passed as parameters** — the library never reads env
|
|
87
|
+
vars or `.env` files itself; that's the caller's job. Supply the key (and any
|
|
88
|
+
custom base URL) directly on `ProviderConfig`:
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from jev_relevance import JevRelevanceRetrieverBuilder, ProviderConfig
|
|
92
|
+
|
|
93
|
+
retriever = (
|
|
94
|
+
JevRelevanceRetrieverBuilder()
|
|
95
|
+
.with_base_retriever(my_vector_retriever)
|
|
96
|
+
.with_provider(
|
|
97
|
+
ProviderConfig(name="openrouter", api_key=os.environ["OPENROUTER_API_KEY"])
|
|
98
|
+
)
|
|
99
|
+
.build()
|
|
100
|
+
)
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## Configuration
|
|
104
|
+
|
|
105
|
+
Everything is available through the builder chain:
|
|
106
|
+
|
|
107
|
+
| Builder method | Default | Meaning |
|
|
108
|
+
| --- | --- | --- |
|
|
109
|
+
| `.with_question_mode("noul" \| "score")` | `noul` | Noul = binary "answers or not"; score = 0..3 rubric |
|
|
110
|
+
| `.with_threshold(0.5)` | `0.5` | Noul keep-if `score >= threshold` |
|
|
111
|
+
| `.with_score_threshold(2.0)` | `2.0` | Score-mode keep-if `grade >= threshold` |
|
|
112
|
+
| `.with_scoring_mode("per_chunk" \| "batched")` | `per_chunk` | One call per chunk, or one call for all |
|
|
113
|
+
| `.with_top_k(20)` | `20` | Max results returned after filtering |
|
|
114
|
+
| `.with_max_concurrency(8)` | `8` | Concurrent Jev calls in per-chunk mode |
|
|
115
|
+
| `.with_fail_open(True)` | `True` | Return unfiltered candidates on scoring error |
|
|
116
|
+
| `.with_classifier(classifier)` | auto | Inject a ready `TypeSafeClassifier` |
|
|
117
|
+
| `.with_provider(config)` | `typesafe` | Which provider config to build from |
|
|
118
|
+
|
|
119
|
+
The same values can be passed directly to `JevRelevanceConfig`, which validates
|
|
120
|
+
every field (pydantic) so invalid values fail fast at construction time.
|
|
121
|
+
|
|
122
|
+
Metadata attached to returned documents:
|
|
123
|
+
|
|
124
|
+
- `relevance`: the Jev value (noul probability 0..1, or score grade 0..3)
|
|
125
|
+
- `relevance_confidence`: Jev's confidence, score mode only
|
|
126
|
+
|
|
127
|
+
## Question modes
|
|
128
|
+
|
|
129
|
+
- **noul** — "Does this passage answer the query?" Binary yes/no probability.
|
|
130
|
+
The default; best for a strict relevance gate.
|
|
131
|
+
- **score** — "How well does this passage answer the query?" Rubric of 0..3
|
|
132
|
+
(`Off-topic`, `Tangential`, `Partly answers`, `Fully answers`), plus Jev's
|
|
133
|
+
confidence.
|
|
134
|
+
|
|
135
|
+
## Examples
|
|
136
|
+
|
|
137
|
+
See [`examples/drop_in.py`](examples/drop_in.py) for a runnable swap-in with a
|
|
138
|
+
naive keyword retriever:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
set TYPESAFE_API_KEY=your-key
|
|
142
|
+
python examples/drop_in.py --provider typesafe
|
|
143
|
+
|
|
144
|
+
set OPENROUTER_API_KEY=your-key
|
|
145
|
+
python examples/drop_in.py --provider openrouter
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
## Benchmarks & dashboard
|
|
149
|
+
|
|
150
|
+
The [jev-relevance-evals](https://github.com/saksham-malhotra-27/jev-relevance-evals)
|
|
151
|
+
companion repo holds the BEIR/NFCorpus benchmark harness and a Streamlit
|
|
152
|
+
dashboard. It scores BM25 candidates three ways on the same frozen pool — plain
|
|
153
|
+
keyword retrieval, the Jev relevance filter (drop-in retriever), and a cheap
|
|
154
|
+
OpenRouter LLM judge — and writes full IR metrics plus per-call cost to JSON.
|
|
155
|
+
On nfcorpus at the default 0.5 threshold, Jev roughly doubles precision at the
|
|
156
|
+
cost of some recall; see the companion repo for the full results tables and
|
|
157
|
+
interactive per-query drill-downs.
|
|
158
|
+
|
|
159
|
+
## Development
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
pip install -e ".[dev]"
|
|
163
|
+
python -m pytest
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Tests cover builder validation, provider model-mapping, per-chunk/batched
|
|
167
|
+
scoring, fail-open behavior, and async parity using a mocked classifier — no
|
|
168
|
+
API keys required.
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# jev-relevance
|
|
2
|
+
|
|
3
|
+
A drop-in relevance filter for LangChain RAG retrievers, powered by
|
|
4
|
+
[TypeSafe Jev](https://typesafe.ai). Wrap any `BaseRetriever`, and Jev decides
|
|
5
|
+
which retrieved chunks actually answer the query. No chain plumbing, no prompt
|
|
6
|
+
engineering, one (or a few) API calls.
|
|
7
|
+
|
|
8
|
+
```python
|
|
9
|
+
from jev_relevance import JevRelevanceRetrieverBuilder
|
|
10
|
+
|
|
11
|
+
retriever = (
|
|
12
|
+
JevRelevanceRetrieverBuilder()
|
|
13
|
+
.with_base_retriever(my_vector_retriever) # any LangChain BaseRetriever
|
|
14
|
+
.with_top_k(5)
|
|
15
|
+
.build()
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
docs = retriever.invoke("What is the capital of France?")
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Because `JevRelevanceRetriever` **is a** `BaseRetriever`, you can swap it into an
|
|
22
|
+
existing chain without touching anything else. It returns only the chunks that
|
|
23
|
+
clear the relevance threshold, sorted by Jev score, with the score attached in
|
|
24
|
+
`Document.metadata["relevance"]`.
|
|
25
|
+
|
|
26
|
+
## Features
|
|
27
|
+
|
|
28
|
+
- **Drop-in**: subclass of `BaseRetriever`; `astream`, `with_retriever`, agent
|
|
29
|
+
and chain wiring all keep working.
|
|
30
|
+
- **Two scoring modes**:
|
|
31
|
+
- `per_chunk` (default): one Jev call per query-passage pair, bounded by
|
|
32
|
+
`max_concurrency`. Most accurate.
|
|
33
|
+
- `batched`: every chunk judged in a single request for the "sub-second"
|
|
34
|
+
latency case (max 32 chunks).
|
|
35
|
+
- **Two providers**: TypeSafe (`typesafe`) and OpenRouter (`openrouter`), plus a
|
|
36
|
+
factory that lets you register more without touching the core.
|
|
37
|
+
- **fail-open**: on scoring errors you get the unfiltered candidates instead of
|
|
38
|
+
empty context (configurable).
|
|
39
|
+
- **Zero magic numbers**: every default lives in `constants.py` and every
|
|
40
|
+
behavior is builder-parameterized.
|
|
41
|
+
|
|
42
|
+
## Installation
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install jev-relevance
|
|
46
|
+
# or, from this repo:
|
|
47
|
+
python -m venv .venv && .venv\Scripts\activate
|
|
48
|
+
pip install -e ".[dev]"
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Python 3.10+. The library itself only needs `langchain-core` and
|
|
52
|
+
`langchain-typesafe`.
|
|
53
|
+
|
|
54
|
+
## Providers
|
|
55
|
+
|
|
56
|
+
| Provider | Default model | Base URL |
|
|
57
|
+
| --- | --- | --- |
|
|
58
|
+
| `typesafe` | `jev-latest` | `https://api.typesafe.ai` |
|
|
59
|
+
| `openrouter` | `~typesafe/jev-latest` | `https://openrouter.ai/api` |
|
|
60
|
+
|
|
61
|
+
Credentials are **always passed as parameters** — the library never reads env
|
|
62
|
+
vars or `.env` files itself; that's the caller's job. Supply the key (and any
|
|
63
|
+
custom base URL) directly on `ProviderConfig`:
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from jev_relevance import JevRelevanceRetrieverBuilder, ProviderConfig
|
|
67
|
+
|
|
68
|
+
retriever = (
|
|
69
|
+
JevRelevanceRetrieverBuilder()
|
|
70
|
+
.with_base_retriever(my_vector_retriever)
|
|
71
|
+
.with_provider(
|
|
72
|
+
ProviderConfig(name="openrouter", api_key=os.environ["OPENROUTER_API_KEY"])
|
|
73
|
+
)
|
|
74
|
+
.build()
|
|
75
|
+
)
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Configuration
|
|
79
|
+
|
|
80
|
+
Everything is available through the builder chain:
|
|
81
|
+
|
|
82
|
+
| Builder method | Default | Meaning |
|
|
83
|
+
| --- | --- | --- |
|
|
84
|
+
| `.with_question_mode("noul" \| "score")` | `noul` | Noul = binary "answers or not"; score = 0..3 rubric |
|
|
85
|
+
| `.with_threshold(0.5)` | `0.5` | Noul keep-if `score >= threshold` |
|
|
86
|
+
| `.with_score_threshold(2.0)` | `2.0` | Score-mode keep-if `grade >= threshold` |
|
|
87
|
+
| `.with_scoring_mode("per_chunk" \| "batched")` | `per_chunk` | One call per chunk, or one call for all |
|
|
88
|
+
| `.with_top_k(20)` | `20` | Max results returned after filtering |
|
|
89
|
+
| `.with_max_concurrency(8)` | `8` | Concurrent Jev calls in per-chunk mode |
|
|
90
|
+
| `.with_fail_open(True)` | `True` | Return unfiltered candidates on scoring error |
|
|
91
|
+
| `.with_classifier(classifier)` | auto | Inject a ready `TypeSafeClassifier` |
|
|
92
|
+
| `.with_provider(config)` | `typesafe` | Which provider config to build from |
|
|
93
|
+
|
|
94
|
+
The same values can be passed directly to `JevRelevanceConfig`, which validates
|
|
95
|
+
every field (pydantic) so invalid values fail fast at construction time.
|
|
96
|
+
|
|
97
|
+
Metadata attached to returned documents:
|
|
98
|
+
|
|
99
|
+
- `relevance`: the Jev value (noul probability 0..1, or score grade 0..3)
|
|
100
|
+
- `relevance_confidence`: Jev's confidence, score mode only
|
|
101
|
+
|
|
102
|
+
## Question modes
|
|
103
|
+
|
|
104
|
+
- **noul** — "Does this passage answer the query?" Binary yes/no probability.
|
|
105
|
+
The default; best for a strict relevance gate.
|
|
106
|
+
- **score** — "How well does this passage answer the query?" Rubric of 0..3
|
|
107
|
+
(`Off-topic`, `Tangential`, `Partly answers`, `Fully answers`), plus Jev's
|
|
108
|
+
confidence.
|
|
109
|
+
|
|
110
|
+
## Examples
|
|
111
|
+
|
|
112
|
+
See [`examples/drop_in.py`](examples/drop_in.py) for a runnable swap-in with a
|
|
113
|
+
naive keyword retriever:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
set TYPESAFE_API_KEY=your-key
|
|
117
|
+
python examples/drop_in.py --provider typesafe
|
|
118
|
+
|
|
119
|
+
set OPENROUTER_API_KEY=your-key
|
|
120
|
+
python examples/drop_in.py --provider openrouter
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Benchmarks & dashboard
|
|
124
|
+
|
|
125
|
+
The [jev-relevance-evals](https://github.com/saksham-malhotra-27/jev-relevance-evals)
|
|
126
|
+
companion repo holds the BEIR/NFCorpus benchmark harness and a Streamlit
|
|
127
|
+
dashboard. It scores BM25 candidates three ways on the same frozen pool — plain
|
|
128
|
+
keyword retrieval, the Jev relevance filter (drop-in retriever), and a cheap
|
|
129
|
+
OpenRouter LLM judge — and writes full IR metrics plus per-call cost to JSON.
|
|
130
|
+
On nfcorpus at the default 0.5 threshold, Jev roughly doubles precision at the
|
|
131
|
+
cost of some recall; see the companion repo for the full results tables and
|
|
132
|
+
interactive per-query drill-downs.
|
|
133
|
+
|
|
134
|
+
## Development
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
pip install -e ".[dev]"
|
|
138
|
+
python -m pytest
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Tests cover builder validation, provider model-mapping, per-chunk/batched
|
|
142
|
+
scoring, fail-open behavior, and async parity using a mocked classifier — no
|
|
143
|
+
API keys required.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "jev-relevance"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Drop-in relevance filter for LangChain RAG retrievers powered by TypeSafe Jev."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
authors = [{ name = "Jev Relevance Contributors" }]
|
|
13
|
+
keywords = [
|
|
14
|
+
"langchain",
|
|
15
|
+
"rag",
|
|
16
|
+
"retriever",
|
|
17
|
+
"relevance",
|
|
18
|
+
"rerank",
|
|
19
|
+
"classifier",
|
|
20
|
+
"jev",
|
|
21
|
+
"typesafe",
|
|
22
|
+
]
|
|
23
|
+
classifiers = [
|
|
24
|
+
"Programming Language :: Python :: 3",
|
|
25
|
+
"Programming Language :: Python :: 3.10",
|
|
26
|
+
"Programming Language :: Python :: 3.11",
|
|
27
|
+
"Programming Language :: Python :: 3.12",
|
|
28
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
29
|
+
]
|
|
30
|
+
dependencies = [
|
|
31
|
+
"langchain-core>=1.0,<2",
|
|
32
|
+
"langchain-typesafe>=0.0.1a3,<1",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
[project.optional-dependencies]
|
|
36
|
+
dev = [
|
|
37
|
+
"pytest>=8",
|
|
38
|
+
"pytest-asyncio>=0.23",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
[project.urls]
|
|
42
|
+
Homepage = "https://github.com/saksham-malhotra-27/jev-relevance"
|
|
43
|
+
Repository = "https://github.com/saksham-malhotra-27/jev-relevance"
|
|
44
|
+
"Bug Tracker" = "https://github.com/saksham-malhotra-27/jev-relevance/issues"
|
|
45
|
+
|
|
46
|
+
[tool.setuptools.packages.find]
|
|
47
|
+
where = ["src"]
|
|
48
|
+
|
|
49
|
+
[tool.setuptools.package-data]
|
|
50
|
+
jev_relevance = ["py.typed"]
|
|
51
|
+
|
|
52
|
+
[tool.pytest.ini_options]
|
|
53
|
+
testpaths = ["tests"]
|
|
54
|
+
asyncio_mode = "auto"
|
|
55
|
+
|
|
56
|
+
[tool.coverage.run]
|
|
57
|
+
source = ["jev_relevance"]
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""jev-relevance: a drop-in relevance filter for LangChain RAG retrievers.
|
|
2
|
+
|
|
3
|
+
Wrap any ``BaseRetriever``; Jev scores every candidate in one bounded pass and
|
|
4
|
+
the wrapper returns only the chunks that answer the query, ranked by relevance.
|
|
5
|
+
|
|
6
|
+
Basic usage (TypeSafe Provider):
|
|
7
|
+
|
|
8
|
+
from jev_relevance import JevRelevanceRetrieverBuilder
|
|
9
|
+
|
|
10
|
+
retriever = (
|
|
11
|
+
JevRelevanceRetrieverBuilder()
|
|
12
|
+
.with_base_retriever(my_vector_retriever)
|
|
13
|
+
.with_top_k(5)
|
|
14
|
+
.build()
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
OpenRouter Provider (api_key passed as a param):
|
|
18
|
+
|
|
19
|
+
from jev_relevance import JevRelevanceRetrieverBuilder
|
|
20
|
+
from jev_relevance.config import ProviderConfig
|
|
21
|
+
|
|
22
|
+
retriever = (
|
|
23
|
+
JevRelevanceRetrieverBuilder()
|
|
24
|
+
.with_base_retriever(my_vector_retriever)
|
|
25
|
+
.with_provider(ProviderConfig(name="openrouter", api_key="sk-..."))
|
|
26
|
+
.build()
|
|
27
|
+
)
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from jev_relevance.builder import JevRelevanceRetrieverBuilder
|
|
31
|
+
from jev_relevance.config import (
|
|
32
|
+
JevRelevanceConfig,
|
|
33
|
+
ProviderConfig,
|
|
34
|
+
ProviderName,
|
|
35
|
+
QuestionMode,
|
|
36
|
+
ScoringMode,
|
|
37
|
+
)
|
|
38
|
+
from jev_relevance.constants import Provider
|
|
39
|
+
from jev_relevance.provider import ProviderFactory, build_classifier
|
|
40
|
+
from jev_relevance.retriever import JevRelevanceRetriever
|
|
41
|
+
from jev_relevance.scoring import (
|
|
42
|
+
build_noul_question,
|
|
43
|
+
build_state,
|
|
44
|
+
)
|
|
45
|
+
from jev_relevance.strategies import (
|
|
46
|
+
BatchedScorer,
|
|
47
|
+
DocumentScore,
|
|
48
|
+
PerChunkScorer,
|
|
49
|
+
ScoringStrategy,
|
|
50
|
+
build_scoring_strategy,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
__version__ = "0.1.0"
|
|
54
|
+
|
|
55
|
+
__all__ = [
|
|
56
|
+
"JevRelevanceRetriever",
|
|
57
|
+
"JevRelevanceRetrieverBuilder",
|
|
58
|
+
"JevRelevanceConfig",
|
|
59
|
+
"ProviderConfig",
|
|
60
|
+
"ProviderFactory",
|
|
61
|
+
"build_classifier",
|
|
62
|
+
"build_scoring_strategy",
|
|
63
|
+
"build_state",
|
|
64
|
+
"build_noul_question",
|
|
65
|
+
"DocumentScore",
|
|
66
|
+
"ScoringStrategy",
|
|
67
|
+
"PerChunkScorer",
|
|
68
|
+
"BatchedScorer",
|
|
69
|
+
"Provider",
|
|
70
|
+
"ProviderName",
|
|
71
|
+
"QuestionMode",
|
|
72
|
+
"ScoringMode",
|
|
73
|
+
"__version__",
|
|
74
|
+
]
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""Fluent builder that assembles a fully-wired :class:`JevRelevanceRetriever`.
|
|
2
|
+
|
|
3
|
+
The builder owns the composition order:
|
|
4
|
+
|
|
5
|
+
1. a validated :class:`JevRelevanceConfig` is built (or edited),
|
|
6
|
+
2. a classifier is produced by the provider factory (or supplied directly),
|
|
7
|
+
3. a :class:`ScoringStrategy` wraps the classifier for the chosen scoring mode,
|
|
8
|
+
4. the strategy and config are handed to the retriever, which wraps the caller's
|
|
9
|
+
``base_retriever``.
|
|
10
|
+
|
|
11
|
+
Every knob has a named default from :mod:`jev_relevance.constants`, so the
|
|
12
|
+
zero-argument path ``JevRelevanceRetrieverBuilder().with_base_retriever(...).build()``
|
|
13
|
+
already behaves sensibly.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from langchain_core.retrievers import BaseRetriever
|
|
21
|
+
from langchain_typesafe import TypeSafeClassifier
|
|
22
|
+
|
|
23
|
+
from jev_relevance.config import JevRelevanceConfig, ProviderConfig
|
|
24
|
+
from jev_relevance.provider import build_classifier
|
|
25
|
+
from jev_relevance.retriever import JevRelevanceRetriever
|
|
26
|
+
from jev_relevance.strategies import build_scoring_strategy
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class JevRelevanceRetrieverBuilder:
|
|
30
|
+
"""Build :class:`JevRelevanceRetriever` instances from validated settings.
|
|
31
|
+
|
|
32
|
+
Examples:
|
|
33
|
+
>>> builder = (
|
|
34
|
+
... JevRelevanceRetrieverBuilder()
|
|
35
|
+
... .with_base_retriever(my_vector_retriever)
|
|
36
|
+
... .with_top_k(5)
|
|
37
|
+
... .with_threshold(0.6)
|
|
38
|
+
... )
|
|
39
|
+
>>> retriever = builder.build()
|
|
40
|
+
|
|
41
|
+
Use OpenRouter as the Jev provider (instead of TypeSafe default). The
|
|
42
|
+
api_key is a parameter; the library never reads env vars itself:
|
|
43
|
+
|
|
44
|
+
>>> from jev_relevance.config import ProviderConfig
|
|
45
|
+
>>> builder.with_provider(
|
|
46
|
+
... ProviderConfig(name="openrouter", api_key="sk-...")
|
|
47
|
+
... )
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
def __init__(self) -> None:
|
|
51
|
+
self._config: JevRelevanceConfig = JevRelevanceConfig()
|
|
52
|
+
self._classifier: TypeSafeClassifier | None = None
|
|
53
|
+
self._classifier_builder: Any | None = None
|
|
54
|
+
self._base_retriever: BaseRetriever | None = None
|
|
55
|
+
|
|
56
|
+
# ------------------------------------------------------------------
|
|
57
|
+
# Configuration setters (chainable)
|
|
58
|
+
# ------------------------------------------------------------------
|
|
59
|
+
|
|
60
|
+
def with_base_retriever(self, retriever: BaseRetriever) -> "JevRelevanceRetrieverBuilder":
|
|
61
|
+
"""Set the underlying retrieval engine this filter wraps."""
|
|
62
|
+
if not isinstance(retriever, BaseRetriever):
|
|
63
|
+
raise TypeError(
|
|
64
|
+
f"base_retriever must be a BaseRetriever, got {type(retriever).__name__}."
|
|
65
|
+
)
|
|
66
|
+
self._base_retriever = retriever
|
|
67
|
+
return self
|
|
68
|
+
|
|
69
|
+
def with_config(self, config: JevRelevanceConfig) -> "JevRelevanceRetrieverBuilder":
|
|
70
|
+
"""Replace the working config with a fully validated one."""
|
|
71
|
+
if not isinstance(config, JevRelevanceConfig):
|
|
72
|
+
raise TypeError(
|
|
73
|
+
f"config must be a JevRelevanceConfig, got {type(config).__name__}."
|
|
74
|
+
)
|
|
75
|
+
self._config = config
|
|
76
|
+
return self
|
|
77
|
+
|
|
78
|
+
def with_question_mode(self, question_mode: str) -> "JevRelevanceRetrieverBuilder":
|
|
79
|
+
"""Choose ``"noul"`` (yes/no probability) or ``"score"`` (0..3 rubric)."""
|
|
80
|
+
self._config = self._config.model_copy(
|
|
81
|
+
update={"question_mode": question_mode}
|
|
82
|
+
)
|
|
83
|
+
return self
|
|
84
|
+
|
|
85
|
+
def with_scoring_mode(self, scoring_mode: str) -> "JevRelevanceRetrieverBuilder":
|
|
86
|
+
"""Choose ``"per_chunk"`` (one call per pair) or ``"batched"`` (one call)."""
|
|
87
|
+
self._config = self._config.model_copy(
|
|
88
|
+
update={"scoring_mode": scoring_mode}
|
|
89
|
+
)
|
|
90
|
+
return self
|
|
91
|
+
|
|
92
|
+
def with_threshold(self, threshold: float) -> "JevRelevanceRetrieverBuilder":
|
|
93
|
+
"""Set the noul-mode relevance probability cutoff (0..1)."""
|
|
94
|
+
self._config = self._config.model_copy(update={"threshold": threshold})
|
|
95
|
+
return self
|
|
96
|
+
|
|
97
|
+
def with_score_threshold(self, score_threshold: float) -> "JevRelevanceRetrieverBuilder":
|
|
98
|
+
"""Set the score-mode grade cutoff (0..3)."""
|
|
99
|
+
self._config = self._config.model_copy(
|
|
100
|
+
update={"score_threshold": score_threshold}
|
|
101
|
+
)
|
|
102
|
+
return self
|
|
103
|
+
|
|
104
|
+
def with_top_k(self, top_k: int | None) -> "JevRelevanceRetrieverBuilder":
|
|
105
|
+
"""Cap how many documents are returned after filtering (``None`` = all)."""
|
|
106
|
+
self._config = self._config.model_copy(update={"top_k": top_k})
|
|
107
|
+
return self
|
|
108
|
+
|
|
109
|
+
def with_max_concurrency(self, max_concurrency: int) -> "JevRelevanceRetrieverBuilder":
|
|
110
|
+
"""Set maximum in-flight Jev calls in per-chunk mode."""
|
|
111
|
+
self._config = self._config.model_copy(
|
|
112
|
+
update={"max_concurrency": max_concurrency}
|
|
113
|
+
)
|
|
114
|
+
return self
|
|
115
|
+
|
|
116
|
+
def with_fail_open(self, fail_open: bool) -> "JevRelevanceRetrieverBuilder":
|
|
117
|
+
"""Let scoring failures fall back to unfiltered candidates."""
|
|
118
|
+
self._config = self._config.model_copy(update={"fail_open": fail_open})
|
|
119
|
+
return self
|
|
120
|
+
|
|
121
|
+
# ------------------------------------------------------------------
|
|
122
|
+
# Classifier wiring (chainable)
|
|
123
|
+
# ------------------------------------------------------------------
|
|
124
|
+
|
|
125
|
+
def with_classifier(
|
|
126
|
+
self, classifier: TypeSafeClassifier
|
|
127
|
+
) -> "JevRelevanceRetrieverBuilder":
|
|
128
|
+
"""Inject an already-constructed classifier (any provider/Typesafe client).
|
|
129
|
+
|
|
130
|
+
Use this when you have custom clients, proxies, or want to reuse a
|
|
131
|
+
long-lived connection pool.
|
|
132
|
+
"""
|
|
133
|
+
if not isinstance(classifier, TypeSafeClassifier):
|
|
134
|
+
raise TypeError(
|
|
135
|
+
f"classifier must be a TypeSafeClassifier, got {type(classifier).__name__}."
|
|
136
|
+
)
|
|
137
|
+
self._classifier = classifier
|
|
138
|
+
return self
|
|
139
|
+
|
|
140
|
+
def with_provider(
|
|
141
|
+
self, provider_config: ProviderConfig
|
|
142
|
+
) -> "JevRelevanceRetrieverBuilder":
|
|
143
|
+
"""Build the classifier through the provider factory from ``provider_config``."""
|
|
144
|
+
if not isinstance(provider_config, ProviderConfig):
|
|
145
|
+
raise TypeError(
|
|
146
|
+
f"provider_config must be a ProviderConfig, got "
|
|
147
|
+
f"{type(provider_config).__name__}."
|
|
148
|
+
)
|
|
149
|
+
self._classifier_builder = lambda: build_classifier(provider_config)
|
|
150
|
+
self._classifier = None
|
|
151
|
+
return self
|
|
152
|
+
|
|
153
|
+
# ------------------------------------------------------------------
|
|
154
|
+
# Construction
|
|
155
|
+
# ------------------------------------------------------------------
|
|
156
|
+
|
|
157
|
+
def build(self) -> JevRelevanceRetriever:
|
|
158
|
+
"""Assemble and return the configured retriever.
|
|
159
|
+
|
|
160
|
+
Raises:
|
|
161
|
+
ValueError: If no base retriever was set.
|
|
162
|
+
The classifier is constructed lazily so providers without a present
|
|
163
|
+
env-var key only fail at build time, not import time.
|
|
164
|
+
"""
|
|
165
|
+
if self._base_retriever is None:
|
|
166
|
+
raise ValueError(
|
|
167
|
+
"Cannot build JevRelevanceRetriever without a base retriever. "
|
|
168
|
+
"Call with_base_retriever(...) first."
|
|
169
|
+
)
|
|
170
|
+
classifier = self._classifier
|
|
171
|
+
if classifier is None:
|
|
172
|
+
if self._classifier_builder is not None:
|
|
173
|
+
classifier = self._classifier_builder()
|
|
174
|
+
else:
|
|
175
|
+
classifier = build_classifier(ProviderConfig.for_provider("typesafe"))
|
|
176
|
+
strategy = build_scoring_strategy(classifier, self._config)
|
|
177
|
+
return JevRelevanceRetriever(
|
|
178
|
+
base_retriever=self._base_retriever,
|
|
179
|
+
config=self._config,
|
|
180
|
+
strategy=strategy,
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
__all__ = ["JevRelevanceRetrieverBuilder"]
|