pyprocessors-jev 1.6.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyprocessors_jev-1.6.1/.gitignore +140 -0
- pyprocessors_jev-1.6.1/.gitmodules +4 -0
- pyprocessors_jev-1.6.1/.python-version +1 -0
- pyprocessors_jev-1.6.1/PKG-INFO +227 -0
- pyprocessors_jev-1.6.1/README.md +183 -0
- pyprocessors_jev-1.6.1/Taskfile.yml +33 -0
- pyprocessors_jev-1.6.1/pyproject.toml +104 -0
- pyprocessors_jev-1.6.1/src/pyprocessors_jev/__init__.py +3 -0
- pyprocessors_jev-1.6.1/src/pyprocessors_jev/client.py +68 -0
- pyprocessors_jev-1.6.1/src/pyprocessors_jev/jev.py +454 -0
- pyprocessors_jev-1.6.1/submodules/python-archetype/.claude-plugin/marketplace.json +18 -0
- pyprocessors_jev-1.6.1/submodules/python-archetype/.claude-plugin/plugin.json +15 -0
- pyprocessors_jev-1.6.1/submodules/python-archetype/HOWTO.md +204 -0
- pyprocessors_jev-1.6.1/submodules/python-archetype/README.md +120 -0
- pyprocessors_jev-1.6.1/submodules/python-archetype/resources/Stages.yml +114 -0
- pyprocessors_jev-1.6.1/submodules/python-archetype/resources/Taskfile.yml +300 -0
- pyprocessors_jev-1.6.1/submodules/python-archetype/skills/new-plugin/SKILL.md +132 -0
- pyprocessors_jev-1.6.1/submodules/python-archetype/skills/new-plugin/references/types.md +37 -0
- pyprocessors_jev-1.6.1/submodules/python-archetype/tools/check-marketplace.py +237 -0
- pyprocessors_jev-1.6.1/submodules/python-archetype/vars/pythonPipeline.groovy +628 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
pip-wheel-metadata/
|
|
24
|
+
share/python-wheels/
|
|
25
|
+
*.egg-info/
|
|
26
|
+
.installed.cfg
|
|
27
|
+
*.egg
|
|
28
|
+
MANIFEST
|
|
29
|
+
|
|
30
|
+
# PyInstaller
|
|
31
|
+
# Usually these files are written by a python script from a template
|
|
32
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
33
|
+
*.manifest
|
|
34
|
+
*.spec
|
|
35
|
+
|
|
36
|
+
# Installer logs
|
|
37
|
+
pip-log.txt
|
|
38
|
+
pip-delete-this-directory.txt
|
|
39
|
+
|
|
40
|
+
# Unit test / coverage reports
|
|
41
|
+
htmlcov/
|
|
42
|
+
.tox/
|
|
43
|
+
.nox/
|
|
44
|
+
.coverage
|
|
45
|
+
.coverage.*
|
|
46
|
+
.cache
|
|
47
|
+
nosetests.xml
|
|
48
|
+
coverage.xml
|
|
49
|
+
results.xml
|
|
50
|
+
*.cover
|
|
51
|
+
*.py,cover
|
|
52
|
+
.hypothesis/
|
|
53
|
+
.pytest_cache/
|
|
54
|
+
|
|
55
|
+
# Translations
|
|
56
|
+
*.mo
|
|
57
|
+
*.pot
|
|
58
|
+
|
|
59
|
+
# Django stuff:
|
|
60
|
+
*.log
|
|
61
|
+
local_settings.py
|
|
62
|
+
db.sqlite3
|
|
63
|
+
db.sqlite3-journal
|
|
64
|
+
|
|
65
|
+
# Flask stuff:
|
|
66
|
+
instance/
|
|
67
|
+
.webassets-cache
|
|
68
|
+
|
|
69
|
+
# Scrapy stuff:
|
|
70
|
+
.scrapy
|
|
71
|
+
|
|
72
|
+
# Sphinx documentation
|
|
73
|
+
docs/_build/
|
|
74
|
+
|
|
75
|
+
# PyBuilder
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv : .python-version est SUIVI ici, il porte la version d'interpréteur qu'uv installe
|
|
86
|
+
|
|
87
|
+
# pipenv
|
|
88
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
89
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
90
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
91
|
+
# install all needed dependencies.
|
|
92
|
+
#Pipfile.lock
|
|
93
|
+
|
|
94
|
+
# celery beat schedule file
|
|
95
|
+
celerybeat-schedule
|
|
96
|
+
|
|
97
|
+
# SageMath parsed files
|
|
98
|
+
*.sage.py
|
|
99
|
+
|
|
100
|
+
# Environments
|
|
101
|
+
.env
|
|
102
|
+
.venv
|
|
103
|
+
env/
|
|
104
|
+
venv/
|
|
105
|
+
ENV/
|
|
106
|
+
env.bak/
|
|
107
|
+
venv.bak/
|
|
108
|
+
|
|
109
|
+
# Spyder project settings
|
|
110
|
+
.spyderproject
|
|
111
|
+
.spyproject
|
|
112
|
+
|
|
113
|
+
# Rope project settings
|
|
114
|
+
.ropeproject
|
|
115
|
+
|
|
116
|
+
# mkdocs documentation
|
|
117
|
+
/site
|
|
118
|
+
|
|
119
|
+
# mypy
|
|
120
|
+
.mypy_cache/
|
|
121
|
+
.dmypy.json
|
|
122
|
+
dmypy.json
|
|
123
|
+
|
|
124
|
+
# Pyre type checker
|
|
125
|
+
.pyre/
|
|
126
|
+
|
|
127
|
+
# Specific
|
|
128
|
+
.idea/
|
|
129
|
+
.groovylintrc.json
|
|
130
|
+
.emailNotif
|
|
131
|
+
uv.lock
|
|
132
|
+
|
|
133
|
+
# SBOMs
|
|
134
|
+
**/sbom*.json
|
|
135
|
+
**/trivy*.html
|
|
136
|
+
**/audit*.json
|
|
137
|
+
**/audit*.md
|
|
138
|
+
|
|
139
|
+
# local caches for tools/eval_mappings.py
|
|
140
|
+
.cache/
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.12
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pyprocessors-jev
|
|
3
|
+
Version: 1.6.1
|
|
4
|
+
Summary: Processor based on the Jev System One API
|
|
5
|
+
Project-URL: Homepage, https://bitbucket.org/kairntech/pyprocessors_jev
|
|
6
|
+
Author-email: Olivier Terrier <olivier.terrier@kairntech.com>
|
|
7
|
+
License: MIT
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Information Technology
|
|
11
|
+
Classifier: Intended Audience :: System Administrators
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Topic :: Software Development
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Requires-Python: >=3.12
|
|
20
|
+
Requires-Dist: log-with-context
|
|
21
|
+
Requires-Dist: pydantic<3.0,>=2.0
|
|
22
|
+
Requires-Dist: pymultirole-plugins<1.7.0,>=1.6.0
|
|
23
|
+
Requires-Dist: python-singleton-metaclasses
|
|
24
|
+
Requires-Dist: requests
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: bump2version; extra == 'dev'
|
|
27
|
+
Requires-Dist: pre-commit; extra == 'dev'
|
|
28
|
+
Provides-Extra: docs
|
|
29
|
+
Requires-Dist: lxml-html-clean; extra == 'docs'
|
|
30
|
+
Requires-Dist: myst-parser; extra == 'docs'
|
|
31
|
+
Requires-Dist: sphinx; extra == 'docs'
|
|
32
|
+
Requires-Dist: sphinx-rtd-theme; extra == 'docs'
|
|
33
|
+
Requires-Dist: sphinxcontrib-apidoc; extra == 'docs'
|
|
34
|
+
Provides-Extra: sbom
|
|
35
|
+
Requires-Dist: cyclonedx-bom; extra == 'sbom'
|
|
36
|
+
Requires-Dist: pip-audit; extra == 'sbom'
|
|
37
|
+
Provides-Extra: test
|
|
38
|
+
Requires-Dist: dirty-equals; extra == 'test'
|
|
39
|
+
Requires-Dist: pytest; extra == 'test'
|
|
40
|
+
Requires-Dist: pytest-cov; extra == 'test'
|
|
41
|
+
Requires-Dist: pytest-dotenv; extra == 'test'
|
|
42
|
+
Requires-Dist: ruff; extra == 'test'
|
|
43
|
+
Description-Content-Type: text/markdown
|
|
44
|
+
|
|
45
|
+
# pyprocessors-jev
|
|
46
|
+
|
|
47
|
+
Processor based on the [Jev](https://typesafe.ai) *System One* API: it asks typed questions about a
|
|
48
|
+
document and gets back **calibrated probabilities** instead of text, so nothing has to be parsed out
|
|
49
|
+
of a completion.
|
|
50
|
+
|
|
51
|
+
Two providers, one protocol (`POST {base_url}/v1/systemone`):
|
|
52
|
+
|
|
53
|
+
| entry point | provider | base url | model | api key |
|
|
54
|
+
|-------------|----------|----------|-------|---------|
|
|
55
|
+
| `jev` | hosted TypeSafe | `JEV_API_BASE`, else `TYPESAFE_BASE_URL`, else `https://api.typesafe.ai` | `JEV_MODEL`, else `jev-latest` | `JEV_API_KEY`, else `TYPESAFE_API_KEY` |
|
|
56
|
+
| `openjev` | self-hosted [Open-Jev](https://github.com/Zefan-Cai/Open-Jev) | `OPENJEV_API_BASE`, else `http://127.0.0.1:8791` | `OPENJEV_MODEL`, else `open-jev` | `OPENJEV_API_KEY` (usually none) |
|
|
57
|
+
|
|
58
|
+
## What it can and cannot do
|
|
59
|
+
|
|
60
|
+
Jev *chooses* and *rates*; it does not write. That bounds the four outputs:
|
|
61
|
+
|
|
62
|
+
| output | how | not possible |
|
|
63
|
+
|--------|-----|--------------|
|
|
64
|
+
| `categories` | `choice` over the project labels, or one `noul` per label | — |
|
|
65
|
+
| `metadata` | typed values only: `noul` → boolean, `choice` → key of a closed set, `score` → number | free-form extraction (dates, amounts, names) — use `pyprocessors_openai_completion` |
|
|
66
|
+
| `annotations` | filter existing candidate spans | create spans: no offsets come back |
|
|
67
|
+
| `altTexts` | select among the texts already on the document (rerank / judge) | generate a summary or a translation |
|
|
68
|
+
|
|
69
|
+
Because the API answers **many questions in one call**, classifying a document and filling its
|
|
70
|
+
metadata costs one HTTP request — there is no `--- METADATA ---` section to split off.
|
|
71
|
+
|
|
72
|
+
## Usage
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
from pymultirole_plugins.v1.schema import Document
|
|
76
|
+
from pyprocessors_jev.jev import JevProcessor, JevParameters
|
|
77
|
+
|
|
78
|
+
processor = JevProcessor()
|
|
79
|
+
parameters = JevParameters(
|
|
80
|
+
labels={"billing": "Payments, invoicing, refunds", "technical": "Bugs, outages, integrations"},
|
|
81
|
+
instructions="Which team should handle this?",
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
docs = processor.process([Document(text="Help! My payouts have been failing for 3 days.")], parameters)
|
|
85
|
+
for cat in docs[0].categories:
|
|
86
|
+
print(cat.labelName, cat.score, cat.properties)
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Options
|
|
90
|
+
|
|
91
|
+
| Option | Default | Description |
|
|
92
|
+
|--------|---------|-------------|
|
|
93
|
+
| `base_url` | provider default (see above) | Jev endpoint base url |
|
|
94
|
+
| `model` | provider default (see above) | model route |
|
|
95
|
+
| `function` | `add_categories` | question built from `labels`: `add_categories` (one `choice`), `add_multilabel_categories` (one `noul` per label, same call), `filter_annotations` (one `noul` per annotation, offsets preserved), `select_altText` (`choice` over the alternative texts). **Ignored as soon as `questions` is defined.** |
|
|
96
|
+
| `instructions` | `Choose the best category for this text.` | instructions of the question built from `function` |
|
|
97
|
+
| `labels` | – | label name → description mapping, injected from the project label set. The description is what Jev reads to decide; the categories written on the document carry **`labelName` only**, never `label` — a description has no place on the document, and the project label set already holds the display name. |
|
|
98
|
+
| `questions` | – | the general case: one JSON question per key, each declaring `type`, `instructions`, `criteria` and `target` (`categories`, `metadata`, `altTexts`). A `criteria` of `$labels` is replaced by `labels`. Takes precedence over `function`. |
|
|
99
|
+
| `state_altText` | – | send that alternative text as the state instead of the document text; falls back on the text with a warning when it is missing |
|
|
100
|
+
| `decision_altText` | – | keep questions, answers and token usage in that alternative text — the audit trail |
|
|
101
|
+
| `selected_altText` | `selected` | alternative text receiving the winner of `select_altText` |
|
|
102
|
+
| `threshold` | `0.5` | probability below which an answer is dropped (a category, or an annotation kept by `filter_annotations`). Lower it below the winner-takes-all point and a `choice` becomes a multilabel decision: every label above the bar becomes a category, sorted by decreasing probability |
|
|
103
|
+
| `keep_best` | `false` | when no answer of a `choice` reaches `threshold`, keep the most probable one anyway, so the document never comes back without a category. A no-op with 3 labels (the winner is mechanically above 1/3), a safety net with a large label set |
|
|
104
|
+
| `max_chars` | `0` | truncate the state, `0` sends it whole |
|
|
105
|
+
| `timeout` | `60.0` | HTTP timeout, in seconds |
|
|
106
|
+
| `max_retries` | `3` | retries of a throttled (429) or overloaded (529) call, exponential backoff, obeying `Retry-After` |
|
|
107
|
+
|
|
108
|
+
Answers land in a predictable shape: a category carries the **probability** as its `score` and the
|
|
109
|
+
question `confidence` in its `properties`; a `noul` in metadata writes a boolean under the question
|
|
110
|
+
id plus its probability under `<id>_probability`; a `choice` and a `score` write their value plus
|
|
111
|
+
`<id>_confidence`. A question whose criteria are unknown labels, or an answer naming a label outside
|
|
112
|
+
`labels`, is dropped with a warning rather than invented.
|
|
113
|
+
|
|
114
|
+
The official `typesafe_sdk` is deliberately not a dependency: it cannot talk to an Open-Jev server,
|
|
115
|
+
which would mean two code paths for one protocol.
|
|
116
|
+
|
|
117
|
+
## Development
|
|
118
|
+
|
|
119
|
+
The build is driven by [Task](https://taskfile.dev) and [uv](https://docs.astral.sh/uv/),
|
|
120
|
+
with the shared stages coming from the `python-archetype` submodule.
|
|
121
|
+
|
|
122
|
+
### Getting started
|
|
123
|
+
|
|
124
|
+
The stages live in a Git submodule, so **clone with `--recurse-submodules`**:
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
git clone --recurse-submodules git@bitbucket.org:kairntech/pyprocessors_jev.git
|
|
128
|
+
cd pyprocessors_jev
|
|
129
|
+
sh -c "$(curl -sSL https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin
|
|
130
|
+
task
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Already cloned without it? The submodule directory is empty, and `task` fails on:
|
|
134
|
+
|
|
135
|
+
```
|
|
136
|
+
task: No Taskfile found at ".../submodules/python-archetype/resources/Taskfile.yml"
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
which means exactly that, and nothing worse:
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
git submodule update --init
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
**Task is the only manual prerequisite.** An archetype cannot bootstrap itself: uv and the
|
|
146
|
+
Python interpreter install themselves on demand (every task that runs uv depends on an
|
|
147
|
+
internal `install-python` task), but the thing that runs them does not. Make sure
|
|
148
|
+
`~/.local/bin` is on your `PATH` — that is where `task` and `uv` both land.
|
|
149
|
+
|
|
150
|
+
### Running the pipeline
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
task stages # print the pipeline stages, in order
|
|
154
|
+
task # run the pipeline up to (but excluding) py:publish
|
|
155
|
+
task -- --skip-tests # same, without the test stage
|
|
156
|
+
task up-to -- py:lint # run the pipeline up to and including one stage
|
|
157
|
+
task jenkins # run every stage, exactly what Jenkins runs
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
`task` with no argument is safe by construction: it runs every stage but the last, and that
|
|
161
|
+
bound is computed from the `STAGES` list rather than written down. The last stage is the
|
|
162
|
+
only one with an effect outside your machine.
|
|
163
|
+
|
|
164
|
+
`STAGES`, declared once in `Taskfile.yml`, is the single definition of the pipeline order —
|
|
165
|
+
so what you run locally is what Jenkins runs.
|
|
166
|
+
|
|
167
|
+
### Individual stages
|
|
168
|
+
|
|
169
|
+
| Task | Description |
|
|
170
|
+
|---------------------------------|-------------------------------------------------------|
|
|
171
|
+
| `task py:sync` | Install the project and its dependencies (uv sync) |
|
|
172
|
+
| `task py:lint` | `ruff check` and `ruff format --check` |
|
|
173
|
+
| `task py:format` | Reformat the code with ruff |
|
|
174
|
+
| `task py:test` | Run the test suite |
|
|
175
|
+
| `task py:test-marker -- <m>` | Run the tests carrying one pytest marker |
|
|
176
|
+
| `task py:sbom` | Generate a CycloneDX SBOM of the resolved environment |
|
|
177
|
+
| `task py:check-vulnerabilities` | Check for known CVEs |
|
|
178
|
+
| `task py:check-updates` | Check for dependency updates |
|
|
179
|
+
| `task py:build` | Build the wheel and sdist (uv build) |
|
|
180
|
+
| `task py:publish` | Publish the distributions (uv publish) |
|
|
181
|
+
| `task py:version-file` | Print the path of the file carrying `__version__` |
|
|
182
|
+
| `task py:set-version VERSION=x` | Write that version into it |
|
|
183
|
+
|
|
184
|
+
`uv.lock` is not versioned here, so `py:sync` always resolves from scratch (`--upgrade`):
|
|
185
|
+
a stale lock lying around on a machine would otherwise make you test and audit versions the
|
|
186
|
+
CI never sees.
|
|
187
|
+
|
|
188
|
+
### Tests, and where the api key goes
|
|
189
|
+
|
|
190
|
+
The unit tests never open a socket: the `recorder` fixture replaces `JevClient.system_one`,
|
|
191
|
+
so the whole suite runs without a key and without a server. They check what the processor
|
|
192
|
+
*sends* (the questions built, `$labels` substituted, one call for every label) and how it
|
|
193
|
+
*reads back* the typed answers — not that Jev answers well. That last part is the job of the
|
|
194
|
+
single `integration` test.
|
|
195
|
+
|
|
196
|
+
Keys for that one live in `tests/.env`, which `.gitignore` keeps out of git
|
|
197
|
+
(`pytest-dotenv` loads it, same convention as `pyprocessors_openai_completion`):
|
|
198
|
+
|
|
199
|
+
```dotenv
|
|
200
|
+
# tests/.env
|
|
201
|
+
JEV_API_KEY=sk-...
|
|
202
|
+
# or, for a self-hosted server:
|
|
203
|
+
OPENJEV_API_BASE=http://127.0.0.1:8791
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
A key sitting there does **not** make `task py:test` hit the network: `addopts` carries
|
|
207
|
+
`-m 'not integration'`, so the default run stays hermetic and the live test is asked for
|
|
208
|
+
explicitly (the `-m` of the command line wins over `addopts`):
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
task py:test-marker -- integration
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
Without a key and without `OPENJEV_API_BASE`, that command skips instead of failing.
|
|
215
|
+
|
|
216
|
+
### Measuring a real label set
|
|
217
|
+
|
|
218
|
+
`tests/eval/` holds an evaluation harness for the Cairn question classifier: a frozen
|
|
219
|
+
dev/holdout split over 574 manually labelled questions, five label-description variants
|
|
220
|
+
with what each one scored, paired McNemar comparison, calibration and coverage curves, and
|
|
221
|
+
the saved model outputs so the numbers can be rechecked without spending tokens. It is not
|
|
222
|
+
part of the test suite — no file there is named `test_*`, so `task py:test` ignores it.
|
|
223
|
+
See `tests/eval/README.md`.
|
|
224
|
+
|
|
225
|
+
`tests/test_cairn_routing.py` pins the delivery configuration of that project — twelve real
|
|
226
|
+
questions of the corpus, the probabilities the API actually answered for them, and what the
|
|
227
|
+
processor must make of them at `threshold=0.25`. It runs offline, like the rest of the suite.
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# pyprocessors-jev
|
|
2
|
+
|
|
3
|
+
Processor based on the [Jev](https://typesafe.ai) *System One* API: it asks typed questions about a
|
|
4
|
+
document and gets back **calibrated probabilities** instead of text, so nothing has to be parsed out
|
|
5
|
+
of a completion.
|
|
6
|
+
|
|
7
|
+
Two providers, one protocol (`POST {base_url}/v1/systemone`):
|
|
8
|
+
|
|
9
|
+
| entry point | provider | base url | model | api key |
|
|
10
|
+
|-------------|----------|----------|-------|---------|
|
|
11
|
+
| `jev` | hosted TypeSafe | `JEV_API_BASE`, else `TYPESAFE_BASE_URL`, else `https://api.typesafe.ai` | `JEV_MODEL`, else `jev-latest` | `JEV_API_KEY`, else `TYPESAFE_API_KEY` |
|
|
12
|
+
| `openjev` | self-hosted [Open-Jev](https://github.com/Zefan-Cai/Open-Jev) | `OPENJEV_API_BASE`, else `http://127.0.0.1:8791` | `OPENJEV_MODEL`, else `open-jev` | `OPENJEV_API_KEY` (usually none) |
|
|
13
|
+
|
|
14
|
+
## What it can and cannot do
|
|
15
|
+
|
|
16
|
+
Jev *chooses* and *rates*; it does not write. That bounds the four outputs:
|
|
17
|
+
|
|
18
|
+
| output | how | not possible |
|
|
19
|
+
|--------|-----|--------------|
|
|
20
|
+
| `categories` | `choice` over the project labels, or one `noul` per label | — |
|
|
21
|
+
| `metadata` | typed values only: `noul` → boolean, `choice` → key of a closed set, `score` → number | free-form extraction (dates, amounts, names) — use `pyprocessors_openai_completion` |
|
|
22
|
+
| `annotations` | filter existing candidate spans | create spans: no offsets come back |
|
|
23
|
+
| `altTexts` | select among the texts already on the document (rerank / judge) | generate a summary or a translation |
|
|
24
|
+
|
|
25
|
+
Because the API answers **many questions in one call**, classifying a document and filling its
|
|
26
|
+
metadata costs one HTTP request — there is no `--- METADATA ---` section to split off.
|
|
27
|
+
|
|
28
|
+
## Usage
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from pymultirole_plugins.v1.schema import Document
|
|
32
|
+
from pyprocessors_jev.jev import JevProcessor, JevParameters
|
|
33
|
+
|
|
34
|
+
processor = JevProcessor()
|
|
35
|
+
parameters = JevParameters(
|
|
36
|
+
labels={"billing": "Payments, invoicing, refunds", "technical": "Bugs, outages, integrations"},
|
|
37
|
+
instructions="Which team should handle this?",
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
docs = processor.process([Document(text="Help! My payouts have been failing for 3 days.")], parameters)
|
|
41
|
+
for cat in docs[0].categories:
|
|
42
|
+
print(cat.labelName, cat.score, cat.properties)
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Options
|
|
46
|
+
|
|
47
|
+
| Option | Default | Description |
|
|
48
|
+
|--------|---------|-------------|
|
|
49
|
+
| `base_url` | provider default (see above) | Jev endpoint base url |
|
|
50
|
+
| `model` | provider default (see above) | model route |
|
|
51
|
+
| `function` | `add_categories` | question built from `labels`: `add_categories` (one `choice`), `add_multilabel_categories` (one `noul` per label, same call), `filter_annotations` (one `noul` per annotation, offsets preserved), `select_altText` (`choice` over the alternative texts). **Ignored as soon as `questions` is defined.** |
|
|
52
|
+
| `instructions` | `Choose the best category for this text.` | instructions of the question built from `function` |
|
|
53
|
+
| `labels` | – | label name → description mapping, injected from the project label set. The description is what Jev reads to decide; the categories written on the document carry **`labelName` only**, never `label` — a description has no place on the document, and the project label set already holds the display name. |
|
|
54
|
+
| `questions` | – | the general case: one JSON question per key, each declaring `type`, `instructions`, `criteria` and `target` (`categories`, `metadata`, `altTexts`). A `criteria` of `$labels` is replaced by `labels`. Takes precedence over `function`. |
|
|
55
|
+
| `state_altText` | – | send that alternative text as the state instead of the document text; falls back on the text with a warning when it is missing |
|
|
56
|
+
| `decision_altText` | – | keep questions, answers and token usage in that alternative text — the audit trail |
|
|
57
|
+
| `selected_altText` | `selected` | alternative text receiving the winner of `select_altText` |
|
|
58
|
+
| `threshold` | `0.5` | probability below which an answer is dropped (a category, or an annotation kept by `filter_annotations`). Lower it below the winner-takes-all point and a `choice` becomes a multilabel decision: every label above the bar becomes a category, sorted by decreasing probability |
|
|
59
|
+
| `keep_best` | `false` | when no answer of a `choice` reaches `threshold`, keep the most probable one anyway, so the document never comes back without a category. A no-op with 3 labels (the winner is mechanically above 1/3), a safety net with a large label set |
|
|
60
|
+
| `max_chars` | `0` | truncate the state, `0` sends it whole |
|
|
61
|
+
| `timeout` | `60.0` | HTTP timeout, in seconds |
|
|
62
|
+
| `max_retries` | `3` | retries of a throttled (429) or overloaded (529) call, exponential backoff, obeying `Retry-After` |
|
|
63
|
+
|
|
64
|
+
Answers land in a predictable shape: a category carries the **probability** as its `score` and the
|
|
65
|
+
question `confidence` in its `properties`; a `noul` in metadata writes a boolean under the question
|
|
66
|
+
id plus its probability under `<id>_probability`; a `choice` and a `score` write their value plus
|
|
67
|
+
`<id>_confidence`. A question whose criteria are unknown labels, or an answer naming a label outside
|
|
68
|
+
`labels`, is dropped with a warning rather than invented.
|
|
69
|
+
|
|
70
|
+
The official `typesafe_sdk` is deliberately not a dependency: it cannot talk to an Open-Jev server,
|
|
71
|
+
which would mean two code paths for one protocol.
|
|
72
|
+
|
|
73
|
+
## Development
|
|
74
|
+
|
|
75
|
+
The build is driven by [Task](https://taskfile.dev) and [uv](https://docs.astral.sh/uv/),
|
|
76
|
+
with the shared stages coming from the `python-archetype` submodule.
|
|
77
|
+
|
|
78
|
+
### Getting started
|
|
79
|
+
|
|
80
|
+
The stages live in a Git submodule, so **clone with `--recurse-submodules`**:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
git clone --recurse-submodules git@bitbucket.org:kairntech/pyprocessors_jev.git
|
|
84
|
+
cd pyprocessors_jev
|
|
85
|
+
sh -c "$(curl -sSL https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin
|
|
86
|
+
task
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Already cloned without it? The submodule directory is empty, and `task` fails on:
|
|
90
|
+
|
|
91
|
+
```
|
|
92
|
+
task: No Taskfile found at ".../submodules/python-archetype/resources/Taskfile.yml"
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
which means exactly that, and nothing worse:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
git submodule update --init
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
**Task is the only manual prerequisite.** An archetype cannot bootstrap itself: uv and the
|
|
102
|
+
Python interpreter install themselves on demand (every task that runs uv depends on an
|
|
103
|
+
internal `install-python` task), but the thing that runs them does not. Make sure
|
|
104
|
+
`~/.local/bin` is on your `PATH` — that is where `task` and `uv` both land.
|
|
105
|
+
|
|
106
|
+
### Running the pipeline
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
task stages # print the pipeline stages, in order
|
|
110
|
+
task # run the pipeline up to (but excluding) py:publish
|
|
111
|
+
task -- --skip-tests # same, without the test stage
|
|
112
|
+
task up-to -- py:lint # run the pipeline up to and including one stage
|
|
113
|
+
task jenkins # run every stage, exactly what Jenkins runs
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
`task` with no argument is safe by construction: it runs every stage but the last, and that
|
|
117
|
+
bound is computed from the `STAGES` list rather than written down. The last stage is the
|
|
118
|
+
only one with an effect outside your machine.
|
|
119
|
+
|
|
120
|
+
`STAGES`, declared once in `Taskfile.yml`, is the single definition of the pipeline order —
|
|
121
|
+
so what you run locally is what Jenkins runs.
|
|
122
|
+
|
|
123
|
+
### Individual stages
|
|
124
|
+
|
|
125
|
+
| Task | Description |
|
|
126
|
+
|---------------------------------|-------------------------------------------------------|
|
|
127
|
+
| `task py:sync` | Install the project and its dependencies (uv sync) |
|
|
128
|
+
| `task py:lint` | `ruff check` and `ruff format --check` |
|
|
129
|
+
| `task py:format` | Reformat the code with ruff |
|
|
130
|
+
| `task py:test` | Run the test suite |
|
|
131
|
+
| `task py:test-marker -- <m>` | Run the tests carrying one pytest marker |
|
|
132
|
+
| `task py:sbom` | Generate a CycloneDX SBOM of the resolved environment |
|
|
133
|
+
| `task py:check-vulnerabilities` | Check for known CVEs |
|
|
134
|
+
| `task py:check-updates` | Check for dependency updates |
|
|
135
|
+
| `task py:build` | Build the wheel and sdist (uv build) |
|
|
136
|
+
| `task py:publish` | Publish the distributions (uv publish) |
|
|
137
|
+
| `task py:version-file` | Print the path of the file carrying `__version__` |
|
|
138
|
+
| `task py:set-version VERSION=x` | Write that version into it |
|
|
139
|
+
|
|
140
|
+
`uv.lock` is not versioned here, so `py:sync` always resolves from scratch (`--upgrade`):
|
|
141
|
+
a stale lock lying around on a machine would otherwise make you test and audit versions the
|
|
142
|
+
CI never sees.
|
|
143
|
+
|
|
144
|
+
### Tests, and where the api key goes
|
|
145
|
+
|
|
146
|
+
The unit tests never open a socket: the `recorder` fixture replaces `JevClient.system_one`,
|
|
147
|
+
so the whole suite runs without a key and without a server. They check what the processor
|
|
148
|
+
*sends* (the questions built, `$labels` substituted, one call for every label) and how it
|
|
149
|
+
*reads back* the typed answers — not that Jev answers well. That last part is the job of the
|
|
150
|
+
single `integration` test.
|
|
151
|
+
|
|
152
|
+
Keys for that one live in `tests/.env`, which `.gitignore` keeps out of git
|
|
153
|
+
(`pytest-dotenv` loads it, same convention as `pyprocessors_openai_completion`):
|
|
154
|
+
|
|
155
|
+
```dotenv
|
|
156
|
+
# tests/.env
|
|
157
|
+
JEV_API_KEY=sk-...
|
|
158
|
+
# or, for a self-hosted server:
|
|
159
|
+
OPENJEV_API_BASE=http://127.0.0.1:8791
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
A key sitting there does **not** make `task py:test` hit the network: `addopts` carries
|
|
163
|
+
`-m 'not integration'`, so the default run stays hermetic and the live test is asked for
|
|
164
|
+
explicitly (the `-m` of the command line wins over `addopts`):
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
task py:test-marker -- integration
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Without a key and without `OPENJEV_API_BASE`, that command skips instead of failing.
|
|
171
|
+
|
|
172
|
+
### Measuring a real label set
|
|
173
|
+
|
|
174
|
+
`tests/eval/` holds an evaluation harness for the Cairn question classifier: a frozen
|
|
175
|
+
dev/holdout split over 574 manually labelled questions, five label-description variants
|
|
176
|
+
with what each one scored, paired McNemar comparison, calibration and coverage curves, and
|
|
177
|
+
the saved model outputs so the numbers can be rechecked without spending tokens. It is not
|
|
178
|
+
part of the test suite — no file there is named `test_*`, so `task py:test` ignores it.
|
|
179
|
+
See `tests/eval/README.md`.
|
|
180
|
+
|
|
181
|
+
`tests/test_cairn_routing.py` pins the delivery configuration of that project — twelve real
|
|
182
|
+
questions of the corpus, the probabilities the API actually answered for them, and what the
|
|
183
|
+
processor must make of them at `threshold=0.25`. It runs offline, like the rest of the suite.
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
version: '3'
|
|
2
|
+
|
|
3
|
+
includes:
|
|
4
|
+
# Les tasks Python, sous le namespace 'py:'.
|
|
5
|
+
py:
|
|
6
|
+
taskfile: submodules/python-archetype/resources/Taskfile.yml
|
|
7
|
+
vars:
|
|
8
|
+
# Le module qui porte __version__. Déclaré ICI et non dans le Jenkinsfile, pour
|
|
9
|
+
# que 'task py:set-version' et 'task py:version-file' servent aussi au
|
|
10
|
+
# développeur : le bump de version n'était jusqu'ici reproductible que par la CI.
|
|
11
|
+
#
|
|
12
|
+
# Chemin depuis la racine, et pas nom de module : l'archétype en fait
|
|
13
|
+
# '{{.MODULE}}/__init__.py'. Ce dépôt est en layout src/, comme les autres
|
|
14
|
+
# dépôts pymultirole.
|
|
15
|
+
MODULE: src/pyprocessors_jev
|
|
16
|
+
# uv.lock n'est pas versionné ici (il est dans .gitignore) : sync doit résoudre à
|
|
17
|
+
# neuf, sinon un lock périmé traînant sur un poste fait tester et auditer autre
|
|
18
|
+
# chose que ce que produit la CI.
|
|
19
|
+
LOCK_VERSIONED: "false"
|
|
20
|
+
# Le lanceur de pipeline, à plat, pour que 'task', 'task up-to' et
|
|
21
|
+
# 'task jenkins' soient au premier niveau.
|
|
22
|
+
_:
|
|
23
|
+
taskfile: submodules/python-archetype/resources/Stages.yml
|
|
24
|
+
flatten: true
|
|
25
|
+
vars:
|
|
26
|
+
# La seule définition de l'ordre du pipeline. pythonPipeline joue ces stages, plus
|
|
27
|
+
# les siens qui n'ont de sens que dans Jenkins (init, version, py:publish), et son
|
|
28
|
+
# checkStageDrift() vérifie que cette liste en est un sous-ensemble ORDONNÉ.
|
|
29
|
+
STAGES: >-
|
|
30
|
+
py:sync py:lint py:test
|
|
31
|
+
py:sbom py:check-vulnerabilities
|
|
32
|
+
py:build py:publish
|
|
33
|
+
TEST_STAGE: py:test
|