ellf-cli 21.0.5__tar.gz → 21.0.7__tar.gz

{ellf_cli-21.0.5/ellf_cli.egg-info → ellf_cli-21.0.7}/PKG-INFO

@@ -1,18 +1,18 @@
 Metadata-Version: 2.4
 Name: ellf-cli
-Version: 21.0.5
+Version: 21.0.7
 Summary: Ellf Command Line Interface
 Home-page: https://prodi.gy
 Author: ExplosionAI GmbH
 Author-email: contact@explosion.ai
 License: All rights reserved
 License-File: LICENSE
+Requires-Dist: certifi>=2026.10.19
 Requires-Dist: cryptography>=35.0.0
 Requires-Dist: ellf-broker-sdk
 Requires-Dist: ellf-pam-sdk
 Requires-Dist: email-validator
 Requires-Dist: httpx
-Requires-Dist: httpx
 Requires-Dist: keyring>=24.0.0
 Requires-Dist: packaging
 Requires-Dist: pkginfo>=1.7

{ellf_cli-21.0.5 → ellf_cli-21.0.7}/ellf_cli/auth.py

@@ -673,14 +673,20 @@ class AuthStateImpl:
         last_check = 0.0
         while time.monotonic() < deadline:
             if time.monotonic() > (last_check + int(interval)):
-                token_response = httpx.post(
-                    token_url,
-                    json={
-                        "grant_type": OIDC_DEVICE_GRANT_TYPE,
-                        "device_code": device_code,
-                    },
-                    timeout=AUTH_REQUEST_TIMEOUT_SECONDS,
-                )
+                try:
+                    token_response = httpx.post(
+                        token_url,
+                        json={
+                            "grant_type": OIDC_DEVICE_GRANT_TYPE,
+                            "device_code": device_code,
+                        },
+                        timeout=AUTH_REQUEST_TIMEOUT_SECONDS,
+                    )
+                except httpx.TransportError:
+                    logger.debug("Device token poll failed; retrying", exc_info=True)
+                    last_check = time.monotonic()
+                    time.sleep(1)
+                    continue
                 last_check = time.monotonic()
                 token_data = self._safe_json(token_response, token_url)
                 if token_response.status_code == 200:

{ellf_cli-21.0.5 → ellf_cli-21.0.7}/ellf_cli/ellf.json

@@ -1,7 +1,7 @@
 {
   "prog": "ellf",
   "help": "Ellf Command Line Interface.",
-  "version": "21.0.5",
+  "version": "21.0.7",
   "extra_key": "_extra",
   "commands": {
     "actions": {

{ellf_cli-21.0.5 → ellf_cli-21.0.7}/ellf_cli/ellf_skills/skills/ellf-annotate/SKILL.coding.md

@@ -1,6 +1,6 @@
 ---
 name: ellf-annotate
-description: "Set up and launch annotation from the coding environment once the methodology, schema, and data are ready. Use when the user wants to start annotation on the Ellf cluster, run a local Prodigy annotation server, choose between built-in Ellf and built-in Prodigy recipes, verify that annotation is ready to launch, preview the annotation setup, or determine whether a new custom recipe is needed. Also trigger on \"I want to annotate\", \"start a local Prodigy server\", \"launch annotation on the cluster\", \"which recipe should I use\", \"can I run this locally\", \"set up review\", or \"help me run annotation\". This skill is the annotation readiness gatekeeper: it applies the audit checklist before launch, prefers built-in workflows when possible, and delegates to `/ellf-prodigy` when a new custom recipe must be implemented."
+description: "Set up and launch annotation from the coding environment once the methodology, schema, and data are ready. Use when the user wants to start annotation on the Ellf cluster, run a local Prodigy annotation server, choose between built-in Ellf and built-in Prodigy recipes, verify that annotation is ready to launch, preview the annotation setup, or determine whether a new custom recipe is needed. Also trigger on \"I want to annotate\", \"start a local Prodigy server\", \"launch annotation on the cluster\", \"which recipe should I use\", \"can I run this locally\", \"set up review\", \"settle disagreements\", \"make the annotation UI easier to use\", or \"help me run annotation\". This skill is the annotation readiness gatekeeper: it applies the audit checklist before launch, prefers built-in workflows when possible, and delegates to `/ellf-prodigy` when a new custom recipe must be implemented."
 argument-hint: "[describe what you want to annotate]"
 allowed-tools: Bash(ellf info defaults:*), Bash(echo:*)
 ---
@@ -24,6 +24,7 @@ You are responsible for:
 - deciding whether an existing custom recipe is sufficient
 - deciding when a new custom recipe is required
 - determining where the user wants annotation to run
+- making the interface work for annotators: checking it with the real data and suggesting layout fixes before anyone labels
 - launching the annotation process
 - verifying that it actually starts and is reachable
 
@@ -43,6 +44,7 @@ Do not launch until you have confirmed:
 - the recipe choice actually matches the task
 - the input data is ready
 - the output dataset or storage target is clear
+- annotators can read the longest examples and reach every option (see "Annotator experience" below)
 
 If the audit fails:
 - route methodological problems to `/ellf-project`
@@ -64,7 +66,13 @@ Use the bundled references as the default selection catalog:
 Key selection rules:
 - prefer `.correct` when the user has existing model or LLM predictions to correct; prefer `.manual` when starting from scratch
 - prefer `.teach` for active learning when labels are a good fit for binary annotation
-- keep `review` strictly for adjudicating existing annotations across datasets or annotators — not as a substitute for `.correct`
+- use `review` to settle disagreements between annotators or datasets (see "Settling disagreements" below), not as a substitute for `.correct`
+
+### Settling disagreements
+
+When two annotators, or two passes, labelled the same examples and the user needs one answer per example, that is what `review` is for. Give it the datasets to compare: it groups each example's answers, shows each annotator's answer, and saves the settled one to its own dataset. With auto-accept on, the examples everyone agreed on are saved without being shown, so the reviewer only sees the conflicts. Check `ellf tasks create review --help` for the exact args. It only reads structured datasets and takes no annotation instructions, so write the settling rule in the plan and tell the reviewer.
+
+A plain labelling task over the disputed examples, with no earlier answers shown, is something else: a third blind answer, not a settlement. It avoids anchoring on the earlier answers, but the reviewer can't see what the disagreement was. If the user wants that, say what it costs and let them choose; default to `review`.
 
 ### Existing custom recipe
 
@@ -119,6 +127,15 @@ ellf-dev run <recipe> [args]
 
 For trusted built-in recipes, skip local validation. For custom recipes or anything recently changed, run at least the form preview before cluster launch.
 
+## Annotator experience
+
+People label faster and more consistently when the interface fits the data. Don't wait for the user to say the layout is awkward: check it yourself before launch and suggest fixes.
+
+- Look at it with real data, the longest examples included. For a built-in recipe on the cluster, create the first task, open its annotation URL and look (take a screenshot if you can), or ask the user to open it and say what's awkward. For a custom recipe, use `ellf-dev run`.
+- Go through "Annotator experience" in `${SKILL_DIR}/references/annotation_audit.md` and propose concrete changes: CSS through the task's Prodigy config (`global_css`), flagging (`show_flag`), keyboard shortcuts, a different interface, or splitting the question.
+- A task's settings can't be changed after it's created, so agree the layout on the first task before creating the rest. Keep the agreed Prodigy config in one file in the project and pass it to every task over the same kind of data, rather than retyping it.
+- When a task restates rules that already exist (guidelines, a label scheme), copy the wording or link to it. Shortened or paraphrased rules change the answers people give.
+
 ## Launch
 
 ### 1. Confirm unresolved values
@@ -153,6 +170,8 @@ or `prodigy <recipe> ...` if that is the working command in the environment.
 
 **Other environment:** clarify the execution model before proceeding.
 
+If this kind of task will be created again (each round, batch or week), put the steps in a script in the project: building the input, publishing it, creating the task with the shared config, assigning annotators. Run it from there and name it in the plan, so the next round doesn't depend on commands someone has to rewrite.
+
 ## Verification after launch
 
 A launch is not complete until you verify that the annotation process is usable.
@@ -196,7 +215,7 @@ When you finish:
 
 | File | What it covers | When to read |
 |------|---------------|--------------|
-| `${SKILL_DIR}/references/annotation_audit.md` | Readiness checklist: objective, schema, recipe fit, data, dataset | Before every launch |
+| `${SKILL_DIR}/references/annotation_audit.md` | Readiness checklist: objective, schema, recipe fit, settling disagreements, annotator experience, data, dataset | Before every launch |
 | `${SKILL_DIR}/references/builtin_ellf_annotation_recipes.md` | Built-in Ellf task and agent recipes with supported workflows | Recipe selection for cluster |
 | `${SKILL_DIR}/references/builtin_prodigy_recipes.md` | Built-in standalone Prodigy recipes | Recipe selection for local |
 | `${SKILL_DIR}/references/label_schemes.md` | Label scheme (`labels` asset) format, design and use by tasks and agents. Built-in `ner`, `spans` and `textcat` only | Labels need definitions, building them from annotated data, or writing guidelines |

{ellf_cli-21.0.5 → ellf_cli-21.0.7}/ellf_cli/ellf_skills/skills/ellf-annotate/references/annotation_audit.coding.md

@@ -29,12 +29,17 @@ over time, it's business logic — factor it out.
 
 ### Label set size
 
-More than 6-8 labels per pass creates cognitive overload. Annotator consistency
-drops, inter-annotator agreement falls, and the model receives noisier signal.
-
-**If > 6 labels**: recommend multi-pass (labels-first iteration). Each pass
-focuses on 1-3 related labels. Use model predictions to pre-annotate subsequent
-passes.
+For spans and entities, more than 6-8 labels per pass creates cognitive
+overload. Annotator consistency drops, inter-annotator agreement falls, and the
+model receives noisier signal. Recommend multi-pass (labels-first iteration):
+each pass focuses on 1-3 related labels, and model predictions pre-annotate
+the later passes.
+
+Picking one category per text is easier: annotators compare whole options
+rather than scanning the text for spans. A longer list can work in one pass if
+every label has a description and all the options fit on screen. Watch the
+pilot's flags and agreement: if they show confusion between labels, group them
+or split the question.
 
 ### Label definitions
 
@@ -61,11 +66,14 @@ rules, it should be.
 | Red flag | What to do |
 |---|---|
 | Labels encode business logic | Escalate to ellf-project |
-| > 8 labels in one pass | Recommend multi-pass, or escalate if labels need redesign |
+| > 8 span or entity labels in one pass | Recommend multi-pass, or escalate if labels need redesign |
 | Composite labels (COMPENSATED_EXECUTIVE) | Factor into components (PERSON + COMPENSATION + rules) |
 | Labels require external knowledge | Factor out — model learns language patterns only |
 | Annotation mirrors output schema | Redesign — annotation schema ≠ output schema |
 | Labels without definitions, or definitions only in someone's head | Write a label scheme before launch |
+| Task instructions paraphrase existing guidelines | Copy the wording or link to it |
+| Disagreements settled in a new blind labelling task | Use `review`, or tell the user it's a third blind answer |
+| Long texts or many options left in the default layout | Check the interface with real data and fix it before launch |
 
 ---
 
@@ -95,14 +103,20 @@ the situation is non-standard or the user needs deeper discussion.
 ### Single vs multi-pass
 
 - **≤ 5 labels**: single pass is fine
-- **6-8 labels**: consider multi-pass, recommend if annotators report difficulty
-- **> 8 labels**: multi-pass required (labels-first iteration)
+- **6-8 span or entity labels**: consider multi-pass, recommend if annotators report difficulty
+- **> 8 span or entity labels**: multi-pass required (labels-first iteration)
+- **One category per text**: see "Label set size" above
 
 ### Quality control
 
-- **Always**: validation callbacks for schema constraints
+- **Always**: let the interface enforce schema constraints (e.g. exclusive
+  choice for one label per text); a custom recipe can add a `validate_answer`
+  callback for the rest
+- **Pilots and reference sets**: flagging on, so annotators can mark the
+  examples they hesitated on
 - **Recommended**: gold-standard examples for onboarding new annotators
-- **For teams**: review queue for disagreements
+- **For teams**: settle disagreements with `review` (see "Settling
+  disagreements" in the skill)
 
 ---
 
@@ -118,12 +132,46 @@ These catch cases where a different recipe would be significantly more efficient
 | User picks `spans.manual` but has a trained spans model | Suggest `spans.correct` |
 | Large dataset, no model assistance | Suggest active learning recipe (`.teach` variants) |
 | Complex UI needs (custom blocks, multiple views) | Delegate to ellf-prodigy for custom recipe |
+| Two annotators' datasets need one answer per example | `review` over both datasets, with auto-accept so only conflicts are shown |
+| Disputed examples put in a new `textcat` or `ner` task to settle | `review`, unless the user wants a blind third answer |
+
+---
+
+## Annotator experience
+
+Check the interface with real data before launch, the longest examples
+included, and propose a fix for anything that fails:
+
+- **Text is readable.** Long texts are left-aligned at a normal reading size
+  and scroll inside the card if needed, rather than pushing the options or
+  buttons off screen. Prose shouldn't be centred, the default for short texts.
+- **Every option is visible without scrolling.** Many options can go in a
+  grid. Keyboard shortcuts work.
+- **Labels explain themselves.** With a label scheme, annotators see each
+  label's description when they hover it (built-in `ner`, `spans`, `textcat`).
+- **Annotators can say they're unsure.** Flagging is on for pilots and
+  reference sets. When one click submits, the flag has to come first: say so
+  in the instructions.
+- **Submitting is deliberate.** Submitting on a single click saves time per
+  example, but there's no second look; make sure annotators know.
+- **An example can be discussed.** An ID or source shows in the meta, so
+  annotators and the user can point to one.
+- **Instructions are short and on the task** (H key), and repeat existing
+  rules word for word.
+- **Nothing shown that shouldn't be.** Hide model suggestions and other
+  annotators' answers when the labels are a reference or a blind check; show
+  them when correcting is the point.
+
+Most fixes go in the task's Prodigy config: CSS in `global_css`, `show_flag`,
+keyboard settings. Agree the layout with the user on the first task, keep the
+config in one file in the project, and reuse it for every task over the same
+kind of data.
 
 ---
 
 ## Escalation Triggers
 
-Escalate to **ellf-project** (via Skill tool) when:
+Escalate to **ellf-project** (`/ellf-project`) when:
 
 - Labels encode business logic and need redesign
 - Task decomposition is needed (complex task → simpler sub-tasks)

{ellf_cli-21.0.5 → ellf_cli-21.0.7}/ellf_cli/ellf_skills/skills/ellf-annotate/references/builtin_ellf_annotation_recipes.coding.md

@@ -159,11 +159,33 @@ that fit the task type. Their label settings add:
 `hf_model_agent` runs a fine-tuned model and uses neither instructions nor label
 definitions.
 
+## Jev: confident answers by machine, the rest by people
+
+If the cluster lists the `textcat_jev` action, it labels an input asset for text
+classification with TypeSafe's Jev and saves only the answers at or above
+`--min-confidence` (default 0.9). It's an action, not an agent: it runs once over the
+input rather than taking tasks from a queue. Pair it with a `textcat` task over the same
+input and labels that has the Jev dataset in `--settings.exclude`: annotators then only
+see the texts Jev was unsure about.
+
+```bash
+ellf actions create textcat_jev --input <asset> --label-scheme <labels_asset> \
+  --labels-exclusive --instructions "<what the labels are for>" \
+  --output <jev_dataset> --api-key <secret>
+ellf tasks create textcat --input <asset> --label-scheme <labels_asset> \
+  --labels-exclusive --settings.exclude <jev_dataset> ...
+```
+
+It needs a TypeSafe API key as an Ellf secret, and sends the texts to TypeSafe. Before
+trusting its threshold, evaluate it on a human-labelled set with `--min-confidence 0`:
+`${SKILL_DIR}/../ellf-project/references/jev.md`.
+
 ## Selection rules
 
 - Prefer a built-in task recipe whenever it fits without bending the schema.
-- Use `review` when the user is correcting, adjudicating, or reviewing existing
-  annotations rather than creating annotations from scratch.
+- Use `review` to settle disagreements or review existing annotations across
+  annotators or datasets, rather than to create annotations from scratch or to
+  correct a model's suggestions.
 - If the user needs a custom interface, multi-view workflow, unusual routing,
   or annotation logic the built-ins cannot express cleanly, delegate to
   `ellf-prodigy`.

{ellf_cli-21.0.5 → ellf_cli-21.0.7}/ellf_cli/ellf_skills/skills/ellf-annotate/references/label_schemes.coding.md

@@ -117,7 +117,8 @@ The export is structured JSONL, one row per answer: the text is in `input.text`,
 the annotation in `user_anns` (`user_anns.spans` for spans, `user_anns.label` for
 text classification, `user_anns.accept` for choice; a key missing there comes from
 `server_anns`, the suggestion as served), the answer in `answer`, and the flag in
-`extras.flagged`. Rows don't record who annotated them.
+`extras.flagged`, a key that is only present on flagged rows. Who answered and when is in
+`extras._annotator_id`, `extras._session_id` and `extras._timestamp`.
 
 Derive the scheme from the exported JSONL:
 - **Labels and order:** the labels used in the data, most frequent first.

{ellf_cli-21.0.5 → ellf_cli-21.0.7}/ellf_cli/ellf_skills/skills/ellf-ops/SKILL.coding.md

@@ -25,7 +25,7 @@ You execute ready operations. You do not:
 - implement new recipe code
 
 If the request is not operationally ready:
-- use `/ellf-annotate` for annotation readiness and recipe choice
+- use `/ellf-annotate` for annotation readiness and recipe choice, and before creating a labelling task people will work in (recipe, settling disagreements, interface), unless it repeats one that already passed
 - use `/ellf-train` for training strategy
 - use `/ellf-project` for planning
 - use `/ellf-prodigy` for custom recipe implementation

{ellf_cli-21.0.5 → ellf_cli-21.0.7}/ellf_cli/ellf_skills/skills/ellf-ops/references.coding/data_infra_cli.md

@@ -28,6 +28,13 @@ ellf datasets delete "<name>"
 ellf datasets export "<name>" --output <path>   # exports as JSONL
 ```
 
+An exported row is structured, not flat Prodigy JSON: the text is in `input.text`, the
+annotation in `user_anns` (`spans`, `label` or `accept`; a missing key falls back to
+`server_anns`), the answer (`accept`, `reject`, `ignore`) in `answer`, and the
+annotator, session and time in `extras._annotator_id`, `extras._session_id` and
+`extras._timestamp`. Flags are in `extras.flagged`, present only when set, so look
+there before reporting that nothing was flagged.
+
 ## Files
 
 Manage files on the cluster's storage bucket directly.

{ellf_cli-21.0.5 → ellf_cli-21.0.7}/ellf_cli/ellf_skills/skills/ellf-project/SKILL.coding.md

@@ -65,6 +65,8 @@ For long-lived sub-topics, prefer creating a separate named plan over appending
 
 If no plan exists yet, create one as soon as the problem statement is clear.
 
+**Script what will run again.** When a step will be repeated (every round, batch or week), write it as a script in the project's `scripts/`, run it from there, and name it in the plan. One-off lookups can stay inline. A routine in the plan should point to scripts, not to commands someone has to rewrite.
+
 ### Linking to Ellf concepts
 
 Project plans are rendered as Markdown, and references to real Ellf concepts — tasks, actions, agents, assets, datasets, recipes, packages, and other plans — display as **inline concept links**: chips with the concept's icon (and a live status indicator for jobs) that link straight to the concept's page in the app. Use them wherever the plan names a concept that already exists, so the plan stays a navigable map of the project rather than a flat document.
@@ -179,7 +181,9 @@ Read this additional reference when relevant:
 - `${SKILL_DIR}/references/annotation_agents.md`
   Use when the user has limited annotation capacity, wants LLM-assisted annotation, or is considering agent-heavy data development.
 - `${SKILL_DIR}/references/label_schemes.md`
-  Use when designing labels, turning an agreed label set into a `labels` asset, building a scheme from annotated data or a pilot, or planning guidelines for labels that already exist.
+  Use when designing labels, turning an agreed label set into a `labels` asset, building a scheme from annotated data or a pilot, reviewing what a pilot's labels and flags say about the scheme, or planning guidelines for labels that already exist. It also describes what an exported dataset row looks like.
+- `${SKILL_DIR}/references/jev.md`
+  Use when the user mentions Jev or TypeSafe, or the cluster lists `textcat_jev` and the job is text classification: what the action does, when it fits, and what's specific to evaluating it.
 
 These references are your strategic foundation. Apply them naturally. Do not cite them as documents unless the user explicitly asks.
 
@@ -214,6 +218,8 @@ Use Explosion knowledge base (KB) via the `query_knowledge_base` tool when you n
 Use KB results to ground your advice. When you find a relevant case study or best practice, weave it naturally into your recommendation.
 Never dump raw KB output.
 
+When the user names a model, vendor or tool you don't recognise, check `ellf recipes list` before saying you don't know it: Ellf may integrate it as a built-in recipe. Read the recipe's description and arguments (`ellf actions create <recipe> --help`, or `tasks` / `agents` for those kinds) and advise from that.
+
 ## Plan lifecycle
 
 ### Starting a plan
@@ -330,6 +336,13 @@ If something relevant surfaces, mention it briefly — it builds credibility and
 calibrates the conversation: "This is similar to a skills extraction project
 where the key was to separate entity recognition from taxonomy mapping."
 
+**Weigh other framings before recommending one.** The way the user phrased the
+problem is one framing among several. Lay out two or three, and for each say
+which business outcome it would move and what it would need (labels, data,
+people). If the data argues for a different framing, say so. Then pick one, say
+why, and record the rejected ones
+in the decision log. Keep this short; it's a paragraph, not a survey.
+
 **Form a rough pipeline shape early.** Sketch which parts are rule-based, which
 need ML, which might use an LLM. Share this sketch — even if it's wrong, it
 gives the user something concrete to react to. This is more valuable than
@@ -431,10 +444,23 @@ evaluation. Apply all of it. Don't let a project plan leave this section empty.
 **How to run this part of the conversation:**
 
 - Make sure every component in the pipeline sketch has an evaluation plan.
+- Name the business outcome the system should move (time saved, cost, errors
+  that reach customers) and how it will be measured once it runs. Accuracy on
+  the test set is how you choose a system, not how you know it helped.
 - Push for a held-out test set to be set aside before training starts.
 - Insist on a rule-based baseline before evaluating any model.
 - For LLM/generative components: recommend A/B evaluation. Consult the KB for
   Prodigy-specific workflows.
+- For a classifier that answers with a confidence and leaves the texts it's
+  unsure about to people (a spaCy `textcat` model's scores, an LLM's
+  probabilities, Jev), the threshold is part of the system:
+  - run it on the evaluation set with no threshold, so every answer and its
+    confidence is kept;
+  - choose the threshold from how accuracy and the share of texts it answers
+    change across thresholds, against a bar agreed beforehand, not from one
+    accuracy number;
+  - record the model version, the threshold and the curve it was chosen from
+    in the plan, and pin the version if results have to be reproducible.
 
 ### 7. Plan Deployment
 

ellf_cli-21.0.7/ellf_cli/ellf_skills/skills/ellf-project/references/jev.coding.md

@@ -0,0 +1,66 @@
+# Jev — Validating It on the User's Data
+
+## What It Is
+
+Jev is TypeSafe's "System One" model. It doesn't generate text: it answers typed
+questions about a text with calibrated probabilities. On Ellf it runs as the built-in
+`textcat_jev` action ("Classify text with Jev"), which labels an input asset for text
+classification and keeps only the answers Jev is confident about. Check that the
+cluster lists it (`ellf recipes list`) before planning around it.
+
+What the action does:
+- **Modes** follow the labels: one label is a yes/no question; several exclusive labels
+  (`--labels-exclusive`) are one choice question with at most 255 options; several
+  non-exclusive labels are one yes/no question per label, asked in the same request.
+- **Label definitions** are sent as the questions' criteria. Pass the labels as a
+  `labels` asset (`--label-scheme`) so Jev gets each label's description, look-alikes
+  and examples, not just the names. `--instructions` adds context about what the
+  labels are for ("Decide whether the review reports a product defect").
+- **Confidence**: each answer comes with a confidence. For non-exclusive labels it's
+  the confidence of the label Jev is least sure about. The answer, the confidence and
+  every label's probability are saved in the example's `meta.jev`, along with the
+  model that answered.
+- **Routing to people**: answers below `--min-confidence` (default 0.9) aren't saved.
+  A `textcat` task over the same input and labels with the Jev dataset in
+  `--settings.exclude` then shows annotators only the texts Jev was unsure about.
+  The action builds its examples like that task does, so the hashes match.
+- **Reruns** skip texts already in the output dataset, so a rerun after a failure
+  only asks about the rest.
+- **Privacy**: the texts are sent to TypeSafe's API. Raise this before the user sends
+  anything sensitive.
+- **Setup**: a TypeSafe API key stored as an Ellf secret (`--api-key`). A bad key, no
+  credits or an unknown model stop the action with the reason.
+
+Don't add claims about Jev beyond this — pricing, benchmark numbers, company news.
+If the user asks, say so, and make the case that their own evaluation set answers
+the question that matters.
+
+## When It Fits
+
+- The job is text classification: routing, triage, tagging, filtering.
+- The user wants a share of texts handled automatically and the rest left to people,
+  rather than one model that has to be right about everything.
+- The labels can be defined in words. Jev reads the definitions; it can't learn a
+  distinction nobody can write down.
+
+It doesn't do span extraction (NER, spans) or generation.
+
+## Evaluating It
+
+Evaluate it like any classifier that answers with a confidence (see the
+evaluation strategy): a human-labelled set Jev never touches, no threshold on the
+evaluation run, and the threshold chosen from the curve. What's specific to the
+action:
+
+- "No threshold" means `--min-confidence 0`, into its own dataset. With the
+  default 0.9, the answers Jev was unsure about are never saved. Each answer's
+  confidence is in `meta.jev`.
+- Pin `--model` to the name Jev reports in `meta.jev` rather than `jev-latest`.
+- Reruns skip texts already in the output dataset, so a run with different labels,
+  definitions or instructions needs a new output dataset.
+
+## In the Plan
+
+Besides what the evaluation strategy records, note the label scheme and
+instructions Jev was given, and keep Jev's answers in their own dataset, separate
+from the human answers.

{ellf_cli-21.0.5 → ellf_cli-21.0.7}/ellf_cli/ellf_skills/skills/ellf-project/references/label_schemes.coding.md

@@ -117,7 +117,8 @@ The export is structured JSONL, one row per answer: the text is in `input.text`,
 the annotation in `user_anns` (`user_anns.spans` for spans, `user_anns.label` for
 text classification, `user_anns.accept` for choice; a key missing there comes from
 `server_anns`, the suggestion as served), the answer in `answer`, and the flag in
-`extras.flagged`. Rows don't record who annotated them.
+`extras.flagged`, a key that is only present on flagged rows. Who answered and when is in
+`extras._annotator_id`, `extras._session_id` and `extras._timestamp`.
 
 Derive the scheme from the exported JSONL:
 - **Labels and order:** the labels used in the data, most frequent first.

{ellf_cli-21.0.5 → ellf_cli-21.0.7}/ellf_cli/pypi_auth.py

@@ -36,6 +36,7 @@ import httpx
 import keyring.backend
 import keyring.credentials
 import keyring.errors
+from packaging.version import InvalidVersion, Version
 
 from .url import URL
 
@@ -52,9 +53,10 @@ REQUEST_TIMEOUT_SECONDS = 15.0
 # so a token that is valid when pip asks doesn't expire mid-download.
 TOKEN_EXPIRY_MARGIN_SECONDS = 60.0
 
-# pip only understands `keyring-provider` in pip.conf from 23.1 on;
-# older pips fail on *every* invocation when the key is present.
-MIN_PIP_VERSION = (23, 1)
+# pip only understands `keyring-provider` in pip.conf from 23.1 on, and
+# pips before 26.2.2 vendor a certifi that trusts the compromised eMudhra
+# intermediates, so the index token they send could be intercepted.
+MIN_PIP_VERSION = Version("26.2.2")
 
 
 def _request_credentials(pam_url: URL, api_token: str) -> httpx.Response:
@@ -271,15 +273,13 @@ def check_venv_supports_keyring(venv_dir: Path) -> Optional[str]:
             "load the ellf keyring backend — install ellf-cli into it and "
             "re-run 'ellf login'"
         )
-    pip_version = res.stdout.strip().split(".")
     try:
-        major_minor = (int(pip_version[0]), int(pip_version[1]))
-    except (ValueError, IndexError):
+        pip_version = Version(res.stdout.strip())
+    except InvalidVersion:
         return f"could not parse pip version {res.stdout.strip()!r}"
-    if major_minor < MIN_PIP_VERSION:
-        needed = ".".join(str(p) for p in MIN_PIP_VERSION)
+    if pip_version < MIN_PIP_VERSION:
         return (
-            f"pip {res.stdout.strip()} is too old for keyring auth; "
-            f"upgrade to pip>={needed} in the virtualenv"
+            f"pip {pip_version} is too old for keyring auth; "
+            f"upgrade to pip>={MIN_PIP_VERSION} in the virtualenv"
         )
     return None

ellf_cli-21.0.7/ellf_cli/version/21.0.5_2026-10-21-09-28-47_patch_fabd5cfe-2d04-4215-abf1-58fba226acb7

@@ -0,0 +1,3 @@
+21.0.5
+patch
+fabd5cfe-2d04-4215-abf1-58fba226acb7

ellf_cli-21.0.7/ellf_cli/version/21.0.6_2026-10-22-16-07-37_patch_d2bb27d7-6304-4c99-abbc-46d3dd7e363b

@@ -0,0 +1,3 @@
+21.0.6
+patch
+d2bb27d7-6304-4c99-abbc-46d3dd7e363b

{ellf_cli-21.0.5 → ellf_cli-21.0.7/ellf_cli.egg-info}/PKG-INFO

@@ -1,18 +1,18 @@
 Metadata-Version: 2.4
 Name: ellf-cli
-Version: 21.0.5
+Version: 21.0.7
 Summary: Ellf Command Line Interface
 Home-page: https://prodi.gy
 Author: ExplosionAI GmbH
 Author-email: contact@explosion.ai
 License: All rights reserved
 License-File: LICENSE
+Requires-Dist: certifi>=2026.10.19
 Requires-Dist: cryptography>=35.0.0
 Requires-Dist: ellf-broker-sdk
 Requires-Dist: ellf-pam-sdk
 Requires-Dist: email-validator
 Requires-Dist: httpx
-Requires-Dist: httpx
 Requires-Dist: keyring>=24.0.0
 Requires-Dist: packaging
 Requires-Dist: pkginfo>=1.7

{ellf_cli-21.0.5 → ellf_cli-21.0.7}/ellf_cli.egg-info/SOURCES.txt

@@ -139,6 +139,7 @@ ellf_cli/ellf_skills/skills/ellf-project/SKILL.coding.md
 ellf_cli/ellf_skills/skills/ellf-project/references/annotation_agents.md
 ellf_cli/ellf_skills/skills/ellf-project/references/consulting_patterns.md
 ellf_cli/ellf_skills/skills/ellf-project/references/explosion_strategy.md
+ellf_cli/ellf_skills/skills/ellf-project/references/jev.coding.md
 ellf_cli/ellf_skills/skills/ellf-project/references/label_schemes.assistant.md
 ellf_cli/ellf_skills/skills/ellf-project/references/label_schemes.coding.md
 ellf_cli/ellf_skills/skills/ellf-support/SKILL.assistant.md
@@ -342,6 +343,8 @@ ellf_cli/version/21.0.1_2026-10-16-16-06-10_patch_e20b4ad2-b87f-478e-b4f4-323c8b
 ellf_cli/version/21.0.2_2026-10-19-13-54-31_patch_bf091df4-00bb-43c8-9117-51047944e305
 ellf_cli/version/21.0.3_2026-10-19-14-58-29_patch_9b985381-1b1d-4531-b242-b02cea2484c5
 ellf_cli/version/21.0.4_2026-10-20-17-08-51_patch_38d02583-cb5c-4adc-b5d7-614b483acf19
+ellf_cli/version/21.0.5_2026-10-21-09-28-47_patch_fabd5cfe-2d04-4215-abf1-58fba226acb7
+ellf_cli/version/21.0.6_2026-10-22-16-07-37_patch_d2bb27d7-6304-4c99-abbc-46d3dd7e363b
 ellf_cli/version/__init__.py
 ellf_cli/version/read_version.py
 tests/test_agents.py

{ellf_cli-21.0.5 → ellf_cli-21.0.7}/ellf_cli.egg-info/requires.txt

@@ -1,9 +1,9 @@
+certifi>=2026.10.19
 cryptography>=35.0.0
 ellf-broker-sdk
 ellf-pam-sdk
 email-validator
 httpx
-httpx
 keyring>=24.0.0
 packaging
 pkginfo>=1.7

{ellf_cli-21.0.5 → ellf_cli-21.0.7}/tests/test_auth.py

@@ -134,7 +134,13 @@ class TestPamHostResolution:
 class TestDeviceLogin:
     """Polling for the browser's approval in ``ellf login``."""
 
-    def _run(self, monkeypatch, polls: Sequence[tuple[int, dict]], *, expires_in=600):
+    def _run(
+        self,
+        monkeypatch,
+        polls: Sequence[tuple[int, dict] | Exception],
+        *,
+        expires_in=600,
+    ):
         import httpx
 
         from ellf_cli import auth as auth_module
@@ -156,7 +162,10 @@ class TestDeviceLogin:
                     "expires_in": expires_in,
                 }
                 return httpx.Response(200, json=body)
-            status, body = next(responses)
+            poll = next(responses)
+            if isinstance(poll, Exception):
+                raise poll
+            status, body = poll
             return httpx.Response(status, json=body)
 
         def fake_sleep(seconds):
@@ -205,3 +214,11 @@ class TestDeviceLogin:
         approved = (200, {"id_token": "id"})
         tokens, _, _ = self._run(monkeypatch, [flaky, approved])
         assert tokens == ("id", None)
+
+    def test_keeps_polling_through_network_errors(self, monkeypatch):
+        import httpx
+
+        dropped = httpx.ReadTimeout("The read operation timed out")
+        approved = (200, {"id_token": "id"})
+        tokens, _, _ = self._run(monkeypatch, [dropped, approved])
+        assert tokens == ("id", None)