pyannotators-entityfishing 1.6.83__tar.gz → 1.6.85__tar.gz

{pyannotators_entityfishing-1.6.83 → pyannotators_entityfishing-1.6.85}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pyannotators-entityfishing
-Version: 1.6.83
+Version: 1.6.85
 Summary: Annotator based on entity-fishing
 Project-URL: Homepage, https://github.com/oterrier/pyannotators_entityfishing/
 Author-email: Olivier Terrier <olivier.terrier@kairntech.com>

{pyannotators_entityfishing-1.6.83 → pyannotators_entityfishing-1.6.85}/src/pyannotators_entityfishing/__init__.py

@@ -1,3 +1,3 @@
 """Annotator based on entity-fishing"""
 
-__version__ = "1.6.83"
+__version__ = "1.6.85"

{pyannotators_entityfishing-1.6.83 → pyannotators_entityfishing-1.6.85}/src/pyannotators_entityfishing/candidates.py

@@ -29,6 +29,9 @@ WIKIDATA = "wikidata"
 # Written by pyprocessors_coreference (ADR-0001, §5), when that step ran before this one.
 ENTITY_GROUP = "entity_group"
 CANONICAL_FORM = "canonical_form"
+# On an ambiguous partial name ("Antonelli" ending both "Kimi Antonelli" and another full
+# name): the groups it may belong to. Its candidates are then the union of theirs and its own.
+ENTITY_GROUP_CANDIDATES = "entity_group_candidates"
 
 
 class EntityFishingCandidatesParameters(ProcessorParameters):
@@ -122,15 +125,33 @@ def add_candidates(
         senses = {form: [s for s in ss if not 0 < s.get("prob_c", 0.0) < min_prior] for form, ss in senses.items()}
         chosen = {key: next((senses[f] for f in fs if senses.get(f)), [])[:top_k] for key, fs in forms.items()}
         concepts = client.get_kb_concepts_by_page({s["pageid"] for ss in chosen.values() for s in ss}, lang)
+        terms_of = {key: candidate_terms(chosen[key], concepts, queries.get(key[0]), wiki_props) for key in groups}
+        by_group_id = {key[2]: key for key in groups if key[1] == "group"}
         for key, members in groups.items():
-            label = key[0]
-            terms = candidate_terms(chosen[key], concepts, queries.get(label), wiki_props)
+            terms = terms_of[key]
+            possible = [by_group_id[g] for g in (members[0].properties or {}).get(ENTITY_GROUP_CANDIDATES) or [] if g in by_group_id]
+            if possible:
+                # An ambiguous partial name: the decision step chooses among the candidates of
+                # every group it may belong to, the full names first, and its own.
+                terms = round_robin([terms_of[k] for k in possible] + [terms], top_k)
             for a in members:
                 # Replace the Wikidata candidates, keep the terms of any other lexicon (AFP...).
                 kept = [t for t in (a.terms or []) if t.lexicon != WIKIDATA]
                 a.terms = kept + [t.model_copy(deep=True) for t in terms]
 
 
+def round_robin(lists: list[list[Term]], top_k: int) -> list[Term]:
+    """The first of each list, then the second of each... without repeating a QID, up to
+    top_k: every possible group keeps its best candidates."""
+    out, seen = [], set()
+    for rank in range(max((len(ts) for ts in lists), default=0)):
+        for ts in lists:
+            if rank < len(ts) and ts[rank].identifier not in seen and len(out) < top_k:
+                seen.add(ts[rank].identifier)
+                out.append(ts[rank])
+    return out
+
+
 def group_mentions(document: Document, labels: list[str] | None) -> dict[tuple, list[Annotation]]:
     """The mentions of the same entity: by `entity_group` when the grouping step ran,
     else by label and normalized surface. Never across labels."""

pyannotators_entityfishing-1.6.85/tools/build_rerank_data.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Build the fine-tuning set of the reranker from Jev's decisions on AFP dispatches.
+
+ADR-0001 §9 Q7, SHERPA-3054: there are no reference links on AFP, so Jev's confident
+decisions stand in for them. Jev becomes a provisional teacher; the student inherits its
+errors, which is why only confident answers are kept and why the evaluation stays on the
+MSNBC and AQUAINT gold links (tools/eval_rerank.py), never on AFP.
+
+Inputs, kept in .cache/afp-runs/ (gitignored; regenerating them costs ~14 M Jev tokens):
+  fr_pipeline_ef_jev.jsonl         plan pipeline_ef_jev on project afp_ner_fr_2 (5,059 dispatches)
+  fr_docs.json                     the texts of those dispatches
+  xx_pipeline_ef_jev_noner.jsonl   plan pipeline_ef_jev_noner on project afp_ner_xx (641 dispatches, 6 languages)
+  the texts of the latter are read from the multilingual test corpus (--xx-corpus).
+
+One example per decided group: the context of its first mention (same window and [ ]
+marks as the evaluation), the descriptions of its candidates ("name. definition", as the
+evaluation without facts), and the target: the index of Jev's choice, or -1 for NONE.
+
+Kept: answers whose winning probability is at least --min-prob, all NONE and all groups
+with several candidates, and a --single-share sample of the one-candidate groups Jev
+accepted (the easy ones: 26,600 of 41,500). Documents are split into train and
+validation by a hash of their identifier.
+
+    .venv/bin/python tools/build_rerank_data.py            # writes .cache/rerank-train/{train,val}.jsonl
+    .venv/bin/python tools/build_rerank_data.py --runs fr_pipeline_ef_jev.jsonl --out .cache/rerank-train-fr
+        # without the 641 REM test dispatches (the xx run), so that the model can be measured on them
+"""
+
+from __future__ import annotations
+
+import argparse
+import glob
+import hashlib
+import json
+import random
+import sys
+from collections import Counter
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from eval_rerank import mark  # noqa: E402
+
+ROOT = Path(__file__).resolve().parent.parent
+RUNS = ROOT / ".cache" / "afp-runs"
+OUT = ROOT / ".cache" / "rerank-train"
+XX_CORPUS = "/media/olivier/DATA/corpora/AFP/REM/corpus-test-depeches/flair"
+
+
+def texts(xx_corpus: str) -> dict[str, tuple[str, str]]:
+    """Identifier → (language, text) of every dispatch of the two runs."""
+    out = {d["identifier"]: ("fr", d["text"]) for d in json.loads((RUNS / "fr_docs.json").read_text())}
+    for f in glob.glob(f"{xx_corpus}/*.json"):
+        d = json.loads(Path(f).read_text())
+        out[d["identifier"]] = (d["metadata"]["language"], d["text"])
+    return out
+
+
+def examples(run: Path, docs: dict[str, tuple[str, str]], min_prob: float):
+    for line in run.open():
+        r = json.loads(line)
+        j = r["decision"]
+        if not j or r["identifier"] not in docs:
+            continue
+        lang, text = docs[r["identifier"]]
+        for k, q in j["questions"].items():
+            a = j["answers"][k]
+            choice = a["choice"]
+            if a["probabilities"].get(choice, 0.0) < min_prob:
+                continue
+            mention = q["instructions"]["mention"]
+            first = next((m for m in r["annotations"] if m["labelName"] == mention["type"] and m["text"] in mention["surfaces"]), None)
+            if first is None:
+                continue
+            qids = [c for c in q["criteria"] if c != "NONE"]
+            cands = []
+            for qid in qids:
+                c = q["criteria"][qid]
+                name, definition = (c, "") if isinstance(c, str) else (c["name"], c.get("definition", ""))
+                cands.append(f"{name}. {definition}")
+            yield {
+                "doc": r["identifier"],
+                "lang": lang,
+                "type": mention["type"],
+                "context": mark(text, first["start"], first["end"]),
+                "qids": qids,
+                "candidates": cands,
+                "target": qids.index(choice) if choice != "NONE" else -1,
+                "jev_prob": a["probabilities"][choice],
+            }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--xx-corpus", default=XX_CORPUS)
+    parser.add_argument("--min-prob", type=float, default=0.9)
+    parser.add_argument("--single-share", type=float, default=0.2, help="share of the accepted one-candidate groups kept (default 0.2)")
+    parser.add_argument("--val-share", type=float, default=0.05)
+    parser.add_argument("--seed", type=int, default=3054)
+    parser.add_argument("--runs", nargs="+", default=["fr_pipeline_ef_jev.jsonl", "xx_pipeline_ef_jev_noner.jsonl"], help="runs of .cache/afp-runs to draw from, in this order")
+    parser.add_argument("--out", type=Path, default=OUT, help="output directory (default .cache/rerank-train)")
+    args = parser.parse_args()
+    out = args.out
+
+    docs = texts(args.xx_corpus)
+    rng = random.Random(args.seed)
+    out.mkdir(parents=True, exist_ok=True)
+    files = {split: (out / f"{split}.jsonl").open("w") for split in ("train", "val")}
+    stats = Counter()
+    for run in args.runs:
+        for ex in examples(RUNS / run, docs, args.min_prob):
+            kind = "none" if ex["target"] < 0 else ("single" if len(ex["qids"]) == 1 else "multi")
+            if kind == "single" and rng.random() >= args.single_share:
+                continue
+            split = "val" if int(hashlib.md5(ex["doc"].encode()).hexdigest(), 16) % 1000 < args.val_share * 1000 else "train"
+            files[split].write(json.dumps(ex, ensure_ascii=False) + "\n")
+            stats[split, kind] += 1
+            stats[split, "pairs"] += len(ex["qids"])
+            stats[split, ex["lang"]] += 1
+    for f in files.values():
+        f.close()
+    for split in ("train", "val"):
+        kinds = {k: stats[split, k] for k in ("multi", "single", "none")}
+        langs = {k[1]: v for k, v in stats.items() if k[0] == split and k[1] in ("fr", "en", "de", "es", "pt", "ar")}
+        print(f"{split}: {sum(kinds.values())} groups {kinds}, {stats[split, 'pairs']} pairs, by language {langs}")
+    print(f"written to {out}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pyannotators_entityfishing-1.6.85/tools/check_rerank_endpoint.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Check a served reranker against the scores of the evaluation: same model, same numbers?
+
+The `rerank` processor of pyprocessors_jev reads log-odds back from a `/rerank` endpoint
+(vLLM serving tools/merge_rerank.py's output, or LiteLLM in front of it). This sends the
+same (context, candidate) pairs as tools/eval_rerank.py --adapter, and compares the
+log-odds with those it cached. vLLM serves in bf16 where the evaluation ran in fp32, so
+small gaps are expected; the decisions must not move.
+
+    .venv/bin/python tools/check_rerank_endpoint.py --base-url http://<host>:<port>/v1 --docs 5
+"""
+
+from __future__ import annotations
+
+import argparse
+import math
+import sys
+from pathlib import Path
+
+import requests
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import eval_linking as el  # noqa: E402
+import eval_rerank as er  # noqa: E402
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--base-url", required=True)
+    parser.add_argument("--model", default="qwen3-reranker-afp-jev")
+    parser.add_argument("--label", default="qwen@qwen3-afp-jev", help="the cached scores to compare with")
+    parser.add_argument("--docs", type=int, default=5)
+    parser.add_argument("--api-key")
+    args = parser.parse_args()
+    headers = {"Authorization": f"Bearer {args.api_key}"} if args.api_key else {}
+    cache = el.cached("rerank.json")
+
+    gaps, same, surfaces = [], 0, 0
+    for doc in el.load_corpus("msnbc")[: args.docs]:
+        cands = el.candidates_of(doc, 8, el.EF_DEFAULT, 0.0)
+        cached = cache[f"{args.label}/k8/p0.0/{doc.corpus}/{doc.name}"]
+        for s, cs in cands.items():
+            if not cs:
+                continue
+            docs = [er.describe(c, facts=False) for c in cs]
+            resp = requests.post(args.base_url.rstrip("/") + "/rerank", json={"model": args.model, "query": er.context(doc, s), "documents": docs}, headers=headers, timeout=60)
+            resp.raise_for_status()
+            by_index = {r["index"]: r["relevance_score"] for r in resp.json()["results"]}
+            served = [math.log(max(p, 1e-7) / max(1 - p, 1e-7)) for p in (by_index[i] for i in range(len(docs)))]
+            gaps += [abs(a - b) for a, b in zip(served, cached[s], strict=True)]
+            same += max(range(len(docs)), key=served.__getitem__) == max(range(len(docs)), key=cached[s].__getitem__)
+            surfaces += 1
+    gaps.sort()
+    print(f"{surfaces} surfaces, {len(gaps)} pairs: log-odds gap median {gaps[len(gaps) // 2]:.3f}, max {gaps[-1]:.3f}; same best candidate {same}/{surfaces}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

{pyannotators_entityfishing-1.6.83 → pyannotators_entityfishing-1.6.85}/tools/eval_linking.py

@@ -23,6 +23,14 @@ jev           term lookup (top K) + one Jev Choice per distinct surface, one cal
     .venv/bin/python tools/eval_linking.py                              # both corpora, min_prior 0.01 (Jev calls billed once, then cached)
     .venv/bin/python tools/eval_linking.py --min-prior 0                # the first evaluation, replayed from the cache
     .venv/bin/python tools/eval_linking.py --bench 8                    # latency and throughput, real calls
+    JEV_API_BASE=http://127.0.0.1:8009 .venv/bin/python tools/eval_linking.py --model cygnet --min-prior 0   # a self-hosted Jev-compatible endpoint, no key
+    JEV_API_BASE=http://127.0.0.1:8009 .venv/bin/python tools/eval_linking.py --model cygnet --min-prior 0 --fragments 1   # one call per surface, state = its sentences +-1
+
+--fragments W replaces the whole text by fragments: one call per distinct surface, whose
+state is the sentences holding its mentions, W sentences on each side. Cygnet re-reads
+the state for every question, so this is where its tokens go; Jev reads it once per call.
+The answers are cached under the model name "<model>+fragW", which
+tools/recalibrate_linking.py --model cygnet+frag1 replays.
 
 Jev answers are cached per (model, document) in .cache/linking/jev.json, so a re-run
 costs nothing; --refresh-jev forces new calls. The key is read from JEV_API_KEY or
@@ -49,7 +57,9 @@ ROOT = Path(__file__).resolve().parent.parent
 CACHE = ROOT / ".cache" / "linking"
 RAW = "https://raw.githubusercontent.com/kermitt2/entity-fishing/master/data/corpus/corpus-long"
 EF_DEFAULT = "https://entityfishing.kairntech.com"
-JEV_URL = "https://api.typesafe.ai/v1/systemone"
+# Any endpoint that speaks the Jev protocol: the hosted TypeSafe API by default, or a
+# self-hosted one (Cygnet, Open-Jev) through JEV_API_BASE, as in pyprocessors_jev.
+JEV_BASE = os.getenv("JEV_API_BASE", os.getenv("TYPESAFE_BASE_URL", "https://api.typesafe.ai"))
 WIKIPEDIA_API = "https://en.wikipedia.org/w/api.php"
 USER_AGENT = "kairntech-eval-linking/0.1 (olivier.terrier@kairntech.com)"
 NONE = "NONE"
@@ -255,27 +265,62 @@ def build_questions(cands: dict[str, list[dict]]) -> tuple[list[str], dict]:
     return surfaces, questions
 
 
+def sentence_spans(text: str) -> list[tuple[int, int]]:
+    """Sentences as (start, end) offsets: a line break, or [.!?] then a space before a capital or a quote."""
+    spans, start = [], 0
+    for m in re.finditer(r"\n+|(?<=[.!?])[\"'”’)]*\s+(?=[A-Z\"“‘(])", text):
+        if text[start : m.start()].strip():
+            spans.append((start, m.start()))
+        start = m.end()
+    if text[start:].strip():
+        spans.append((start, len(text)))
+    return spans
+
+
+def fragment(doc: Doc, surface: str, window: int) -> str:
+    """The sentences holding the mentions of `surface`, `window` sentences on each side, gaps marked by [...]."""
+    spans = sentence_spans(doc.text)
+    hit = {i for m in doc.mentions if m.surface == surface for i, (s, e) in enumerate(spans) if s < m.end and m.start < e}
+    keep = sorted({j for i in hit for j in range(i - window, i + window + 1) if 0 <= j < len(spans)})
+    parts, prev = [], None
+    for j in keep:
+        if prev is not None and j != prev + 1:
+            parts.append("[...]")
+        parts.append(doc.text[spans[j][0] : spans[j][1]].strip())
+        prev = j
+    return "\n".join(parts)
+
+
 def call_jev(text: str, questions: dict, model: str, key: str) -> dict:
-    resp = session.post(JEV_URL, json={"model": model, "state": {"text": text}, "questions": questions}, headers={"Authorization": f"Bearer {key}"}, timeout=300)
+    headers = {"Authorization": f"Bearer {key}"} if key else {}
+    resp = session.post(JEV_BASE.rstrip("/") + "/v1/systemone", json={"model": model, "state": {"text": text}, "questions": questions}, headers=headers, timeout=300)
     resp.raise_for_status()
     return resp.json()
 
 
-def jev(doc: Doc, cands: dict[str, list[dict]], model: str, key: str, refresh: bool, k: int, min_prior: float) -> tuple[dict, dict]:
+def jev(doc: Doc, cands: dict[str, list[dict]], model: str, key: str, refresh: bool, k: int, min_prior: float, fragments: int = -1) -> tuple[dict, dict]:
     cache = cached("jev.json")
     # The answers depend on the candidates asked: the key carries K and min_prior. The
     # first evaluation (K=8, no min_prior) keeps its original key, so it replays for free.
-    ck = f"{model}/{doc.corpus}/{doc.name}" if (k, min_prior) == (8, 0.0) else f"{model}/k{k}/p{min_prior}/{doc.corpus}/{doc.name}"
+    tag = f"{model}+frag{fragments}" if fragments >= 0 else model
+    ck = f"{tag}/{doc.corpus}/{doc.name}" if (k, min_prior) == (8, 0.0) else f"{tag}/k{k}/p{min_prior}/{doc.corpus}/{doc.name}"
     if ck in cache and not refresh:
         return cache[ck]["answers"], cache[ck]["usage"]
     surfaces, questions = build_questions(cands)
     if not questions:
         return {}, {}
-    result = call_jev(doc.text, questions, model, key)
-    answers = {surfaces[int(q[1:])]: a for q, a in result["answers"].items()}
-    cache[ck] = {"answers": answers, "usage": result.get("usage", {})}
+    if fragments < 0:
+        result = call_jev(doc.text, questions, model, key)
+        answers = {surfaces[int(q[1:])]: a for q, a in result["answers"].items()}
+        usage = result.get("usage", {})
+    else:  # one call per surface, on its own fragment
+        with ThreadPoolExecutor(8) as pool:
+            results = list(pool.map(lambda q: call_jev(fragment(doc, surfaces[int(q[1:])], fragments), {q: questions[q]}, model, key), questions))
+        answers = {surfaces[int(q[1:])]: a for r in results for q, a in r["answers"].items()}
+        usage = {u: sum(r.get("usage", {}).get(u, 0) for r in results) for u in ("input_tokens", "output_tokens")}
+    cache[ck] = {"answers": answers, "usage": usage}
     save("jev.json", cache)
-    return answers, result.get("usage", {})
+    return answers, usage
 
 
 # --------------------------------------------------------------------------- scoring
@@ -390,6 +435,7 @@ def main() -> int:
     parser.add_argument("--min-prior", type=float, default=0.01, help="drop senses with 0 < prob_c < this, as the processor (default 0.01; 0 replays the first evaluation)")
     parser.add_argument("--model", default="jev-latest")
     parser.add_argument("--refresh-jev", action="store_true")
+    parser.add_argument("--fragments", type=int, default=-1, metavar="W", help="one call per surface, state = the sentences of its mentions, W on each side (default -1: the whole text, one call)")
     parser.add_argument("--no-jev", action="store_true", help="skip the Jev calls (candidates and baselines only; with --bench, time entity-fishing only)")
     parser.add_argument("--bench", type=int, default=0, metavar="N", help="time real calls (no cache) on N documents, half from each corpus")
     parser.add_argument("--concurrency", type=int, nargs="+", default=[1, 8], help="Jev concurrencies measured by --bench")
@@ -402,7 +448,7 @@ def main() -> int:
                 k, v = line.split("=", 1)
                 os.environ.setdefault(k.strip(), v.strip().strip("'\""))
     key = os.getenv("JEV_API_KEY") or os.getenv("TYPESAFE_API_KEY")
-    if not key and not args.no_jev:
+    if not key and not args.no_jev and "JEV_API_BASE" not in os.environ:
         print("No JEV_API_KEY: use --no-jev", file=sys.stderr)
         return 1
 
@@ -419,14 +465,19 @@ def main() -> int:
     scores: dict[str, Score] = {"disambiguate": Score(), "prior": Score()}
     for t in THRESHOLDS:
         scores[f"jev conf>={t}"] = Score()
-    cand_recall, usage = Counter(), Counter()
+    cand_recall, usage, state_chars = Counter(), Counter(), Counter()
 
     for n, doc in enumerate(docs, 1):
         print(f"# {n}/{len(docs)} {doc.corpus}/{doc.name} ({len(doc.mentions)} mentions)", file=sys.stderr)
         ef_links = disambiguate(doc, args.ef, args.min_selector)
         cands = candidates_of(doc, args.k, args.ef, args.min_prior)
-        answers, u = ({}, {}) if args.no_jev else jev(doc, cands, args.model, key, args.refresh_jev, args.k, args.min_prior)
+        answers, u = ({}, {}) if args.no_jev else jev(doc, cands, args.model, key, args.refresh_jev, args.k, args.min_prior, args.fragments)
         usage.update(u)
+        n_questions = sum(1 for cs in cands.values() if cs)
+        state_chars["text"] += len(doc.text)
+        state_chars["text x questions"] += len(doc.text) * n_questions
+        if args.fragments >= 0:
+            state_chars["fragments"] += sum(len(fragment(doc, s, args.fragments)) for s, cs in cands.items() if cs)
         for m in doc.mentions:
             scores["disambiguate"].add(m, ef_links.get((m.start, m.end)))
             cs = cands.get(m.surface, [])
@@ -437,7 +488,8 @@ def main() -> int:
             a = answers.get(m.surface)
             for t in THRESHOLDS:
                 pred = None
-                if a and a["choice"] != NONE and a["confidence"] >= t:
+                # Cygnet returns no confidence: the probability of the choice stands for it (they coincide for Jev).
+                if a and a["choice"] != NONE and a.get("confidence", a["probabilities"].get(a["choice"], 0.0)) >= t:
                     pred = a["choice"]
                 scores[f"jev conf>={t}"].add(m, pred)
             if a and m.qid:  # keep_ambiguity: the gold QID among the 2 most probable non-NONE options
@@ -446,6 +498,9 @@ def main() -> int:
 
     mentions = sum(len(d.mentions) for d in docs)
     print(f"\n{len(docs)} documents, {mentions} mentions scored ({unresolved} gold titles with no QID dropped)")
+    if args.fragments >= 0:
+        reread, frag = state_chars["text x questions"], state_chars["fragments"]
+        print(f"state characters: whole text {state_chars['text']}, re-read per question {reread}, fragments (W={args.fragments}) {frag} = {frag / max(reread, 1):.0%} of the re-read text")
     print(f"candidate recall @K={args.k}, min_prior={args.min_prior}: {cand_recall['in_candidates']}/{cand_recall['gold']} = {cand_recall['in_candidates'] / max(cand_recall['gold'], 1):.1%}  (ceiling of the term-lookup chain)\n")
     for name, s in scores.items():
         if name.startswith("jev") and args.no_jev: