pyannotators-entityfishing 1.6.67__tar.gz → 1.6.71__tar.gz

{pyannotators_entityfishing-1.6.67 → pyannotators_entityfishing-1.6.71}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pyannotators-entityfishing
-Version: 1.6.67
+Version: 1.6.71
 Summary: Annotator based on entity-fishing
 Project-URL: Homepage, https://github.com/oterrier/pyannotators_entityfishing/
 Author-email: Olivier Terrier <olivier.terrier@kairntech.com>
@@ -89,24 +89,34 @@ for ann in docs[0].annotations:
 The build is driven by [Task](https://taskfile.dev) and [uv](https://docs.astral.sh/uv/),
 with the shared stages coming from the `python-archetype` submodule.
 
-### Prerequisites
+### Getting started
 
-**Task is the only manual prerequisite.** An archetype cannot bootstrap itself: uv and the
-Python interpreter install themselves on demand (every task that runs uv depends on an
-internal `install-python` task), but the thing that runs them does not.
+The stages live in a Git submodule, so **clone with `--recurse-submodules`**:
 
 ```bash
+git clone --recurse-submodules git@bitbucket.org:kairntech/pyannotators_entityfishing.git
+cd pyannotators_entityfishing
 sh -c "$(curl -sSL https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin
+task
 ```
 
-Make sure `~/.local/bin` is on your `PATH` — that is where `task` and `uv` both land.
+Already cloned without it? The submodule directory is empty, and `task` fails on:
+
+```
+task: No Taskfile found at ".../submodules/python-archetype/resources/Taskfile.yml"
+```
 
-Then fetch the shared stages:
+which means exactly that, and nothing worse:
 
 ```bash
 git submodule update --init
 ```
 
+**Task is the only manual prerequisite.** An archetype cannot bootstrap itself: uv and the
+Python interpreter install themselves on demand (every task that runs uv depends on an
+internal `install-python` task), but the thing that runs them does not. Make sure
+`~/.local/bin` is on your `PATH` — that is where `task` and `uv` both land.
+
 ### Running the pipeline
 
 ```bash

{pyannotators_entityfishing-1.6.67 → pyannotators_entityfishing-1.6.71}/README.md

@@ -41,24 +41,34 @@ for ann in docs[0].annotations:
 The build is driven by [Task](https://taskfile.dev) and [uv](https://docs.astral.sh/uv/),
 with the shared stages coming from the `python-archetype` submodule.
 
-### Prerequisites
+### Getting started
 
-**Task is the only manual prerequisite.** An archetype cannot bootstrap itself: uv and the
-Python interpreter install themselves on demand (every task that runs uv depends on an
-internal `install-python` task), but the thing that runs them does not.
+The stages live in a Git submodule, so **clone with `--recurse-submodules`**:
 
 ```bash
+git clone --recurse-submodules git@bitbucket.org:kairntech/pyannotators_entityfishing.git
+cd pyannotators_entityfishing
 sh -c "$(curl -sSL https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin
+task
 ```
 
-Make sure `~/.local/bin` is on your `PATH` — that is where `task` and `uv` both land.
+Already cloned without it? The submodule directory is empty, and `task` fails on:
+
+```
+task: No Taskfile found at ".../submodules/python-archetype/resources/Taskfile.yml"
+```
 
-Then fetch the shared stages:
+which means exactly that, and nothing worse:
 
 ```bash
 git submodule update --init
 ```
 
+**Task is the only manual prerequisite.** An archetype cannot bootstrap itself: uv and the
+Python interpreter install themselves on demand (every task that runs uv depends on an
+internal `install-python` task), but the thing that runs them does not. Make sure
+`~/.local/bin` is on your `PATH` — that is where `task` and `uv` both land.
+
 ### Running the pipeline
 
 ```bash

{pyannotators_entityfishing-1.6.67 → pyannotators_entityfishing-1.6.71}/src/pyannotators_entityfishing/__init__.py

@@ -1,3 +1,3 @@
 """Annotator based on entity-fishing"""
 
-__version__ = "1.6.67"
+__version__ = "1.6.71"

pyannotators_entityfishing-1.6.71/tools/el_jev.py

@@ -0,0 +1,118 @@
+"""Prototype : NER amont -> candidats entity-fishing -> désambiguïsation Jev.
+
+Clé lue dans JEV_API_KEY (comme tests/.env) ou TYPESAFE_API_KEY ; sans clé, affiche la
+requête Jev au lieu de l'envoyer. Voir docs/entity-fishing-disambiguation-candidates_fr.md.
+
+    .venv/bin/python tools/el_jev.py "Macron à Washington" "Macron:PERSON,Washington:LOCATION" fr
+"""
+
+import json
+import os
+import re
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+import mongoquery
+
+EF = "https://entityfishing.kairntech.com/service"
+API_KEY = os.getenv("JEV_API_KEY") or os.getenv("TYPESAFE_API_KEY")
+JEV = "https://api.typesafe.ai/v1/systemone"
+TOP_K = 8
+NONE = "NONE"
+
+# Filtres de type : les mappings mongoquery de l'annotateur (tests/data_v2/<Label>.json),
+# appliqués au JSON de /kb/concept. Un type NER sans mapping (EVENT) n'est pas filtré.
+MAPPINGS = Path(__file__).resolve().parent.parent / "tests" / "data_v2"
+NER_TO_LABEL = {"PERSON": "Person", "LOCATION": "Location", "ORGANIZATION": "Organization"}
+FILTERS = {ner: mongoquery.Query(json.loads((MAPPINGS / f"{label}.json").read_text())) for ner, label in NER_TO_LABEL.items()}
+
+
+def get(url):
+    with urllib.request.urlopen(url, timeout=60) as r:
+        return json.load(r)
+
+
+def senses(term, lang="en"):
+    return get(f"{EF}/kb/term/{urllib.parse.quote(term)}?lang={lang}").get("senses", [])
+
+
+def concept(pageid, lang="en"):
+    return get(f"{EF}/kb/concept/{pageid}?lang={lang}")
+
+
+def first_sentence(wikitext):
+    t = re.sub(r"\[\[(?:[^|\]]*\|)?([^\]]*)\]\]", r"\1", wikitext)
+    t = re.sub(r"'''?|\{\{[^}]*\}\}|\([^)]*\)", "", t)
+    return re.split(r"(?<=\.)\s", re.sub(r"\s+", " ", t).strip())[0]
+
+
+def candidates(mention, ner_type, lang="en"):
+    out = []
+    for s in senses(mention, lang)[:TOP_K]:
+        c = concept(s["pageid"], lang)
+        st = c.get("statements", [])
+        if ner_type in FILTERS and not FILTERS[ner_type].match(c):
+            continue
+        facts = {}
+        for x in st:
+            if x["propertyId"] in ("P31", "P106", "P27", "P641", "P585") and x.get("valueName"):
+                facts.setdefault(x["propertyName"], []).append(x["valueName"])
+            if x["propertyId"] == "P569":
+                facts["date of birth"] = [x["value"]["time"][1:11]]
+        defs = c.get("definitions") or [{}]
+        out.append(
+            {
+                "qid": c["wikidataId"],
+                "label": s["preferred"],
+                "prior": s["prob_c"],
+                "description": first_sentence(defs[0].get("definition", "")),
+                "facts": {k: v[:3] for k, v in facts.items()},
+            }
+        )
+    return out
+
+
+def jev_request(text, mentions):
+    """Une Choice par mention, toutes dans le même appel (évaluées en parallèle)."""
+    questions = {}
+    for i, m in enumerate(mentions):
+        criteria = {c["qid"]: {"name": c["label"], "description": c["description"], "facts": c["facts"]} for c in m["candidates"]}
+        criteria[NONE] = "None of the listed entities is the one meant by the mention"
+        questions[f"m{i}"] = {
+            "type": "choice",
+            "instructions": {
+                "mention": {"surface": m["surface"], "ner_type": m["type"]},
+                "question": "In `text`, which knowledge-base entity does `mention.surface` refer to?",
+            },
+            "criteria": criteria,
+        }
+    return {"model": "jev-latest", "state": {"text": text}, "questions": questions}
+
+
+def call_jev(payload):
+    req = urllib.request.Request(JEV, data=json.dumps(payload).encode(), headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"})
+    with urllib.request.urlopen(req, timeout=60) as r:
+        return json.load(r)
+
+
+if __name__ == "__main__":
+    import sys
+
+    text = sys.argv[1] if len(sys.argv) > 1 else "Ronaldo at the World Cup"
+    spec = sys.argv[2] if len(sys.argv) > 2 else "Ronaldo:PERSON,World Cup:EVENT"
+    lang = sys.argv[3] if len(sys.argv) > 3 else "en"
+    ner = [m.rsplit(":", 1) for m in spec.split(",")]
+    mentions = [{"surface": s, "type": t, "candidates": candidates(s, t, lang)} for s, t in ner]
+    payload = jev_request(text, mentions)
+    if not API_KEY:
+        print(json.dumps(payload, indent=1, ensure_ascii=False))
+        sys.exit()
+    answers = call_jev(payload)["answers"]
+    for i, m in enumerate(mentions):
+        a = answers[f"m{i}"]
+        prior = {c["qid"]: c["prior"] for c in m["candidates"]}
+        print(f"{m['surface']} -> {a['choice']} (conf {a['confidence']:.2f})")
+        for qid, p in sorted(a["probabilities"].items(), key=lambda kv: -kv[1]):
+            label = {c["qid"]: c["label"] for c in m["candidates"]}.get(qid, "")
+            print(f"   {qid:10} jev={p:.3f} prior={prior.get(qid, 0):.3f}  {label}")