pyannotators-entityfishing 1.6.64__tar.gz → 1.6.71__tar.gz

{pyannotators_entityfishing-1.6.64 → pyannotators_entityfishing-1.6.71}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pyannotators-entityfishing
-Version: 1.6.64
+Version: 1.6.71
 Summary: Annotator based on entity-fishing
 Project-URL: Homepage, https://github.com/oterrier/pyannotators_entityfishing/
 Author-email: Olivier Terrier <olivier.terrier@kairntech.com>
@@ -89,24 +89,34 @@ for ann in docs[0].annotations:
 The build is driven by [Task](https://taskfile.dev) and [uv](https://docs.astral.sh/uv/),
 with the shared stages coming from the `python-archetype` submodule.
 
-### Prerequisites
+### Getting started
 
-**Task is the only manual prerequisite.** An archetype cannot bootstrap itself: uv and the
-Python interpreter install themselves on demand (every task that runs uv depends on an
-internal `install-python` task), but the thing that runs them does not.
+The stages live in a Git submodule, so **clone with `--recurse-submodules`**:
 
 ```bash
+git clone --recurse-submodules git@bitbucket.org:kairntech/pyannotators_entityfishing.git
+cd pyannotators_entityfishing
 sh -c "$(curl -sSL https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin
+task
 ```
 
-Make sure `~/.local/bin` is on your `PATH` — that is where `task` and `uv` both land.
+Already cloned without it? The submodule directory is empty, and `task` fails on:
+
+```
+task: No Taskfile found at ".../submodules/python-archetype/resources/Taskfile.yml"
+```
 
-Then fetch the shared stages:
+which means exactly that, and nothing worse:
 
 ```bash
 git submodule update --init
 ```
 
+**Task is the only manual prerequisite.** An archetype cannot bootstrap itself: uv and the
+Python interpreter install themselves on demand (every task that runs uv depends on an
+internal `install-python` task), but the thing that runs them does not. Make sure
+`~/.local/bin` is on your `PATH` — that is where `task` and `uv` both land.
+
 ### Running the pipeline
 
 ```bash

{pyannotators_entityfishing-1.6.64 → pyannotators_entityfishing-1.6.71}/README.md

@@ -41,24 +41,34 @@ for ann in docs[0].annotations:
 The build is driven by [Task](https://taskfile.dev) and [uv](https://docs.astral.sh/uv/),
 with the shared stages coming from the `python-archetype` submodule.
 
-### Prerequisites
+### Getting started
 
-**Task is the only manual prerequisite.** An archetype cannot bootstrap itself: uv and the
-Python interpreter install themselves on demand (every task that runs uv depends on an
-internal `install-python` task), but the thing that runs them does not.
+The stages live in a Git submodule, so **clone with `--recurse-submodules`**:
 
 ```bash
+git clone --recurse-submodules git@bitbucket.org:kairntech/pyannotators_entityfishing.git
+cd pyannotators_entityfishing
 sh -c "$(curl -sSL https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin
+task
 ```
 
-Make sure `~/.local/bin` is on your `PATH` — that is where `task` and `uv` both land.
+Already cloned without it? The submodule directory is empty, and `task` fails on:
+
+```
+task: No Taskfile found at ".../submodules/python-archetype/resources/Taskfile.yml"
+```
 
-Then fetch the shared stages:
+which means exactly that, and nothing worse:
 
 ```bash
 git submodule update --init
 ```
 
+**Task is the only manual prerequisite.** An archetype cannot bootstrap itself: uv and the
+Python interpreter install themselves on demand (every task that runs uv depends on an
+internal `install-python` task), but the thing that runs them does not. Make sure
+`~/.local/bin` is on your `PATH` — that is where `task` and `uv` both land.
+
 ### Running the pipeline
 
 ```bash

{pyannotators_entityfishing-1.6.64 → pyannotators_entityfishing-1.6.71}/src/pyannotators_entityfishing/__init__.py

@@ -1,3 +1,3 @@
 """Annotator based on entity-fishing"""
 
-__version__ = "1.6.64"
+__version__ = "1.6.71"

{pyannotators_entityfishing-1.6.64 → pyannotators_entityfishing-1.6.71}/submodules/python-archetype/vars/pythonPipeline.groovy

@@ -211,6 +211,28 @@ def call(Map config = [:]) {
                 }
             }
 
+            stage('py:sbom') {
+                steps {
+                    sh 'task py:sbom'
+                }
+            }
+
+            stage('py:check-vulnerabilities') {
+                steps {
+                    script {
+                        // pip-audit ne voit que les paquets Python, jamais une couche OS : ce
+                        // stage ne remplace pas un scan d'image, il couvre les dépendances.
+                        if (env._BLOCK_ON_VULNS == 'true') {
+                            sh 'task py:check-vulnerabilities'
+                        } else {
+                            catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
+                                sh 'task py:check-vulnerabilities'
+                            }
+                        }
+                    }
+                }
+            }
+
             stage('images:base') {
                 when { expression { return stageDeclared('images:base') } }
                 steps {
@@ -254,28 +276,6 @@ def call(Map config = [:]) {
                 }
             }
 
-            stage('py:sbom') {
-                steps {
-                    sh 'task py:sbom'
-                }
-            }
-
-            stage('py:check-vulnerabilities') {
-                steps {
-                    script {
-                        // pip-audit ne voit que les paquets Python, jamais une couche OS : ce
-                        // stage ne remplace pas un scan d'image, il couvre les dépendances.
-                        if (env._BLOCK_ON_VULNS == 'true') {
-                            sh 'task py:check-vulnerabilities'
-                        } else {
-                            catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
-                                sh 'task py:check-vulnerabilities'
-                            }
-                        }
-                    }
-                }
-            }
-
             stage('version') {
                 when {
                     allOf {
@@ -545,7 +545,17 @@ Either add it here, or remove it from STAGES — not two lists drifting in silen
 """
 }
 
-/** The stages this pipeline runs, in order. Kept next to checkStageDrift so both change together. */
+/**
+ * The stages this pipeline runs, in order. Kept next to checkStageDrift so both change together.
+ *
+ * L'ordre suit une seule regle : tout ce qui peut dire non passe avant ce qu'on ne peut pas
+ * defaire. 'version' pousse un commit et 'py:publish' envoie sur PyPI, qui est immuable -- ces
+ * deux-la sont les seuls a avoir un effet hors de l'agent, donc ils restent en queue, apres
+ * les portes. Les portes Python (py:sbom, py:check-vulnerabilities) et les portes d'image
+ * (images:scan) n'ont aucune dependance entre elles : aucun depot ne declare a la fois une roue
+ * et des images. Les premieres sont donc groupees avec py:test, pour que les stages py:* d'un
+ * depot a roue soient contigus et que les colonnes images:*, vides chez lui, ne les coupent pas.
+ */
 @groovy.transform.Field
 List<String> JENKINS_STAGES = [
     'init',
@@ -553,12 +563,12 @@ List<String> JENKINS_STAGES = [
     'py:lock-check',
     'py:lint',
     'py:test',
+    'py:sbom',
+    'py:check-vulnerabilities',
     'images:base',
     'images:flavors',
     'backport-proof',
     'images:scan',
-    'py:sbom',
-    'py:check-vulnerabilities',
     'version',
     'py:build',
     'py:publish',

pyannotators_entityfishing-1.6.71/tools/el_jev.py

@@ -0,0 +1,118 @@
+"""Prototype : NER amont -> candidats entity-fishing -> désambiguïsation Jev.
+
+Clé lue dans JEV_API_KEY (comme tests/.env) ou TYPESAFE_API_KEY ; sans clé, affiche la
+requête Jev au lieu de l'envoyer. Voir docs/entity-fishing-disambiguation-candidates_fr.md.
+
+    .venv/bin/python tools/el_jev.py "Macron à Washington" "Macron:PERSON,Washington:LOCATION" fr
+"""
+
+import json
+import os
+import re
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+import mongoquery
+
+EF = "https://entityfishing.kairntech.com/service"
+API_KEY = os.getenv("JEV_API_KEY") or os.getenv("TYPESAFE_API_KEY")
+JEV = "https://api.typesafe.ai/v1/systemone"
+TOP_K = 8
+NONE = "NONE"
+
+# Filtres de type : les mappings mongoquery de l'annotateur (tests/data_v2/<Label>.json),
+# appliqués au JSON de /kb/concept. Un type NER sans mapping (EVENT) n'est pas filtré.
+MAPPINGS = Path(__file__).resolve().parent.parent / "tests" / "data_v2"
+NER_TO_LABEL = {"PERSON": "Person", "LOCATION": "Location", "ORGANIZATION": "Organization"}
+FILTERS = {ner: mongoquery.Query(json.loads((MAPPINGS / f"{label}.json").read_text())) for ner, label in NER_TO_LABEL.items()}
+
+
+def get(url):
+    with urllib.request.urlopen(url, timeout=60) as r:
+        return json.load(r)
+
+
+def senses(term, lang="en"):
+    return get(f"{EF}/kb/term/{urllib.parse.quote(term)}?lang={lang}").get("senses", [])
+
+
+def concept(pageid, lang="en"):
+    return get(f"{EF}/kb/concept/{pageid}?lang={lang}")
+
+
+def first_sentence(wikitext):
+    t = re.sub(r"\[\[(?:[^|\]]*\|)?([^\]]*)\]\]", r"\1", wikitext)
+    t = re.sub(r"'''?|\{\{[^}]*\}\}|\([^)]*\)", "", t)
+    return re.split(r"(?<=\.)\s", re.sub(r"\s+", " ", t).strip())[0]
+
+
+def candidates(mention, ner_type, lang="en"):
+    out = []
+    for s in senses(mention, lang)[:TOP_K]:
+        c = concept(s["pageid"], lang)
+        st = c.get("statements", [])
+        if ner_type in FILTERS and not FILTERS[ner_type].match(c):
+            continue
+        facts = {}
+        for x in st:
+            if x["propertyId"] in ("P31", "P106", "P27", "P641", "P585") and x.get("valueName"):
+                facts.setdefault(x["propertyName"], []).append(x["valueName"])
+            if x["propertyId"] == "P569":
+                facts["date of birth"] = [x["value"]["time"][1:11]]
+        defs = c.get("definitions") or [{}]
+        out.append(
+            {
+                "qid": c["wikidataId"],
+                "label": s["preferred"],
+                "prior": s["prob_c"],
+                "description": first_sentence(defs[0].get("definition", "")),
+                "facts": {k: v[:3] for k, v in facts.items()},
+            }
+        )
+    return out
+
+
+def jev_request(text, mentions):
+    """Une Choice par mention, toutes dans le même appel (évaluées en parallèle)."""
+    questions = {}
+    for i, m in enumerate(mentions):
+        criteria = {c["qid"]: {"name": c["label"], "description": c["description"], "facts": c["facts"]} for c in m["candidates"]}
+        criteria[NONE] = "None of the listed entities is the one meant by the mention"
+        questions[f"m{i}"] = {
+            "type": "choice",
+            "instructions": {
+                "mention": {"surface": m["surface"], "ner_type": m["type"]},
+                "question": "In `text`, which knowledge-base entity does `mention.surface` refer to?",
+            },
+            "criteria": criteria,
+        }
+    return {"model": "jev-latest", "state": {"text": text}, "questions": questions}
+
+
+def call_jev(payload):
+    req = urllib.request.Request(JEV, data=json.dumps(payload).encode(), headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"})
+    with urllib.request.urlopen(req, timeout=60) as r:
+        return json.load(r)
+
+
+if __name__ == "__main__":
+    import sys
+
+    text = sys.argv[1] if len(sys.argv) > 1 else "Ronaldo at the World Cup"
+    spec = sys.argv[2] if len(sys.argv) > 2 else "Ronaldo:PERSON,World Cup:EVENT"
+    lang = sys.argv[3] if len(sys.argv) > 3 else "en"
+    ner = [m.rsplit(":", 1) for m in spec.split(",")]
+    mentions = [{"surface": s, "type": t, "candidates": candidates(s, t, lang)} for s, t in ner]
+    payload = jev_request(text, mentions)
+    if not API_KEY:
+        print(json.dumps(payload, indent=1, ensure_ascii=False))
+        sys.exit()
+    answers = call_jev(payload)["answers"]
+    for i, m in enumerate(mentions):
+        a = answers[f"m{i}"]
+        prior = {c["qid"]: c["prior"] for c in m["candidates"]}
+        print(f"{m['surface']} -> {a['choice']} (conf {a['confidence']:.2f})")
+        for qid, p in sorted(a["probabilities"].items(), key=lambda kv: -kv[1]):
+            label = {c["qid"]: c["label"] for c in m["candidates"]}.get(qid, "")
+            print(f"   {qid:10} jev={p:.3f} prior={prior.get(qid, 0):.3f}  {label}")