pyannotators-entityfishing 1.6.154__tar.gz → 1.6.158__tar.gz

{pyannotators_entityfishing-1.6.154 → pyannotators_entityfishing-1.6.158}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pyannotators-entityfishing
-Version: 1.6.154
+Version: 1.6.158
 Summary: Annotator based on entity-fishing
 Project-URL: Homepage, https://github.com/oterrier/pyannotators_entityfishing/
 Author-email: Olivier Terrier <olivier.terrier@kairntech.com>

{pyannotators_entityfishing-1.6.154 → pyannotators_entityfishing-1.6.158}/src/pyannotators_entityfishing/__init__.py

@@ -1,3 +1,3 @@
 """Annotator based on entity-fishing"""
 
-__version__ = "1.6.154"
+__version__ = "1.6.158"

{pyannotators_entityfishing-1.6.154 → pyannotators_entityfishing-1.6.158}/src/pyannotators_entityfishing/candidates.py

@@ -23,6 +23,8 @@ import threading
 import time
 import unicodedata
 from collections import OrderedDict, defaultdict
+from concurrent.futures import ThreadPoolExecutor
+from functools import lru_cache
 from typing import cast
 
 import mongoquery
@@ -92,6 +94,14 @@ FILTERED = LRUCache(int(os.getenv("APP_EF_CANDIDATE_CACHE_SIZE", "50000")), CACH
 def clear_caches() -> None:
     CONCEPTS.clear()
     FILTERED.clear()
+    get_client.cache_clear()
+
+
+@lru_cache(maxsize=8)
+def get_client(ef_uri: str, pool_size: int) -> EntityFishingClient:
+    """One client per service, kept from one batch to the next: its connections stay open, and its
+    pool_size threads cap the calls to entity-fishing of every document of the process together."""
+    return EntityFishingClient(base_url=ef_uri, pool_size=pool_size)
 
 
 class EntityFishingCandidatesParameters(ProcessorParameters):
@@ -162,9 +172,15 @@ class EntityFishingCandidatesParameters(ProcessorParameters):
     )
     ef_pool_size: int = Field(
         int(os.getenv("APP_EF_CLIENT_POOLSIZE", "8")),
-        description="Number of parallel HTTP connections to entity-fishing",
+        description="Number of calls to entity-fishing at the same time, all the documents processed together",
         json_schema_extra={"extra": "internal"},
     )
+    concurrency: int = Field(
+        1,
+        description="""How many documents are processed at the same time. Their calls to entity-fishing
+        still go at most ef_pool_size at a time.""",
+        json_schema_extra={"extra": "advanced"},
+    )
 
 
 class EntityFishingCandidatesProcessor(ProcessorBase):
@@ -185,7 +201,7 @@ class EntityFishingCandidatesProcessor(ProcessorBase):
                 raise AttributeError(f"Metadata language '{lang}' is required and must be in {SUPPORTED_LANGUAGES}")
             langs.append(lang)
         mapping = {k: json.loads(v) for k, v in params.mapped_labels.items()} if params.mapped_labels else {}
-        client = EntityFishingClient(base_url=params.ef_uri, pool_size=params.ef_pool_size)
+        client = get_client(params.ef_uri, params.ef_pool_size)
         add_candidates(
             documents,
             langs,
@@ -200,6 +216,7 @@ class EntityFishingCandidatesProcessor(ProcessorBase):
             comma_separated_to_list(params.wikidata_facts),
             params.facts_values,
             params.facts_max_values,
+            params.concurrency,
         )
         return documents
 
@@ -222,18 +239,21 @@ def add_candidates(
     wiki_facts: list[str] | None = None,
     facts_values: PropertyValues = PropertyValues.label,
     facts_max_values: int = 3,
+    concurrency: int = 1,
 ) -> None:
     """Write the Wikidata candidates of every mention in `annotation.terms`, in place."""
     wiki_facts = wiki_facts or []
     queries = {label: mongoquery.Query(condition) for label, condition in mapping.items()}
     hashes = {label: hashlib.sha1(json.dumps(condition, sort_keys=True).encode()).hexdigest() for label, condition in mapping.items()}
     ef = client.base_url
-    failures: list[tuple[str, str, str]] = getattr(client, "failures", [])
-    for document, lang in zip(documents, langs, strict=True):
-        failed_before = len(failures)
+
+    def add_document(document: Document, lang: str) -> None:
+        # The calls this thread lost, so those of this document only.
+        failures: list[tuple[str, str, str]] = getattr(client, "failures", [])
+        failures.clear()
         groups = group_mentions(document, labels, mapping)
         if not groups:
-            continue
+            return
         # One term lookup per group, on its longest form; the surface when that finds nothing.
         forms = {key: lookup_forms(members) for key, members in groups.items()}
 
@@ -312,7 +332,15 @@ def add_candidates(
                 # Replace the Wikidata candidates, keep the terms of any other lexicon (AFP...).
                 kept = [t for t in (a.terms or []) if t.lexicon != WIKIDATA]
                 a.terms = kept + [t.model_copy(deep=True) for t in mine]
-        report_failures(document, failures[failed_before:])
+        report_failures(document, failures)
+
+    workers = min(max(concurrency, 1), len(documents))
+    if workers > 1:
+        with ThreadPoolExecutor(max_workers=workers) as pool:
+            list(pool.map(add_document, documents, langs))
+    else:
+        for document, lang in zip(documents, langs, strict=True):
+            add_document(document, lang)
 
 
 def report_failures(document: Document, lost: list[tuple[str, str, str]]) -> None:
@@ -320,6 +348,7 @@ def report_failures(document: Document, lost: list[tuple[str, str, str]]) -> Non
     they concern may have lost candidates, and the document is worth processing again. A document
     processed again without failure drops the report of the earlier run."""
     metadata = {k: v for k, v in (document.metadata or {}).items() if k != ERROR_METADATA}
+    lost = list(dict.fromkeys(lost))  # a page fetched again by a later step fails again
     if lost:
         calls = ", ".join(f"{kind} {key}" for kind, _, key in lost)
         metadata[ERROR_METADATA] = f"{len(lost)} entity-fishing calls failed, candidates may be missing: {calls}"[:MAX_ERROR_CHARS]

{pyannotators_entityfishing-1.6.154 → pyannotators_entityfishing-1.6.158}/src/pyannotators_entityfishing/ef_client.py

@@ -1,6 +1,9 @@
 import hashlib
+import json
 import logging
 import os
+import re
+import threading
 import time
 from collections.abc import Iterable
 from concurrent.futures import as_completed
@@ -25,6 +28,18 @@ VERIFY_SSL = os.getenv("APP_EF_VERIFY_SSL", "true").lower() not in ("0", "false"
 RETRY = Retry(total=4, backoff_factor=1, status_forcelist=(429, 500, 502, 503, 504), allowed_methods=None, raise_on_status=False)
 # (connect, read) seconds for a term lookup or a concept: without one, a stuck entity-fishing holds the document forever
 KB_TIMEOUT = (10, 60)
+# EF does not escape the quotes of a property name ("Apple Maps "auid" identifier", P10046), which breaks the JSON of
+# every concept holding that property: the White House, Georgia... Workaround until kermitt2/entity-fishing#177 is fixed
+PROPERTY_NAME = re.compile(r'("propertyName" : ")(.*?)(", "valueType")')
+
+
+def concept_json(resp: requests.Response) -> dict:
+    """The concept of a kb response, its unescaped property names escaped if it does not parse as is."""
+    try:
+        return resp.json()
+    except ValueError:
+        # ponytail: repairs propertyName only; drop concept_json once kermitt2/entity-fishing#177 is fixed
+        return json.loads(PROPERTY_NAME.sub(lambda m: m[1] + re.sub(r'(?<!\\)"', r"\"", m[2]) + m[3], resp.text))
 
 
 class EntityFishingClient:
@@ -79,14 +94,22 @@ class EntityFishingClient:
         self.ksession.verify = VERIFY_SSL
         self.fksession = FuturesSession(session=self.ksession, max_workers=pool_size)
 
-        # the calls that failed even after the retries, ("term" or "concept", lang, term or page id): the
-        # candidates step reports them on the document, and does not cache what they left out
-        self.failures: list[tuple[str, str, str]] = []
+        self._local = threading.local()
 
         self.disamb_url = "/service/disambiguate/"
         self.kb_url = "/service/kb/concept/"
         self.term_url = "/service/kb/term/"
 
+    @property
+    def failures(self) -> list[tuple[str, str, str]]:
+        """The calls that failed even after the retries, ("term" or "concept", lang, term or page id): the
+        candidates step reports them on the document, and does not cache what they left out. One list per
+        thread, as the failures are recorded in the calling thread: documents processed at the same time
+        through one client each see only their own."""
+        if not hasattr(self._local, "failures"):
+            self._local.failures = []
+        return self._local.failures
+
     def _build_disamb_query(self, text: str, sentences: list[dict], lang: str, minSelectorScore: float, maxTermFrequency: float | None, short_text: bool = False) -> dict:
         text = text.replace("\r\n", " \n")
         disamb_query = {
@@ -228,7 +251,7 @@ class EntityFishingClient:
         try:
             resp = self.ksession.get(self.base_url + self.kb_url + qid)
             if resp.ok:
-                return resp.json()
+                return concept_json(resp)
             else:
                 resp.raise_for_status()
         except Exception:
@@ -244,7 +267,7 @@ class EntityFishingClient:
             try:
                 resp = future.result()
                 if resp.ok:
-                    concept = resp.json()
+                    concept = concept_json(resp)
                     if "wikidataId" in concept:
                         concepts[concept["wikidataId"]] = concept
                 else:
@@ -288,7 +311,7 @@ class EntityFishingClient:
             try:
                 resp = future.result()
                 if resp.ok:
-                    concepts[futures[future]] = resp.json()
+                    concepts[futures[future]] = concept_json(resp)
                 elif resp.status_code != 404:  # 404: a page the KB does not hold, not a failure
                     resp.raise_for_status()
             except Exception:

pyannotators_entityfishing-1.6.158/tools/bench_plan.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Throughput of one or more Sherpa plans on the same documents of a project.
+
+Each plan runs twice on the same documents: the first run fills the caches (entity-fishing
+concepts and candidates, vLLM prefixes), the second is the one measured. The documents are
+sent without their annotations: the plan runs the NER itself.
+
+    SHERPA_TOKEN=... python3 tools/bench_plan.py --sherpa https://viper.kairntech.com --project afp_ner_xx \\
+        --plans pipeline_ner_old pipeline_ner_new --limit 200 --batch 10 --jobs 1
+
+--batch is what each processor receives at once: a processor's `concurrency` only helps above 1.
+"""
+
+import argparse
+import os
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+from sherpa_plan_project import annotate, documents, login
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--sherpa", default="http://localhost:7070")
+    parser.add_argument("--project", required=True)
+    parser.add_argument("--plans", nargs="+", required=True)
+    parser.add_argument("--limit", type=int, default=200, help="first N documents")
+    parser.add_argument("--batch", type=int, default=10, help="documents per annotate call")
+    parser.add_argument("--jobs", type=int, default=1, help="annotate calls at the same time")
+    parser.add_argument("--parallelize", action="store_true", help="Sherpa sends each document of a batch to each step in its own call, all at once")
+    args = parser.parse_args()
+    token, email, password = os.getenv("SHERPA_TOKEN"), os.getenv("SHERPA_EMAIL"), os.getenv("SHERPA_PASSWORD")
+    if not token and not (email and password):
+        print("set SHERPA_TOKEN, or SHERPA_EMAIL and SHERPA_PASSWORD", file=sys.stderr)
+        return 1
+    auth = {"X-Kairntech-key": token} if token else {"Authorization": f"Bearer {login(args.sherpa, email, password)}"}
+    headers = {**auth, "Accept": "application/json"}
+
+    docs = documents(args.sherpa, args.project, headers)
+    print(f"{len(docs)} documents in {args.project}", flush=True)
+    docs = [{k: d[k] for k in ("identifier", "title", "text", "metadata") if d.get(k) is not None} for d in docs if d.get("text")][: args.limit]
+    if not docs:
+        print("no document with a text", file=sys.stderr)
+        return 1
+    batches = [docs[i : i + args.batch] for i in range(0, len(docs), args.batch)]
+
+    for plan in args.plans:
+        for run in ("warm-up", "measured"):
+            start = time.time()
+            with ThreadPoolExecutor(args.jobs) as pool:
+                list(pool.map(lambda batch, plan=plan: annotate(args.sherpa, args.project, plan, headers, batch, args.parallelize), batches))
+            elapsed = time.time() - start
+            print(f"{plan} [{run}] {len(docs)} documents in {elapsed:.1f} s -> {len(docs) / elapsed:.1f} documents/s", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

{pyannotators_entityfishing-1.6.154 → pyannotators_entityfishing-1.6.158}/tools/sherpa_plan_project.py

@@ -47,10 +47,10 @@ def to_input(doc: dict) -> dict:
     return out
 
 
-def annotate(sherpa: str, project: str, plan: str, headers: dict, batch: list[dict]) -> list[dict]:
+def annotate(sherpa: str, project: str, plan: str, headers: dict, batch: list[dict], parallelize: bool = False) -> list[dict]:
     r = requests.post(
         f"{sherpa}/api/annotate/projects/{project}/annotators/{plan}/_annotate_documents",
-        params={"inlineLabels": True, "inlineText": True},
+        params={"inlineLabels": True, "inlineText": True, "parallelize": str(parallelize).lower()},
         json=batch,
         headers=headers,
         timeout=1800,