pyannotators-entityfishing 1.6.151__tar.gz → 1.6.156__tar.gz

{pyannotators_entityfishing-1.6.151 → pyannotators_entityfishing-1.6.156}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pyannotators-entityfishing
-Version: 1.6.151
+Version: 1.6.156
 Summary: Annotator based on entity-fishing
 Project-URL: Homepage, https://github.com/oterrier/pyannotators_entityfishing/
 Author-email: Olivier Terrier <olivier.terrier@kairntech.com>

{pyannotators_entityfishing-1.6.151 → pyannotators_entityfishing-1.6.156}/src/pyannotators_entityfishing/__init__.py

@@ -1,3 +1,3 @@
 """Annotator based on entity-fishing"""
 
-__version__ = "1.6.151"
+__version__ = "1.6.156"

{pyannotators_entityfishing-1.6.151 → pyannotators_entityfishing-1.6.156}/src/pyannotators_entityfishing/candidates.py

@@ -320,6 +320,7 @@ def report_failures(document: Document, lost: list[tuple[str, str, str]]) -> Non
     they concern may have lost candidates, and the document is worth processing again. A document
     processed again without failure drops the report of the earlier run."""
     metadata = {k: v for k, v in (document.metadata or {}).items() if k != ERROR_METADATA}
+    lost = list(dict.fromkeys(lost))  # a page fetched again by a later step fails again
     if lost:
         calls = ", ".join(f"{kind} {key}" for kind, _, key in lost)
         metadata[ERROR_METADATA] = f"{len(lost)} entity-fishing calls failed, candidates may be missing: {calls}"[:MAX_ERROR_CHARS]

{pyannotators_entityfishing-1.6.151 → pyannotators_entityfishing-1.6.156}/src/pyannotators_entityfishing/ef_client.py

@@ -1,6 +1,8 @@
 import hashlib
+import json
 import logging
 import os
+import re
 import time
 from collections.abc import Iterable
 from concurrent.futures import as_completed
@@ -25,6 +27,18 @@ VERIFY_SSL = os.getenv("APP_EF_VERIFY_SSL", "true").lower() not in ("0", "false"
 RETRY = Retry(total=4, backoff_factor=1, status_forcelist=(429, 500, 502, 503, 504), allowed_methods=None, raise_on_status=False)
 # (connect, read) seconds for a term lookup or a concept: without one, a stuck entity-fishing holds the document forever
 KB_TIMEOUT = (10, 60)
+# EF does not escape the quotes of a property name ("Apple Maps "auid" identifier", P10046), which breaks the JSON of
+# every concept holding that property: the White House, Georgia... Workaround until kermitt2/entity-fishing#177 is fixed
+PROPERTY_NAME = re.compile(r'("propertyName" : ")(.*?)(", "valueType")')
+
+
+def concept_json(resp: requests.Response) -> dict:
+    """The concept of a kb response, its unescaped property names escaped if it does not parse as is."""
+    try:
+        return resp.json()
+    except ValueError:
+        # ponytail: repairs propertyName only; drop concept_json once kermitt2/entity-fishing#177 is fixed
+        return json.loads(PROPERTY_NAME.sub(lambda m: m[1] + re.sub(r'(?<!\\)"', r"\"", m[2]) + m[3], resp.text))
 
 
 class EntityFishingClient:
@@ -228,7 +242,7 @@ class EntityFishingClient:
         try:
             resp = self.ksession.get(self.base_url + self.kb_url + qid)
             if resp.ok:
-                return resp.json()
+                return concept_json(resp)
             else:
                 resp.raise_for_status()
         except Exception:
@@ -244,7 +258,7 @@ class EntityFishingClient:
             try:
                 resp = future.result()
                 if resp.ok:
-                    concept = resp.json()
+                    concept = concept_json(resp)
                     if "wikidataId" in concept:
                         concepts[concept["wikidataId"]] = concept
                 else:
@@ -288,11 +302,11 @@ class EntityFishingClient:
             try:
                 resp = future.result()
                 if resp.ok:
-                    concepts[futures[future]] = resp.json()
+                    concepts[futures[future]] = concept_json(resp)
                 elif resp.status_code != 404:  # 404: a page the KB does not hold, not a failure
                     resp.raise_for_status()
             except Exception:
-                logging.warning("An exception was thrown!", exc_info=True)
+                logging.warning("EF kb concept of page %s (%s) failed", futures[future], lang, exc_info=True)
                 self.failures.append(("concept", lang, str(futures[future])))
         logger.debug("EF get kb %d concepts by page duration %0.3fs", len(pageids), time.time() - start)
         return concepts