pyannotators-entityfishing 1.6.134__tar.gz → 1.6.136__tar.gz

{pyannotators_entityfishing-1.6.134 → pyannotators_entityfishing-1.6.136}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pyannotators-entityfishing
-Version: 1.6.134
+Version: 1.6.136
 Summary: Annotator based on entity-fishing
 Project-URL: Homepage, https://github.com/oterrier/pyannotators_entityfishing/
 Author-email: Olivier Terrier <olivier.terrier@kairntech.com>

{pyannotators_entityfishing-1.6.134 → pyannotators_entityfishing-1.6.136}/src/pyannotators_entityfishing/__init__.py

@@ -1,3 +1,3 @@
 """Annotator based on entity-fishing"""
 
-__version__ = "1.6.134"
+__version__ = "1.6.136"

{pyannotators_entityfishing-1.6.134 → pyannotators_entityfishing-1.6.136}/tools/build_poi_lexicons.py

@@ -35,6 +35,10 @@ tools/poi_common_words.json overrides the rule, per language: the forms to keep
 --clean-only applies it to the OUTDIR/poi_<lang>.json already written, without Wikidata
 (the sitelink counts come from OUTDIR/_entities.jsonl).
 
+Both modes end by writing OUTDIR/poi_cities.json from the six lexicons: QID -> cityId, countryId and cityName, the
+city name per language. The REM formatter (pyformatters_json_rem) embeds a copy, to give its city to a
+POI the gazetteer did not match but entity-fishing linked: "Portimão" for its racing circuit.
+
 Step 1 lists the items with SPARQL (OUTDIR/_items.json); step 2 fetches them with
 wbgetentities (OUTDIR/_entities.jsonl). Both are kept, so a rerun resumes; a rerun with a
 lower --min-sitelinks needs _items.json deleted.
@@ -239,6 +243,26 @@ def clean(path: Path, lang: str, dictionaries: Path, famous: set[str], generic:
     print(f"{path}: {len(lexicon)} concepts, {entries} named by a common word and {forms} aliases (common words, codes) dropped", file=sys.stderr)
 
 
+def city_table(lexicons: dict[str, list[dict]]) -> dict[str, dict]:
+    """QID -> GeoNames IDs of the city and the country, and the city name per language, of every POI of the lexicons."""
+    table: dict[str, dict] = {}
+    for lang, lexicon in lexicons.items():
+        for e in lexicon:
+            props = e["properties"]
+            poi = table.setdefault(e["identifier"], {})
+            poi.update({k: props[k] for k in ("cityId", "countryId") if props.get(k)})
+            if props.get("cityName"):
+                poi.setdefault("cityName", {})[lang] = props["cityName"]
+    return dict(sorted(table.items(), key=lambda item: int(item[0][1:])))
+
+
+def write_city_table(outdir: Path) -> None:
+    table = city_table({lang: json.loads((outdir / f"poi_{lang}.json").read_text()) for lang in LANGS})
+    path = outdir / "poi_cities.json"
+    path.write_text(json.dumps(table, ensure_ascii=False, separators=(",", ":")) + "\n")
+    print(f"{path}: {len(table)} POI, {sum('cityId' in p for p in table.values())} with a city", file=sys.stderr)
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     parser.add_argument("outdir", type=Path)
@@ -260,6 +284,7 @@ def main() -> None:
         famous = {e["id"] for e in map(json.loads, cache.open()) if len(e.get("sitelinks", {})) >= args.keep_sitelinks}
         for lang in LANGS:
             clean(args.outdir / f"poi_{lang}.json", lang, args.dictionaries, famous, generic)
+        write_city_table(args.outdir)
         return
     args.outdir.mkdir(parents=True, exist_ok=True)
     classes, orgs = mapping_lists()
@@ -337,6 +362,7 @@ def main() -> None:
         print(f"{path}: {len(lexicon)} concepts, {dropped} named after a city dropped", file=sys.stderr)
         if args.dictionaries:
             clean(path, lang, args.dictionaries, famous, generic)
+    write_city_table(args.outdir)
 
 
 if __name__ == "__main__":