pyprocessors-coreference 1.6.1__py3-none-any.whl

pyprocessors_coreference/__init__.py

@@ -0,0 +1,3 @@
+"""Processor grouping the mentions of the same entity"""
+
+__version__ = "1.6.1"

pyprocessors_coreference/coreference.py

@@ -0,0 +1,174 @@
+import re
+import unicodedata
+from collections import defaultdict
+from typing import cast
+
+from pydantic import BaseModel, Field
+from pymultirole_plugins.v1.processor import ProcessorBase, ProcessorParameters
+from pymultirole_plugins.v1.schema import Annotation, Document
+
+# Honorifics in front of a person name, stripped before comparing names: in AFP
+# dispatches "M. Macron" follows "Emmanuel Macron". Compared case-insensitively.
+DEFAULT_HONORIFICS = ["M.", "MM.", "Mme", "Mmes", "Mlle", "Mlles", "Me", "Dr", "Dr.", "Pr", "Pr.", "Mr", "Mr.", "Mrs", "Mrs.", "Ms", "Ms.", "Miss", "Sir", "Dame", "Lord", "Lady"]
+
+# Properties written on every grouped annotation (ADR-0001, §5).
+ENTITY_GROUP = "entity_group"
+CANONICAL_FORM = "canonical_form"
+MENTION_TYPE = "mention_type"
+# Only on an ambiguous partial name: the groups it may belong to.
+ENTITY_GROUP_CANDIDATES = "entity_group_candidates"
+
+# The family of every person label: the NER's "person" and the AFP lexicon's
+# "afpperson" annotate the same span, and must land in the same group.
+PERSON_FAMILY = "\x00person"
+
+APOSTROPHES = str.maketrans({"’": "'", "‘": "'", "ʼ": "'"})
+
+
+class CoreferenceParameters(ProcessorParameters):
+    """Parameters of the coreference processor"""
+
+    labels: list[str] | None = Field(
+        None,
+        description="""Labels of the mentions to group. If empty, the mentions of every label are grouped.<br/>
+        Mentions are only grouped with mentions of the same label, except person labels, which all form one family.""",
+        json_schema_extra={"extra": "label"},
+    )
+    person_labels: list[str] | None = Field(
+        None,
+        description="""Labels of person names. They form one family (person and afpperson mentions are grouped together),
+        and a partial name ("Macron") joins the full name it ends ("Emmanuel Macron").<br/>
+        If empty, every label whose name contains "person" (person, afpperson...).""",
+        json_schema_extra={"extra": "label"},
+    )
+    honorifics: list[str] = Field(
+        DEFAULT_HONORIFICS,
+        description="Honorifics ignored in front of a person name when comparing names (M., Mme, Dr...)",
+        json_schema_extra={"extra": "advanced"},
+    )
+
+
+class CoreferenceProcessor(ProcessorBase):
+    """Group the mentions of the same entity, before entity linking.
+
+    Tier 1 of the mention grouping of ADR-0001 (pyannotators_entityfishing): names
+    only. Every annotation of a grouped label gets an `entity_group` (the same id for
+    every mention of the entity in the document) and a `canonical_form` (the longest
+    form of the group, which the candidate generation looks up):
+
+    - same normalized surface and same label → same group; all person labels count
+      as one (the NER's "person" and the AFP lexicon's "afpperson" annotate the same
+      span);
+    - person labels only: a partial name joins the full name it ends, wherever the
+      full name is in the document ("Macron" joins "Emmanuel Macron");
+    - a first name alone never joins ("Emmanuel" stays on its own);
+    - an ambiguous partial name ("Macron" with both Emmanuel and Brigitte Macron)
+      keeps a group of its own, and lists the possible groups in
+      `entity_group_candidates` for the linking step to decide.
+    """
+
+    def process(self, documents: list[Document], parameters: ProcessorParameters) -> list[Document]:
+        params = cast(CoreferenceParameters, parameters)
+        honorifics = {h.casefold() for h in params.honorifics}
+        for document in documents:
+            group_document(document, params, honorifics)
+        return documents
+
+    @classmethod
+    def get_model(cls) -> type[BaseModel]:
+        return CoreferenceParameters
+
+
+def group_document(document: Document, params: CoreferenceParameters, honorifics: set[str]) -> None:
+    text = document.text or ""
+    mentions = [a for a in (document.annotations or []) if a.labelName and (not params.labels or a.labelName in params.labels)]
+    if not mentions:
+        return
+
+    # 1. Same label, same normalized surface: one surface group.
+    surfaces: dict[tuple[str, tuple[str, ...]], list[Annotation]] = defaultdict(list)
+    for a in sorted(mentions, key=lambda a: (a.start, a.end)):
+        person = is_person_label(a.labelName, params)
+        tokens = name_tokens(surface_of(a, text), honorifics if person else set())
+        if tokens:
+            surfaces[(PERSON_FAMILY if person else a.labelName, tokens)].append(a)
+
+    # 2. Person labels: a surface that is a proper suffix of longer surfaces joins
+    # them when they are all one entity. Longest first, so that "Emmanuel Macron" has
+    # already joined "Président Emmanuel Macron" when "Macron" is considered.
+    parent = {key: key for key in surfaces}
+
+    def root(key):
+        while parent[key] != key:
+            parent[key] = parent[parent[key]]
+            key = parent[key]
+        return key
+
+    ambiguous: dict[tuple, set] = {}
+    for key in sorted(surfaces, key=lambda k: -len(k[1])):
+        family, tokens = key
+        if family != PERSON_FAMILY:
+            continue
+        longer = {root(other) for other in surfaces if other[0] == PERSON_FAMILY and is_proper_suffix(tokens, other[1])}
+        longer.discard(root(key))
+        if len(longer) == 1:
+            parent[root(key)] = longer.pop()
+        elif longer:
+            ambiguous[key] = longer
+
+    # 3. Ids in order of first appearance, canonical form = longest mention of the group.
+    members: dict[tuple, list[Annotation]] = defaultdict(list)
+    for key, annotations in surfaces.items():
+        members[root(key)].extend(annotations)
+    ordered = sorted(members, key=lambda r: min(a.start for a in members[r]))
+    group_id = {r: f"g{i}" for i, r in enumerate(ordered, start=1)}
+    canonical = {r: canonical_form(members[r], text, honorifics if r[0] == PERSON_FAMILY else set()) for r in ordered}
+
+    for key, annotations in surfaces.items():
+        r = root(key)
+        for a in annotations:
+            properties = dict(a.properties or {})
+            properties[ENTITY_GROUP] = group_id[r]
+            properties[CANONICAL_FORM] = canonical[r]
+            properties[MENTION_TYPE] = "name"
+            if key in ambiguous:
+                properties[ENTITY_GROUP_CANDIDATES] = sorted((group_id[c] for c in ambiguous[key]), key=lambda g: int(g[1:]))
+            else:
+                properties.pop(ENTITY_GROUP_CANDIDATES, None)
+            a.properties = properties
+
+
+def is_person_label(label: str, params: CoreferenceParameters) -> bool:
+    if params.person_labels:
+        return label in params.person_labels
+    return "person" in label.casefold()
+
+
+def surface_of(a: Annotation, text: str) -> str:
+    return a.text if a.text is not None else text[a.start : a.end]
+
+
+def name_tokens(surface: str, honorifics: set[str]) -> tuple[str, ...]:
+    """Normalized tokens of a surface: NFKC, one apostrophe, case-folded, and the
+    leading honorifics dropped (only for person names, where the set is not empty)."""
+    tokens = unicodedata.normalize("NFKC", surface).translate(APOSTROPHES).casefold().split()
+    while len(tokens) > 1 and tokens[0] in honorifics:
+        tokens = tokens[1:]
+    return tuple(tokens)
+
+
+def strip_honorifics(surface: str, honorifics: set[str]) -> str:
+    words = re.sub(r"\s+", " ", surface).strip().split(" ")
+    while len(words) > 1 and words[0].casefold() in honorifics:
+        words = words[1:]
+    return " ".join(words)
+
+
+def is_proper_suffix(tokens: tuple[str, ...], other: tuple[str, ...]) -> bool:
+    return len(tokens) < len(other) and other[len(other) - len(tokens) :] == tokens
+
+
+def canonical_form(annotations: list[Annotation], text: str, honorifics: set[str]) -> str:
+    """The longest form of the group: most words, then most characters, then the first."""
+    forms = [strip_honorifics(surface_of(a, text), honorifics) for a in sorted(annotations, key=lambda a: a.start)]
+    return max(forms, key=lambda f: (len(f.split()), len(f)))

pyprocessors_coreference-1.6.1.dist-info/METADATA

@@ -0,0 +1,131 @@
+Metadata-Version: 2.5
+Name: pyprocessors-coreference
+Version: 1.6.1
+Summary: Processor grouping the mentions of the same entity
+Project-URL: Homepage, https://bitbucket.org/kairntech/pyprocessors_coreference
+Author-email: Olivier Terrier <olivier.terrier@kairntech.com>
+License: MIT
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: System Administrators
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.12
+Requires-Dist: pymultirole-plugins<1.7.0,>=1.6.0
+Provides-Extra: sbom
+Requires-Dist: cyclonedx-bom; extra == 'sbom'
+Requires-Dist: pip-audit; extra == 'sbom'
+Provides-Extra: test
+Requires-Dist: pip; extra == 'test'
+Requires-Dist: pytest; extra == 'test'
+Requires-Dist: ruff; extra == 'test'
+Description-Content-Type: text/markdown
+
+# pyprocessors-coreference
+
+Processor grouping the mentions of the same entity.
+
+Ce dépôt enregistre un processeur dans le groupe d'entry-points `pyprocessors.plugins` :
+
+| Plugin | Classe |
+|--------|--------|
+| `coreference` | `pyprocessors_coreference.coreference:CoreferenceProcessor` |
+
+Il regroupe les mentions d'une même entité **avant** le linking, pour qu'elles soient
+liées une seule fois, à partir de leur forme la plus longue. C'est l'étape de
+regroupement des mentions de l'ADR-0001 de `pyannotators_entityfishing`
+([`docs/adr/`](docs/adr/0001-linking-candidats-entityfishing-decision-jev.md)), palier 1 :
+les noms seulement.
+
+## Ce qu'il fait
+
+Pour chaque document, entre mentions de même label :
+
+- **même surface normalisée → même groupe.** La normalisation ignore la casse, les
+  espaces, le style des apostrophes, et les civilités en tête d'un nom de personne
+  (« M. », « Mme », « Dr »…) ;
+- **tous les labels de personne forment une seule famille** : le `person` du NER et
+  l'`afpperson` des lexiques AFP annotent le même span, et vont dans le même groupe ;
+- **un nom partiel de personne rejoint le nom complet dont il est la fin**, que le
+  nom complet soit avant ou après lui dans le document : « Macron » et « M. Macron »
+  rejoignent « Emmanuel Macron » ;
+- **un prénom seul ne rejoint jamais un nom complet** : « Emmanuel » reste à part ;
+- **un nom partiel ambigu n'est pas fusionné** : avec « Emmanuel Macron » et
+  « Brigitte Macron » dans le texte, « Macron » garde son propre groupe, et liste les
+  groupes possibles pour que l'étape de linking tranche ;
+- **la règle du nom partiel ne vaut que pour les personnes** : « Airbus » ne rejoint pas
+  « Airbus Defence and Space ».
+
+Sur chaque annotation d'un label regroupé, il écrit dans `properties` :
+
+| Propriété | Valeur |
+|-----------|--------|
+| `entity_group` | l'identifiant du groupe dans le document : `g1`, `g2`… dans l'ordre de première apparition |
+| `canonical_form` | la forme la plus longue du groupe, sans civilité : c'est elle qu'on cherche dans la base de connaissances |
+| `mention_type` | `name` (les périphrases et les pronoms, `nominal` et `pronoun`, viendront avec les paliers 2 et 3) |
+| `entity_group_candidates` | seulement sur un nom partiel ambigu : les groupes auxquels il peut appartenir |
+
+Les autres propriétés de l'annotation sont gardées, et traiter deux fois un document
+donne le même résultat. La portée est le document entier, ce qui convient à une
+dépêche ; les documents longs sont une question ouverte (SHERPA-3048).
+
+## Requirements
+
+- Python 3.12+
+- `pymultirole_plugins` (>=1.6.0,<1.7.0)
+
+## Installation
+
+```
+pip install pyprocessors-coreference
+```
+
+## Usage
+
+```python
+from pymultirole_plugins.v1.schema import Annotation, Document
+
+from pyprocessors_coreference.coreference import CoreferenceParameters, CoreferenceProcessor
+
+text = "Emmanuel Macron est arrivé. M. Macron a parlé."
+document = Document(
+    text=text,
+    annotations=[
+        Annotation(start=0, end=15, labelName="person", text="Emmanuel Macron"),
+        Annotation(start=28, end=37, labelName="person", text="M. Macron"),
+    ],
+)
+[document] = CoreferenceProcessor().process([document], CoreferenceParameters())
+# Both annotations: entity_group "g1", canonical_form "Emmanuel Macron".
+```
+
+### Parameters
+
+| Paramètre | Défaut | Description |
+|-----------|--------|-------------|
+| `labels` | vide | Labels des mentions à regrouper. Vide : tous les labels. |
+| `person_labels` | vide | Labels de personne : ils forment une seule famille, et la règle du nom partiel s'y applique. Vide : tout label dont le nom contient « person » (`person`, `afpperson`…). |
+| `honorifics` | `M.`, `Mme`, `Dr`, `Mr`… | Civilités ignorées en tête d'un nom de personne. Paramètre avancé. |
+
+## Development
+
+Le build est piloté par [Task](https://taskfile.dev) et [uv](https://docs.astral.sh/uv/),
+les stages venant du submodule `python-archetype` :
+
+```
+git submodule update --init
+task stages          # les stages du pipeline, dans l'ordre
+task                 # tout sauf le dernier stage — donc tout sauf la publication
+task -- --skip-tests # idem, sans le stage de test
+task up-to -- py:lint
+task jenkins         # tout, exactement ce que joue Jenkins
+task --list
+```
+
+`uv.lock` n'est pas versionné ici, donc `py:sync` résout les dépendances à neuf.

pyprocessors_coreference-1.6.1.dist-info/RECORD

@@ -0,0 +1,6 @@
+pyprocessors_coreference/__init__.py,sha256=FhyAPdjVtDKqUrcvJaY5vUCeqGcWK2UlYYOf3nQ_yY0,80
+pyprocessors_coreference/coreference.py,sha256=4rENXHvwliOCRMuO9BS-K9WUlLE55YaDGby-SDcgipI,7980
+pyprocessors_coreference-1.6.1.dist-info/METADATA,sha256=7TrpLF88egi26V1467WL1RrYo_moKqGcrehg0Le2w7k,5660
+pyprocessors_coreference-1.6.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+pyprocessors_coreference-1.6.1.dist-info/entry_points.txt,sha256=Ko1I0ZLLs3U0D2GDEZ_kzHN6OfsAjpwJNb7BCpWL-J4,95
+pyprocessors_coreference-1.6.1.dist-info/RECORD,,

pyprocessors_coreference-1.6.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

pyprocessors_coreference-1.6.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pyprocessors.plugins]
+coreference = pyprocessors_coreference.coreference:CoreferenceProcessor