crossref-matcher 1.2.1__tar.gz → 1.2.2__tar.gz

{crossref_matcher-1.2.1/crossref_matcher.egg-info → crossref_matcher-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crossref-matcher
-Version: 1.2.1
+Version: 1.2.2
 Summary: Crossref's matching API
 Home-page: https://gitlab.com/crossref/marple
 Author: Crossref

crossref_matcher-1.2.2/crossref_matcher/resources/VERSION.txt

@@ -0,0 +1 @@
+1.2.2

{crossref_matcher-1.2.1 → crossref_matcher-1.2.2}/crossref_matcher/resources/plugins.json

@@ -11,7 +11,6 @@
         "crossref_matcher.strategies.reference.sbmv",
         "crossref_matcher.strategies.reference.sbmv_orig",
         "crossref_matcher.strategies.reference.sbmv_unstructured",
-        "crossref_matcher.strategies.funder.funder_single_search_01",
         "crossref_matcher.strategies.funder.funder_name_to_ror_search",
         "crossref_example_matching_strategy"
     ],

{crossref_matcher-1.2.1 → crossref_matcher-1.2.2}/crossref_matcher/strategies/funder/funder_name_to_ror_search/strategy.py

@@ -1,7 +1,7 @@
 import os
 import re
-from crossref_matcher.matching.utils import get_resource_path
 from typing import Iterable
+from crossref_matcher.matching.utils import get_resource_path
 from dataclasses import dataclass
 import unidecode
 from crossref_matcher.strategies import Strategy, MatchTask
@@ -53,6 +53,7 @@ class FunderNameToRorSearch(Strategy):
     index = "organization"
     min_score = 96
 
+    # Matches acronyms: sequences of 2 or more consecutive uppercase ASCII letters as whole words (e.g., 'NASA', 'UNESCO').
     pattern_acronym = re.compile(r"(?P<acronym>\b[A-Z]{2,}\b)")
 
     def __init__(self, es_client=ES_CLIENT, max_candidates=18, extra_output=False):
@@ -117,6 +118,12 @@ class FunderNameToRorSearch(Strategy):
     def match(self, input_data):
         self.input_string_rep = StringRep(input_data)
         self.fund_countries = self.get_countries(self.input_string_rep)
+
+        # Before anything, check for exact acronym match (very conservatively)
+        acronym_result = self.acronym_substrategy_match(self.input_string_rep)
+        if acronym_result:
+            return [acronym_result]
+
         candidates = self.get_candidates(input_data, max_candidates=self.max_candidates)
         # store initial candidates for help with debugging/evaluation
         self.initial_candidates = [
@@ -200,7 +207,14 @@ class FunderNameToRorSearch(Strategy):
         )["hits"]["hits"]
 
     def _exclusion_acronym_common_ending(self, fund: StringRep, name: StringRep):
-        # returns True if this `name` should be excluded
+        """
+        If the ROR name (`name`) being considered ends in a common ending
+        (e.g., "foundation", "research center") and contains an acronym
+        (e.g., "NASA"), then exclude it if the acronym(s) do not also occur
+        in the fund name (`fund`).
+
+        Returns True if this `name` should be excluded.
+        """
         acronyms = self.pattern_acronym.findall(name.original)
         if not acronyms:
             return False
@@ -416,3 +430,27 @@ class FunderNameToRorSearch(Strategy):
     def get_countries(self, s: StringRep) -> list[str]:
         codes = self.get_country_codes(s)
         return [self.to_region(c) for c in codes]
+
+    def acronym_substrategy_match(self, s: StringRep) -> dict | None:
+        stripped = s.original.strip()
+        whitelist = ["CNPq"]
+        # Must be 1 word, all uppercase, and 4 or more letters
+        if re.fullmatch(r"[A-Z]{4,}", stripped) or stripped in whitelist:
+            r = self.es_client.search(
+                body={
+                    "size": 10,
+                    "query": {
+                        "bool": {"filter": {"term": {"acronyms": {"value": stripped}}}}
+                    },
+                },
+                index=self.index,
+            )["hits"]["hits"]
+
+            # there must be one and only one match
+            if r and len(r) == 1:
+                return {
+                    "id": r[0]["_id"],
+                    "confidence": 1,
+                    "strategies": [self.strategy],
+                }
+        return None

{crossref_matcher-1.2.1 → crossref_matcher-1.2.2/crossref_matcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crossref-matcher
-Version: 1.2.1
+Version: 1.2.2
 Summary: Crossref's matching API
 Home-page: https://gitlab.com/crossref/marple
 Author: Crossref

{crossref_matcher-1.2.1 → crossref_matcher-1.2.2}/crossref_matcher.egg-info/SOURCES.txt

@@ -80,10 +80,6 @@ crossref_matcher/strategies/affiliation/single_search/strategy.py
 crossref_matcher/strategies/funder/__init__.py
 crossref_matcher/strategies/funder/funder_name_to_ror_search/__init__.py
 crossref_matcher/strategies/funder/funder_name_to_ror_search/strategy.py
-crossref_matcher/strategies/funder/funder_single_search_01/__init__.py
-crossref_matcher/strategies/funder/funder_single_search_01/strategy.py
-crossref_matcher/strategies/funder/funder_single_search_02/__init__.py
-crossref_matcher/strategies/funder/funder_single_search_02/strategy.py
 crossref_matcher/strategies/healthcheck/__init__.py
 crossref_matcher/strategies/healthcheck/healthcheck/__init__.py
 crossref_matcher/strategies/healthcheck/healthcheck/strategy.py

crossref_matcher-1.2.1/crossref_matcher/resources/VERSION.txt

@@ -1 +0,0 @@
-1.2.1

crossref_matcher-1.2.1/crossref_matcher/strategies/funder/funder_single_search_01/strategy.py

@@ -1,481 +0,0 @@
-import os
-import re
-from crossref_matcher.matching.utils import get_resource_path
-from typing import Iterable
-from dataclasses import dataclass
-import unidecode
-import unicodedata
-from functools import cached_property
-from crossref_matcher.strategies import Strategy, MatchTask
-
-from crossref_matcher.matching.search import ES_CLIENT
-
-from rapidfuzz import fuzz
-
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-def matches_exactly(name, substring, ignore_case=True):
-    if ignore_case:
-        try:
-            name = name.lower_
-        except AttributeError:
-            name = name.lower()
-        substring = substring.lower()
-    return re.search(rf"\b{re.escape(substring)}\b", name) is not None
-
-
-class StringRep(str):
-    # This is meant to standardize all of the manipulations of the name strings
-    def __new__(cls, s: str):
-        obj = super().__new__(cls, s)
-        obj.original = str(s)
-        return obj
-
-    def __str__(self):
-        return self.original
-
-    def __repr__(self):
-        return f"StringRep({self.original!r})"
-
-    def __len__(self):
-        return len(self.original)
-
-    @cached_property
-    def lower_(self):
-        return re.sub(r"\s+", " ", self.original.lower())
-
-    @cached_property
-    def lower_alpha(self):
-        s = unidecode.unidecode(self.original).strip()
-        return re.sub(
-            r"\s+", " ", re.sub("[^a-z]", " ", s.lower())
-        )  # Lowercase and keep only letters
-
-    @cached_property
-    def alpha(self):
-        s = unidecode.unidecode(self.original).strip()
-        return re.sub(r"\s+", " ", re.sub("[^a-zA-Z]", " ", s))  # Keep only letters
-
-    def check_latin_chars(self, s):
-        for ch in s:
-            if ch.isalpha():
-                if "LATIN" not in unicodedata.name(ch):
-                    return False
-        return True
-
-    @cached_property
-    def normalized(self):
-        return self._normalize(self.original)
-
-    def _normalize(self, s):
-        """Normalize a string for matching.
-
-        Args:
-            s (str): The string to normalize.
-
-        Returns:
-            str: The normalized string.
-        """
-        if self.check_latin_chars(s):
-            # if these are Latin characters, remove accent marks
-            s = re.sub(r"\s+", " ", unidecode.unidecode(s).strip().lower())
-        else:
-            s = re.sub(r"\s+", " ", s.strip().lower())
-
-        # Replace specific abbreviations with their full forms
-        s = re.sub(
-            "(?<![a-z])univ$",
-            "university",
-            re.sub(
-                r"(?<![a-z])univ[\. ]",
-                "university ",
-                re.sub(r"(?<![a-z])u\.(?! ?[a-z]\.)", "university ", s),
-            ),
-        )
-        s = re.sub(
-            "(?<![a-z])lab$",
-            "laboratory",
-            re.sub("(?<![a-z])lab[^a-z]", "laboratory ", s),
-        )
-        s = re.sub(
-            "(?<![a-z])inst$",
-            "institute",
-            re.sub("(?<![a-z])inst[^a-z]", "institute ", s),
-        )
-        s = re.sub(
-            "(?<![a-z])tech$",
-            "technology",
-            re.sub("(?<![a-z])tech[^a-z]", "technology ", s),
-        )
-        s = re.sub(r"(?<![a-z])u\. ?s\.", "united states", s)
-        s = re.sub("&", " and ", re.sub("&amp;", " and ", s))
-        s = re.sub("^the ", "", s)
-
-        s = re.sub(r"\s+", " ", s.strip().lower())
-        return s
-
-
-@dataclass
-class CandidateMatch:
-    # Represents a candidate match in the search
-    candidate: dict
-    name: StringRep
-    score: float
-    start: int
-    end: int
-
-    def trimmed_output(self):
-        return {
-            "ror_id": self.candidate["_id"],
-            "name": self.name.original,
-            "score": self.score,
-            "start": self.start,
-            "end": self.end,
-        }
-
-
-class FunderSingleSearch01(Strategy):
-    id = "funder-single-search-01"
-    task = MatchTask.FUNDER
-    description = "Heuristic-based strategy; it performs a single candidate search against ES and uses a number of heuristics to validate the candidates."
-
-    index = "organization"
-    min_score = 96
-
-    pattern_acronym = re.compile(r"(?P<acronym>\b[A-Z]{2,}\b)")
-
-    def __init__(self, es_client=ES_CLIENT, max_candidates=18):
-        self.es_client = es_client
-        self.max_candidates = max_candidates
-        if self.es_client is None:
-            if os.getenv("PYTEST_CURRENT_TEST"):
-                logger.warning(
-                    "this strategy normally requires an elasticsearch client, but pytest is running, so bypassing this error"
-                )
-            else:
-                raise RuntimeError("this strategy requires an elasticsearch client")
-
-        self.countries = []
-        with open(
-            get_resource_path(
-                "crossref_matcher.resources.data.countries", "countries2.txt"
-            ),
-            "r",
-            encoding="utf-8",
-        ) as file:
-            lines = [
-                line.strip().split()
-                for line in file
-                if not line.startswith("#") and line.strip()
-            ]
-            self.countries = [(line[0], " ".join(line[1:])) for line in lines]
-
-        self.common_ror_endings = set()
-        with open(
-            get_resource_path(
-                "crossref_matcher.resources.data.common_ror_endings",
-                "common_ror_endings.txt",
-            ),
-            "r",
-            encoding="utf-8",
-        ) as file:
-            for line in file:
-                line = line.strip()
-                if line:
-                    self.common_ror_endings.add(line)
-
-        self.fund_countries = []  # can be filled in during match()
-        self.initial_candidates = []  # can be filled in during match()
-        # self.word_counter = Counter()  # can be filled in during match()
-
-    def match(self, input_data, extra=False):
-        self.input_string_rep = StringRep(input_data)
-        self.fund_countries = self.get_countries(self.input_string_rep)
-        candidates = self.get_candidates(input_data, max_candidates=self.max_candidates)
-        # store initial candidates for help with debugging/evaluation
-        self.initial_candidates = [
-            {"id": c["_id"], "elasticsearch_score": c["_score"]} for c in candidates
-        ]
-        candidates = [c for c in candidates if self.eligible_candidates_filter(c)]
-
-        # self.word_counter = Counter()
-        # for candidate in candidates:
-        #     this_candidate_words = set()
-        #     for candidate_name in candidate["_source"]["names"]:
-        #         name = candidate_name["name"]
-        #         # count each word only once per candidate
-        #         this_candidate_words.update(re.split(r"\b", name.lower()))
-        #     self.word_counter.update(this_candidate_words)
-        # common_words = set(word for word, freq in self.word_counter.most_common() if freq > 1 and len(word) > 1)
-
-        candidates = [self.score(self.input_string_rep, c) for c in candidates]
-        all_scored_candidates = candidates
-        candidates = [c for c in candidates if c.score >= self.min_score]
-
-        matched = self.choose_candidate(self.input_string_rep, candidates)
-        if matched is None:
-            return []
-        if (
-            self.fund_countries
-            and self.to_region(matched.candidate["_source"]["country"])
-            not in self.fund_countries
-        ):
-            return []
-
-        ret = {
-            "id": matched.candidate["_id"],
-            "confidence": min(12, matched.candidate["_score"]) / 12,
-            "strategies": [self.strategy],
-        }
-        if extra is True:
-            candidates_out = [c.trimmed_output() for c in all_scored_candidates]
-            candidates_out.sort(key=lambda c: c["score"], reverse=True)
-            ret.update(
-                {
-                    "matched_name": matched.name,
-                    "alignment_score": matched.score,
-                    "fund_countries": self.fund_countries,
-                    "all_scored_candidates": candidates_out,
-                    "elasticsearch_score": matched.candidate["_score"],
-                }
-            )
-        return [ret]
-
-    def eligible_candidates_filter(self, candidate) -> bool:
-        if candidate["_source"]["status"] not in ["active", "inactive"]:
-            return False
-        elif (
-            self.fund_countries
-            and self.to_region(candidate["_source"]["country"])
-            not in self.fund_countries
-        ):
-            return False
-        else:
-            return True
-
-    def get_candidates(self, fund, max_candidates=20):
-        return self.es_client.search(
-            body={
-                "size": max_candidates,
-                # "size": 10,
-                "query": {
-                    "nested": {
-                        "path": "names",
-                        "score_mode": "max",
-                        "query": {"match": {"names.name": {"query": fund}}},
-                    }
-                },
-            },
-            index=self.index,
-        )["hits"]["hits"]
-
-    def _exclusion_acronym_common_ending(self, fund: StringRep, name: StringRep):
-        # returns True if this `name` should be excluded
-        acronyms = self.pattern_acronym.findall(name.original)
-        if not acronyms:
-            return False
-        for ending in self.common_ror_endings:
-            # use a regex to see if `name` ends in `ending`
-            if re.search(rf"\b{re.escape(ending)}$", name.lower_):
-                # acronyms from `name` must also occur in `fund` (not fuzzy)
-                if not all(matches_exactly(fund, acronym) for acronym in acronyms):
-                    return True
-        return False
-
-    def candidate_name_match_exclusion(self, fund: StringRep, name: StringRep):
-        # returns True if this `name` should be excluded
-        if (
-            # names that are too general to match:
-            name.lower_
-            in [
-                "university school",
-                "university hospital",
-                "development fund",
-                "korea university",  # there are two korea universities (one in japan, one in south korea)
-            ]
-            # names that are too general to match unless a country is matched:
-            or (
-                not self.fund_countries
-                and name.lower_
-                in [
-                    "computing center",
-                    "center for international studies",
-                    "central government",
-                    "centre for international studies",
-                    "department of education",
-                    "department of conservation",
-                    "marie curie",
-                    "department of the environment",
-                    "department of veterans affairs",
-                    "development research center",
-                    "education trust",
-                    "heritage fund",
-                    "hospital research foundation",
-                    "medical research council",
-                    "medical research foundation",
-                    "ministry of agriculture, food and rural affairs",
-                    "ministry of education",
-                    "ministry of science",
-                    "national council of science and technology",
-                    "national institute of health research",
-                    "national research institute",
-                    "national science foundation",
-                    "national science fund",
-                    "prostate cancer research",
-                    "research promotion foundation",
-                    "startup foundation",
-                    "the alliance",
-                    "l’alliance",
-                    "alliance",
-                    "the arc",
-                    "the data lab",
-                    "the george",
-                    "the research council",
-                    "the research network",
-                    "the royal",
-                    "the tech",
-                    "the thousand",
-                    "the trust",
-                    "the way",
-                    "union government",
-                    "waste management",
-                    "r core",
-                    "creative foundation",
-                    "fonds de la recherche scientifique",
-                    "fund for scientific research",
-                    "fonds national de la recherche",
-                    "foundation for strategic research",
-                ]
-            )
-            or len(name) >= len(fund) + 4
-            or len(name) < 5
-            or (" " not in name.original and fund.lower_ != name.lower_)
-            or (" " not in fund.original and fund.lower_ != name.lower_)
-            or self._exclusion_acronym_common_ending(fund, name)
-        ):
-            return True
-        return False
-
-    def score(self, fund: StringRep, candidate) -> CandidateMatch:
-        best = CandidateMatch(
-            candidate=candidate, name=StringRep(""), score=0, start=-1, end=-1
-        )
-        for candidate_name in candidate["_source"]["names"]:
-            name = StringRep(candidate_name["name"])
-            if self.candidate_name_match_exclusion(fund, name):
-                continue
-            alignment = fuzz.partial_ratio_alignment(fund.normalized, name.normalized)
-            if alignment.score > best.score:
-                best = CandidateMatch(
-                    candidate=candidate,
-                    name=name,
-                    score=alignment.score,
-                    start=alignment.src_start,
-                    end=alignment.src_end,
-                )
-        return best
-
-    def is_better(
-        self, fund: StringRep, candidate: CandidateMatch, other: CandidateMatch
-    ) -> bool:
-        score = 0
-        if "univ" in candidate.name.lower_ and "univ" not in other.name.lower_:
-            score += 1
-        if "univ" not in candidate.name.lower_ and "univ" in other.name.lower_:
-            score -= 1
-        c_diff = abs(len(candidate.name) - len(fund))
-        o_diff = abs(len(other.name) - len(fund))
-        if o_diff - c_diff > 4:
-            score += 1
-        if c_diff - o_diff > 4:
-            score -= 1
-        if candidate.start > other.end:
-            score += 1
-        if other.start > candidate.end:
-            score -= 1
-        if candidate.score > 99 and other.score < 99:
-            score += 1
-        if candidate.score < 99 and other.score > 99:
-            score -= 1
-        return score > 0
-
-    def rescore(self, fund: StringRep, candidates: Iterable[CandidateMatch]):
-        new_scores = []
-        for candidate in candidates:
-            ns = 0
-            for other in candidates:
-                if self.is_better(fund, candidate, other):
-                    ns += 1
-            new_scores.append(ns)
-        for c, ns in zip(candidates, new_scores):
-            c.score = ns
-            yield c
-
-    def last_non_overlapping(self, candidates):
-        matched = None
-        for candidate in candidates:
-            overlap = False
-            for other in candidates:
-                if candidate.candidate["_id"] == other.candidate["_id"]:
-                    continue
-                if (
-                    candidate.start <= other.start <= candidate.end
-                    or candidate.start <= other.end <= candidate.end
-                    or other.start <= candidate.start <= other.end
-                    or other.start <= candidate.end <= other.end
-                ):
-                    overlap = True
-            if not overlap:
-                matched = candidate
-        return matched
-
-    def choose_candidate(self, fund: StringRep, candidates: Iterable[CandidateMatch]):
-        if not candidates:
-            return None
-
-        if len(candidates) == 1:
-            return candidates[0]
-
-        rescored = list(self.rescore(fund, candidates))
-        top_score = max([c.score for c in rescored])
-        top_scored = [c for c in rescored if c.score == top_score]
-
-        if len(top_scored) == 1:
-            return top_scored[0]
-
-        return self.last_non_overlapping(top_scored)
-
-    def to_region(self, c: str) -> str:
-        return {
-            "GB": "GB-UK",
-            "UK": "GB-UK",
-            "CN": "CN-HK-TW",
-            "HK": "CN-HK-TW",
-            "TW": "CN-HK-TW",
-            "PR": "US-PR",
-            "US": "US-PR",
-        }.get(c, c)
-
-    def get_country_codes(self, s: StringRep) -> list[str]:
-        _lower = re.sub(r"\s+", " ", unidecode.unidecode(s.original).strip().lower())
-        codes = []
-        for code, name in self.countries:
-            if re.search("[^a-z]", name):
-                # name contains not only letters (e.g., a space)
-                score = fuzz.partial_ratio(name, _lower)
-            elif len(name) == 2:
-                score = max(
-                    [fuzz.ratio(name.upper(), t) for t in s.alpha.split()] + [0]
-                )
-            else:
-                score = max([fuzz.ratio(name, t) for t in s.lower_alpha.split()] + [0])
-            if score >= 90:
-                codes.append(code.upper())
-        return list(set(codes))
-
-    def get_countries(self, s: StringRep) -> list[str]:
-        codes = self.get_country_codes(s)
-        return [self.to_region(c) for c in codes]