crossref-matcher 1.3.2__tar.gz → 1.4.0__tar.gz

{crossref_matcher-1.3.2 → crossref_matcher-1.4.0}/MANIFEST.in

@@ -4,3 +4,4 @@ recursive-include crossref_matcher/strategies *.txt
 recursive-include crossref_matcher/resources/data/models *.pkl *.gz *.local
 recursive-include crossref_matcher/resources/data/common_ror_endings *.txt
 recursive-include crossref_matcher/resources/data/countries *.txt
+recursive-include crossref_matcher/resources/data/funders *.csv.gz

{crossref_matcher-1.3.2/crossref_matcher.egg-info → crossref_matcher-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crossref-matcher
-Version: 1.3.2
+Version: 1.4.0
 Summary: Crossref's matching API
 Home-page: https://gitlab.com/crossref/marple
 Author: Crossref

{crossref_matcher-1.3.2 → crossref_matcher-1.4.0}/README.md

@@ -22,6 +22,8 @@ Looking for the source code to the matching strategies?
   - Funder Name to ROR Search strategy -- [Source Code](crossref_matcher/strategies/funder/funder_name_to_ror_search/strategy.py)
 - Affiliation matching
   - Single Search strategy -- [Source Code](crossref_matcher/strategies/affiliation/single_search/strategy.py)
+- Preprint matching
+  - SBMV (Search-Based Matching with Validation) -- [Source Code](crossref_matcher/strategies/preprint/sbmv/strategy.py)
 
 ## Terminology
 
@@ -119,6 +121,22 @@ Some strategies need an OpenSearch index to match. [indexes directory](crossref_
 
 Use ES_HOST env var to point Marple to the OpenSearch cluster.
 
+#### Bootstrapping indexes for integration testing
+
+The `organization` (ROR) and `grant` indexes can each be created and populated from a pinned, static dataset hosted in a public Crossref S3 bucket. This needs no AWS credentials and no local data files, so it works from inside a bare container — it is how the [integration harness](https://gitlab.com/crossref/integration) populates OpenSearch on startup:
+
+```sh
+python -m crossref_matcher.indexes.ror_organizations.create_index
+python -m crossref_matcher.indexes.ror_organizations.index_data --integration
+
+python -m crossref_matcher.indexes.grants.create_index
+python -m crossref_matcher.indexes.grants.index_data --integration
+```
+
+`ror_organizations.create_index` creates a dated index (`organization-local-<time>`) behind the `organization` alias, and `index_data --integration` loads it through the alias. The `update_ror_index` DAG builds each new ROR version into its own dated index behind the same alias; see [`ror_organizations/update.py`](crossref_matcher/indexes/ror_organizations/update.py). To rebuild at the current ROR version (for example, after adding a document field), trigger the DAG with its `force` param, or run `index_data --force`.
+
+The grants dataset is a pre-built export of the finished index documents, so the `--integration` path needs no source tables, no ROR dump and no `pyarrow`. Regenerate and republish it with [`scripts/export_grant_index_docs.py`](scripts/export_grant_index_docs.py); see [data/grants_awards/README.md](data/grants_awards/README.md).
+
 ### How to run
 
 Run
@@ -178,6 +196,12 @@ To enable the check locally (as a non-blocking warning):
 git config core.hooksPath .githooks
 ```
 
+### Updating a strategy in production
+
+For the step-by-step process of shipping a change to an existing strategy
+(editing the strategy code, bumping its version, and merging to `main`), see
+[how_to_update_a_strategy.md](crossref_matcher/strategies/how_to_update_a_strategy.md).
+
 ### Dependencies
 
 Requirements are defined in `setup.py`. Requirements files with exact versions can be updated with `pip-compile`, which is a part of `pip-tools`. First run `pip install pip-tools`. Then:
@@ -192,7 +216,7 @@ CI (see [.gitlab-ci.yml](.gitlab-ci.yml)) deploys from two long-lived branches:
 
 | Branch / trigger | Deploys to | Jobs |
 |---|---|---|
-| `dev` | **Research** environment | ECS deploy (`marple-research`), Airflow DAG data sync to `s3://airflow.crossref.org` (research-only bucket), `X.Y.Z-dev.N` prerelease tags |
+| `dev` | **Research** environment | ECS deploy (`marple-research`), Airflow DAG data sync to the research Airflow DAG bucket (set by the `research`-scoped `AIRFLOW_DAG_BUCKET` CI/CD variable in GitLab; the research DAG runs on the Airflow 3 MWAA environment), `X.Y.Z-dev.N` prerelease tags |
 | `main` | **Staging** environment | ECS deploy (`marple-staging`), GitLab-registry image for the integration project, release tags via semantic-release |
 | Stable `vX.Y.Z` tags | PyPI | `crossref-matcher` published via OIDC trusted publishing (prerelease tags are not published) |
 
@@ -276,5 +300,5 @@ features still in flight:
 4. Disable force-push again.
 
 The reset push re-deploys research and re-syncs the Airflow DAG data in
-`s3://airflow.crossref.org` to the reset state (the sync uses `--delete`, so
-files added by abandoned experiments are removed, not just left stale).
+the research Airflow DAG bucket to the reset state (the sync uses `--delete`,
+so files added by abandoned experiments are removed, not just left stale).

{crossref_matcher-1.3.2 → crossref_matcher-1.4.0}/crossref_matcher/MatchTask.py

@@ -10,6 +10,7 @@ class MatchTask(Enum):
     PREPRINT = ("preprint", "Matching journal articles to preprints.")
     AFFILIATION = ("affiliation", "Matching affiliations to ROR IDs.")
     FUNDER = ("funder", "Matching funders to ROR IDs.")
+    GRANT = ("grant", "Matching grant awards to DOIs.")
     OTHER = ("other", "A generic matching task.")
     HEALTHCHECK = (
         "healthcheck",

{crossref_matcher-1.3.2 → crossref_matcher-1.4.0}/crossref_matcher/app.py

@@ -9,6 +9,7 @@ from crossref_matcher import MatchTask, __version__
 from crossref_matcher.evaluation.api_routes import router as evaluation_router
 from crossref_matcher.matching import schemas
 from crossref_matcher.strategies import (
+    InvalidInputError,
     Strategy,
     get_default_strategy,
     list_strategies,
@@ -196,6 +197,8 @@ def match(
                 items=items, strategy=strategy_id, target_data=target_data
             )
         )
+    except InvalidInputError as e:
+        raise HTTPException(status_code=422, detail=str(e))
     except NotImplementedError:
         raise HTTPException(
             status_code=400,
@@ -203,9 +206,22 @@ def match(
         )
 
 
+# Replaces FastAPI's own 422 entry, whose schema covers only parameter
+# validation errors and not the message an InvalidInputError carries.
+MATCH_RESPONSES = {
+    422: {
+        "model": schemas.ErrorResponse,
+        "description": "A parameter is missing or malformed, or the strategy "
+        "can't use the input. For the input, `detail` is a message describing "
+        "what the strategy expects.",
+    },
+}
+
+
 @app.get(
     "/match",
     response_model=schemas.MatchedItemsResponse,
+    responses=MATCH_RESPONSES,
     tags=["Matching"],
     description="Match input to identifiers.",
     response_class=AsciiJSONResponse,
@@ -221,6 +237,7 @@ async def match_get(
 @app.post(
     "/match",
     response_model=schemas.MatchedItemsResponse,
+    responses=MATCH_RESPONSES,
     tags=["Matching"],
     description="Match input to identifiers.",
     response_class=AsciiJSONResponse,

{crossref_matcher-1.3.2 → crossref_matcher-1.4.0}/crossref_matcher/evaluation/all_links.py

@@ -126,7 +126,23 @@ def get_all_links(items: Iterable[dict]) -> list[dict]:
         if len(item["output"]) == 0:
             this_result_true = ["no_match"]
         else:
-            this_result_true = item["output"]
+            this_result_true = list(item["output"])
+
+        # `matched` (and its relaxed/alternate counterpart) can contain more
+        # entries than there are true links, e.g. a strategy that returns
+        # several candidate matches for a single expected link.
+        # construct_matched_list only pairs up to len(this_result_true)
+        # matches and silently drops the rest, so pad the true list with
+        # "no_match" placeholders to give every matched entry its own row
+        # (which then correctly resolves to FP via res_cat).
+        n_extra = max(
+            0,
+            len(item["matched"]) - len(this_result_true),
+            len(get_matched_alternate(item)) - len(this_result_true),
+        )
+        if n_extra:
+            this_result_true = this_result_true + ["no_match"] * n_extra
+
         alternates_negative = [
             x[1] for x in item.get("alternates", []) if x[0] == "no_match"
         ]
@@ -139,6 +155,7 @@ def get_all_links(items: Iterable[dict]) -> list[dict]:
             get_matched_alternate(item),
             alternates_negative=alternates_negative,
         )
+        no_match_count = 0
         for i, this_row_true in enumerate(this_result_true):
             if this_row_true in this_result_matched:
                 this_row_matched = this_row_true
@@ -150,7 +167,14 @@ def get_all_links(items: Iterable[dict]) -> list[dict]:
             else:
                 this_row_matched_relaxed = this_result_matched_relaxed[0]
             this_result_matched_relaxed.remove(this_row_matched_relaxed)
-            unique_id = f"""{item["seq_no"]:06}_{this_row_true}"""
+            if this_row_true == "no_match":
+                # Multiple padded/no-output rows can share the same item, so
+                # disambiguate beyond the first to keep unique_id unique.
+                suffix = f"_{no_match_count}" if no_match_count else ""
+                unique_id = f"""{item["seq_no"]:06}_no_match{suffix}"""
+                no_match_count += 1
+            else:
+                unique_id = f"""{item["seq_no"]:06}_{this_row_true}"""
             target_in_initial_candidates = get_candidate_position(
                 item["extra"].get("initial_candidates", None), this_row_true
             )

crossref_matcher-1.4.0/crossref_matcher/indexes/aliased_index.py

@@ -0,0 +1,278 @@
+"""Build an OpenSearch index under a dated name and serve it through an alias.
+
+Each build creates ``<alias>-<suffix>``, loads and validates it, then moves
+the alias onto it in one atomic ``_aliases`` call, so readers and writers
+using the alias name never see a half-built index. Older dated indexes are
+pruned, keeping a few for rollback.
+
+If a concrete index (not an alias) still sits under the alias name, the
+build migrates it: the concrete index is cloned to a dated rollback copy,
+and the swap removes it and adds the alias in the same ``_aliases`` call, so
+the name keeps resolving throughout.
+
+Each build is created carrying the marker alias ``<alias>.unswapped``, and
+the swap removes the marker in the same ``_aliases`` call that adds the
+alias. A build that still carries the marker never went live: its process
+was killed before cleanup could run, or it was kept with ``keep_failed``.
+Pruning deletes such builds and keeps the newest rollback copies.
+
+Every index named ``<alias>-*`` that carries no alias is treated as a
+rollback copy of that alias and may be pruned, so that prefix must not be
+used for anything else.
+"""
+
+import logging
+import time
+from typing import Callable
+
+logger = logging.getLogger(__name__)
+
+# Rollback copies kept per alias, besides the live index.
+KEEP_PREVIOUS = 2
+
+HEALTH_POLL_SECONDS = 1.0
+HEALTH_WAIT_SECONDS = 300.0
+
+
+class ConcreteIndexError(RuntimeError):
+    """A concrete index sits under the alias name and no migration was asked
+    for."""
+
+
+def dated_index_name(alias: str, suffix: str) -> str:
+    return f"{alias}-{suffix}"
+
+
+def unswapped_alias(alias: str) -> str:
+    """The marker alias carried by builds of ``alias`` that were never live."""
+    return f"{alias}.unswapped"
+
+
+def concrete_index_exists(es_client, alias: str) -> bool:
+    """Return True if ``alias`` names a concrete index rather than an alias."""
+    return not es_client.indices.exists_alias(name=alias) and es_client.indices.exists(
+        index=alias
+    )
+
+
+def aliased_indexes(es_client, alias: str) -> list[str]:
+    """Return the indexes ``alias`` points to, or an empty list if there is no
+    such alias."""
+    if not es_client.indices.exists_alias(name=alias):
+        return []
+    return sorted(es_client.indices.get_alias(name=alias))
+
+
+def wait_for_index(
+    es_client,
+    index: str,
+    wait_seconds: float = HEALTH_WAIT_SECONDS,
+    poll_seconds: float = HEALTH_POLL_SECONDS,
+) -> None:
+    """Poll until ``index`` is at least yellow.
+
+    opensearch-py's ``cluster.health(..., timeout=...)`` clashes with the
+    client's own request timeout, so this polls instead of asking the server
+    to wait.
+    """
+    deadline = time.monotonic() + wait_seconds
+    while True:
+        status = es_client.cluster.health(index=index)["status"]
+        if status in ("green", "yellow"):
+            return
+        if time.monotonic() >= deadline:
+            raise TimeoutError(f"Index {index!r} still {status} after {wait_seconds}s")
+        time.sleep(poll_seconds)
+
+
+def build_and_swap(
+    es_client,
+    alias: str,
+    index_body: dict,
+    suffix: str,
+    load: Callable[[str], None],
+    validate: Callable[[str, str | None], None],
+    *,
+    migrate_suffix: str | None = None,
+    keep_failed: bool = False,
+    keep_previous: int = KEEP_PREVIOUS,
+) -> str:
+    """Build ``<alias>-<suffix>``, validate it and point ``alias`` at it.
+
+    Two builds of one alias must not run at once. The first to swap prunes
+    the other's index, which still carries the marker alias.
+
+    Args:
+        es_client: OpenSearch client.
+        alias: The name readers and writers use.
+        index_body: Settings and mappings for the new index.
+        suffix: Version or date that makes the new index name unique.
+        load: Called with the new index name to fill it.
+        validate: Called with the new index name and the live index name
+            (``None`` if nothing is live yet). Raise to stop the swap.
+        migrate_suffix: Needed only when a concrete index sits under
+            ``alias``. The concrete index is cloned to
+            ``<alias>-<migrate_suffix>`` as the rollback copy, so this should
+            be the version that index holds.
+        keep_failed: Keep the new index if loading or validation fails,
+            for inspection. By default it is deleted. A kept index still
+            carries the marker, so the next successful build prunes it.
+        keep_previous: Rollback copies to keep after the swap.
+
+    Returns:
+        The name of the new index, which ``alias`` now points to.
+
+    Raises:
+        ConcreteIndexError: A concrete index sits under ``alias`` and
+            ``migrate_suffix`` is ``None``.
+        ValueError: The new index name or the rollback copy's name is
+            already taken.
+    """
+    new_index = dated_index_name(alias, suffix)
+    marker = unswapped_alias(alias)
+    if es_client.indices.exists(index=new_index):
+        raise ValueError(f"Index {new_index!r} already exists")
+
+    migrating = concrete_index_exists(es_client, alias)
+    if migrating:
+        if migrate_suffix is None:
+            raise ConcreteIndexError(
+                f"{alias!r} is a concrete index; pass migrate_suffix to migrate it"
+            )
+        clone = dated_index_name(alias, migrate_suffix)
+        if clone == new_index:
+            raise ValueError(
+                f"Rollback copy and new index would both be named {new_index!r}"
+            )
+        if es_client.indices.exists(index=clone):
+            raise ValueError(f"Index {clone!r} already exists")
+        live_indexes = []
+        live = alias
+    else:
+        live_indexes = aliased_indexes(es_client, alias)
+        if len(live_indexes) > 1:
+            raise ValueError(
+                f"Alias {alias!r} points to several indexes: {live_indexes}"
+            )
+        live = live_indexes[0] if live_indexes else None
+
+    if migrating:
+        _clone_for_rollback(es_client, alias, clone)
+
+    try:
+        logger.info(f"Creating index {new_index!r}")
+        body = {
+            **index_body,
+            "aliases": {**index_body.get("aliases", {}), marker: {}},
+        }
+        es_client.indices.create(index=new_index, body=body)
+        wait_for_index(es_client, new_index)
+        load(new_index)
+        es_client.indices.refresh(index=new_index)
+        validate(new_index, live)
+    except Exception:
+        logger.exception(f"Build of {new_index!r} failed; {alias!r} is unchanged")
+        if not keep_failed and es_client.indices.exists(index=new_index):
+            es_client.indices.delete(index=new_index)
+        if migrating:
+            # The concrete index stays live, so a later run makes a fresh copy.
+            es_client.indices.delete(index=clone)
+        raise
+
+    if migrating:
+        actions = [{"remove_index": {"index": alias}}]
+    else:
+        actions = [{"remove": {"index": i, "alias": alias}} for i in live_indexes]
+    actions.append({"remove": {"index": new_index, "alias": marker}})
+    actions.append({"add": {"index": new_index, "alias": alias}})
+    es_client.indices.update_aliases(body={"actions": actions})
+    logger.info(f"Alias {alias!r} now points to {new_index!r} (was {live!r})")
+
+    prune(es_client, alias, keep_previous)
+    return new_index
+
+
+def _clone_for_rollback(es_client, index: str, clone: str) -> None:
+    """Copy the concrete ``index`` to ``clone``.
+
+    ``_clone`` needs a write-blocked source. The block is lifted from the
+    source straight after, so it lasts only as long as the copy. The clone
+    keeps it; ``rollback`` clears it.
+    """
+    logger.info(f"Cloning concrete index {index!r} to {clone!r}")
+    es_client.indices.put_settings(index=index, body={"index.blocks.write": True})
+    try:
+        es_client.indices.clone(index=index, target=clone)
+        wait_for_index(es_client, clone)
+    finally:
+        es_client.indices.put_settings(index=index, body={"index.blocks.write": None})
+
+
+def prune(es_client, alias: str, keep_previous: int = KEEP_PREVIOUS) -> list[str]:
+    """Delete unswapped builds of ``alias``, and the rollback copies beyond
+    the newest ``keep_previous``.
+
+    An unswapped build carries ``unswapped_alias(alias)``. Indexes that carry
+    any other alias are never deleted. Age is the index's creation date,
+    since suffixes such as ROR versions don't sort as strings.
+
+    Returns:
+        The deleted index names.
+    """
+    found = es_client.indices.get_alias(index=f"{alias}-*")
+    marker = unswapped_alias(alias)
+    # Unlike rollback, spare a marked index that also carries another alias.
+    orphans = [
+        name for name, info in found.items() if set(info.get("aliases", {})) == {marker}
+    ]
+    for name in orphans:
+        logger.info(f"Pruning orphan {name!r}, a build that was never live")
+        es_client.indices.delete(index=name)
+    candidates = [name for name, info in found.items() if not info.get("aliases")]
+    if len(candidates) <= keep_previous:
+        return orphans
+    settings = es_client.indices.get_settings(
+        index=",".join(candidates), name="index.creation_date"
+    )
+    candidates.sort(
+        key=lambda name: int(settings[name]["settings"]["index"]["creation_date"]),
+        reverse=True,
+    )
+    to_delete = candidates[keep_previous:]
+    for name in to_delete:
+        logger.info(f"Pruning rollback copy {name!r}")
+        es_client.indices.delete(index=name)
+    return orphans + to_delete
+
+
+def rollback(es_client, alias: str, index: str, *, force: bool = False) -> None:
+    """Point ``alias`` back at an earlier ``index``.
+
+    Clears ``index.blocks.write`` on ``index`` first: a rollback copy made
+    during migration inherits the block, and writes through the alias would
+    fail with it in place.
+
+    Raises:
+        ValueError: ``index`` is an unswapped build (it still carries
+            ``unswapped_alias(alias)``) and ``force`` is False. Such a build
+            was never live, and may not have been fully loaded or validated.
+    """
+    if concrete_index_exists(es_client, alias):
+        raise ConcreteIndexError(f"{alias!r} is a concrete index, not an alias")
+    marker = unswapped_alias(alias)
+    unswapped = index in aliased_indexes(es_client, marker)
+    if unswapped and not force:
+        raise ValueError(
+            f"{index!r} was never live under {alias!r}; pass force=True to use it"
+        )
+    es_client.indices.put_settings(index=index, body={"index.blocks.write": None})
+    actions = [
+        {"remove": {"index": i, "alias": alias}}
+        for i in aliased_indexes(es_client, alias)
+        if i != index
+    ]
+    if unswapped:
+        actions.append({"remove": {"index": index, "alias": marker}})
+    actions.append({"add": {"index": index, "alias": alias}})
+    es_client.indices.update_aliases(body={"actions": actions})
+    logger.info(f"Alias {alias!r} rolled back to {index!r}")

crossref_matcher-1.4.0/crossref_matcher/indexes/grants/create_index.py

@@ -0,0 +1,115 @@
+"""Create and initialize the Crossref grants OpenSearch index.
+
+The index holds one document per grant DOI. Award numbers and funder
+identifiers are stored as ``keyword`` fields (normalized at index time) so that
+matching can be done with cheap, deterministic ``term``/``terms`` queries
+rather than free-text relevance search. See
+``crossref_matcher/indexes/grants/normalize.py`` for the
+normalization rules.
+"""
+
+from opensearchpy import OpenSearch
+from opensearchpy.exceptions import NotFoundError
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+INDEX = "grant"
+
+
+def index_body() -> dict:
+    """Return the OpenSearch settings and mappings for the grants index."""
+    return {
+        "settings": {
+            "number_of_shards": 1,
+        },
+        "mappings": {
+            "_meta": {
+                # Identifies the dataset version indexed, e.g. a snapshot date.
+                # Filled in when indexing.
+                "target_data": "",
+            },
+            "properties": {
+                # Original grant DOI (as deposited); used to build the output id.
+                "doi": {"type": "keyword"},
+                # Normalized (lowercased, resolver-stripped) DOI, for the direct
+                # grant-DOI fast path where the incoming award *is* a grant DOI.
+                "doi_norm": {"type": "keyword"},
+                # Normalized grant award number(s). Queried with the variants of
+                # the incoming award string.
+                "award": {"type": "keyword"},
+                # Normalized Funder Registry (Fundref) DOIs for this grant.
+                "funder_ids": {"type": "keyword"},
+                # Normalized ROR IDs for this grant.
+                "ror_ids": {"type": "keyword"},
+                # ROR IDs hierarchically related to this grant's funders: the
+                # transitive ancestors and descendants (ROR parent/child
+                # relationships) of every ID in ror_ids, excluding ror_ids
+                # itself so the exact and hierarchy confidence tiers stay
+                # disjoint. Funders with no ROR record of their own (e.g. EU
+                # funding programmes like H2020) are canonicalized to their
+                # nearest Fundref ancestor's ROR before ror_ids is built (see
+                # the packaged funder->ROR mapping), so their registry
+                # hierarchy is captured here rather than in a separate field.
+                "ror_family": {"type": "keyword"},
+                # Publisher (member) name, carried through for reporting/debugging.
+                "publisher": {"type": "keyword"},
+            },
+        },
+    }
+
+
+def initialize_index(index: str, es_client: OpenSearch, body: dict) -> dict:
+    """Create the OpenSearch index and return the client response.
+
+    Args:
+        index: Name of the index to create.
+        es_client: OpenSearch client instance.
+        body: Index settings and mappings payload.
+
+    Returns:
+        OpenSearch response payload from index creation.
+    """
+    response = es_client.indices.create(index=index, body=body)
+    return response
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    logging.getLogger("opensearch").setLevel(logging.WARNING)
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Create OpenSearch index for grant matching"
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Force delete and recreate the index if it already exists",
+    )
+    args = parser.parse_args()
+
+    from crossref_matcher.matching.search import ES_CLIENT as es_client
+
+    # Check if index already exists
+    index_exists = es_client.indices.exists(index=INDEX)
+
+    if index_exists and not args.force:
+        logger.info(
+            f"Index '{INDEX}' already exists. Use --force to delete and recreate it."
+        )
+        exit(0)
+
+    # Delete index if it exists and force flag is set
+    if index_exists and args.force:
+        try:
+            response = es_client.indices.delete(index=INDEX)
+            logger.info(f"Deleted existing index: {response}")
+        except NotFoundError:
+            logger.warning("Index not found during deletion")
+
+    index_body_content = index_body()
+
+    response = initialize_index(INDEX, es_client, index_body_content)
+    logger.info(f"Created index '{INDEX}': {response}")