crossref-matcher 1.3.1__tar.gz → 1.3.2__tar.gz

{crossref_matcher-1.3.1/crossref_matcher.egg-info → crossref_matcher-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crossref-matcher
-Version: 1.3.1
+Version: 1.3.2
 Summary: Crossref's matching API
 Home-page: https://gitlab.com/crossref/marple
 Author: Crossref

{crossref_matcher-1.3.1 → crossref_matcher-1.3.2}/README.md

@@ -185,3 +185,96 @@ Requirements are defined in `setup.py`. Requirements files with exact versions c
 * To update `requirements.txt` file: `pip-compile setup.py --extra strategies-core`
 
 * To update `requirements-tests.txt` file: `pip-compile setup.py --extra strategies-core --extra evaluation --extra test -o requirements-tests.txt`
+
+## Branching and deployment
+
+CI (see [.gitlab-ci.yml](.gitlab-ci.yml)) deploys from two long-lived branches:
+
+| Branch / trigger | Deploys to | Jobs |
+|---|---|---|
+| `dev` | **Research** environment | ECS deploy (`marple-research`), Airflow DAG data sync to `s3://airflow.crossref.org` (research-only bucket), `X.Y.Z-dev.N` prerelease tags |
+| `main` | **Staging** environment | ECS deploy (`marple-staging`), GitLab-registry image for the integration project, release tags via semantic-release |
+| Stable `vX.Y.Z` tags | PyPI | `crossref-matcher` published via OIDC trusted publishing (prerelease tags are not published) |
+
+Both branches are protected. `main` is merged to via MR only; `dev`'s
+protection must also allow Developers to push and merge directly
+(**Settings → Repository → Protected branches → `dev` → Allowed to push
+and merge**), since the workflow below pushes to it directly. Deploys are
+automatic on push and are keyed on the commit SHA (`$CI_COMMIT_SHORT_SHA`),
+independent of the semantic version.
+
+### Feature workflow
+
+1. **Branch off `main`** (not `dev`), so the feature can merge to `main`
+   cleanly regardless of what else is being tested on `dev`.
+2. **To test on the research environment**, merge the feature into `dev` and
+   push (no MR needed — review happens at the `main` MR):
+
+   ```sh
+   git checkout dev && git pull
+   git merge --no-ff my-feature
+   git push
+   ```
+
+3. **When validated, open an MR from the feature branch into `main`.**
+   Do not merge `dev` into `main` — `dev` may contain other unfinished work.
+   Use a [conventional commit](https://www.conventionalcommits.org/) message
+   (`feat:`/`fix:`) for the squash/merge commit; semantic-release derives the
+   release version from it.
+4. **After the MR merges, merge `main` back into `dev`** to keep the branches
+   converged. This isn't strictly required — semantic-release's prerelease
+   branches skip the "behind the release branch" check — but without it,
+   `dev`'s next prerelease can recompute a version `main` already released,
+   or a confusing `vX.Y.Z-dev.N` alongside an already-shipped `vX.Y.Z`:
+
+   ```sh
+   git checkout dev && git pull
+   git merge origin/main
+   git push
+   ```
+
+### Versioning
+
+Releases are cut by semantic-release from conventional commit messages
+(`fix:` → patch, `feat:` → minor, `BREAKING CHANGE` → major; other types cut
+no release). On `dev`, release-worthy commits produce prerelease tags like
+`v1.3.0-dev.1` (configured in [.releaserc.json](.releaserc.json)). Commits
+without a `feat:`/`fix:` still deploy — only the tag is skipped, and
+`VERSION.txt` keeps its `0.0.dev0` placeholder. To identify what is running
+in an environment, use the commit SHA (the ECS image tag), not the version
+string.
+
+### Resetting a polluted `dev` back to `main`
+
+`dev` is disposable: `main` never depends on its history. If `dev` has
+accumulated abandoned experiments, reset it to `main` and re-merge only the
+features still in flight:
+
+1. Temporarily enable force-push for `dev` in GitLab
+   (**Settings → Repository → Protected branches → `dev` → Allowed to force push**).
+2. Delete any `vX.Y.Z-dev.N` prerelease tags cut on the old `dev` — after the
+   reset their commits are unreachable from the branch, so semantic-release
+   won't see them and will try to recreate the same tag, failing every
+   subsequent release-worthy `dev` push until they're gone:
+
+   ```sh
+   git tag -l 'v*-dev.*' | xargs -r git push origin --delete
+   git tag -l 'v*-dev.*' | xargs -r git tag -d
+   ```
+
+3. Reset and push:
+
+   ```sh
+   git fetch origin
+   git checkout dev
+   git reset --hard origin/main
+   git push --force-with-lease origin dev
+   git merge --no-ff still-in-flight-feature   # re-merge anything still being tested
+   git push
+   ```
+
+4. Disable force-push again.
+
+The reset push re-deploys research and re-syncs the Airflow DAG data in
+`s3://airflow.crossref.org` to the reset state (the sync uses `--delete`, so
+files added by abandoned experiments are removed, not just left stale).

crossref_matcher-1.3.2/crossref_matcher/indexes/ror_organizations/opensearch_utils.py

@@ -0,0 +1,44 @@
+"""Shared OpenSearch client utilities for ROR index DAG tasks."""
+
+import base64
+import boto3
+
+REGION = "us-east-1"
+INDEX = "organization"
+
+
+def get_secret(secret_name: str) -> str | bytes:
+    """Retrieve a secret value from AWS Secrets Manager."""
+    session = boto3.session.Session()
+    client = session.client(service_name="secretsmanager", region_name=REGION)
+
+    get_secret_value_response = client.get_secret_value(SecretId=secret_name)
+    if "SecretString" in get_secret_value_response:
+        return get_secret_value_response["SecretString"]
+    else:
+        return base64.b64decode(get_secret_value_response["SecretBinary"])
+
+
+def get_es_client(host: str | None = None):
+    """Create an authenticated OpenSearch client using AWS credentials."""
+    from opensearchpy import (
+        OpenSearch,
+        RequestsHttpConnection,
+        AWSV4SignerAuth,
+    )
+
+    if host is None:
+        host = get_secret("es-host-matching")
+
+    credentials = boto3.Session().get_credentials()
+    auth = AWSV4SignerAuth(credentials, REGION, "es")
+
+    return OpenSearch(
+        hosts=[{"host": host, "port": 443}],
+        http_auth=auth,
+        use_ssl=True,
+        verify_certs=True,
+        connection_class=RequestsHttpConnection,
+        pool_maxsize=20,
+        timeout=120,
+    )

crossref_matcher-1.3.2/crossref_matcher/resources/VERSION.txt

@@ -0,0 +1 @@
+1.3.2

{crossref_matcher-1.3.1 → crossref_matcher-1.3.2/crossref_matcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crossref-matcher
-Version: 1.3.1
+Version: 1.3.2
 Summary: Crossref's matching API
 Home-page: https://gitlab.com/crossref/marple
 Author: Crossref

{crossref_matcher-1.3.1 → crossref_matcher-1.3.2}/crossref_matcher.egg-info/SOURCES.txt

@@ -28,7 +28,7 @@ crossref_matcher/indexes/preprint_matching/indexer_service.py
 crossref_matcher/indexes/ror_organizations/__init__.py
 crossref_matcher/indexes/ror_organizations/create_index.py
 crossref_matcher/indexes/ror_organizations/index_data.py
-crossref_matcher/indexes/ror_organizations/ror_opensearch_update_dag.py
+crossref_matcher/indexes/ror_organizations/opensearch_utils.py
 crossref_matcher/indexes/ror_organizations/ror_zenodo_api.py
 crossref_matcher/matching/__init__.py
 crossref_matcher/matching/http_utils.py

crossref_matcher-1.3.1/crossref_matcher/indexes/ror_organizations/ror_opensearch_update_dag.py

@@ -1,124 +0,0 @@
-"""Airflow DAG to keep the ROR OpenSearch index up to date."""
-
-import base64
-import boto3
-
-from airflow.decorators import dag, task
-from datetime import datetime
-from datetime import timedelta
-from opensearchpy import (
-    OpenSearch,
-    RequestsHttpConnection,
-    AWSV4SignerAuth,
-    NotFoundError,
-)
-
-import logging
-from typing import Any
-
-from crossref_matcher.indexes.ror_organizations.ror_zenodo_api import (
-    get_ror_data_dump_metadata,
-)
-from crossref_matcher.indexes.ror_organizations.create_index import (
-    index_body,
-    initialize_index,
-)
-from crossref_matcher.indexes.ror_organizations.index_data import (
-    download_ror_data_and_update_index,
-)
-
-logger = logging.getLogger(__name__)
-
-REGION = "us-east-1"
-INDEX = "organization"
-INDEX_BATCH_SIZE = 400
-
-
-def get_secret(secret_name: str) -> str | bytes:
-    """Retrieve a secret value from AWS Secrets Manager."""
-    session = boto3.session.Session()
-    client = session.client(service_name="secretsmanager", region_name=REGION)
-
-    get_secret_value_response = client.get_secret_value(SecretId=secret_name)
-    if "SecretString" in get_secret_value_response:
-        return get_secret_value_response["SecretString"]
-    else:
-        return base64.b64decode(get_secret_value_response["SecretBinary"])
-
-
-def get_es_client() -> OpenSearch:
-    """Create an authenticated OpenSearch client using AWS credentials."""
-    credentials = boto3.Session().get_credentials()
-    auth = AWSV4SignerAuth(credentials, REGION, "es")
-
-    return OpenSearch(
-        hosts=[{"host": get_secret("es-host-matching"), "port": 443}],
-        http_auth=auth,
-        use_ssl=True,
-        verify_certs=True,
-        connection_class=RequestsHttpConnection,
-        pool_maxsize=20,
-    )
-
-
-@dag(
-    schedule_interval="@daily",
-    catchup=False,
-    dagrun_timeout=timedelta(hours=10),
-    start_date=datetime(2026, 1, 3),
-    tags=["experiment", "ror", "matching"],
-)
-def update_ror_index():
-    @task()
-    def fetch_ror_metadata() -> dict[str, Any]:
-        """Fetch metadata for the latest ROR data dump from Zenodo."""
-        return get_ror_data_dump_metadata()
-
-    @task.short_circuit()
-    def is_update_needed(metadata: dict[str, Any]) -> bool:
-        """Return True if the index checksum differs from the latest dump."""
-        ror_checksum = metadata["checksum"]
-        es_client = get_es_client()
-
-        # # FOR TESTING: RUN THIS TO FORCE UPDATE, OTHERWISE COMMENT OUT
-        # index_metadata = {
-        #     "target_data": "",
-        #     "ror_data_dump_checksum": "",
-        #     "ror_data_dump_doi": "",
-        # }
-        # body = {"_meta": index_metadata}
-        # es_client.indices.put_mapping(index=INDEX, body=body)
-
-        try:
-            mapping = es_client.indices.get_mapping(index=INDEX)
-        except NotFoundError:
-            return True
-        meta = mapping[INDEX]["mappings"].get("_meta", {})
-        current_checksum = meta.get("ror_data_dump_checksum", "")
-        return current_checksum != ror_checksum
-
-    @task()
-    def create_index_if_not_exists():
-        es_client = get_es_client()
-
-        index_exists = es_client.indices.exists(index=INDEX)
-
-        if not index_exists:
-            index_body_content = index_body()
-            response = initialize_index(INDEX, es_client, index_body_content)
-            logger.info(f"Created index '{INDEX}': {response}")
-
-    @task
-    def download_and_update(metadata: dict[str, Any]) -> None:
-        """Download the ROR data dump and update the index."""
-        es_client = get_es_client()
-        download_ror_data_and_update_index(metadata, es_client)
-
-    metadata = fetch_ror_metadata()
-    update_needed = is_update_needed(metadata)
-    # if update_needed is false, the DAG will stop here
-
-    update_needed >> create_index_if_not_exists() >> download_and_update(metadata)
-
-
-workflow = update_ror_index()

crossref_matcher-1.3.1/crossref_matcher/resources/VERSION.txt

@@ -1 +0,0 @@
-1.3.1