mex-common 0.62.0__py3-none-any.whl

mex/__init__.py

@@ -0,0 +1,3 @@
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

mex/common/__init__.py

@@ -0,0 +1,8 @@
+from mex.common.identity.backend_api import BackendApiIdentityProvider
+from mex.common.identity.memory import MemoryIdentityProvider
+from mex.common.identity.registry import register_provider
+from mex.common.types import IdentityProvider
+
+# register the default providers shipped with mex-common
+register_provider(IdentityProvider.MEMORY, MemoryIdentityProvider)
+register_provider(IdentityProvider.BACKEND, BackendApiIdentityProvider)

mex/common/backend_api/connector.py

@@ -0,0 +1,399 @@
+from typing import Any, TypeVar
+from urllib.parse import urljoin
+
+from requests.exceptions import HTTPError
+
+from mex.common.connector import HTTPConnector
+from mex.common.identity.models import Identity
+from mex.common.models import (
+    AnyExtractedModel,
+    AnyMergedModel,
+    AnyPreviewModel,
+    AnyRuleSetRequest,
+    AnyRuleSetResponse,
+    ExtractedOrganization,
+    ExtractedPerson,
+    ItemsContainer,
+    PaginatedItemsContainer,
+    PreviewModelTypeAdapter,
+    RuleSetResponseTypeAdapter,
+)
+from mex.common.settings import BaseSettings
+from mex.common.types import Identifier, MergedPrimarySourceIdentifier
+
+_IngestibleModelT = TypeVar(
+    "_IngestibleModelT", bound=AnyExtractedModel | AnyRuleSetResponse
+)
+
+
+class BackendApiConnector(HTTPConnector):
+    """Connector class to handle interaction with the Backend API."""
+
+    API_VERSION = "v0"
+
+    def _check_availability(self) -> None:
+        """Send a GET request to verify the API is available."""
+        self.request("GET", "_system/check")
+
+    def _set_authentication(self) -> None:
+        """Set the backend API key to all session headers."""
+        settings = BaseSettings.get()
+        self.session.headers["X-API-Key"] = settings.backend_api_key.get_secret_value()
+
+    def _set_url(self) -> None:
+        """Set the backend api url with the version path."""
+        settings = BaseSettings.get()
+        self.url = urljoin(str(settings.backend_api_url), self.API_VERSION)
+
+    def fetch_extracted_items(
+        self,
+        query_string: str | None,
+        stable_target_id: str | None,
+        entity_type: list[str] | None,
+        skip: int,
+        limit: int,
+    ) -> PaginatedItemsContainer[AnyExtractedModel]:
+        """Fetch extracted items that match the given set of filters.
+
+        Args:
+            query_string: Full-text search query
+            stable_target_id: The item's stableTargetId
+            entity_type: The item's entityType
+            skip: How many items to skip for pagination
+            limit: How many items to return in one page
+
+        Raises:
+            HTTPError: If search was not accepted, crashes or times out
+
+        Returns:
+            One page of extracted items and the total count that was matched
+        """
+        response = self.request(
+            method="GET",
+            endpoint="extracted-item",
+            params={
+                "q": query_string,
+                "stableTargetId": stable_target_id,
+                "entityType": entity_type,
+                "skip": str(skip),
+                "limit": str(limit),
+            },
+        )
+        return PaginatedItemsContainer[AnyExtractedModel].model_validate(response)
+
+    def fetch_merged_items(
+        self,
+        query_string: str | None,
+        entity_type: list[str] | None,
+        had_primary_source: list[str] | None,
+        skip: int,
+        limit: int,
+    ) -> PaginatedItemsContainer[AnyMergedModel]:
+        """Fetch merged items that match the given set of filters.
+
+        Args:
+            query_string: Full-text search query
+            entity_type: The items' entityType
+            had_primary_source: The items' hadPrimarySource
+            skip: How many items to skip for pagination
+            limit: How many items to return in one page
+
+        Raises:
+            HTTPError: If search was not accepted, crashes or times out
+
+        Returns:
+            One page of merged items and the total count that was matched
+        """
+        response = self.request(
+            method="GET",
+            endpoint="merged-item",
+            params={
+                "q": query_string,
+                "entityType": entity_type,
+                "hadPrimarySource": had_primary_source,
+                "skip": str(skip),
+                "limit": str(limit),
+            },
+        )
+        return PaginatedItemsContainer[AnyMergedModel].model_validate(response)
+
+    def get_merged_item(
+        self,
+        identifier: str,
+    ) -> AnyMergedModel:
+        """Return one merged item for the given `identifier`.
+
+        Args:
+            identifier: The merged item's identifier
+
+        Raises:
+            HTTPError: If no merged item was found
+
+        Returns:
+            A single merged item
+        """
+        # TODO(ND): stop-gap until backend has proper get merged item endpoint (MX-1669)
+        response = self.request(
+            method="GET",
+            endpoint="merged-item",
+            params={
+                "identifier": identifier,
+                "limit": "1",
+            },
+        )
+        response_model = PaginatedItemsContainer[AnyMergedModel].model_validate(
+            response
+        )
+        try:
+            return response_model.items[0]
+        except IndexError:
+            msg = "merged item was not found"
+            raise HTTPError(msg) from None
+
+    def preview_merged_item(
+        self,
+        stable_target_id: str,
+        rule_set: AnyRuleSetRequest,
+    ) -> AnyPreviewModel:
+        """Return a preview for merging the given rule-set with stored extracted items.
+
+        Args:
+            stable_target_id: The extracted items' `stableTargetId`
+            rule_set: A rule-set to use for previewing
+
+        Raises:
+            HTTPError: If preview produces errors, crashes or times out
+
+        Returns:
+            A single merged item
+        """
+        response = self.request(
+            method="POST",
+            endpoint=f"preview-item/{stable_target_id}",
+            payload=rule_set,
+        )
+        return PreviewModelTypeAdapter.validate_python(response)
+
+    def fetch_preview_items(
+        self,
+        query_string: str | None,
+        entity_type: list[str] | None,
+        had_primary_source: list[str] | None,
+        skip: int,
+        limit: int,
+    ) -> PaginatedItemsContainer[AnyPreviewModel]:
+        """Fetch merged item previews that match the given set of filters.
+
+        Args:
+            query_string: Full-text search query
+            entity_type: The items' entityType
+            had_primary_source: The items' hadPrimarySource
+            skip: How many items to skip for pagination
+            limit: How many items to return in one page
+
+        Raises:
+            HTTPError: If search was not accepted, crashes or times out
+
+        Returns:
+            One page of preview items and the total count that was matched
+        """
+        response = self.request(
+            method="GET",
+            endpoint="preview-item",
+            params={
+                "q": query_string,
+                "entityType": entity_type,
+                "hadPrimarySource": had_primary_source,
+                "skip": str(skip),
+                "limit": str(limit),
+            },
+        )
+        return PaginatedItemsContainer[AnyPreviewModel].model_validate(response)
+
+    def create_rule_set(
+        self,
+        rule_set: AnyRuleSetRequest,
+    ) -> AnyRuleSetResponse:
+        """Create a new rule set.
+
+        Args:
+            rule_set: New rule-set to create
+
+        Raises:
+            HTTPError: If the rule-set did not validate
+
+        Returns:
+            The newly created rule-set
+        """
+        response = self.request(method="POST", endpoint="rule-set", payload=rule_set)
+        return RuleSetResponseTypeAdapter.validate_python(response)
+
+    def get_rule_set(
+        self,
+        stable_target_id: str,
+    ) -> AnyRuleSetResponse:
+        """Return a triple of rules for the given `stableTargetId`.
+
+        Args:
+            stable_target_id: The merged item's identifier
+
+        Raises:
+            HTTPError: If no rule-set was found
+
+        Returns:
+            A set of three rules
+        """
+        response = self.request(
+            method="GET",
+            endpoint=f"rule-set/{stable_target_id}",
+        )
+        return RuleSetResponseTypeAdapter.validate_python(response)
+
+    def update_rule_set(
+        self, stable_target_id: str, rule_set: AnyRuleSetRequest
+    ) -> AnyRuleSetResponse:
+        """Update an existing rule set.
+
+        Args:
+            stable_target_id: The merged item's identifier
+            rule_set: The new rule-set contents
+
+        Raises:
+            HTTPError: If no rule-set was found
+
+        Returns:
+            A set of three rules
+        """
+        response = self.request(
+            method="PUT", endpoint=f"rule-set/{stable_target_id}", payload=rule_set
+        )
+        return RuleSetResponseTypeAdapter.validate_python(response)
+
+    def search_organization_in_wikidata(
+        self,
+        q: str,
+        offset: int = 0,
+        limit: int = 10,
+    ) -> PaginatedItemsContainer[ExtractedOrganization]:
+        """Search for organizations in wikidata.
+
+        Args:
+            q: Wikidata item ID or full URL
+            offset: The starting index for pagination
+            limit: The maximum number of results to return
+
+        Returns:
+            Paginated list of ExtractedOrganizations
+        """
+        response = self.request(
+            method="GET",
+            endpoint="wikidata",
+            params={"q": q, "offset": str(offset), "limit": str(limit)},
+        )
+        return PaginatedItemsContainer[ExtractedOrganization].model_validate(response)
+
+    def search_person_in_ldap(
+        self,
+        q: str,
+        offset: int = 0,
+        limit: int = 10,
+    ) -> PaginatedItemsContainer[ExtractedPerson]:
+        """Search for persons in LDAP.
+
+        Args:
+            q: The name of the person to be searched
+            offset: The starting index for pagination
+            limit: The maximum number of results to return
+
+        Returns:
+            Paginated list of ExtractedPersons
+        """
+        response = self.request(
+            method="GET",
+            endpoint="ldap",
+            params={"q": q, "offset": str(offset), "limit": str(limit)},
+        )
+        return PaginatedItemsContainer[ExtractedPerson].model_validate(response)
+
+    def search_person_in_orcid(
+        self,
+        q: str,
+        offset: int = 0,
+        limit: int = 10,
+    ) -> PaginatedItemsContainer[ExtractedPerson]:
+        """Search for persons in orcid.
+
+        Args:
+            q: The name of the person to be searched
+            offset: The starting index for pagination
+            limit: The maximum number of results to return
+
+        Returns:
+            Paginated list of ExtractedPersons
+        """
+        response = self.request(
+            method="GET",
+            endpoint="orcid",
+            params={"q": q, "offset": str(offset), "limit": str(limit)},
+        )
+        return PaginatedItemsContainer[ExtractedPerson].model_validate(response)
+
+    def assign_identity(
+        self,
+        had_primary_source: MergedPrimarySourceIdentifier,
+        identifier_in_primary_source: str,
+    ) -> Identity:
+        """Find an Identity in a database or assign a new one."""
+        response = self.request(
+            "POST",
+            "identity",
+            {
+                "hadPrimarySource": had_primary_source,
+                "identifierInPrimarySource": identifier_in_primary_source,
+            },
+        )
+        return Identity.model_validate(response)
+
+    def fetch_identities(
+        self,
+        had_primary_source: Identifier | None = None,
+        identifier_in_primary_source: str | None = None,
+        stable_target_id: Identifier | None = None,
+    ) -> ItemsContainer[Identity]:
+        """Find Identity instances matching the given filters.
+
+        Either provide `stableTargetId` or `hadPrimarySource`
+        and `identifierInPrimarySource` together to get a unique result.
+        """
+        connector = BackendApiConnector.get()
+        response = connector.request(
+            "GET",
+            "identity",
+            params={
+                "hadPrimarySource": had_primary_source,
+                "identifierInPrimarySource": identifier_in_primary_source,
+                "stableTargetId": stable_target_id,
+            },
+        )
+        return ItemsContainer[Identity].model_validate(response)
+
+    def ingest(
+        self,
+        ingestible_models: list[_IngestibleModelT],
+        **kwargs: Any,  # noqa: ANN401
+    ) -> None:
+        """Post extracted models or rule-sets to the backend in bulk.
+
+        Args:
+            ingestible_models: Extracted models or rule-sets to ingest
+            kwargs: Further keyword arguments passed to `requests`
+
+        Raises:
+            HTTPError: If post was not accepted, crashes or times out
+        """
+        self.request(
+            method="POST",
+            endpoint="ingest",
+            payload=ItemsContainer[_IngestibleModelT](items=ingestible_models),
+            **kwargs,
+        )

mex/common/cli.py

@@ -0,0 +1,188 @@
+import json
+import pdb  # noqa: T100
+import sys
+from bdb import BdbQuit
+from collections.abc import Callable
+from functools import partial
+from textwrap import dedent
+from traceback import format_exc
+from typing import Any
+
+import click
+from click import Command, Option
+from click.core import ParameterSource
+from click.exceptions import Abort, Exit
+from pydantic.fields import FieldInfo
+
+from mex.common.connector import CONNECTOR_STORE
+from mex.common.logging import logger
+from mex.common.settings import SETTINGS_STORE, BaseSettings
+from mex.common.transform import MExEncoder
+
+HELP_TEMPLATE = """
+{doc}
+
+Acceptable configuration sources sorted by priority:
+(1) command line arguments and options
+(2) environment variables
+(3) dotenv file located at {env_file}
+(4) default values from settings model
+"""
+
+
+def _field_to_parameters(name: str, field: FieldInfo) -> list[str]:
+    """Convert a field of a pydantic settings class into parameter declarations.
+
+    The field's name and alias are considered. Underscores are replaced with dashes
+    and single character parameters have two leading dashes while single character
+    parameters have just one.
+
+    Args:
+        name: name of the Field
+        field: Field of a Settings definition class
+
+    Returns:
+        List of parameter declaring strings
+    """
+    names = [name] + ([field.alias] if field.alias else [])
+    names = [n.replace("_", "-") for n in names]
+    dashes = ["--" if len(n) > 1 else "-" for n in names]
+    return [f"{d}{n}" for d, n in zip(dashes, names, strict=False)]
+
+
+def _field_to_option(name: str, settings_cls: type[BaseSettings]) -> Option:
+    """Convert a field of a pydantic settings class into a click option.
+
+    Args:
+        name: name of the Field
+        settings_cls: Base settings class or a subclass of it
+
+    Returns:
+        Option: click Option with appropriate attributes
+    """
+    # normalize field type to be compatible with advanced string types
+    # https://pydantic-docs.helpmanual.io/usage/types/#pydantic-types
+    # complex fields or type unions are always interpreted as strings
+    # and add support for SecretStr fields with correct default values
+    # https://pydantic-docs.helpmanual.io/usage/types/#secret-types
+    field = settings_cls.model_fields[name]
+
+    if field.annotation in (int, bool, float):
+        field_type: Any = field.annotation
+    else:
+        field_type = str
+
+    if field.is_required():
+        default = None
+    elif field.annotation in (int, bool, float):
+        default = field.default
+    else:
+        default = json.dumps(field.default, cls=MExEncoder).strip('"')
+
+    return Option(
+        _field_to_parameters(name, field),
+        default=default,
+        envvar=settings_cls.get_env_name(name),
+        help=field.description,
+        is_flag=field.annotation is bool and field.default is False,
+        show_default=True,
+        show_envvar=True,
+        type=field_type,
+        required=field.is_required(),
+    )
+
+
+def _callback(
+    func: Callable[[], None],
+    settings_cls: type[BaseSettings],
+    **cli_settings: str,
+) -> None:
+    """Run the decorated function in the current click context.
+
+    When `cli_settings` specify debug mode and an exception occurs,
+    jump into post mortem debugging and raise exception.
+
+    Args:
+        func: Entry point function for a cli
+        settings_cls: Base settings class or a subclass of it
+        cli_settings: Parsed settings in string format
+
+    Raises:
+        Exception: Any uncaught exception when in debug mode
+        SysExit: With exit code 0 or 1
+    """
+    # get current click context.
+    context = click.get_current_context()
+
+    # ensure all singletons are reset.
+    context.call_on_close(CONNECTOR_STORE.reset)
+    context.call_on_close(SETTINGS_STORE.reset)
+
+    # load settings from parameters and store it globally.
+    settings = settings_cls.model_validate(
+        {
+            key: value
+            for key, value in cli_settings.items()
+            if context.get_parameter_source(key) == ParameterSource.COMMANDLINE
+        }
+    )
+    SETTINGS_STORE.push(settings)
+
+    # otherwise print loaded settings in pretty way and continue.
+    logger.info(click.style(dedent(f"    {func.__doc__}"), fg="green"))
+    logger.info(click.style(f"{settings.text()}\n", fg="bright_cyan"))
+
+    # now try to execute the decorated function.
+    try:
+        func()
+    except (Abort, BdbQuit, Exit, KeyboardInterrupt):  # pragma: no cover
+        context.exit(130)
+    except Exception:
+        # an error occurred, let's print the traceback
+        logger.error(click.style(format_exc(), fg="red"))
+        if settings.debug:  # pragma: no cover
+            # if we are in debug mode, jump into interactive debugging.
+            pdb.post_mortem(sys.exc_info()[2])
+            raise
+        # if not in debug mode, exit with code 1.
+        logger.error("exit")
+        context.exit(1)
+
+    # all good, exit with code 0.
+    logger.info("done")
+    context.exit(0)
+
+
+def entrypoint(
+    settings_cls: type[BaseSettings],
+) -> Callable[[Callable[[], None]], Command]:
+    """Decorate given function to mark it as a cli entrypoint.
+
+    The decorator takes one argument `settings_cls` that is either
+    `mex.common.settings.BaseSettings` or a subclass thereof. The decorated function
+    must not require any positional arguments and does not need to return anything.
+
+    Running an `entrypoint` will print a summary on startup, register settings and
+    connector singletons globally and provide error handling as well as debugging.
+
+    Args:
+        settings_cls: Settings class that should be instantiated globally.
+
+    Returns:
+        Callable: The decorated function with initialized settings.
+    """
+
+    def decorator(func: Callable[[], None]) -> Command:
+        return Command(
+            func.__name__,
+            help=HELP_TEMPLATE.format(
+                doc=func.__doc__, env_file=settings_cls.model_config.get("env_file")
+            ),
+            callback=partial(_callback, func, settings_cls),
+            params=[
+                _field_to_option(name, settings_cls)
+                for name in settings_cls.model_fields
+            ],
+        )
+
+    return decorator

mex/common/connector/__init__.py

@@ -0,0 +1,8 @@
+from mex.common.connector.base import CONNECTOR_STORE, BaseConnector
+from mex.common.connector.http import HTTPConnector
+
+__all__ = (
+    "CONNECTOR_STORE",
+    "BaseConnector",
+    "HTTPConnector",
+)

mex/common/connector/base.py

@@ -0,0 +1,50 @@
+from abc import ABCMeta, abstractmethod
+from contextlib import ExitStack, closing
+from typing import Self, cast, final
+
+from mex.common.context import SingletonStore
+from mex.common.transform import dromedary_to_snake
+
+
+class _ConnectorStore(SingletonStore["BaseConnector"]):
+    """Thin wrapper for storing thread-local singletons of connectors."""
+
+    def reset(self) -> None:
+        """Close all connectors and remove them from the singleton store."""
+        with ExitStack() as stack:
+            for connector in self:
+                stack.push(closing(connector))
+        super().reset()
+
+    def metrics(self) -> dict[str, int]:
+        """Generate metrics about all active connectors."""
+        return {
+            f"{dromedary_to_snake(connector.__class__.__name__)}_{metric_key}": value
+            for connector in self
+            for metric_key, value in connector.metrics().items()
+        }
+
+
+CONNECTOR_STORE = _ConnectorStore()
+
+
+class BaseConnector(metaclass=ABCMeta):
+    """Base class for connectors that are handled as singletons."""
+
+    @final
+    @classmethod
+    def get(cls) -> Self:
+        """Get the singleton instance for this class from the store."""
+        return cast("Self", CONNECTOR_STORE.load(cls))
+
+    @abstractmethod
+    def __init__(self) -> None:  # pragma: no cover
+        """Create a new connector instance."""
+
+    def metrics(self) -> dict[str, int]:  # pragma: no cover
+        """Generate metrics about connector usage."""
+        return {}
+
+    @abstractmethod
+    def close(self) -> None:  # pragma: no cover
+        """Close the connector's underlying sockets."""