destiny_sdk 0.6.0__py3-none-any.whl → 0.7.2__py3-none-any.whl

destiny_sdk/client.py

@@ -1,11 +1,22 @@
 """Send authenticated requests to Destiny Repository."""
 
+import sys
 import time
 from collections.abc import Generator
 
 import httpx
-from pydantic import UUID4, HttpUrl
+from msal import (
+    ConfidentialClientApplication,
+    ManagedIdentityClient,
+    PublicClientApplication,
+    UserAssignedManagedIdentity,
+)
+from pydantic import UUID4, HttpUrl, TypeAdapter
 
+from destiny_sdk.auth import create_signature
+from destiny_sdk.core import sdk_version
+from destiny_sdk.identifiers import IdentifierLookup
+from destiny_sdk.references import Reference, ReferenceSearchResult
 from destiny_sdk.robots import (
     EnhancementRequestRead,
     RobotEnhancementBatch,
@@ -13,8 +24,10 @@ from destiny_sdk.robots import (
     RobotEnhancementBatchResult,
     RobotResult,
 )
+from destiny_sdk.search import AnnotationFilter
 
-from .auth import create_signature
+python_version = ".".join(map(str, sys.version_info[:3]))
+user_agent = f"python@{python_version}/destiny-sdk@{sdk_version}"
 
 
 class HMACSigningAuth(httpx.Auth):
@@ -53,7 +66,7 @@ class HMACSigningAuth(httpx.Auth):
         yield request
 
 
-class Client:
+class RobotClient:
     """
     Client for interaction with the Destiny API.
 
@@ -71,7 +84,10 @@ class Client:
         """
         self.session = httpx.Client(
             base_url=str(base_url).removesuffix("/").removesuffix("/v1") + "/v1",
-            headers={"Content-Type": "application/json"},
+            headers={
+                "Content-Type": "application/json",
+                "User-Agent": user_agent,
+            },
             auth=HMACSigningAuth(secret_key=secret_key, client_id=client_id),
         )
 
@@ -114,7 +130,11 @@ class Client:
         return RobotEnhancementBatchRead.model_validate(response.json())
 
     def poll_robot_enhancement_batch(
-        self, robot_id: UUID4, limit: int = 10, timeout: int = 60
+        self,
+        robot_id: UUID4,
+        limit: int = 10,
+        lease: str | None = None,
+        timeout: int = 60,
     ) -> RobotEnhancementBatch | None:
         """
         Poll for a robot enhancement batch.
@@ -125,13 +145,20 @@ class Client:
         :type robot_id: UUID4
         :param limit: The maximum number of pending enhancements to return
         :type limit: int
+        :param lease: The duration to lease the pending enhancements for,
+            in ISO 8601 duration format eg PT10M. If not provided the repository will
+            use a default lease duration.
+        :type lease: str | None
         :return: The RobotEnhancementBatch object from the response, or None if no
             batches available
         :rtype: destiny_sdk.robots.RobotEnhancementBatch | None
         """
+        params = {"robot_id": str(robot_id), "limit": limit}
+        if lease:
+            params["lease"] = lease
         response = self.session.post(
             "/robot-enhancement-batches/",
-            params={"robot_id": str(robot_id), "limit": limit},
+            params=params,
             timeout=timeout,
         )
         # HTTP 204 No Content indicates no batches available
@@ -140,3 +167,435 @@ class Client:
 
         response.raise_for_status()
         return RobotEnhancementBatch.model_validate(response.json())
+
+    def renew_robot_enhancement_batch_lease(
+        self, robot_enhancement_batch_id: UUID4, lease_duration: str | None = None
+    ) -> None:
+        """
+        Renew the lease for a robot enhancement batch.
+
+        Signs the request with the client's secret key.
+
+        :param robot_enhancement_batch_id: The ID of the robot enhancement batch
+        :type robot_enhancement_batch_id: UUID4
+        :param lease_duration: The duration to lease the pending enhancements for,
+            in ISO 8601 duration format eg PT10M. If not provided the repository will
+            use a default lease duration.
+        :type lease_duration: str | None
+        """
+        response = self.session.post(
+            f"/robot-enhancement-batches/{robot_enhancement_batch_id}/renew-lease/",
+            params={"lease": lease_duration} if lease_duration else None,
+        )
+        response.raise_for_status()
+
+
+# Backward compatibility
+Client = RobotClient
+
+
+class OAuthMiddleware(httpx.Auth):
+    """
+    Auth middleware that handles OAuth2 token retrieval and refresh.
+
+    This is generally used in conjunction with
+    :class:`OAuthClient <libs.sdk.src.destiny_sdk.client.OAuthClient>`.
+
+    Supports three authentication flows:
+
+    **Public Client Application (human login)**
+
+    Initial login will be interactive through a browser window. Subsequent token
+    retrievals will use cached tokens and refreshes where possible, and only prompt
+    for login again if necessary.
+
+    .. code-block:: python
+
+        auth = OAuthMiddleware(
+            azure_client_id="client-id",
+            azure_application_id="login-url",
+            azure_tenant_id="tenant-id",
+        )
+
+    **Confidential Client Application (client credentials)**
+
+    Suitable for service-to-service authentication where no user interaction is
+    possible or desired. Reach out if you need help setting up a confidential client
+    application. The secret must be stored securely.
+
+    .. code-block:: python
+
+        auth = OAuthMiddleware(
+            azure_client_id="client-id",
+            azure_application_id="application-id",
+            azure_login_url="login-url",
+            azure_client_secret="your-azure-client-secret",
+        )
+
+    **Azure Managed Identity**
+
+    Suitable for Azure environments that have had API permissions provisioned for
+    their managed identity. Note that the ``azure_client_id`` here is the client ID of
+    the managed identity, not the repository.
+
+    .. code-block:: python
+
+        auth = OAuthMiddleware(
+            azure_client_id="your-managed-identity-client-id",
+            azure_application_id="application-id",
+            use_managed_identity=True,
+        )
+
+    """
+
+    def __init__(
+        self,
+        azure_client_id: str,
+        azure_application_id: str,
+        azure_login_url: HttpUrl | str | None = None,
+        azure_client_secret: str | None = None,
+        *,
+        use_managed_identity: bool = False,
+    ) -> None:
+        """
+        Initialize the auth middleware.
+
+        :param tenant_id: The OAuth2 tenant ID.
+        :type tenant_id: str
+        :param client_id: The OAuth2 client ID.
+        :type client_id: str
+        :param application_id: The application ID for the Destiny API.
+        :type application_id: str
+        :param azure_login_url: The Azure login URL.
+        :type azure_login_url: str
+        :param azure_client_secret: The Azure client secret.
+        :type azure_client_secret: str | None
+        :param use_managed_identity: Whether to use managed identity for authentication
+        :type use_managed_identity: bool
+        """
+        if use_managed_identity:
+            if (
+                any(
+                    [
+                        azure_login_url,
+                        azure_client_secret,
+                    ]
+                )
+                or not azure_client_id
+            ):
+                msg = (
+                    "azure_login_url and azure_client_secret must not be provided "
+                    "when using managed identity authentication"
+                )
+                raise ValueError(msg)
+            self._oauth_app = ManagedIdentityClient(
+                UserAssignedManagedIdentity(client_id=azure_client_id),
+                http_client=httpx.Client(),
+            )
+            self._get_token = self._get_token_from_managed_identity
+        elif azure_client_secret:
+            if not azure_login_url:
+                msg = (
+                    "azure_login_url must be provided "
+                    "when not using managed identity authentication"
+                )
+                raise ValueError(msg)
+            self._oauth_app = ConfidentialClientApplication(
+                client_id=azure_client_id,
+                authority=str(azure_login_url),
+                client_credential=azure_client_secret,
+            )
+            self._get_token = self._get_token_from_confidential_client
+        else:
+            if not azure_login_url:
+                msg = (
+                    "azure_login_url must be provided "
+                    "when not using managed identity authentication"
+                )
+                raise ValueError(msg)
+            self._oauth_app = PublicClientApplication(
+                azure_client_id,
+                authority=str(azure_login_url),
+                client_credential=None,
+            )
+            self._get_token = self._get_token_from_public_client
+
+        self._scope = f"api://{azure_application_id}/.default"
+        self._account = None
+
+    def _parse_token(self, msal_response: dict) -> str:
+        """
+        Parse the OAuth2 token from an MSAL response.
+
+        :param msal_response: The MSAL response containing the token.
+        :type msal_response: dict
+        :return: The OAuth2 token.
+        :rtype: str
+        """
+        if not msal_response.get("access_token"):
+            msg = (
+                "Failed to acquire access token: "
+                f"{msal_response.get('error', 'Unknown error')}"
+            )
+            raise RuntimeError(msg)
+
+        return msal_response["access_token"]
+
+    def _get_token_from_public_client(self, *, force_refresh: bool = False) -> str:
+        """
+        Get an OAuth2 token from a PublicClientApplication.
+
+        :param force_refresh: Whether to force a token refresh.
+        :type force_refresh: bool
+        :return: The OAuth2 token.
+        :rtype: str
+        """
+        if not isinstance(self._oauth_app, PublicClientApplication):
+            msg = "oauth_app must be a PublicClientApplication for this method"
+            raise TypeError(msg)
+
+        # Uses msal cache if possible, else interactive login
+        result = self._oauth_app.acquire_token_silent(
+            scopes=[self._scope],
+            account=self._account,
+            force_refresh=force_refresh,
+        )
+        if not result:
+            result = self._oauth_app.acquire_token_interactive(scopes=[self._scope])
+
+        access_token = self._parse_token(result)
+
+        # After first login, cache the account for silent token acquisition
+        if not self._account and (accounts := self._oauth_app.get_accounts()):
+            self._account = accounts[0]
+
+        return access_token
+
+    def _get_token_from_confidential_client(
+        self,
+        *,
+        force_refresh: bool = False,  # noqa: ARG002 MSAL will handle refreshing
+    ) -> str:
+        """
+        Get an OAuth2 token from a ConfidentialClientApplication.
+
+        :param force_refresh: Whether to force a token refresh.
+        :type force_refresh: bool
+        :return: The OAuth2 token.
+        :rtype: str
+        """
+        if not isinstance(self._oauth_app, ConfidentialClientApplication):
+            msg = "oauth_app must be a ConfidentialClientApplication for this method"
+            raise TypeError(msg)
+
+        # Uses msal cache if possible, else client credentials flow
+        result = self._oauth_app.acquire_token_for_client(scopes=[self._scope])
+
+        return self._parse_token(result)
+
+    def _get_token_from_managed_identity(
+        self,
+        *,
+        force_refresh: bool = False,  # noqa: ARG002 MSAL will handle refreshing
+    ) -> str:
+        """
+        Get an OAuth2 token from a ManagedIdentityClient.
+
+        :param force_refresh: Whether to force a token refresh.
+        :type force_refresh: bool
+        :return: The OAuth2 token.
+        :rtype: str
+        """
+        if not isinstance(self._oauth_app, ManagedIdentityClient):
+            msg = "oauth_app must be a ManagedIdentityClient for this method"
+            raise TypeError(msg)
+
+        result = self._oauth_app.acquire_token_for_client(
+            resource=self._scope.removesuffix("/.default")
+        )
+
+        return self._parse_token(result)
+
+    def auth_flow(
+        self, request: httpx.Request
+    ) -> Generator[httpx.Request, httpx.Response]:
+        """
+        Add OAuth2 token to request and handle token refresh on expiration.
+
+        :param request: The request to authenticate.
+        :type request: httpx.Request
+        :yield: Authenticated request with token refresh handling.
+        :rtype: Generator[httpx.Request, httpx.Response]
+        """
+        # Add initial token
+        token = self._get_token()
+        request.headers["Authorization"] = f"Bearer {token}"
+
+        response = yield request
+
+        # Check if token expired and retry once with fresh token
+        if response.status_code == httpx.codes.UNAUTHORIZED:
+            try:
+                json_response: dict = response.json()
+                error_detail: str = json_response.get("detail", {})
+            except ValueError:
+                error_detail = ""
+
+            if error_detail == "Token has expired.":
+                # Force refresh token and retry
+                token = self._get_token(force_refresh=True)
+                request.headers["Authorization"] = f"Bearer {token}"
+                yield request
+
+
+class OAuthClient:
+    """
+    Client for interaction with the Destiny API using OAuth2.
+
+    This will apply the provided authentication, usually
+    :class:`OAuthMiddleware <libs.sdk.src.destiny_sdk.client.OAuthMiddleware>`,
+    to all requests. Some API endpoints are supported directly through methods on this
+    class, while others can be accessed through the underlying ``httpx`` client.
+
+    Example usage:
+
+    .. code-block:: python
+
+        from destiny_sdk.client import OAuthClient, OAuthMiddleware
+
+        client = OAuthClient(
+            base_url="https://destiny-repository.example.com",
+            auth=OAuthMiddleware(...),
+        )
+
+        # Supported method
+        response = client.search(query="example")
+
+        # Unsupported method, use underlying httpx client
+        response = client.get_client().get("/system/healthcheck/")
+    """
+
+    def __init__(
+        self,
+        base_url: HttpUrl | str,
+        auth: httpx.Auth | None = None,
+    ) -> None:
+        """
+        Initialize the client.
+
+        :param base_url: The base URL for the Destiny Repository API.
+        :type base_url: HttpUrl
+        :param auth: The middleware for authentication. If not provided, only
+            unauthenticated requests can be made. This should almost always be an
+            instance of ``OAuthMiddleware``, unless you need to create a custom auth
+            class.
+        :type auth: httpx.Auth | None
+        """
+        self._client = httpx.Client(
+            base_url=str(base_url).removesuffix("/").removesuffix("/v1") + "/v1",
+            headers={
+                "Content-Type": "application/json",
+                "User-Agent": user_agent,
+            },
+        )
+
+        if auth:
+            self._client.auth = auth
+
+    def _raise_for_status(self, response: httpx.Response) -> None:
+        """
+        Raise an error if the response status is not successful.
+
+        :param response: The HTTP response to check.
+        :type response: httpx.Response
+        :raises httpx.HTTPStatusError: If the response status is not successful.
+        """
+        try:
+            response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            msg = (
+                f"Error response {exc.response.status_code} from "
+                f"{exc.request.url}: {exc.response.text}"
+            )
+            raise httpx.HTTPStatusError(
+                msg, request=exc.request, response=exc.response
+            ) from exc
+
+    def search(  # noqa: PLR0913
+        self,
+        query: str,
+        start_year: int | None = None,
+        end_year: int | None = None,
+        annotations: list[str | AnnotationFilter] | None = None,
+        sort: str | None = None,
+        page: int = 1,
+    ) -> ReferenceSearchResult:
+        """
+        Send a search request to the Destiny Repository API.
+
+        See also: :ref:`search-procedure`.
+
+        :param query: The search query string.
+        :type query: str
+        :param start_year: The start year for filtering results.
+        :type start_year: int | None
+        :param end_year: The end year for filtering results.
+        :type end_year: int | None
+        :param annotations: A list of annotation filters to apply.
+        :type annotations: list[str | libs.sdk.src.destiny_sdk.search.AnnotationFilter] | None
+        :param sort: The sort order for the results.
+        :type sort: str | None
+        :param page: The page number of results to retrieve.
+        :type page: int
+        :return: The response from the API.
+        :rtype: libs.sdk.src.destiny_sdk.references.ReferenceSearchResult
+        """  # noqa: E501
+        params = {"q": query, "page": page}
+        if start_year:
+            params["start_year"] = start_year
+        if end_year:
+            params["end_year"] = end_year
+        if annotations:
+            params["annotation"] = [str(annotation) for annotation in annotations]
+        if sort:
+            params["sort"] = sort
+        response = self._client.get(
+            "/references/search/",
+            params=params,
+        )
+        self._raise_for_status(response)
+        return ReferenceSearchResult.model_validate(response.json())
+
+    def lookup(
+        self,
+        identifiers: list[str | IdentifierLookup],
+    ) -> list[Reference]:
+        """
+        Lookup references by identifiers.
+
+        See also: :ref:`lookup-procedure`.
+
+        :param identifiers: The identifiers to look up.
+        :type identifiers: list[str | libs.sdk.src.destiny_sdk.identifiers.IdentifierLookup]
+        :return: The list of references matching the identifiers.
+        :rtype: list[libs.sdk.src.destiny_sdk.references.Reference]
+        """  # noqa: E501
+        response = self._client.get(
+            "/references/",
+            params={
+                "identifier": ",".join([str(identifier) for identifier in identifiers])
+            },
+        )
+        self._raise_for_status(response)
+        return TypeAdapter(list[Reference]).validate_python(response.json())
+
+    def get_client(self) -> httpx.Client:
+        """
+        Get the underlying ``httpx`` client.
+
+        This can be used to make custom requests not covered by the SDK methods.
+
+        :return: The underlying ``httpx`` client with authentication attached.
+        :rtype: `httpx.Client <https://www.python-httpx.org/advanced/clients/>`_
+        """
+        return self._client

destiny_sdk/core.py

@@ -1,11 +1,18 @@
 """Core classes for the Destiny SDK, not exposed to package users."""
 
+from importlib.metadata import PackageNotFoundError, version
 from typing import Self
 
 from pydantic import BaseModel, Field
 
 from destiny_sdk.search import SearchResultPage, SearchResultTotal
 
+try:
+    sdk_version = version("destiny-sdk")
+except PackageNotFoundError:
+    sdk_version = "unknown"
+
+
 # These are non-standard newline characters that are not escaped by model_dump_json().
 # We want jsonl files to have empirical new lines so they can be streamed line by line.
 # Hence we replace each occurrence with standard new lines.

destiny_sdk/enhancements.py

@@ -2,9 +2,9 @@
 
 import datetime
 from enum import StrEnum, auto
-from typing import Annotated, Literal
+from typing import Annotated, Any, Literal, Self
 
-from pydantic import UUID4, BaseModel, Field, HttpUrl
+from pydantic import UUID4, BaseModel, Field, HttpUrl, model_validator
 
 from destiny_sdk.core import _JsonlFileInputMixIn
 from destiny_sdk.visibility import Visibility
@@ -25,6 +25,8 @@ class EnhancementType(StrEnum):
     """A free-form enhancement for tagging with labels."""
     LOCATION = auto()
     """Locations where the reference can be found."""
+    RAW = auto()
+    """A free form enhancement for arbitrary/unstructured data."""
     FULL_TEXT = auto()
     """The full text of the reference. (To be implemented)"""
 
@@ -145,22 +147,33 @@ class AnnotationType(StrEnum):
     """
 
 
-class ScoreAnnotation(BaseModel):
-    """
-    An annotation which represents the score for a label.
+class BaseAnnotation(BaseModel):
+    """Base class for annotations, defining the minimal required fields."""
 
-    This is similar to a BooleanAnnotation, but lacks a boolean determination
-    as to the application of the label.
-    """
-
-    annotation_type: Literal[AnnotationType.SCORE] = AnnotationType.SCORE
     scheme: str = Field(
         description="An identifier for the scheme of annotation",
         examples=["openalex:topic", "pubmed:mesh"],
+        pattern=r"^[^/]+$",  # No slashes allowed
     )
     label: str = Field(
         description="A high level label for this annotation like the name of the topic",
     )
+
+    @property
+    def qualified_label(self) -> str:
+        """The qualified label for this annotation."""
+        return f"{self.scheme}/{self.label}"
+
+
+class ScoreAnnotation(BaseAnnotation):
+    """
+    An annotation which represents the score for a label.
+
+    This is similar to a BooleanAnnotation, but lacks a boolean determination
+    as to the application of the label.
+    """
+
+    annotation_type: Literal[AnnotationType.SCORE] = AnnotationType.SCORE
     score: float = Field(description="""Score for this annotation""")
     data: dict = Field(
         default_factory=dict,
@@ -171,7 +184,7 @@ class ScoreAnnotation(BaseModel):
     )
 
 
-class BooleanAnnotation(BaseModel):
+class BooleanAnnotation(BaseAnnotation):
     """
     An annotation is a way of tagging the content with a label of some kind.
 
@@ -180,13 +193,6 @@ class BooleanAnnotation(BaseModel):
     """
 
     annotation_type: Literal[AnnotationType.BOOLEAN] = AnnotationType.BOOLEAN
-    scheme: str = Field(
-        description="An identifier for the scheme of the annotation",
-        examples=["openalex:topic", "pubmed:mesh"],
-    )
-    label: str = Field(
-        description="A high level label for this annotation like the name of the topic",
-    )
     value: bool = Field(description="""Boolean flag for this annotation""")
     score: float | None = Field(
         None, description="A confidence score for this annotation"
@@ -295,12 +301,45 @@ class LocationEnhancement(BaseModel):
     )
 
 
+class RawEnhancement(BaseModel):
+    """
+    An enhancement for storing raw/arbitrary/unstructured data.
+
+    Data in these enhancements is intended for future conversion into structured form.
+
+    This enhancement accepts any fields passed in to `data`. These enhancements cannot
+    be created by robots.
+    """
+
+    enhancement_type: Literal[EnhancementType.RAW] = EnhancementType.RAW
+    source_export_date: datetime.datetime = Field(
+        description="Date the enhancement data was retrieved."
+    )
+    description: str = Field(
+        description="Description of the data to aid in future refinement."
+    )
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional metadata to aid in future structuring of raw data",
+    )
+    data: Any = Field(description="Unstructured data for later processing.")
+
+    @model_validator(mode="after")
+    def forbid_no_data(self) -> Self:
+        """Prevent a raw enhancement from being created with no data."""
+        if not self.data:
+            msg = "data must be populated on a raw enhancement."
+            raise ValueError(msg)
+        return self
+
+
 #: Union type for all enhancement content types.
 EnhancementContent = Annotated[
     BibliographicMetadataEnhancement
     | AbstractContentEnhancement
     | AnnotationEnhancement
-    | LocationEnhancement,
+    | LocationEnhancement
+    | RawEnhancement,
     Field(discriminator="enhancement_type"),
 ]
 

destiny_sdk/identifiers.py

@@ -17,8 +17,14 @@ class ExternalIdentifierType(StrEnum):
 
     DOI = auto()
     """A DOI (Digital Object Identifier) which is a unique identifier for a document."""
+    ERIC = auto()
+    """An ERIC (Education Resources Information Identifier) ID which is a unique
+    identifier for a document in ERIC.
+    """
     PM_ID = auto()
     """A PubMed ID which is a unique identifier for a document in PubMed."""
+    PRO_QUEST = auto()
+    """A ProQuest ID which is a unqiue identifier for a document in ProQuest."""
     OPEN_ALEX = auto()
     """An OpenAlex ID which is a unique identifier for a document in OpenAlex."""
     OTHER = auto()
@@ -41,8 +47,64 @@ class DOIIdentifier(BaseModel):
     def remove_doi_url(cls, value: str) -> str:
         """Remove the URL part of the DOI if it exists."""
         return (
-            value.removeprefix("http://doi.org/")
-            .removeprefix("https://doi.org/")
+            value.removeprefix("http://")
+            .removeprefix("https://")
+            .removeprefix("doi.org/")
+            .removeprefix("dx.doi.org/")
+            .removeprefix("doi:")
+            .strip()
+        )
+
+
+class ProQuestIdentifier(BaseModel):
+    """An external identifier representing a ProQuest ID."""
+
+    identifier: str = Field(
+        description="The ProQuest id of the reference", pattern=r"[0-9]+$"
+    )
+    identifier_type: Literal[ExternalIdentifierType.PRO_QUEST] = Field(
+        ExternalIdentifierType.PRO_QUEST, description="The type of identifier used."
+    )
+
+    @field_validator("identifier", mode="before")
+    @classmethod
+    def remove_proquest_url(cls, value: str) -> str:
+        """Remove the URL part of the ProQuest id if it exists."""
+        return (
+            value.removeprefix("http://")
+            .removeprefix("https://")
+            .removeprefix("search.proquest.com/")
+            .removeprefix("www.proquest.com/")
+            .removeprefix("docview/")
+            .strip()
+        )
+
+
+class ERICIdentifier(BaseModel):
+    """
+    An external identifier representing an ERIC Number.
+
+    An ERIC Number is defined as a unqiue identifiying number preceeded by
+    EJ (for a journal article) or ED (for a non-journal document).
+    """
+
+    identifier: str = Field(
+        description="The ERIC Number of the reference.", pattern=r"E[D|J][0-9]+$"
+    )
+    identifier_type: Literal[ExternalIdentifierType.ERIC] = Field(
+        ExternalIdentifierType.ERIC, description="The type of identifier used."
+    )
+
+    @field_validator("identifier", mode="before")
+    @classmethod
+    def remove_eric_url(cls, value: str) -> str:
+        """Remove the URL part of the ERIC ID if it exists."""
+        return (
+            value.removeprefix("http://")
+            .removeprefix("https://")
+            .removeprefix("eric.ed.gov/?id=")
+            .removeprefix("files.eric.ed.gov/fulltext/")
+            .removesuffix(".pdf")
             .strip()
         )
 
@@ -71,8 +133,11 @@ class OpenAlexIdentifier(BaseModel):
     def remove_open_alex_url(cls, value: str) -> str:
         """Remove the OpenAlex URL if it exists."""
         return (
-            value.removeprefix("http://openalex.org/")
-            .removeprefix("https://openalex.org/")
+            value.removeprefix("http://")
+            .removeprefix("https://")
+            .removeprefix("openalex.org/")
+            .removeprefix("explore.openalex.org/")
+            .removeprefix("works/")
             .strip()
         )
 
@@ -91,7 +156,12 @@ class OtherIdentifier(BaseModel):
 
 #: Union type for all external identifiers.
 ExternalIdentifier = Annotated[
-    DOIIdentifier | PubMedIdentifier | OpenAlexIdentifier | OtherIdentifier,
+    DOIIdentifier
+    | ERICIdentifier
+    | PubMedIdentifier
+    | ProQuestIdentifier
+    | OpenAlexIdentifier
+    | OtherIdentifier,
     Field(discriminator="identifier_type"),
 ]
 
@@ -190,3 +260,11 @@ class IdentifierLookup(BaseModel):
         if self.identifier_type is None:
             return UUID4(self.identifier)
         return ExternalIdentifierAdapter.validate_python(self.model_dump())
+
+    def __repr__(self) -> str:
+        """Serialize the identifier lookup to a string."""
+        return self.serialize()
+
+    def __str__(self) -> str:
+        """Serialize the identifier lookup to a string."""
+        return self.serialize()

destiny_sdk/parsers/eppi_parser.py

@@ -1,7 +1,10 @@
 """Parser for a EPPI JSON export file."""
 
+from datetime import datetime
 from typing import Any
 
+from pydantic import ValidationError
+
 from destiny_sdk.enhancements import (
     AbstractContentEnhancement,
     AbstractProcessType,
@@ -13,12 +16,16 @@ from destiny_sdk.enhancements import (
     BooleanAnnotation,
     EnhancementContent,
     EnhancementFileInput,
+    RawEnhancement,
 )
 from destiny_sdk.identifiers import (
     DOIIdentifier,
+    ERICIdentifier,
     ExternalIdentifier,
-    ExternalIdentifierType,
+    OpenAlexIdentifier,
+    ProQuestIdentifier,
 )
+from destiny_sdk.parsers.exceptions import ExternalIdentifierNotFoundError
 from destiny_sdk.references import ReferenceFileInput
 from destiny_sdk.visibility import Visibility
 
@@ -30,9 +37,17 @@ class EPPIParser:
     See example here: https://eppi.ioe.ac.uk/cms/Portals/35/Maps/Examples/example_orignal.json
     """
 
-    version = "1.0"
+    version = "2.0"
 
-    def __init__(self, tags: list[str] | None = None) -> None:
+    def __init__(
+        self,
+        *,
+        tags: list[str] | None = None,
+        include_raw_data: bool = False,
+        source_export_date: datetime | None = None,
+        data_description: str | None = None,
+        raw_enhancement_excludes: list[str] | None = None,
+    ) -> None:
         """
         Initialize the EPPIParser with optional tags.
 
@@ -42,20 +57,75 @@ class EPPIParser:
         """
         self.tags = tags or []
         self.parser_source = f"destiny_sdk.eppi_parser@{self.version}"
+        self.include_raw_data = include_raw_data
+        self.source_export_date = source_export_date
+        self.data_description = data_description
+        self.raw_enhancement_excludes = (
+            raw_enhancement_excludes if raw_enhancement_excludes else []
+        )
+
+        if self.include_raw_data and not all(
+            (
+                self.source_export_date,
+                self.data_description,
+            )
+        ):
+            msg = (
+                "Cannot include raw data enhancements without "
+                "source_export_date, data_description, and raw_enhancement_metadata"
+            )
+            raise RuntimeError(msg)
 
     def _parse_identifiers(
         self, ref_to_import: dict[str, Any]
     ) -> list[ExternalIdentifier]:
         identifiers = []
         if doi := ref_to_import.get("DOI"):
-            identifiers.append(
-                DOIIdentifier(
-                    identifier=doi,
-                    identifier_type=ExternalIdentifierType.DOI,
-                )
+            doi_identifier = self._parse_doi(doi=doi)
+            if doi_identifier:
+                identifiers.append(doi_identifier)
+
+        if url := ref_to_import.get("URL"):
+            identifier = self._parse_url_to_identifier(url=url)
+            if identifier:
+                identifiers.append(identifier)
+
+        if not identifiers:
+            msg = (
+                "No known external identifiers found for Reference data "
+                f"with DOI: '{doi if doi else None}' "
+                f"and URL: '{url if url else None}'."
             )
+            raise ExternalIdentifierNotFoundError(detail=msg)
+
         return identifiers
 
+    def _parse_doi(self, doi: str) -> DOIIdentifier | None:
+        """Attempt to parse a DOI from a string."""
+        try:
+            doi = doi.strip()
+            return DOIIdentifier(identifier=doi)
+        except ValidationError:
+            return None
+
+    def _parse_url_to_identifier(self, url: str) -> ExternalIdentifier | None:
+        """Attempt to parse an external identifier from a url string."""
+        url = url.strip()
+        identifier_cls = None
+        if "eric" in url:
+            identifier_cls = ERICIdentifier
+        elif "proquest" in url:
+            identifier_cls = ProQuestIdentifier
+        elif "openalex" in url:
+            identifier_cls = OpenAlexIdentifier
+        else:
+            return None
+
+        try:
+            return identifier_cls(identifier=url)
+        except ValidationError:
+            return None
+
     def _parse_abstract_enhancement(
         self, ref_to_import: dict[str, Any]
     ) -> EnhancementContent | None:
@@ -107,6 +177,23 @@ class EPPIParser:
             authorship=authorships if authorships else None,
         )
 
+    def _parse_raw_enhancement(
+        self, ref_to_import: dict[str, Any], raw_enhancement_metadata: dict[str, Any]
+    ) -> EnhancementContent | None:
+        """Add Reference data as a raw enhancement."""
+        raw_enhancement_data = ref_to_import.copy()
+
+        # Remove any keys that should be excluded
+        for exclude in self.raw_enhancement_excludes:
+            raw_enhancement_data.pop(exclude, None)
+
+        return RawEnhancement(
+            source_export_date=self.source_export_date,
+            description=self.data_description,
+            metadata=raw_enhancement_metadata,
+            data=raw_enhancement_data,
+        )
+
     def _create_annotation_enhancement(self) -> EnhancementContent | None:
         if not self.tags:
             return None
@@ -124,8 +211,11 @@ class EPPIParser:
         )
 
     def parse_data(
-        self, data: dict, source: str | None = None, robot_version: str | None = None
-    ) -> list[ReferenceFileInput]:
+        self,
+        data: dict,
+        source: str | None = None,
+        robot_version: str | None = None,
+    ) -> tuple[list[ReferenceFileInput], list[dict]]:
         """
         Parse an EPPI JSON export dict and return a list of ReferenceFileInput objects.
 
@@ -140,33 +230,55 @@ class EPPIParser:
 
         """
         parser_source = source if source is not None else self.parser_source
+
+        if self.include_raw_data:
+            codesets = [codeset.get("SetId") for codeset in data.get("CodeSets", [])]
+            raw_enhancement_metadata = {"codeset_ids": codesets}
+
         references = []
+        failed_refs = []
         for ref_to_import in data.get("References", []):
-            enhancement_contents = [
-                content
-                for content in [
-                    self._parse_abstract_enhancement(ref_to_import),
-                    self._parse_bibliographic_enhancement(ref_to_import),
-                    self._create_annotation_enhancement(),
+            try:
+                enhancement_contents = [
+                    content
+                    for content in [
+                        self._parse_abstract_enhancement(ref_to_import),
+                        self._parse_bibliographic_enhancement(ref_to_import),
+                        self._create_annotation_enhancement(),
+                    ]
+                    if content
                 ]
-                if content
-            ]
 
-            enhancements = [
-                EnhancementFileInput(
-                    source=parser_source,
-                    visibility=Visibility.PUBLIC,
-                    content=content,
-                    robot_version=robot_version,
-                )
-                for content in enhancement_contents
-            ]
+                if self.include_raw_data:
+                    raw_enhancement = self._parse_raw_enhancement(
+                        ref_to_import=ref_to_import,
+                        raw_enhancement_metadata=raw_enhancement_metadata,
+                    )
 
-            references.append(
-                ReferenceFileInput(
-                    visibility=Visibility.PUBLIC,
-                    identifiers=self._parse_identifiers(ref_to_import),
-                    enhancements=enhancements,
+                    if raw_enhancement:
+                        enhancement_contents.append(raw_enhancement)
+
+                enhancements = [
+                    EnhancementFileInput(
+                        source=parser_source,
+                        visibility=Visibility.PUBLIC,
+                        content=content,
+                        robot_version=robot_version,
+                    )
+                    for content in enhancement_contents
+                ]
+
+                references.append(
+                    ReferenceFileInput(
+                        visibility=Visibility.PUBLIC,
+                        identifiers=self._parse_identifiers(
+                            ref_to_import=ref_to_import
+                        ),
+                        enhancements=enhancements,
+                    )
                 )
-            )
-        return references
+
+            except ExternalIdentifierNotFoundError:
+                failed_refs.append(ref_to_import)
+
+        return references, failed_refs

destiny_sdk/parsers/exceptions.py

@@ -0,0 +1,17 @@
+"""Custom exceptions for destiny sdk parsers."""
+
+
+class ExternalIdentifierNotFoundError(Exception):
+    """Raised when an reference has no identifiable external identifiers."""
+
+    def __init__(self, detail: str | None = None, *args: object) -> None:
+        """
+        Initialize the ExternalIdentifiersNotFoundError.
+
+        Args:
+            *args: Additional arguments for the exception.
+            **kwargs: Additional keyword arguments for the exception.
+
+        """
+        self.detail = detail or "No detail provided."
+        super().__init__(detail, *args)

destiny_sdk/search.py

@@ -30,6 +30,7 @@ class AnnotationFilter(BaseModel):
 
     scheme: str = Field(
         description="The annotation scheme to filter by.",
+        pattern=r"^[^/]+$",
     )
     label: str | None = Field(
         None,
@@ -42,7 +43,7 @@ class AnnotationFilter(BaseModel):
         le=1.0,
     )
 
-    def serialize(self) -> str:
+    def __repr__(self) -> str:
         """Serialize the annotation filter to a string."""
         annotation = self.scheme
         if self.label:
@@ -50,3 +51,7 @@ class AnnotationFilter(BaseModel):
         if self.score is not None:
             annotation += f"@{self.score}"
         return annotation
+
+    def __str__(self) -> str:
+        """Serialize the annotation filter to a string."""
+        return repr(self)

{destiny_sdk-0.6.0.dist-info → destiny_sdk-0.7.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: destiny_sdk
-Version: 0.6.0
+Version: 0.7.2
 Summary: A software development kit (sdk) to support interaction with the DESTINY repository
 Author-email: Adam Hamilton <adam@futureevidence.org>, Andrew Harvey <andrew@futureevidence.org>, Daniel Breves <daniel@futureevidence.org>, Jack Walmisley <jack@futureevidence.org>, Tim Repke <tim.repke@pik-potsdam.de>
 License-Expression: Apache-2.0
@@ -9,6 +9,7 @@ Requires-Python: ~=3.12
 Requires-Dist: cachetools<6,>=5.5.2
 Requires-Dist: fastapi<0.116,>=0.115.12
 Requires-Dist: httpx<0.29,>=0.28.1
+Requires-Dist: msal>=1.34.0
 Requires-Dist: pydantic<3,>=2.11.3
 Requires-Dist: pytest-asyncio<2,>=1.0.0
 Requires-Dist: pytest-httpx<0.36,>=0.35.0

destiny_sdk-0.7.2.dist-info/RECORD

@@ -0,0 +1,21 @@
+destiny_sdk/__init__.py,sha256=NdSlsPQyDF3TW30_JzbvYMRBRA9iT677iTRWWCMdYOA,382
+destiny_sdk/auth.py,sha256=bY72ywZEcG_67YBd9PrwgWTXkCf58rhLvVEXrtXbWtA,6247
+destiny_sdk/client.py,sha256=nKvS5rRkIpBqv8dVIB57Xsop0UvVz3i875RQxfVSMao,21306
+destiny_sdk/core.py,sha256=E0Wotu9psggK1JRJxbvx3Jc7WEGE6zaz2R2awvRrLz8,2023
+destiny_sdk/enhancements.py,sha256=-4jLm3R0T5UpgCt09CgUfPcnzPOyjdhUZCT1zhEP6sQ,12838
+destiny_sdk/identifiers.py,sha256=I9Q2I35Lg8oyl3uytq1gCGOUu92F9sZSSwzLoCXBJi4,9727
+destiny_sdk/imports.py,sha256=b-rh-dt3NsyLGxqmVzIzKaHiXhbw-3wtAaBN-ZW-i1E,5940
+destiny_sdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+destiny_sdk/references.py,sha256=3Y8gBMTSyZY35S3pB1bnVHMai9RRiGeoGZysNvSo7kk,2553
+destiny_sdk/robots.py,sha256=I_ZvMxwST52e8ovhv0-gPbOB3P9tptbRG0LrkNNOqKo,13463
+destiny_sdk/search.py,sha256=QWQBNEJJnH2o6CGapChKUp6kOZl2Uq3Pxqg1kcu9x-4,1590
+destiny_sdk/visibility.py,sha256=8D44Q868YdScAt6eAFgXXrhonozXnv_Qa5w5yEGMPX8,577
+destiny_sdk/labs/__init__.py,sha256=H4RFPyeelqZ56PagnWPX-JZeWlxPnCZoYHtr4c9SU9Q,180
+destiny_sdk/labs/references.py,sha256=iZisRgGZ5c7X7uTFoe6Q0AwwFMa4yJbIoPUVv_hvOiU,5589
+destiny_sdk/parsers/__init__.py,sha256=d5gS--bXla_0I7e_9wTBnGWMXt2U8b-_ndeprTPe1hk,149
+destiny_sdk/parsers/eppi_parser.py,sha256=_1xnAT0F0o1HKpMWOGQbVS3VPOrhPqyzHDWR3CosWwk,9484
+destiny_sdk/parsers/exceptions.py,sha256=0Sc_M4j560Nqh4SjeP_YrgOUVagdIwWwRz24E6YlZ1k,573
+destiny_sdk-0.7.2.dist-info/METADATA,sha256=cp-onr2xd51yB4M3VnumJxUpn0rAvHWuAfSKe7V2ZqY,2685
+destiny_sdk-0.7.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+destiny_sdk-0.7.2.dist-info/licenses/LICENSE,sha256=6QURU4gvvTjVZ5rfp5amZ6FtFvcpPhAGUjxF5WSZAHI,9138
+destiny_sdk-0.7.2.dist-info/RECORD,,

{destiny_sdk-0.6.0.dist-info → destiny_sdk-0.7.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any

destiny_sdk-0.6.0.dist-info/RECORD

@@ -1,20 +0,0 @@
-destiny_sdk/__init__.py,sha256=NdSlsPQyDF3TW30_JzbvYMRBRA9iT677iTRWWCMdYOA,382
-destiny_sdk/auth.py,sha256=bY72ywZEcG_67YBd9PrwgWTXkCf58rhLvVEXrtXbWtA,6247
-destiny_sdk/client.py,sha256=fTBtuq5emT8ieNtCuCY8Y6xAKZJDLq8sG1WOvmjLz-I,4971
-destiny_sdk/core.py,sha256=PYCYpY72MHXo7iQMHtnXcnCOGn6CUsbYoykHvtQl4Oc,1857
-destiny_sdk/enhancements.py,sha256=SkIlIlWKBN7Z-aXpQiy22SXrU7zVnKxaRb4F5yaFsO8,11503
-destiny_sdk/identifiers.py,sha256=1N2cszBmnQoUeKm54-7MUTO-zTDuvW8U9OjTeAmhWvo,7182
-destiny_sdk/imports.py,sha256=b-rh-dt3NsyLGxqmVzIzKaHiXhbw-3wtAaBN-ZW-i1E,5940
-destiny_sdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-destiny_sdk/references.py,sha256=3Y8gBMTSyZY35S3pB1bnVHMai9RRiGeoGZysNvSo7kk,2553
-destiny_sdk/robots.py,sha256=I_ZvMxwST52e8ovhv0-gPbOB3P9tptbRG0LrkNNOqKo,13463
-destiny_sdk/search.py,sha256=LIPj_h0yMnav_Stp4qRLg1PvZa6h3BV4N2bXwAYZDqA,1447
-destiny_sdk/visibility.py,sha256=8D44Q868YdScAt6eAFgXXrhonozXnv_Qa5w5yEGMPX8,577
-destiny_sdk/labs/__init__.py,sha256=H4RFPyeelqZ56PagnWPX-JZeWlxPnCZoYHtr4c9SU9Q,180
-destiny_sdk/labs/references.py,sha256=iZisRgGZ5c7X7uTFoe6Q0AwwFMa4yJbIoPUVv_hvOiU,5589
-destiny_sdk/parsers/__init__.py,sha256=d5gS--bXla_0I7e_9wTBnGWMXt2U8b-_ndeprTPe1hk,149
-destiny_sdk/parsers/eppi_parser.py,sha256=rEOtt_5Kp3oktFlzRTLZ2x4_7aQ9-ba3FYpkaEnpnvs,5521
-destiny_sdk-0.6.0.dist-info/METADATA,sha256=_xLg34LzOiBZmcjmg6-P6bTKmHIBnArl-0k-Qndftwc,2657
-destiny_sdk-0.6.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-destiny_sdk-0.6.0.dist-info/licenses/LICENSE,sha256=6QURU4gvvTjVZ5rfp5amZ6FtFvcpPhAGUjxF5WSZAHI,9138
-destiny_sdk-0.6.0.dist-info/RECORD,,