destiny_sdk 0.5.1__tar.gz → 0.7.1__tar.gz

{destiny_sdk-0.5.1 → destiny_sdk-0.7.1}/.gitignore

@@ -198,3 +198,10 @@ infra/app/*.py
 .certs
 
 libs/fake_data/*.jsonl
+
+.data
+.env.*
+!.env.example
+.idea/
+
+.test.tmp

{destiny_sdk-0.5.1 → destiny_sdk-0.7.1}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: destiny_sdk
-Version: 0.5.1
+Version: 0.7.1
 Summary: A software development kit (sdk) to support interaction with the DESTINY repository
-Author-email: Adam Hamilton <adam@futureevidence.org>, Andrew Harvey <andrew@futureevidence.org>, Daniel Breves <daniel@futureevidence.org>, Jack Walmisley <jack@futureevidence.org>
+Author-email: Adam Hamilton <adam@futureevidence.org>, Andrew Harvey <andrew@futureevidence.org>, Daniel Breves <daniel@futureevidence.org>, Jack Walmisley <jack@futureevidence.org>, Tim Repke <tim.repke@pik-potsdam.de>
 License-Expression: Apache-2.0
 License-File: LICENSE
 Requires-Python: ~=3.12
@@ -14,6 +14,7 @@ Requires-Dist: pytest-asyncio<2,>=1.0.0
 Requires-Dist: pytest-httpx<0.36,>=0.35.0
 Requires-Dist: pytest<9,>=8.4.0
 Requires-Dist: python-jose<4,>=3.4.0
+Provides-Extra: labs
 Description-Content-Type: text/markdown
 
 # DESTINY SDK
@@ -34,6 +35,13 @@ pip install destiny-sdk
 uv add destiny-sdk
 ```
 
+Some labs functionality may require extra dependencies - these can be installed by:
+
+```sh
+pip install destiny-sdk[labs]
+uv add destiny-sdk --extra labs
+```
+
 ## Development
 
 ### Dependencies

{destiny_sdk-0.5.1 → destiny_sdk-0.7.1}/README.md

@@ -16,6 +16,13 @@ pip install destiny-sdk
 uv add destiny-sdk
 ```
 
+Some labs functionality may require extra dependencies - these can be installed by:
+
+```sh
+pip install destiny-sdk[labs]
+uv add destiny-sdk --extra labs
+```
+
 ## Development
 
 ### Dependencies

{destiny_sdk-0.5.1 → destiny_sdk-0.7.1}/pyproject.toml

@@ -17,6 +17,7 @@ authors = [
     {email = "andrew@futureevidence.org", name = "Andrew Harvey"},
     {email = "daniel@futureevidence.org", name = "Daniel Breves"},
     {email = "jack@futureevidence.org", name = "Jack Walmisley"},
+    {email = "tim.repke@pik-potsdam.de", name = "Tim Repke"},
 ]
 dependencies = [
     "cachetools>=5.5.2,<6",
@@ -33,7 +34,10 @@ license = "Apache-2.0"
 name = "destiny_sdk"
 readme = "README.md"
 requires-python = "~=3.12"
-version = "0.5.1"
+version = "0.7.1"
+
+[project.optional-dependencies]
+labs = []
 
 [tool.pytest.ini_options]
 addopts = ["--color=yes", "--import-mode=importlib", "--verbose"]

{destiny_sdk-0.5.1 → destiny_sdk-0.7.1}/src/destiny_sdk/__init__.py

@@ -8,6 +8,7 @@ from . import (
     imports,
     references,
     robots,
+    search,
     visibility,
 )
 
@@ -19,5 +20,6 @@ __all__ = [
     "imports",
     "references",
     "robots",
+    "search",
     "visibility",
 ]

{destiny_sdk-0.5.1 → destiny_sdk-0.7.1}/src/destiny_sdk/client.py

@@ -114,7 +114,11 @@ class Client:
         return RobotEnhancementBatchRead.model_validate(response.json())
 
     def poll_robot_enhancement_batch(
-        self, robot_id: UUID4, limit: int = 10, timeout: int = 60
+        self,
+        robot_id: UUID4,
+        limit: int = 10,
+        lease: str | None = None,
+        timeout: int = 60,
     ) -> RobotEnhancementBatch | None:
         """
         Poll for a robot enhancement batch.
@@ -125,13 +129,20 @@ class Client:
         :type robot_id: UUID4
         :param limit: The maximum number of pending enhancements to return
         :type limit: int
+        :param lease: The duration to lease the pending enhancements for,
+            in ISO 8601 duration format eg PT10M. If not provided the repository will
+            use a default lease duration.
+        :type lease: str | None
         :return: The RobotEnhancementBatch object from the response, or None if no
             batches available
         :rtype: destiny_sdk.robots.RobotEnhancementBatch | None
         """
+        params = {"robot_id": str(robot_id), "limit": limit}
+        if lease:
+            params["lease"] = lease
         response = self.session.post(
             "/robot-enhancement-batches/",
-            params={"robot_id": str(robot_id), "limit": limit},
+            params=params,
             timeout=timeout,
         )
         # HTTP 204 No Content indicates no batches available
@@ -140,3 +151,24 @@ class Client:
 
         response.raise_for_status()
         return RobotEnhancementBatch.model_validate(response.json())
+
+    def renew_robot_enhancement_batch_lease(
+        self, robot_enhancement_batch_id: UUID4, lease_duration: str | None = None
+    ) -> None:
+        """
+        Renew the lease for a robot enhancement batch.
+
+        Signs the request with the client's secret key.
+
+        :param robot_enhancement_batch_id: The ID of the robot enhancement batch
+        :type robot_enhancement_batch_id: UUID4
+        :param lease_duration: The duration to lease the pending enhancements for,
+            in ISO 8601 duration format eg PT10M. If not provided the repository will
+            use a default lease duration.
+        :type lease_duration: str | None
+        """
+        response = self.session.post(
+            f"/robot-enhancement-batches/{robot_enhancement_batch_id}/renew-lease/",
+            params={"lease": lease_duration} if lease_duration else None,
+        )
+        response.raise_for_status()

{destiny_sdk-0.5.1 → destiny_sdk-0.7.1}/src/destiny_sdk/core.py

@@ -2,7 +2,9 @@
 
 from typing import Self
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
+
+from destiny_sdk.search import SearchResultPage, SearchResultTotal
 
 # These are non-standard newline characters that are not escaped by model_dump_json().
 # We want jsonl files to have empirical new lines so they can be streamed line by line.
@@ -47,3 +49,14 @@ class _JsonlFileInputMixIn(BaseModel):
         :rtype: Self
         """
         return cls.model_validate_json(jsonl)
+
+
+class SearchResultMixIn(BaseModel):
+    """A mixin class for models that represent search results."""
+
+    total: SearchResultTotal = Field(
+        description="The total number of results matching the search criteria.",
+    )
+    page: SearchResultPage = Field(
+        description="Information about the page of results.",
+    )

{destiny_sdk-0.5.1 → destiny_sdk-0.7.1}/src/destiny_sdk/enhancements.py

@@ -2,9 +2,9 @@
 
 import datetime
 from enum import StrEnum, auto
-from typing import Annotated, Literal
+from typing import Annotated, Any, Literal, Self
 
-from pydantic import UUID4, BaseModel, Field, HttpUrl
+from pydantic import UUID4, BaseModel, Field, HttpUrl, model_validator
 
 from destiny_sdk.core import _JsonlFileInputMixIn
 from destiny_sdk.visibility import Visibility
@@ -25,6 +25,8 @@ class EnhancementType(StrEnum):
     """A free-form enhancement for tagging with labels."""
     LOCATION = auto()
     """Locations where the reference can be found."""
+    RAW = auto()
+    """A free form enhancement for arbitrary/unstructured data."""
     FULL_TEXT = auto()
     """The full text of the reference. (To be implemented)"""
 
@@ -145,22 +147,33 @@ class AnnotationType(StrEnum):
     """
 
 
-class ScoreAnnotation(BaseModel):
-    """
-    An annotation which represents the score for a label.
+class BaseAnnotation(BaseModel):
+    """Base class for annotations, defining the minimal required fields."""
 
-    This is similar to a BooleanAnnotation, but lacks a boolean determination
-    as to the application of the label.
-    """
-
-    annotation_type: Literal[AnnotationType.SCORE] = AnnotationType.SCORE
     scheme: str = Field(
         description="An identifier for the scheme of annotation",
         examples=["openalex:topic", "pubmed:mesh"],
+        pattern=r"^[^/]+$",  # No slashes allowed
     )
     label: str = Field(
         description="A high level label for this annotation like the name of the topic",
     )
+
+    @property
+    def qualified_label(self) -> str:
+        """The qualified label for this annotation."""
+        return f"{self.scheme}/{self.label}"
+
+
+class ScoreAnnotation(BaseAnnotation):
+    """
+    An annotation which represents the score for a label.
+
+    This is similar to a BooleanAnnotation, but lacks a boolean determination
+    as to the application of the label.
+    """
+
+    annotation_type: Literal[AnnotationType.SCORE] = AnnotationType.SCORE
     score: float = Field(description="""Score for this annotation""")
     data: dict = Field(
         default_factory=dict,
@@ -171,7 +184,7 @@ class ScoreAnnotation(BaseModel):
     )
 
 
-class BooleanAnnotation(BaseModel):
+class BooleanAnnotation(BaseAnnotation):
     """
     An annotation is a way of tagging the content with a label of some kind.
 
@@ -180,13 +193,6 @@ class BooleanAnnotation(BaseModel):
     """
 
     annotation_type: Literal[AnnotationType.BOOLEAN] = AnnotationType.BOOLEAN
-    scheme: str = Field(
-        description="An identifier for the scheme of the annotation",
-        examples=["openalex:topic", "pubmed:mesh"],
-    )
-    label: str = Field(
-        description="A high level label for this annotation like the name of the topic",
-    )
     value: bool = Field(description="""Boolean flag for this annotation""")
     score: float | None = Field(
         None, description="A confidence score for this annotation"
@@ -295,12 +301,45 @@ class LocationEnhancement(BaseModel):
     )
 
 
+class RawEnhancement(BaseModel):
+    """
+    An enhancement for storing raw/arbitrary/unstructured data.
+
+    Data in these enhancements is intended for future conversion into structured form.
+
+    This enhancement accepts any fields passed in to `data`. These enhancements cannot
+    be created by robots.
+    """
+
+    enhancement_type: Literal[EnhancementType.RAW] = EnhancementType.RAW
+    source_export_date: datetime.datetime = Field(
+        description="Date the enhancement data was retrieved."
+    )
+    description: str = Field(
+        description="Description of the data to aid in future refinement."
+    )
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional metadata to aid in future structuring of raw data",
+    )
+    data: Any = Field(description="Unstructured data for later processing.")
+
+    @model_validator(mode="after")
+    def forbid_no_data(self) -> Self:
+        """Prevent a raw enhancement from being created with no data."""
+        if not self.data:
+            msg = "data must be populated on a raw enhancement."
+            raise ValueError(msg)
+        return self
+
+
 #: Union type for all enhancement content types.
 EnhancementContent = Annotated[
     BibliographicMetadataEnhancement
     | AbstractContentEnhancement
     | AnnotationEnhancement
-    | LocationEnhancement,
+    | LocationEnhancement
+    | RawEnhancement,
     Field(discriminator="enhancement_type"),
 ]
 

{destiny_sdk-0.5.1 → destiny_sdk-0.7.1}/src/destiny_sdk/identifiers.py

@@ -17,8 +17,14 @@ class ExternalIdentifierType(StrEnum):
 
     DOI = auto()
     """A DOI (Digital Object Identifier) which is a unique identifier for a document."""
+    ERIC = auto()
+    """An ERIC (Education Resources Information Identifier) ID which is a unique
+    identifier for a document in ERIC.
+    """
     PM_ID = auto()
     """A PubMed ID which is a unique identifier for a document in PubMed."""
+    PRO_QUEST = auto()
+    """A ProQuest ID which is a unqiue identifier for a document in ProQuest."""
     OPEN_ALEX = auto()
     """An OpenAlex ID which is a unique identifier for a document in OpenAlex."""
     OTHER = auto()
@@ -41,8 +47,64 @@ class DOIIdentifier(BaseModel):
     def remove_doi_url(cls, value: str) -> str:
         """Remove the URL part of the DOI if it exists."""
         return (
-            value.removeprefix("http://doi.org/")
-            .removeprefix("https://doi.org/")
+            value.removeprefix("http://")
+            .removeprefix("https://")
+            .removeprefix("doi.org/")
+            .removeprefix("dx.doi.org/")
+            .removeprefix("doi:")
+            .strip()
+        )
+
+
+class ProQuestIdentifier(BaseModel):
+    """An external identifier representing a ProQuest ID."""
+
+    identifier: str = Field(
+        description="The ProQuest id of the reference", pattern=r"[0-9]+$"
+    )
+    identifier_type: Literal[ExternalIdentifierType.PRO_QUEST] = Field(
+        ExternalIdentifierType.PRO_QUEST, description="The type of identifier used."
+    )
+
+    @field_validator("identifier", mode="before")
+    @classmethod
+    def remove_proquest_url(cls, value: str) -> str:
+        """Remove the URL part of the ProQuest id if it exists."""
+        return (
+            value.removeprefix("http://")
+            .removeprefix("https://")
+            .removeprefix("search.proquest.com/")
+            .removeprefix("www.proquest.com/")
+            .removeprefix("docview/")
+            .strip()
+        )
+
+
+class ERICIdentifier(BaseModel):
+    """
+    An external identifier representing an ERIC Number.
+
+    An ERIC Number is defined as a unqiue identifiying number preceeded by
+    EJ (for a journal article) or ED (for a non-journal document).
+    """
+
+    identifier: str = Field(
+        description="The ERIC Number of the reference.", pattern=r"E[D|J][0-9]+$"
+    )
+    identifier_type: Literal[ExternalIdentifierType.ERIC] = Field(
+        ExternalIdentifierType.ERIC, description="The type of identifier used."
+    )
+
+    @field_validator("identifier", mode="before")
+    @classmethod
+    def remove_eric_url(cls, value: str) -> str:
+        """Remove the URL part of the ERIC ID if it exists."""
+        return (
+            value.removeprefix("http://")
+            .removeprefix("https://")
+            .removeprefix("eric.ed.gov/?id=")
+            .removeprefix("files.eric.ed.gov/fulltext/")
+            .removesuffix(".pdf")
             .strip()
         )
 
@@ -71,8 +133,11 @@ class OpenAlexIdentifier(BaseModel):
     def remove_open_alex_url(cls, value: str) -> str:
         """Remove the OpenAlex URL if it exists."""
         return (
-            value.removeprefix("http://openalex.org/")
-            .removeprefix("https://openalex.org/")
+            value.removeprefix("http://")
+            .removeprefix("https://")
+            .removeprefix("openalex.org/")
+            .removeprefix("explore.openalex.org/")
+            .removeprefix("works/")
             .strip()
         )
 
@@ -91,7 +156,12 @@ class OtherIdentifier(BaseModel):
 
 #: Union type for all external identifiers.
 ExternalIdentifier = Annotated[
-    DOIIdentifier | PubMedIdentifier | OpenAlexIdentifier | OtherIdentifier,
+    DOIIdentifier
+    | ERICIdentifier
+    | PubMedIdentifier
+    | ProQuestIdentifier
+    | OpenAlexIdentifier
+    | OtherIdentifier,
     Field(discriminator="identifier_type"),
 ]
 

destiny_sdk-0.7.1/src/destiny_sdk/labs/__init__.py

@@ -0,0 +1,10 @@
+"""
+Experimental DESTINY SDK.
+
+The DESTINY SDK-labs provides experimental features
+for interacting with DESTINY repository.
+"""
+
+from . import references
+
+__all__ = ["references"]

destiny_sdk-0.7.1/src/destiny_sdk/labs/references.py

@@ -0,0 +1,154 @@
+"""
+Extended Reference SDK.
+
+Extended Reference class for the Destiny SDK
+with added experimental convenience methods and properties.
+"""
+
+from collections.abc import Generator
+from typing import cast
+
+from pydantic import BaseModel, Field
+
+from destiny_sdk.enhancements import (
+    Annotation,
+    AnnotationType,
+    BibliographicMetadataEnhancement,
+    EnhancementType,
+)
+from destiny_sdk.identifiers import ExternalIdentifierType
+from destiny_sdk.references import Reference
+
+
+class LabsReference(BaseModel):
+    """Experimental presenter class for Reference with added convenience methods."""
+
+    reference: Reference = Field(
+        ...,
+        description="The core Reference object",
+    )
+
+    def _get_id(self, identifier_type: ExternalIdentifierType) -> str | int | None:
+        """Fetch an identifier matching the given identifier_type."""
+        for identifier in self.reference.identifiers or []:
+            if identifier.identifier_type == identifier_type:
+                return identifier.identifier
+        return None
+
+    @property
+    def openalex_id(self) -> str | None:
+        """Return an OpenAlex ID for the reference."""
+        return cast(
+            str | None, self._get_id(identifier_type=ExternalIdentifierType.OPEN_ALEX)
+        )
+
+    @property
+    def doi(self) -> str | None:
+        """Return a DOI for the reference."""
+        return cast(
+            str | None, self._get_id(identifier_type=ExternalIdentifierType.DOI)
+        )
+
+    @property
+    def pubmed_id(self) -> int | None:
+        """Return a pubmed ID for the reference."""
+        return cast(
+            int | None, self._get_id(identifier_type=ExternalIdentifierType.PM_ID)
+        )
+
+    @property
+    def abstract(self) -> str | None:
+        """Return an abstract for the reference."""
+        for enhancement in self.reference.enhancements or []:
+            if enhancement.content.enhancement_type == EnhancementType.ABSTRACT:
+                return enhancement.content.abstract
+        return None
+
+    @property
+    def publication_year(self) -> int | None:
+        """Return a publication year for the reference."""
+        for meta in self.it_bibliographics():
+            if meta.publication_year is not None:
+                return meta.publication_year
+        return None
+
+    @property
+    def title(self) -> str | None:
+        """The title of the reference. If multiple are present, return first one."""
+        for meta in self.it_bibliographics():
+            if meta.title is not None:
+                return meta.title
+        return None
+
+    def it_bibliographics(
+        self,
+    ) -> Generator[BibliographicMetadataEnhancement, None, None]:
+        """Iterate bibliographic enhancements."""
+        for enhancement in self.reference.enhancements or []:
+            if enhancement.content.enhancement_type == EnhancementType.BIBLIOGRAPHIC:
+                yield enhancement.content
+
+    def it_annotations(
+        self,
+        source: str | None = None,
+        annotation_type: AnnotationType | None = None,
+        scheme: str | None = None,
+        label: str | None = None,
+    ) -> Generator[Annotation, None, None]:
+        """
+        Iterate annotation enhancements for the given filters.
+
+        :param source: Optional filter for Enhancement.source
+        :param annotation_type: Optional filter for
+                                AnnotationEnhancement.annotation_type
+        :param scheme: Optional filter for Annotation.scheme
+        :param label: Optional filter for Annotation.label
+        """
+        for enhancement in self.reference.enhancements or []:
+            if enhancement.content.enhancement_type == EnhancementType.ANNOTATION:
+                if source is not None and enhancement.source != source:
+                    continue
+                for annotation in enhancement.content.annotations:
+                    if (
+                        annotation_type is not None
+                        and annotation.annotation_type != annotation_type
+                    ):
+                        continue
+                    if scheme is not None and annotation.scheme != scheme:
+                        continue
+                    if label is not None and annotation.label != label:
+                        continue
+                    yield annotation
+
+    def has_bool_annotation(
+        self,
+        source: str | None = None,
+        scheme: str | None = None,
+        label: str | None = None,
+        expected_value: bool = True,  # noqa: FBT001, FBT002
+    ) -> bool | None:
+        """
+        Check if a specific annotation exists and is true.
+
+        :param source: Optional filter for Enhancement.source
+        :param scheme: Optional filter for Annotation.scheme
+        :param label: Optional filter for Annotation.label
+        :param expected_value: Specify expected boolean annotation value
+        :return: Returns the boolean value for the first annotation matching
+                 the filters or None if nothing is found.
+        """
+        if scheme is None and label is None:
+            msg = "Please use at least one of the optional scheme or label filters."
+            raise AssertionError(msg)
+
+        found_annotation = False
+        for annotation in self.it_annotations(
+            source=source,
+            annotation_type=AnnotationType.BOOLEAN,
+            scheme=scheme,
+            label=label,
+        ):
+            if annotation.value == expected_value:
+                return True
+            found_annotation = True
+        return False if found_annotation else None