destiny_sdk 0.1.0__py3-none-any.whl

destiny_sdk/imports.py

@@ -0,0 +1,253 @@
+"""Import process classes for the Destiny SDK."""
+
+import datetime
+from enum import StrEnum, auto
+
+from pydantic import (
+    UUID4,
+    BaseModel,
+    Field,
+    HttpUrl,
+    PastDatetime,
+)
+
+
+class ImportRecordStatus(StrEnum):
+    """
+    Describes the status of an import record.
+
+    - `created`: Created, but no processing has started.
+    - `started`: Processing has started on the batch.
+    - `completed`: Processing has been completed.
+    - `cancelled`: Processing was cancelled by calling the API.
+    """
+
+    CREATED = auto()
+    STARTED = auto()
+    COMPLETED = auto()
+    CANCELLED = auto()
+
+
+class ImportBatchStatus(StrEnum):
+    """
+    Describes the status of an import batch.
+
+    - `created`: Created, but no processing has started.
+    - `started`: Processing has started on the batch.
+    - `completed`: Processing has been completed.
+    - `cancelled`: Processing was cancelled by calling the API.
+    """
+
+    CREATED = auto()
+    STARTED = auto()
+    FAILED = auto()
+    COMPLETED = auto()
+    CANCELLED = auto()
+
+
+class CollisionStrategy(StrEnum):
+    """
+    The strategy to use when an identifier collision is detected.
+
+    Identifier collisions are detected on ``identifier_type`` and ``identifier``
+    (and ``other_identifier_name`` where relevant) already present in the database.
+
+    Enhancement collisions are detected on an entry with matching ``enhancement_type``
+    and ``source`` already being present on the collided reference.
+
+    - `discard`: Do nothing with the incoming reference.
+    - `fail`: Do nothing with the incoming reference and mark it as failed. This
+      allows the importing process to "follow up" on the failure.
+    - `merge_aggressive`: Prioritize the incoming reference's identifiers and
+      enhancements in the merge.
+    - `merge_defensive`: Prioritize the existing reference's identifiers and
+      enhancements in the merge.
+    - `append`: Performs an aggressive merge of identifiers, and an append of
+      enhancements.
+    - `overwrite`: Performs an aggressive merge of identifiers, and an overwrite of
+      enhancements (deleting existing and recreating what is imported). This should
+      be used sparingly and carefully.
+    """
+
+    DISCARD = auto()
+    FAIL = auto()
+    MERGE_AGGRESSIVE = auto()
+    MERGE_DEFENSIVE = auto()
+    APPEND = auto()
+    OVERWRITE = auto()
+
+
+class ImportResultStatus(StrEnum):
+    """
+    Describes the status of an import result.
+
+    - `created`: Created, but no processing has started.
+    - `started`: The reference is currently being processed.
+    - `completed`: The reference has been created.
+    - `partially_failed`: The reference was created but one or more enhancements or
+      identifiers failed to be added. See the result's `failure_details` field for
+      more information.
+    - `failed`: The reference failed to be created. See the result's `failure_details`
+      field for more information.
+    - `cancelled`: Processing was cancelled by calling the API.
+    """
+
+    CREATED = auto()
+    STARTED = auto()
+    COMPLETED = auto()
+    CANCELLED = auto()
+    PARTIALLY_FAILED = auto()
+    FAILED = auto()
+
+
+class _ImportRecordBase(BaseModel):
+    """Base import record class."""
+
+    search_string: str | None = Field(
+        default=None,
+        description="The search string used to produce this import",
+    )
+    searched_at: PastDatetime = Field(
+        default_factory=lambda: datetime.datetime.now(tz=datetime.UTC),
+        description="""
+The timestamp (including timezone) at which the search which produced
+this import was conducted. If no timezone is included, the timestamp
+is assumed to be in UTC.
+        """,
+    )
+    processor_name: str = Field(
+        description="The name of the processor that is importing the data."
+    )
+    processor_version: str = Field(
+        description="The version of the processor that is importing the data."
+    )
+    notes: str | None = Field(
+        default=None,
+        description="""
+Any additional notes regarding the import (eg. reason for importing, known
+issues).
+        """,
+    )
+    expected_reference_count: int = Field(
+        description="""
+The number of references expected to be included in this import.
+-1 is accepted if the number is unknown.
+""",
+        ge=-1,
+    )
+    source_name: str = Field(
+        description="The source of the reference being imported (eg. Open Alex)"
+    )
+
+
+class ImportRecordIn(_ImportRecordBase):
+    """Input for creating an import record."""
+
+
+class ImportRecordRead(_ImportRecordBase):
+    """Core import record class."""
+
+    id: UUID4 = Field(
+        description="The ID of the import record",
+    )
+    status: ImportRecordStatus = Field(
+        ImportRecordStatus.CREATED,
+        description="The status of the import record",
+    )
+    batches: list["ImportBatchRead"] | None = Field(
+        default=None,
+        description="A list of batches for the import record",
+    )
+
+
+class _ImportBatchBase(BaseModel):
+    """The base class for import batches."""
+
+    collision_strategy: CollisionStrategy = Field(
+        default=CollisionStrategy.FAIL,
+        description="""
+The strategy to use for each reference when an identifier collision occurs.
+Default is `fail`, which allows the importing process to "follow up" on the collision.
+        """,
+    )
+    storage_url: HttpUrl = Field(
+        description="""
+The URL at which the set of references for this batch are stored. The file is a jsonl
+with each line formatted according to
+:class:`ReferenceFileInput <libs.sdk.src.destiny_sdk.references.ReferenceFileInput>`.
+    """,
+    )
+    callback_url: HttpUrl | None = Field(
+        default=None,
+        description="""
+The URL to which the processor should send a callback when the batch has been processed.
+        """,
+    )
+
+
+class ImportBatchIn(_ImportBatchBase):
+    """Input for creating an import batch."""
+
+
+class ImportBatchRead(_ImportBatchBase):
+    """Core import batch class."""
+
+    id: UUID4 = Field(
+        description="The ID of the import batch",
+    )
+    status: ImportBatchStatus = Field(
+        default=ImportBatchStatus.CREATED, description="The status of the batch."
+    )
+    import_record_id: UUID4 = Field(
+        description="The ID of the import record this batch is associated with"
+    )
+    import_record: ImportRecordRead | None = Field(
+        default=None, description="The parent import record."
+    )
+    import_results: list["ImportResultRead"] | None = Field(
+        default=None, description="The results from processing the batch."
+    )
+
+
+class ImportBatchSummary(_ImportBatchBase):
+    """A view for an import batch that includes a summary of its results."""
+
+    id: UUID4 = Field(
+        description="""
+The identifier of the batch.
+""",
+    )
+
+    import_batch_id: UUID4 = Field(description="The ID of the batch being summarised")
+
+    import_batch_status: ImportBatchStatus = Field(
+        description="The status of the batch being summarised"
+    )
+
+    results: dict[ImportResultStatus, int] = Field(
+        description="A count of references by their current import status."
+    )
+    failure_details: list[str] | None = Field(
+        description="""
+        The details of the failures that occurred.
+        Each failure will start with `"Entry x"` where x is the line number of the
+        jsonl object attempted to be imported.
+        """,
+    )
+
+
+class ImportResultRead(BaseModel):
+    """Core import result class."""
+
+    id: UUID4 = Field(description="The ID of the import result.")
+    reference_id: UUID4 | None = Field(
+        default=None,
+        description="The ID of the reference created by this import result.",
+    )
+    failure_details: str | None = Field(
+        default=None,
+        description="The details of the failure, if the import result failed.",
+    )
+    import_batch: ImportBatchRead | None = Field(
+        default=None, description="The parent import batch."
+    )

destiny_sdk/references.py

@@ -0,0 +1,45 @@
+"""Reference classes for the Destiny SDK."""
+
+from pydantic import UUID4, BaseModel, Field
+
+from destiny_sdk.core import _JsonlFileInputMixIn
+from destiny_sdk.enhancements import Enhancement, EnhancementFileInput
+from destiny_sdk.identifiers import ExternalIdentifier
+from destiny_sdk.visibility import Visibility
+
+
+class Reference(_JsonlFileInputMixIn, BaseModel):
+    """Core reference class."""
+
+    visibility: Visibility = Field(
+        default=Visibility.PUBLIC,
+        description="The level of visibility of the reference",
+    )
+    id: UUID4 = Field(
+        description="The ID of the reference",
+    )
+    identifiers: list[ExternalIdentifier] | None = Field(
+        default=None,
+        description="A list of `ExternalIdentifiers` for the Reference",
+    )
+    enhancements: list[Enhancement] | None = Field(
+        default=None,
+        description="A list of enhancements for the reference",
+    )
+
+
+class ReferenceFileInput(_JsonlFileInputMixIn, BaseModel):
+    """Enhancement model used to marshall a file input."""
+
+    visibility: Visibility = Field(
+        default=Visibility.PUBLIC,
+        description="The level of visibility of the reference",
+    )
+    identifiers: list[ExternalIdentifier] | None = Field(
+        default=None,
+        description="A list of `ExternalIdentifiers` for the Reference",
+    )
+    enhancements: list[EnhancementFileInput] | None = Field(
+        default=None,
+        description="A list of enhancements for the reference",
+    )

destiny_sdk/robots.py

@@ -0,0 +1,337 @@
+"""Schemas that define inputs/outputs for robots."""
+
+from enum import StrEnum, auto
+from typing import Annotated, Self
+
+from pydantic import UUID4, BaseModel, ConfigDict, Field, HttpUrl, model_validator
+
+from destiny_sdk.core import _JsonlFileInputMixIn
+from destiny_sdk.enhancements import Enhancement
+from destiny_sdk.references import Reference
+
+
+class RobotError(BaseModel):
+    """A record of something going wrong with the robot."""
+
+    message: Annotated[
+        str,
+        Field(
+            description=(
+                "Message which describes the error encountered during processing"
+            )
+        ),
+    ]
+
+
+class LinkedRobotError(_JsonlFileInputMixIn, RobotError):
+    """
+    A record of something going wrong when processing an individual reference.
+
+    Used in results for batch requests - in single requests, the reference
+    id is derived from the request id.
+    """
+
+    reference_id: UUID4 = Field(
+        description="The ID of the reference which caused the error."
+    )
+
+
+class RobotResult(BaseModel):
+    """The result of a robot request which is returned to the repo."""
+
+    request_id: UUID4
+    error: RobotError | None = Field(
+        default=None,
+        description="Error the robot encountered while creating enhancement.",
+    )
+    enhancement: Enhancement | None = Field(
+        default=None, description="An enhancement to create"
+    )
+
+    @model_validator(mode="after")
+    def validate_error_or_enhancement_set(self) -> Self:
+        """Validate that a robot result has either an error or an enhancement set."""
+        if (self.error is None) == (self.enhancement is None):
+            msg = """
+            exactly one of 'error' or 'enhancement' must be provided
+            """
+            raise ValueError(msg)
+        return self
+
+
+class BatchRobotResult(BaseModel):
+    """Used to indicate to the repository that the robot has finished processing."""
+
+    request_id: UUID4
+    error: RobotError | None = Field(
+        default=None,
+        description="""
+Error the robot encountered while creating enhancements. If this field is populated,
+we assume the entire batch or request failed, rather than an individual reference.
+If there was an error with processing an individual reference, it should be passed in
+the result file and this field should be left as None. Vice-versa, if this field is
+None, the repository will assume that the result file is ready for processing.
+""",
+    )
+
+
+class BatchRobotResultValidationEntry(_JsonlFileInputMixIn, BaseModel):
+    """A single entry in the validation result file for a batch enhancement request."""
+
+    reference_id: UUID4 | None = Field(
+        default=None,
+        description=(
+            "The ID of the reference which was enhanced. "
+            "If this is empty, the BatchEnhancementResultEntry could not be parsed."
+        ),
+    )
+    error: str | None = Field(
+        default=None,
+        description=(
+            "Error encountered during the enhancement process for this reference. "
+            "If this is empty, the enhancement was successfully created."
+        ),
+    )
+
+
+class RobotRequest(BaseModel):
+    """An enhancement request from the repo to a robot."""
+
+    id: UUID4
+    reference: Reference  # Reference with selected enhancements
+    extra_fields: (
+        dict | None
+    )  # We need something to pass through the signed url for uploads
+
+
+#: The result for a single reference when processed by a batch enhancement request.
+#: This is a single entry in the result file.
+BatchEnhancementResultEntry = Annotated[
+    Enhancement | LinkedRobotError,
+    Field(),
+]
+
+
+class BatchRobotRequest(BaseModel):
+    """A batch enhancement request from the repo to a robot."""
+
+    id: UUID4
+    reference_storage_url: HttpUrl = Field(
+        description="""
+The URL at which the set of references are stored. The file is a jsonl
+with each line formatted according to
+:class:`Reference <libs.sdk.src.destiny_sdk.references.Reference>`, one
+reference per line.
+Each reference may have identifiers or enhancements attached, as
+required by the robot.
+If the URL expires, a new one can be generated using
+``GET /references/enhancement/batch/<batch_request_id>``.
+"""
+    )
+    result_storage_url: HttpUrl = Field(
+        description="""
+The URL at which the set of enhancements are to be stored. The file is to be a jsonl
+with each line formatted according to
+:class:`BatchEnhancementResultEntry <libs.sdk.src.destiny_sdk.robots.BatchEnhancementResultEntry>`.
+If the URL expires, a new one can be generated using
+``GET /references/enhancement/batch/<batch_request_id>``.
+"""  # noqa: E501
+    )
+    extra_fields: dict | None = Field(
+        default=None,
+        description="Extra fields to pass to the robot. TBC.",
+    )
+
+
+class EnhancementRequestStatus(StrEnum):
+    """
+    The status of an enhancement request.
+
+    **Allowed values**:
+    - `received`: Enhancement request has been received.
+    - `accepted`: Enhancement request has been accepted.
+    - `rejected`: Enhancement request has been rejected.
+    - `failed`: Enhancement failed to create.
+    - `completed`: Enhancement has been created.
+    """
+
+    RECEIVED = auto()
+    ACCEPTED = auto()
+    REJECTED = auto()
+    FAILED = auto()
+    COMPLETED = auto()
+
+
+class _EnhancementRequestBase(BaseModel):
+    """
+    Base enhancement request class.
+
+    An enhancement request is a request to create an enhancement on a reference.
+    It contains the reference and the robot to be used to create the enhancement.
+    """
+
+    reference_id: UUID4 = Field(description="The ID of the reference to be enhanced.")
+    robot_id: UUID4 = Field(
+        description="The robot to be used to create the enhancement."
+    )
+
+    enhancement_parameters: dict | None = Field(
+        default=None, description="Information needed to create the enhancement. TBC."
+    )
+
+
+class EnhancementRequestIn(_EnhancementRequestBase):
+    """The model for requesting an enhancement on specific reference."""
+
+
+class EnhancementRequestRead(_EnhancementRequestBase):
+    """Core enhancement request class."""
+
+    id: UUID4
+    request_status: EnhancementRequestStatus = Field(
+        description="The status of the request to create an enhancement",
+    )
+    error: str | None = Field(
+        default=None,
+        description="Error encountered during the enhancement process",
+    )
+
+
+class BatchEnhancementRequestStatus(StrEnum):
+    """
+    The status of an enhancement request.
+
+    **Allowed values**:
+    - `received`: Enhancement request has been received by the repo.
+    - `accepted`: Enhancement request has been accepted by the robot.
+    - `rejected`: Enhancement request has been rejected by the robot.
+    - `partial_failed`: Some enhancements failed to create.
+    - `failed`: All enhancements failed to create.
+    - `importing`: Enhancements have been received by the repo and are being imported.
+    - `completed`: All enhancements have been created.
+    """
+
+    RECEIVED = auto()
+    ACCEPTED = auto()
+    REJECTED = auto()
+    PARTIAL_FAILED = auto()
+    FAILED = auto()
+    IMPORTING = auto()
+    COMPLETED = auto()
+
+
+class _BatchEnhancementRequestBase(BaseModel):
+    """
+    Base batch enhancement request class.
+
+    A batch enhancement request is a request to create multiple enhancements.
+    """
+
+    robot_id: UUID4 = Field(
+        description="The robot to be used to create the enhancements."
+    )
+    reference_ids: list[UUID4] = Field(
+        description="The IDs of the references to be enhanced."
+    )
+
+
+class BatchEnhancementRequestIn(_BatchEnhancementRequestBase):
+    """The model for requesting multiple enhancements on specific references."""
+
+
+class BatchEnhancementRequestRead(_BatchEnhancementRequestBase):
+    """Core batch enhancement request class."""
+
+    id: UUID4
+    request_status: BatchEnhancementRequestStatus = Field(
+        description="The status of the request to create enhancements",
+    )
+    reference_data_url: HttpUrl | None = Field(
+        default=None,
+        description="""
+The URL at which the set of references are stored. The file is a jsonl with each line
+formatted according to
+:class:`Reference <libs.sdk.src.destiny_sdk.references.Reference>`.
+, one reference per line.
+Each reference may have identifiers or enhancements attached, as
+required by the robot.
+If the URL expires, a new one can be generated using
+``GET /references/enhancement/batch/<batch_request_id>``.
+        """,
+    )
+    result_storage_url: HttpUrl | None = Field(
+        default=None,
+        description="""
+The URL at which the set of enhancements are stored. The file is to be a jsonl
+with each line formatted according to
+:class:`BatchEnhancementResultEntry <libs.sdk.src.destiny_sdk.robots.BatchEnhancementResultEntry>`.
+This field is only relevant to robots.
+If the URL expires, a new one can be generated using
+``GET /references/enhancement/batch/<batch_request_id>``.
+        """,  # noqa: E501
+    )
+    validation_result_url: HttpUrl | None = Field(
+        default=None,
+        description="""
+The URL at which the result of the batch enhancement request is stored.
+This file is a txt file, one line per reference, with either an error
+or a success message.
+If the URL expires, a new one can be generated using
+``GET /references/enhancement/batch/<batch_request_id>``.
+        """,
+    )
+    error: str | None = Field(
+        default=None,
+        description="Error encountered during the enhancement process. This "
+        "is only used if the entire batch enhancement request failed, rather than an "
+        "individual reference. If there was an error with processing an individual "
+        "reference, it is passed in the validation result file.",
+    )
+
+
+class _RobotBase(BaseModel):
+    """
+    Base Robot class.
+
+    A Robot is a provider of enhancements to destiny repository
+    """
+
+    model_config = ConfigDict(extra="forbid")  # Forbid extra fields on robot models
+
+    name: str = Field(description="The name of the robot, must be unique.")
+    base_url: HttpUrl = Field(
+        description="The base url of the robot. The robot must implement endpoints "
+        "base_url/single for the enhancement of single references and "
+        "base_url/batch for batch enhancements of references.",
+    )
+    description: str = Field(
+        description="Description of the enhancement the robot provides."
+    )
+    owner: str = Field(description="The owner/publisher of the robot.")
+
+
+class RobotIn(_RobotBase):
+    """The model for registering a new robot."""
+
+
+class Robot(_RobotBase):
+    """Then model for a registered robot."""
+
+    id: UUID4 = Field(
+        description="The id of the robot provided by destiny repository. "
+        "Used as the client_id when sending HMAC authenticated requests."
+    )
+
+
+class ProvisionedRobot(Robot):
+    """
+    The model for a provisioned robot.
+
+    Used only when a robot is initially created,
+    or when cycling a robot's client_secret.
+    """
+
+    client_secret: str = Field(
+        description="The client secret of the robot, used as the secret key "
+        "when sending HMAC authenticated requests."
+    )

destiny_sdk/visibility.py

@@ -0,0 +1,24 @@
+"""Visibility enum for repository data."""
+
+from enum import StrEnum, auto
+
+
+class Visibility(StrEnum):
+    """
+    The visibility of a data element in the repository.
+
+    This is used to manage whether information should be publicly available or
+    restricted (generally due to copyright constraints from publishers).
+
+    TODO: Implement data governance layer to manage this.
+
+    **Allowed values**:
+
+    - `public`: Visible to the general public without authentication.
+    - `restricted`: Requires authentication to be visible.
+    - `hidden`: Is not visible, but may be passed to data mining processes.
+    """
+
+    PUBLIC = auto()
+    RESTRICTED = auto()
+    HIDDEN = auto()