archivist-py 1.0.0__py3-none-any.whl

archivist/__init__.py

@@ -0,0 +1,100 @@
+"""Preserve the web and retrieve archives with a typed Python API."""
+
+from archivist.core import (
+    Archiver,
+    ArchiveRecord,
+    ArchivistError,
+    AsyncArchiver,
+    AsyncSearcher,
+    AuthenticationError,
+    CaptureFailedError,
+    CaptureJob,
+    InvalidOptionError,
+    InvalidServiceResponseError,
+    InvalidTargetURLError,
+    InvalidURLError,
+    NetworkError,
+    OptionCombinationError,
+    PagedSearchResult,
+    PollingTimeoutError,
+    RateLimitError,
+    Searcher,
+    ServiceError,
+    ServiceStatus,
+    TLSVerificationError,
+)
+from archivist.services.archive_today import (
+    ArchiveTodayClient,
+    ArchiveTodayMemento,
+    ArchiveTodayRecentCapture,
+    ArchiveTodayRecentFeed,
+    ArchiveTodayTimeMap,
+    AsyncArchiveTodayClient,
+)
+from archivist.services.internet_archive import (
+    AsyncInternetArchiveClient,
+    InternetArchiveAccount,
+    InternetArchiveApiKey,
+    InternetArchiveAvailability,
+    InternetArchiveCaptureJob,
+    InternetArchiveCaptureStatus,
+    InternetArchiveCdxRecord,
+    InternetArchiveCdxResult,
+    InternetArchiveClient,
+    InternetArchiveCookies,
+    InternetArchiveFailedStatus,
+    InternetArchivePendingStatus,
+    InternetArchiveSaveOptions,
+    InternetArchiveSuccessStatus,
+    InternetArchiveSystemStatus,
+    InternetArchiveUserStatus,
+)
+
+__version__ = "1.0.0"
+
+__all__ = [
+    "ArchiveRecord",
+    "ArchiveTodayClient",
+    "ArchiveTodayMemento",
+    "ArchiveTodayRecentCapture",
+    "ArchiveTodayRecentFeed",
+    "ArchiveTodayTimeMap",
+    "Archiver",
+    "ArchivistError",
+    "AsyncArchiveTodayClient",
+    "AsyncArchiver",
+    "AsyncInternetArchiveClient",
+    "AsyncSearcher",
+    "AuthenticationError",
+    "CaptureFailedError",
+    "CaptureJob",
+    "InternetArchiveAccount",
+    "InternetArchiveApiKey",
+    "InternetArchiveAvailability",
+    "InternetArchiveCaptureJob",
+    "InternetArchiveCaptureStatus",
+    "InternetArchiveCdxRecord",
+    "InternetArchiveCdxResult",
+    "InternetArchiveClient",
+    "InternetArchiveCookies",
+    "InternetArchiveFailedStatus",
+    "InternetArchivePendingStatus",
+    "InternetArchiveSaveOptions",
+    "InternetArchiveSuccessStatus",
+    "InternetArchiveSystemStatus",
+    "InternetArchiveUserStatus",
+    "InvalidOptionError",
+    "InvalidServiceResponseError",
+    "InvalidTargetURLError",
+    "InvalidURLError",
+    "NetworkError",
+    "OptionCombinationError",
+    "PagedSearchResult",
+    "PollingTimeoutError",
+    "RateLimitError",
+    "Searcher",
+    "ServiceError",
+    "ServiceStatus",
+    "TLSVerificationError",
+    "__version__",
+]

archivist/core/__init__.py

@@ -0,0 +1,52 @@
+"""Shared models, protocols, and exceptions."""
+
+import logging
+
+from archivist.core.errors import (
+    ArchivistError,
+    AuthenticationError,
+    CaptureFailedError,
+    InvalidOptionError,
+    InvalidServiceResponseError,
+    InvalidTargetURLError,
+    InvalidURLError,
+    NetworkError,
+    OptionCombinationError,
+    PollingTimeoutError,
+    RateLimitError,
+    ServiceError,
+    TLSVerificationError,
+)
+from archivist.core.models import (
+    ArchiveRecord,
+    CaptureJob,
+    PagedSearchResult,
+    ServiceStatus,
+)
+from archivist.core.protocols import Archiver, AsyncArchiver, AsyncSearcher, Searcher
+
+logger = logging.getLogger(__name__)
+
+__all__ = [
+    "ArchiveRecord",
+    "Archiver",
+    "ArchivistError",
+    "AsyncArchiver",
+    "AsyncSearcher",
+    "AuthenticationError",
+    "CaptureFailedError",
+    "CaptureJob",
+    "InvalidOptionError",
+    "InvalidServiceResponseError",
+    "InvalidTargetURLError",
+    "InvalidURLError",
+    "NetworkError",
+    "OptionCombinationError",
+    "PagedSearchResult",
+    "PollingTimeoutError",
+    "RateLimitError",
+    "Searcher",
+    "ServiceError",
+    "ServiceStatus",
+    "TLSVerificationError",
+]

archivist/core/_http.py

@@ -0,0 +1,261 @@
+"""Shared HTTP response and exception handling."""
+
+from __future__ import annotations
+
+import logging
+import math
+from collections.abc import Mapping
+from datetime import UTC, datetime
+from email.utils import parsedate_to_datetime
+from inspect import isawaitable
+from typing import Any, Protocol, cast
+
+import niquests
+
+from archivist.core.errors import (
+    AuthenticationError,
+    InvalidServiceResponseError,
+    NetworkError,
+    RateLimitError,
+    TLSVerificationError,
+)
+
+logger = logging.getLogger(__name__)
+
+_HTTP_ERROR_STATUS = 400
+_HTTP_RATE_LIMIT_STATUS = 429
+
+
+class ResponseLike(Protocol):
+    status_code: int
+    headers: Mapping[str, str]
+    text: str
+    url: str
+
+    def json(self) -> object:
+        """Decode the response as JSON."""
+        ...
+
+
+class AsyncJsonResponseLike(Protocol):
+    """Expose fields consumed while decoding an asynchronous JSON response."""
+
+    status_code: int
+
+    def json(self) -> object:
+        """Decode the response as JSON, possibly returning an awaitable."""
+        ...
+
+
+class AsyncTextResponseLike(Protocol):
+    """Expose fields consumed while decoding an asynchronous text response."""
+
+    status_code: int
+    text: object
+
+
+def parse_retry_after(
+    headers: Mapping[str, str],
+) -> tuple[float | datetime | None, str | None]:
+    """Parse Retry-After while preserving an unrecognized raw value."""
+    raw = headers.get("Retry-After") or headers.get("retry-after")
+    if raw is None:
+        return None, None
+    value = raw.strip()
+    try:
+        seconds = float(value)
+    except ValueError:
+        try:
+            date = parsedate_to_datetime(value)
+        except (TypeError, ValueError, OverflowError):
+            return None, raw
+        if date.tzinfo is None:
+            date = date.replace(tzinfo=UTC)
+        return date.astimezone(UTC), raw
+    if not math.isfinite(seconds):
+        return None, raw
+    return max(0.0, seconds), raw
+
+
+def response_json(response: ResponseLike, *, service: str) -> object:
+    """Decode JSON or raise a body-safe service response error."""
+    try:
+        return response.json()
+    except niquests.exceptions.JSONDecodeError:
+        error = None
+    except (niquests.exceptions.RequestException, OSError) as exc:
+        error = translate_request_error(exc, service=service)
+    except (TypeError, ValueError):
+        error = None
+    if error is not None:
+        raise error from None
+    logger.warning("%s returned invalid JSON", service)
+    raise InvalidServiceResponseError(
+        f"{service} returned invalid JSON",
+        service=service,
+        status_code=response.status_code,
+    ) from None
+
+
+def response_mapping(response: ResponseLike, *, service: str) -> Mapping[str, Any]:
+    """Decode a JSON object response."""
+    value = response_json(response, service=service)
+    if not isinstance(value, Mapping):
+        logger.warning("%s returned JSON with an unexpected shape", service)
+        raise InvalidServiceResponseError(
+            f"{service} returned JSON with an unexpected shape",
+            service=service,
+            status_code=response.status_code,
+        )
+    return cast("Mapping[str, Any]", value)
+
+
+async def async_response_json(response: object, *, service: str) -> object:
+    """Decode either a regular or multiplexed niquests async response."""
+    typed = cast("AsyncJsonResponseLike", response)
+    try:
+        value = typed.json()
+        if isawaitable(value):
+            value = await value
+        return value
+    except niquests.exceptions.JSONDecodeError:
+        error = None
+    except (niquests.exceptions.RequestException, OSError) as exc:
+        error = translate_request_error(exc, service=service)
+    except (TypeError, ValueError):
+        error = None
+    if error is not None:
+        raise error from None
+    logger.warning("%s returned invalid JSON", service)
+    raise InvalidServiceResponseError(
+        f"{service} returned invalid JSON",
+        service=service,
+        status_code=typed.status_code,
+    ) from None
+
+
+async def async_response_mapping(
+    response: object, *, service: str
+) -> Mapping[str, Any]:
+    """Decode an async JSON object response."""
+    value = await async_response_json(response, service=service)
+    if not isinstance(value, Mapping):
+        logger.warning("%s returned JSON with an unexpected shape", service)
+        raise InvalidServiceResponseError(
+            f"{service} returned JSON with an unexpected shape",
+            service=service,
+            status_code=cast("AsyncJsonResponseLike", response).status_code,
+        )
+    return cast("Mapping[str, Any]", value)
+
+
+def response_text(response: ResponseLike, *, service: str) -> str:
+    """Read a regular response body and require decoded text."""
+    try:
+        value = response.text
+    except (niquests.exceptions.RequestException, OSError) as exc:
+        error = translate_request_error(exc, service=service)
+    else:
+        error = None
+    if error is not None:
+        raise error from None
+    if not isinstance(value, str):
+        logger.warning("%s returned a non-text response", service)
+        raise InvalidServiceResponseError(
+            f"{service} returned a non-text response",
+            service=service,
+            status_code=response.status_code,
+        )
+    return value
+
+
+async def async_response_text(response: object, *, service: str) -> str:
+    """Read text from either niquests async response representation."""
+    typed = cast("AsyncTextResponseLike", response)
+    try:
+        value = typed.text
+        if isawaitable(value):
+            value = await value
+    except (niquests.exceptions.RequestException, OSError) as exc:
+        error = translate_request_error(exc, service=service)
+    else:
+        error = None
+    if error is not None:
+        raise error from None
+    if not isinstance(value, str):
+        logger.warning("%s returned a non-text response", service)
+        raise InvalidServiceResponseError(
+            f"{service} returned a non-text response",
+            service=service,
+            status_code=typed.status_code,
+        )
+    return value
+
+
+def raise_for_common_status(response: ResponseLike, *, service: str) -> None:
+    """Translate common HTTP failures without retaining response bodies."""
+    status_code = response.status_code
+    if status_code in {401, 403}:
+        logger.warning(
+            "%s rejected request credentials with HTTP %s", service, status_code
+        )
+        raise AuthenticationError(
+            f"{service} rejected the request credentials",
+            service=service,
+            status_code=status_code,
+        )
+    if status_code == _HTTP_RATE_LIMIT_STATUS:
+        retry_after, raw = parse_retry_after(response.headers)
+        logger.warning("%s rate limit reached", service)
+        raise RateLimitError(
+            f"{service} rate limit reached",
+            service=service,
+            status_code=status_code,
+            retry_after=retry_after,
+            retry_after_raw=raw,
+        )
+    if status_code >= _HTTP_ERROR_STATUS:
+        logger.warning("%s returned HTTP %s", service, status_code)
+        raise InvalidServiceResponseError(
+            f"{service} returned HTTP {status_code}",
+            service=service,
+            status_code=status_code,
+        )
+
+
+def translate_request_error(exc: BaseException, *, service: str) -> NetworkError:
+    """Convert a transport exception without copying its potentially sensitive text."""
+    type_name = type(exc).__name__.lower()
+    if "ssl" in type_name or "tls" in type_name:
+        return TLSVerificationError(
+            f"TLS request to {service} failed",
+            service=service,
+            cause_type=type(exc).__name__,
+        )
+    return NetworkError(
+        f"network request to {service} failed",
+        service=service,
+        cause_type=type(exc).__name__,
+    )
+
+
+_SECRET_KEYS = frozenset(
+    {
+        "authorization",
+        "cookie",
+        "capture_cookie",
+        "password",
+        "target_password",
+        "logged-in-sig",
+        "logged-in-user",
+        "secret_key",
+    }
+)
+
+
+def redact_mapping(values: Mapping[str, object]) -> dict[str, object]:
+    """Return a shallow, log-safe copy of request metadata."""
+    return {
+        key: "<redacted>" if key.lower() in _SECRET_KEYS else value
+        for key, value in values.items()
+    }

archivist/core/_urls.py

@@ -0,0 +1,80 @@
+"""URL validation and safe log rendering."""
+
+from __future__ import annotations
+
+import logging
+from urllib.parse import SplitResult, urlsplit, urlunsplit
+
+from archivist.core.errors import InvalidTargetURLError
+
+logger = logging.getLogger(__name__)
+
+_ASCII_CONTROL_LIMIT = 32
+
+
+def _split_http_url(url: str, *, label: str) -> SplitResult:
+    if not isinstance(url, str) or not url:
+        raise InvalidTargetURLError(f"{label} must be a non-empty string")
+    if any(
+        character.isspace() or ord(character) < _ASCII_CONTROL_LIMIT
+        for character in url
+    ):
+        raise InvalidTargetURLError(
+            f"{label} cannot contain whitespace or control characters"
+        )
+    try:
+        parsed = urlsplit(url)
+        _ = parsed.port
+    except ValueError as exc:
+        raise InvalidTargetURLError(f"{label} is not a valid URL") from exc
+    if parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname:
+        raise InvalidTargetURLError(f"{label} must be an absolute HTTP or HTTPS URL")
+    return parsed
+
+
+def validate_target_url(url: str) -> str:
+    """Validate an archive target without changing its wire representation."""
+    _split_http_url(url, label="target URL")
+    return url
+
+
+def validate_cdx_query(query: str) -> str:
+    """Validate an absolute URL or wildcard expression accepted by CDX."""
+    if not isinstance(query, str) or not query:
+        raise InvalidTargetURLError("CDX query must be a non-empty string")
+    if any(
+        character.isspace() or ord(character) < _ASCII_CONTROL_LIMIT
+        for character in query
+    ):
+        raise InvalidTargetURLError(
+            "CDX query cannot contain whitespace or control characters"
+        )
+    if "://" in query:
+        validate_target_url(query)
+    return query
+
+
+def validate_service_url(url: str) -> str:
+    """Validate and normalize a configurable archive service base URL."""
+    parsed = _split_http_url(url, label="service URL")
+    if parsed.username is not None or parsed.password is not None:
+        raise InvalidTargetURLError("service URL cannot contain user information")
+    if "?" in url or "#" in url:
+        raise InvalidTargetURLError("service URL cannot contain a query or fragment")
+    return url.rstrip("/")
+
+
+def sanitize_url_for_log(url: str) -> str:
+    """Remove user information, query parameters, and fragments from a URL."""
+    try:
+        parsed = urlsplit(url)
+        hostname = parsed.hostname
+        if not parsed.scheme or not hostname:
+            return "<invalid-url>"
+        if ":" in hostname and not hostname.startswith("["):
+            hostname = f"[{hostname}]"
+        port = parsed.port
+        netloc = f"{hostname}:{port}" if port is not None else hostname
+        return urlunsplit((parsed.scheme, netloc, parsed.path or "/", "", ""))
+    except (TypeError, ValueError):
+        return "<invalid-url>"

archivist/core/errors.py

@@ -0,0 +1,124 @@
+"""Exceptions raised by Archivist clients."""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+
+logger = logging.getLogger(__name__)
+
+
+class ArchivistError(Exception):
+    """Base class for all package exceptions."""
+
+
+class InvalidTargetURLError(ArchivistError, ValueError):
+    """The target is not an absolute HTTP or HTTPS URL."""
+
+
+InvalidURLError = InvalidTargetURLError
+
+
+class InvalidOptionError(ArchivistError, ValueError):
+    """A client option has an invalid value."""
+
+
+class OptionCombinationError(InvalidOptionError):
+    """Two or more otherwise valid options cannot be used together."""
+
+
+class ServiceError(ArchivistError):
+    """An archive service request failed."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        service: str | None = None,
+        status_code: int | None = None,
+    ) -> None:
+        """Initialize the error with service response context."""
+        super().__init__(message)
+        self.service = service
+        self.status_code = status_code
+
+
+class NetworkError(ServiceError):
+    """A request failed before a usable HTTP response was received."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        service: str | None = None,
+        status_code: int | None = None,
+        cause_type: str | None = None,
+    ) -> None:
+        """Initialize the error with transport failure context."""
+        super().__init__(message, service=service, status_code=status_code)
+        self.cause_type = cause_type
+
+
+class TLSVerificationError(NetworkError):
+    """TLS negotiation or certificate verification failed."""
+
+
+class AuthenticationError(ServiceError):
+    """A service rejected or requires the supplied credentials."""
+
+
+class RateLimitError(ServiceError):
+    """A service refused a request because a rate limit was reached."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        service: str | None = None,
+        status_code: int | None = None,
+        retry_after: float | datetime | None = None,
+        retry_after_raw: str | None = None,
+    ) -> None:
+        """Initialize the error with retry timing information."""
+        super().__init__(message, service=service, status_code=status_code)
+        self.retry_after = retry_after
+        self.retry_after_raw = retry_after_raw
+
+
+class InvalidServiceResponseError(ServiceError):
+    """A service returned an undocumented or malformed response."""
+
+
+class CaptureFailedError(ServiceError):
+    """A service accepted a capture job but later reported failure."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        service: str | None = None,
+        status_code: int | None = None,
+        job_id: str | None = None,
+        service_code: str | None = None,
+    ) -> None:
+        """Initialize the error with failed capture details."""
+        super().__init__(message, service=service, status_code=status_code)
+        self.job_id = job_id
+        self.service_code = service_code
+
+
+class PollingTimeoutError(ServiceError, TimeoutError):
+    """A capture did not reach a terminal state before its deadline."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        service: str | None = None,
+        job_id: str | None = None,
+        timeout: float | None = None,
+    ) -> None:
+        """Initialize the error with polling deadline details."""
+        super().__init__(message, service=service)
+        self.job_id = job_id
+        self.timeout = timeout

archivist/core/models.py

@@ -0,0 +1,83 @@
+"""Service-independent Archivist models."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import Generic, TypeVar
+
+logger = logging.getLogger(__name__)
+
+
+def _as_utc(value: datetime | None) -> datetime | None:
+    if value is None:
+        return None
+    if value.tzinfo is None or value.utcoffset() is None:
+        raise ValueError("timestamps must include timezone information")
+    return value.astimezone(UTC)
+
+
+@dataclass(frozen=True, slots=True)
+class ArchiveRecord:
+    """A capture returned by an archive service."""
+
+    service: str
+    archive_url: str = field(repr=False)
+    original_url: str | None = field(default=None, repr=False)
+    archived_at: datetime | None = None
+    capture_id: str | None = None
+
+    def __post_init__(self) -> None:
+        """Normalize the archive timestamp to UTC."""
+        object.__setattr__(self, "archived_at", _as_utc(self.archived_at))
+
+
+ItemT = TypeVar("ItemT")
+
+
+@dataclass(frozen=True, slots=True)
+class PagedSearchResult(Generic[ItemT]):
+    """One page of service search results."""
+
+    items: tuple[ItemT, ...]
+    page: int = 1
+    page_count: int = 1
+    total_count: int | None = None
+
+    def __post_init__(self) -> None:
+        """Validate pagination metadata."""
+        if self.page < 1:
+            raise ValueError("page must be at least 1")
+        if self.page_count < 1:
+            raise ValueError("page_count must be at least 1")
+        if self.total_count is not None and self.total_count < 0:
+            raise ValueError("total_count cannot be negative")
+
+    def __iter__(self) -> Iterator[ItemT]:
+        """Iterate over the items on this page."""
+        return iter(self.items)
+
+    def __len__(self) -> int:
+        """Return the number of items on this page."""
+        return len(self.items)
+
+
+@dataclass(frozen=True, slots=True)
+class CaptureJob:
+    """An accepted asynchronous capture job."""
+
+    service: str
+    job_id: str
+    target_url: str = field(repr=False)
+    message: str | None = field(default=None, repr=False)
+
+
+@dataclass(frozen=True, slots=True)
+class ServiceStatus:
+    """A service health summary."""
+
+    service: str
+    status: str
+    message: str | None = None