sharefetch 1.0.0__py3-none-any.whl

sharefetch/errors.py

@@ -0,0 +1,57 @@
+"""Exception hierarchy for sharefetch."""
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class SharefetchError(Exception):
+    """Base class for every error this package raises.
+
+    ``status`` carries the HTTP status of the response that produced the error,
+    and is ``None`` for every error that did not come from one. It exists on the
+    base class because a caller several layers above the socket has to tell two
+    refusals apart without reading the text of a message: site resolution probes
+    candidate site paths, and a ``404`` there means the path names no site while
+    a ``403`` means it names one the account cannot read. The two lead to
+    different messages, and only the status separates them.
+
+    The keyword is optional, so every existing raise site is unchanged and
+    records ``None``.
+    """
+
+    def __init__(self, *args: object, status: int | None = None) -> None:
+        super().__init__(*args)
+        self.status = status
+
+
+class ValidationError(SharefetchError):
+    """A command-line option, URL or destination failed boundary validation."""
+
+
+class AuthError(SharefetchError):
+    """Device flow declined, expired, or refused by policy."""
+
+
+class ResolveError(SharefetchError):
+    """A site, library or folder could not be resolved, or was ambiguous."""
+
+
+class EnumerationError(SharefetchError):
+    """A folder listing was refused or malformed."""
+
+
+class TransferError(SharefetchError):
+    """One item failed to transfer. The queue continues."""
+
+
+class VerificationError(SharefetchError):
+    """One item failed verification. The queue continues."""
+
+
+class StateError(SharefetchError):
+    """The state file is unreadable in a way reconciliation cannot absorb."""
+
+
+class ManifestError(SharefetchError):
+    """The manifest could not be written. The downloaded tree is unaffected."""

sharefetch/events.py

@@ -0,0 +1,129 @@
+"""Event dataclasses and the sink protocol the front ends implement.
+
+``engine.run`` emits the events below and nothing else, so the command-line
+front end and the graphical front end observe one run through one vocabulary.
+Each event is frozen, carries only data, and can be queued across threads. The
+module is pure: it opens no socket and reads no clock.
+"""
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Protocol, runtime_checkable
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class Resolved:
+    site_web_url: str
+    drive_name: str
+    folder_path: str
+    drive_id: str
+
+
+@dataclass(frozen=True)
+class DeviceCodeRequested:
+    user_code: str
+    verification_uri: str
+    message: str
+    expires_at: float
+
+
+@dataclass(frozen=True)
+class DeviceCodeCompleted:
+    account: str
+
+
+@dataclass(frozen=True)
+class EnumerationProgress:
+    folders: int
+    files: int
+    total_bytes: int
+
+
+@dataclass(frozen=True)
+class EnumerationComplete:
+    files: int
+    total_bytes: int
+    folders: int
+
+
+@dataclass(frozen=True)
+class Reconciled:
+    counts: dict[str, int]
+
+
+@dataclass(frozen=True)
+class FileStarted:
+    drive_path: str
+    size: int
+    resume_from: int
+
+
+@dataclass(frozen=True)
+class FileProgress:
+    drive_path: str
+    bytes_written: int
+    size: int
+
+
+@dataclass(frozen=True)
+class FileFinished:
+    drive_path: str
+    status: str
+    # A verify.VerificationResult, or None where verification did not run. The
+    # annotation stays untyped because events depends on no other module.
+    verification: Any
+    error: str | None
+
+
+@dataclass(frozen=True)
+class Throttled:
+    seconds: float
+    reason: str
+
+
+@dataclass(frozen=True)
+class RunSummary:
+    transferred: int
+    skipped: int
+    failed: int
+    already_present: int
+    verified: int
+    bytes_transferred: int
+    manifest_path: Path | None
+    hash_algorithm_validated: bool | None
+
+
+@dataclass(frozen=True)
+class RunFinished:
+    summary: RunSummary
+    manifest_path: Path | None
+
+
+@dataclass(frozen=True)
+class LogRecord:
+    level: str
+    message: str
+
+
+Event = (
+    Resolved
+    | DeviceCodeRequested
+    | DeviceCodeCompleted
+    | EnumerationProgress
+    | EnumerationComplete
+    | Reconciled
+    | FileStarted
+    | FileProgress
+    | FileFinished
+    | Throttled
+    | RunFinished
+    | LogRecord
+)
+
+
+@runtime_checkable
+class EventSink(Protocol):
+    def emit(self, event: "Event") -> None: ...

sharefetch/graph.py

@@ -0,0 +1,474 @@
+"""The single HTTP choke point: one session, one pacer, one classification table.
+
+Every request the package makes to Microsoft Graph and to the pre-authenticated
+storage host it redirects to passes through :class:`GraphClient`. Concentrating
+them here makes the traffic discipline enforceable: the inter-request delay,
+the byte-rate ceiling, the throttle hold, the retry ceiling and the response
+classification each exist once and cannot be bypassed by a caller that forgets
+them.
+
+Four properties hold here and are the reason the module exists.
+
+* One pacer. The :class:`~sharefetch.pacing.Pacer` is supplied by the caller and
+  shared by every worker, so a ``429`` met by one worker stops both.
+* The bearer token is attached per request and never to the session. The content
+  endpoint answers with a redirect to a storage host whose URL carries its own
+  pre-authenticated credential in its query string, and ``requests`` drops a
+  per-request ``Authorization`` header when a redirect crosses hosts. Sending the
+  bearer token to storage can be refused outright. The download-URL fallback
+  addresses that host directly, so :meth:`GraphClient.stream`
+  takes ``authenticated=False`` and builds no header at all for it.
+* No credential and no non-Graph query string reaches a log. Every log call in
+  this module passes a URL through :func:`safe_url` first, and
+  :class:`RedactingFilter` is the second line of defence over every handler the
+  front ends install.
+* One forced refresh answers a ``401``. A second ``401`` on the same request
+  raises :class:`~sharefetch.errors.AuthError`, which is terminal for the run.
+
+``sleep`` and ``session`` are injected so the classification table is testable
+without a socket and without elapsing real time.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+from collections.abc import Callable
+from typing import Any
+from urllib.parse import urlsplit, urlunsplit
+
+import requests
+from requests.adapters import HTTPAdapter
+
+from .auth import TokenProvider
+from .config import (
+    GRAPH_HOST,
+    MAX_ATTEMPTS_PER_FILE,
+    REQUEST_TIMEOUT,
+    check_connections,
+)
+from .errors import AuthError, EnumerationError, SharefetchError, TransferError
+from .events import EventSink, Throttled
+from .pacing import Pacer, backoff_delay, parse_retry_after
+
+logger = logging.getLogger(__name__)
+
+# One name per status row, so the classification reads as the table does.
+_STATUS_SUCCESS = frozenset({200, 206})
+_STATUS_OK = 200
+_STATUS_UNAUTHORIZED = 401
+_STATUS_FORBIDDEN = 403
+_STATUS_NOT_FOUND = 404
+_STATUS_RANGE_NOT_SATISFIABLE = 416
+_STATUS_THROTTLED = frozenset({429, 503})
+_STATUS_SERVER_RETRYABLE = frozenset({500, 502, 504})
+
+# Transport failures are answered with backoff and a resume by range. A
+# connect timeout is a subclass of both ConnectionError and Timeout, so it is
+# covered. TooManyRedirects is deliberately absent: it is terminal.
+_TRANSPORT_FAILURES = (
+    requests.ConnectionError,
+    requests.Timeout,
+    requests.exceptions.ChunkedEncodingError,
+    requests.exceptions.ContentDecodingError,
+)
+
+# A refused item is recorded with the first 400 characters of the body.
+# The byte budget bounds the read, because an error body arrives on a streaming
+# response whose full length is not known in advance.
+_BODY_EXCERPT_CHARS = 400
+_BODY_EXCERPT_BYTES = 4096
+
+# Attached to a 403. A successful listing is not evidence that bytes will be
+# served, so the refusal says so rather than leaving the operator to infer it.
+_PERMISSION_NOTE = (
+    "enumeration permission and content permission are granted separately, so a "
+    "successful listing is not evidence that bytes will be served"
+)
+
+# The credential half of each match is replaced, the naming half kept, so a
+# redacted line still says which key was masked.
+_BEARER = re.compile(r"(Bearer\s+)[A-Za-z0-9._~+/=-]{8,}")
+_KEYS = re.compile(
+    r"([\"']?(?:access_token|refresh_token|id_token|device_code)[\"']?\s*[:=]\s*[\"']?)"
+    r"[A-Za-z0-9._~+/=-]{8,}"
+)
+_URL = re.compile(r"https?://[^\s\"'<>]+")
+_MASK = "***"
+
+
+class RestartFromZeroError(SharefetchError):
+    """The requested range will not be served, so the file restarts at offset zero.
+
+    Raised for the two classification rows that mean the same to the caller:
+    a ``200`` answering a request that carried a ``Range`` header, which is a
+    server that ignored the range, and a ``416``, which is a recorded offset
+    past the current length. The response is discarded before the raise, so the
+    caller reopens its ``.part`` file with ``wb`` and requests again without a
+    range.
+    """
+
+    def __init__(self, message: str, status: int) -> None:
+        super().__init__(message, status=status)
+
+
+def safe_url(url: str) -> str:
+    """Return ``url`` with the query and fragment dropped unless the host is Graph.
+
+    The pre-authenticated storage URL carries a credential in its query string,
+    so no non-Graph URL is ever logged whole. A Graph URL keeps its query,
+    because ``$select`` and ``$top`` are addressing rather than credentials.
+    """
+    parts = urlsplit(url)
+    if not parts.query and not parts.fragment:
+        return url
+    if (parts.hostname or "").lower() == GRAPH_HOST:
+        return url
+    return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
+
+
+def redact(text: str) -> str:
+    """Mask every credential in ``text`` and strip every non-Graph query string."""
+    masked = _BEARER.sub(rf"\1{_MASK}", text)
+    masked = _KEYS.sub(rf"\1{_MASK}", masked)
+    return _URL.sub(lambda match: safe_url(match.group(0)), masked)
+
+
+class RedactingFilter(logging.Filter):
+    """Masks credentials and non-Graph query strings on every record that passes.
+
+    Attached by the front ends to every handler they install, so a log line
+    written by this package, by ``requests`` or by ``urllib3`` is covered alike.
+    The record is rewritten in place, which is what makes the mask survive the
+    handler's own formatting.
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        if isinstance(record.msg, str):
+            record.msg = redact(record.msg)
+        args = record.args
+        if isinstance(args, dict):
+            record.args = {
+                key: redact(value) if isinstance(value, str) else value
+                for key, value in args.items()
+            }
+        elif isinstance(args, tuple):
+            record.args = tuple(redact(item) if isinstance(item, str) else item for item in args)
+        return True
+
+
+class GraphClient:
+    """Performs every HTTP request the run makes.
+
+    The client is shared by enumeration and by every transfer worker and is safe
+    to call from all of them: the session is thread-safe for distinct requests,
+    the pacer carries its own lock, and the client holds no per-request state on
+    the instance.
+    """
+
+    def __init__(
+        self,
+        tokens: TokenProvider,
+        pacer: Pacer,
+        connections: int,
+        sink: EventSink,
+        *,
+        session: Any | None = None,
+        sleep: Callable[[float], None] = time.sleep,
+    ) -> None:
+        """Build the client and, unless one is supplied, its pooled session.
+
+        ``connections`` sizes the connection pool to the concurrency limit and
+        is validated here, so a caller that skipped ``config.check_connections``
+        still cannot open a third connection. ``session`` and ``sleep`` exist so
+        a test can drive the classification table without a socket and without
+        elapsing real time; a supplied session is used as given and is never
+        re-mounted.
+        """
+        self._tokens = tokens
+        self._pacer = pacer
+        self._sink = sink
+        self._sleep = sleep
+        self._connections = check_connections(connections)
+        if session is None:
+            session = requests.Session()
+            adapter = HTTPAdapter(
+                pool_connections=self._connections,
+                pool_maxsize=self._connections,
+            )
+            session.mount("https://", adapter)
+            session.mount("http://", adapter)
+        self._session = session
+        self._closed = False
+
+    @property
+    def connections(self) -> int:
+        """The validated concurrency limit the connection pool is sized to."""
+        return self._connections
+
+    def get_json(self, url: str) -> dict[str, Any]:
+        """Return the JSON object at ``url``, having classified the response.
+
+        Raises :class:`~sharefetch.errors.EnumerationError` where the body is
+        not a JSON object, and the errors :meth:`stream` raises for every other
+        classification row. Each of them carries the HTTP status on ``.status``,
+        so a caller probing candidates can tell a ``404`` from a ``403``.
+        """
+        response = self._perform(url, None, stream=False)
+        status = response.status_code
+        try:
+            payload = response.json()
+        except ValueError as exc:
+            raise EnumerationError(
+                f"{safe_url(url)} answered {status} with a body that is not JSON",
+                status=status,
+            ) from exc
+        finally:
+            response.close()
+        if not isinstance(payload, dict):
+            raise EnumerationError(
+                f"{safe_url(url)} answered with a JSON {type(payload).__name__} "
+                "where an object was expected",
+                status=status,
+            )
+        return payload
+
+    def stream(
+        self,
+        url: str,
+        headers: dict[str, str] | None = None,
+        *,
+        authenticated: bool = True,
+    ) -> requests.Response:
+        """Return an open streaming response for ``url``, which the caller closes.
+
+        ``headers`` carries the ``Range`` header when resuming. The
+        ``Authorization`` header is added here and never by the caller, so a
+        redirect crossing hosts drops it.
+
+        ``authenticated=False`` withholds the bearer token for the whole
+        request, including every redirect it follows. It is used for the
+        ``@microsoft.graph.downloadUrl`` fallback: that URL
+        addresses a pre-authenticated storage host, carries its own credential
+        in its query string, and can refuse a bearer token outright. Everything
+        else is unchanged, so the fallback is paced, retried, classified and
+        redacted exactly as a Graph request is.
+
+        Raises :class:`RestartFromZeroError` where the range will not be served,
+        :class:`~sharefetch.errors.AuthError` on a second ``401``, and
+        :class:`~sharefetch.errors.TransferError` where the item is refused or
+        the retry ceiling is exhausted. An unauthenticated request has no token
+        to refresh, so its ``401`` is terminal for the item on the first one.
+        """
+        return self._perform(url, headers, stream=True, authenticated=authenticated)
+
+    def close(self) -> None:
+        """Close the underlying session. Calling it twice is harmless."""
+        if self._closed:
+            return
+        self._closed = True
+        self._session.close()
+        logger.debug("graph client session closed")
+
+    def __enter__(self) -> GraphClient:
+        return self
+
+    def __exit__(self, *_exc: object) -> None:
+        self.close()
+
+    def _perform(
+        self,
+        url: str,
+        headers: dict[str, str] | None,
+        stream: bool,
+        authenticated: bool = True,
+    ) -> requests.Response:
+        """Run one logical request to completion through the classification table.
+
+        Attempts are bounded by ``config.MAX_ATTEMPTS_PER_FILE``. A ``401``
+        carries its own bound of one retry, independent of that ceiling, because
+        a second ``401`` is terminal for the run rather than retryable.
+
+        ``authenticated`` is the only thing the download-URL fallback changes.
+        With it false no token is requested and no ``Authorization`` header is
+        built, so nothing in this method holds one to send.
+        """
+        request_headers = dict(headers or {})
+        ranged = any(name.lower() == "range" for name in request_headers)
+        shown = safe_url(url)
+        unauthorized = 0
+        last_failure = "no attempt completed"
+        last_status: int | None = None
+
+        for attempt in range(1, MAX_ATTEMPTS_PER_FILE + 1):
+            self._pacer.acquire()
+            sent = dict(request_headers)
+            if authenticated:
+                sent["Authorization"] = f"Bearer {self._tokens.get_token()}"
+            try:
+                response = self._session.request(
+                    "GET",
+                    url,
+                    headers=sent,
+                    timeout=REQUEST_TIMEOUT,
+                    stream=stream,
+                    allow_redirects=True,
+                )
+            except _TRANSPORT_FAILURES as exc:
+                last_failure = redact(f"{type(exc).__name__}: {exc}")
+                seconds = backoff_delay(attempt)
+                logger.warning(
+                    "attempt %d of %d for %s failed at the transport layer with %s, "
+                    "backing off %.3fs",
+                    attempt,
+                    MAX_ATTEMPTS_PER_FILE,
+                    shown,
+                    type(exc).__name__,
+                    seconds,
+                )
+                self._wait(seconds)
+                continue
+            except requests.RequestException as exc:
+                # Everything left over, such as TooManyRedirects or a malformed
+                # URL, is a fault a retry cannot fix.
+                raise TransferError(
+                    f"{shown} could not be requested: {redact(f'{type(exc).__name__}: {exc}')}"
+                ) from exc
+
+            status = response.status_code
+            last_status = status
+
+            if status in _STATUS_SUCCESS:
+                if status == _STATUS_OK and ranged:
+                    response.close()
+                    raise RestartFromZeroError(
+                        f"{shown} answered 200 to a request carrying a Range header, "
+                        "so the server ignored the range and the file restarts at zero",
+                        status,
+                    )
+                logger.debug("%s answered %d on attempt %d", shown, status, attempt)
+                return response
+
+            if status == _STATUS_UNAUTHORIZED:
+                response.close()
+                if not authenticated:
+                    # No token was sent, so there is none to refresh. The URL
+                    # carries its own credential and the host rejected it.
+                    raise TransferError(
+                        f"{shown} answered 401 to a request carrying no bearer token, so the "
+                        "credential the URL carries was refused and no refresh can change it",
+                        status=status,
+                    )
+                unauthorized += 1
+                if unauthorized > 1:
+                    raise AuthError(
+                        f"{shown} answered 401 again after a forced token refresh, "
+                        "which is terminal for the run",
+                        status=status,
+                    )
+                logger.warning("%s answered 401, forcing one token refresh and retrying", shown)
+                self._tokens.get_token(force_refresh=True)
+                last_failure = "HTTP 401"
+                continue
+
+            if status in _STATUS_THROTTLED:
+                seconds = self._retry_wait(response, attempt)
+                response.close()
+                reason = f"HTTP {status}"
+                last_failure = reason
+                logger.warning(
+                    "%s answered %d, holding every worker for %.3fs", shown, status, seconds
+                )
+                self._sink.emit(Throttled(seconds=seconds, reason=reason))
+                self._pacer.hold(seconds)
+                continue
+
+            if status in _STATUS_SERVER_RETRYABLE:
+                seconds = self._retry_wait(response, attempt)
+                response.close()
+                last_failure = f"HTTP {status}"
+                logger.warning(
+                    "%s answered %d, retrying in %.3fs (attempt %d of %d)",
+                    shown,
+                    status,
+                    seconds,
+                    attempt,
+                    MAX_ATTEMPTS_PER_FILE,
+                )
+                self._wait(seconds)
+                continue
+
+            if status == _STATUS_RANGE_NOT_SATISFIABLE:
+                excerpt = self._excerpt(response)
+                raise RestartFromZeroError(
+                    f"{shown} answered 416, so the recorded offset is past the current "
+                    f"length and the file restarts at zero: {excerpt}",
+                    status,
+                )
+
+            if status == _STATUS_FORBIDDEN:
+                excerpt = self._excerpt(response)
+                raise TransferError(
+                    f"{shown} answered 403, which is terminal for this item. "
+                    f"{_PERMISSION_NOTE}. Body: {excerpt}",
+                    status=status,
+                )
+
+            if status == _STATUS_NOT_FOUND:
+                excerpt = self._excerpt(response)
+                raise TransferError(
+                    f"{shown} answered 404, so the item vanished since enumeration. "
+                    f"Body: {excerpt}",
+                    status=status,
+                )
+
+            excerpt = self._excerpt(response)
+            raise TransferError(
+                f"{shown} answered {status}, which is terminal for this item. Body: {excerpt}",
+                status=status,
+            )
+
+        raise TransferError(
+            f"{shown} did not succeed in {MAX_ATTEMPTS_PER_FILE} attempts, "
+            f"the last failure was {last_failure}",
+            status=last_status,
+        )
+
+    def _retry_wait(self, response: Any, attempt: int) -> float:
+        """Return the wait for a retryable status, from ``Retry-After`` or backoff.
+
+        ``parse_retry_after`` returns ``None`` where the header is missing or
+        unusable, which is the fall-through to the backoff curve and never a
+        wait of zero.
+        """
+        honoured = parse_retry_after(response.headers.get("Retry-After"))
+        if honoured is not None:
+            return honoured
+        return backoff_delay(attempt)
+
+    def _wait(self, seconds: float) -> None:
+        """Sleep for ``seconds`` where it is positive."""
+        if seconds > 0:
+            self._sleep(seconds)
+
+    def _excerpt(self, response: Any) -> str:
+        """Return a bounded, whitespace-collapsed excerpt of a refused response body.
+
+        The response is closed before returning, because the caller raises on
+        every path that reaches here.
+        """
+        raw = b""
+        try:
+            for piece in response.iter_content(_BODY_EXCERPT_BYTES):
+                raw = piece or b""
+                break
+        except (requests.RequestException, OSError, ValueError) as exc:
+            logger.debug("the refused response body could not be read: %s", type(exc).__name__)
+        finally:
+            response.close()
+        if not raw:
+            return "(empty)"
+        encoding = getattr(response, "encoding", None) or "utf-8"
+        text = raw.decode(encoding, errors="replace")
+        return redact(" ".join(text.split()))[:_BODY_EXCERPT_CHARS]