sharefetch 1.0.0__py3-none-any.whl

sharefetch/hashes.py

@@ -0,0 +1,123 @@
+"""quickXorHash, the content hash Microsoft Graph reports for a drive item.
+
+quickXorHash is a 160-bit shift-and-xor checksum published by Microsoft and
+returned in a drive item's ``file.hashes`` facet. Recomputing it over the bytes
+written to disk is the only content check that works on every file type without
+any knowledge of the format, so it is verification check 2 and the strongest
+evidence the tool has that a transfer delivered the declared bytes.
+
+The algorithm keeps a 160-bit register held in three cells, of which the last
+carries 32 usable bits rather than 64. Each input byte is exclusive-ored into
+the register at a bit position that advances by 11 bits per byte and wraps at
+160, so bytes 160 apart share a position and fold together. The digest is the
+register written little-endian into 20 bytes, with the total input length
+exclusive-ored in as a little-endian 64-bit value over the final 8 bytes.
+
+The module is pure: it opens no socket and reads no clock.
+"""
+
+import base64
+import logging
+from functools import reduce
+from operator import xor
+from pathlib import Path
+
+from .errors import ValidationError
+from .paths import long_path
+
+logger = logging.getLogger(__name__)
+
+_WIDTH_IN_BITS = 160
+_SHIFT = 11
+_CELL_BITS = 64
+# The last cell holds 160 - 128 bits. A byte landing near its top wraps round to
+# the first cell, which is what makes the register 160 bits wide and not 192.
+_BITS_IN_LAST_CELL = _WIDTH_IN_BITS - _CELL_BITS * (_WIDTH_IN_BITS // _CELL_BITS)
+_CELL_COUNT = (_WIDTH_IN_BITS - 1) // _CELL_BITS + 1
+_BYTE_MASK = 0xFF
+_BITS_PER_BYTE = 8
+
+DIGEST_SIZE = 20
+LENGTH_FIELD_BYTES = 8
+
+
+class QuickXorHash:
+    """Incremental quickXorHash over a byte stream.
+
+    Updating in chunks produces the same digest as updating once with the
+    concatenation, at every chunk width, because ``_shifted`` carries the bit
+    position of the next byte across the call boundary.
+    """
+
+    def __init__(self) -> None:
+        self._data = [0] * _CELL_COUNT
+        self._length = 0
+        self._shifted = 0
+
+    def update(self, data: bytes) -> None:
+        """Fold ``data`` into the register and advance the bit position."""
+        if not data:
+            return
+        shift = self._shifted
+        last_cell = _CELL_COUNT - 1
+        for i in range(min(len(data), _WIDTH_IN_BITS)):
+            # Every byte whose index is congruent to i modulo the register width
+            # lands on the same bit position, so they fold before they are
+            # written. Exclusive or is associative, so the result is identical.
+            folded = reduce(xor, data[i::_WIDTH_IN_BITS], 0)
+            if folded == 0:
+                continue
+            cell, offset = divmod((shift + _SHIFT * i) % _WIDTH_IN_BITS, _CELL_BITS)
+            capacity = _BITS_IN_LAST_CELL if cell == last_cell else _CELL_BITS
+            self._data[cell] ^= (folded << offset) & ((1 << capacity) - 1)
+            if offset > capacity - _BITS_PER_BYTE:
+                # The byte straddles the end of this cell. The bits masked off
+                # above continue at the bottom of the next one, wrapping from
+                # the last cell back to the first.
+                spill_cell = 0 if cell == last_cell else cell + 1
+                self._data[spill_cell] ^= folded >> (capacity - offset)
+        self._shifted = (shift + _SHIFT * len(data)) % _WIDTH_IN_BITS
+        self._length += len(data)
+
+    def digest(self) -> bytes:
+        """Return the 20-byte digest, length field included."""
+        out = bytearray(DIGEST_SIZE)
+        for i in range(DIGEST_SIZE):
+            cell, offset = divmod(i * _BITS_PER_BYTE, _CELL_BITS)
+            out[i] = (self._data[cell] >> offset) & _BYTE_MASK
+        for i in range(LENGTH_FIELD_BYTES):
+            out[DIGEST_SIZE - LENGTH_FIELD_BYTES + i] ^= (
+                self._length >> (_BITS_PER_BYTE * i)
+            ) & _BYTE_MASK
+        return bytes(out)
+
+    def b64(self) -> str:
+        """Return the digest as Graph reports it in ``file.hashes.quickXorHash``."""
+        return base64.b64encode(self.digest()).decode("ascii")
+
+
+def quick_xor_hash_file(path: Path, chunk_size: int) -> str:
+    """Return the base64 quickXorHash of the file at ``path``.
+
+    ``chunk_size`` bounds the memory the read holds at once and changes nothing
+    about the result. The file is opened through ``paths.long_path`` so a
+    destination deeper than 260 characters is readable on Windows.
+    """
+    if chunk_size <= 0:
+        raise ValidationError(
+            f"chunk size must be a positive number of bytes, received {chunk_size}"
+        )
+    digest = QuickXorHash()
+    total = 0
+    # long_path returns an extended-length path string, which Path would
+    # normalise away, so the builtin open is the correct call here.
+    with open(long_path(path), "rb") as handle:  # noqa: PTH123
+        while True:
+            block = handle.read(chunk_size)
+            if not block:
+                break
+            digest.update(block)
+            total += len(block)
+    value = digest.b64()
+    logger.debug("hashed %s: %d bytes, quickXorHash %s", path, total, value)
+    return value

sharefetch/listing.py

@@ -0,0 +1,493 @@
+"""Depth-first enumeration of a drive folder tree, one page at a time.
+
+:func:`walk` yields one :class:`RemoteItem` per file beneath the addressed
+folder, in the order Graph returns it, so a resumed run revisits items in a
+stable sequence. A folder contributes no item of its own; it is recursed into.
+
+Four properties hold here and are the reason the module exists.
+
+* The walk is lazy. Each page is fetched only when the consumer has drained the
+  items the previous page produced, so a large tree never has to be held in
+  memory and a consumer that stops early stops the requests too.
+* Every request goes through :class:`~sharefetch.graph.GraphClient`, which
+  carries the pacer, the retry ceiling and the response classification table.
+  This module adds no retry loop of its own.
+* A refusal of one container is recorded and the walk continues: a per-item
+  failure never ends the run. A refusal of the addressed
+  folder itself is fatal, because the run has nothing left to do.
+* ``@microsoft.graph.downloadUrl`` is never named in ``$select``. It is an
+  instance annotation, and naming it makes Graph omit it silently. Where Graph
+  volunteers it, it is carried on the item as ``download_url``, which is used
+  only as a fallback when the content endpoint fails.
+
+No extension allowlist exists anywhere in this package. Every enumerated file is
+yielded whatever its name, its extension, or the absence of one. The only
+filtering is the caller's ``include`` and ``exclude`` globs.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import logging
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+from urllib.parse import quote, urlsplit
+
+from .config import GRAPH_HOST, GRAPH_ROOT, ITEM_SELECT, PAGE_SIZE, check_max_depth
+from .errors import EnumerationError, TransferError, ValidationError
+from .events import EnumerationComplete, EnumerationProgress
+from .graph import GraphClient, safe_url
+
+if TYPE_CHECKING:  # pragma: no cover - resolve is imported for its type only
+    from .resolve import DriveTarget
+
+logger = logging.getLogger(__name__)
+
+__all__ = [
+    "ProgressCallback",
+    "RemoteItem",
+    "SkippedItem",
+    "WalkEvent",
+    "walk",
+]
+
+# The two facets that make an item a container. A package, such as a OneNote
+# notebook, enumerates as a folder and is recursed into, so its sections
+# transfer as individual files.
+_FOLDER_FACET = "folder"
+_PACKAGE_FACET = "package"
+_FILE_FACET = "file"
+
+_NEXT_LINK = "@odata.nextLink"
+
+# The instance annotation the transfer falls back to. It is never named in
+# ``$select``; it is read off the item where Graph volunteered it.
+_DOWNLOAD_URL = "@microsoft.graph.downloadUrl"
+
+# The annotation points at a pre-authenticated storage host whose name varies by
+# tenant and by region, so no host allowlist can be stated here. The scheme can:
+# the credential travels in the query string, and an ``http`` URL would put it
+# on the wire in clear. A value that is not absolute ``https`` is dropped, and
+# the item transfers from the content endpoint alone.
+_DOWNLOAD_URL_SCHEME = "https"
+
+# The depth of the addressed folder itself. A file directly inside it carries
+# this depth, a file one folder down carries this depth plus one, and
+# ``max_depth`` is compared against the depth of the folder being entered.
+_ROOT_DEPTH = 0
+
+
+@dataclass(frozen=True)
+class RemoteItem:
+    """One file on the drive, addressed by its drive-relative path.
+
+    ``drive_path`` is the identity used by the state file, the local path
+    mapping and the manifest: drive-relative, forward slashes, no leading
+    slash. ``depth`` counts folder levels beneath the addressed folder, so a
+    file directly inside it is depth 0.
+
+    ``download_url`` is the ``@microsoft.graph.downloadUrl`` annotation where
+    Graph volunteered one and it is an absolute ``https`` URL, and ``None``
+    otherwise. It is a fallback and is never the first URL tried. It carries a
+    credential in its query string, so it is never logged whole:
+    :func:`sharefetch.graph.safe_url` is applied at every log site.
+    """
+
+    drive_path: str
+    item_id: str
+    name: str
+    size: int
+    etag: str | None
+    ctag: str | None
+    mime_type: str | None
+    quick_xor_hash: str | None
+    depth: int
+    download_url: str | None = None
+
+
+@dataclass(frozen=True)
+class SkippedItem:
+    """An item the walk stepped over, reported so the run can record it skipped.
+
+    Two things reach here: a container the server refuses to enumerate, whose
+    ``reason`` is the server's own refusal, already redacted by
+    :mod:`sharefetch.graph`; and an item carrying no file, folder or package
+    facet, which has neither bytes to fetch nor children to list.
+    ``is_package`` marks a package refusing enumeration, held apart from an
+    ordinary folder the account may list but not read.
+    """
+
+    drive_path: str
+    reason: str
+    is_package: bool
+
+
+WalkEvent = EnumerationProgress | EnumerationComplete | SkippedItem
+ProgressCallback = Callable[[WalkEvent], None]
+
+
+@dataclass
+class _WalkState:
+    """The running counts and the caller's filters, shared by every recursion level."""
+
+    include: tuple[str, ...]
+    exclude: tuple[str, ...]
+    max_depth: int | None
+    on_progress: ProgressCallback | None
+    folders: int = 0
+    files: int = 0
+    total_bytes: int = 0
+
+    def report(self, event: WalkEvent) -> None:
+        """Hand one event to the caller's callback, where one was supplied.
+
+        An exception raised by the callback propagates and ends the walk, which
+        is how a front end cancels an enumeration in progress.
+        """
+        if self.on_progress is not None:
+            self.on_progress(event)
+
+    def progress(self) -> EnumerationProgress:
+        return EnumerationProgress(
+            folders=self.folders,
+            files=self.files,
+            total_bytes=self.total_bytes,
+        )
+
+    def complete(self) -> EnumerationComplete:
+        return EnumerationComplete(
+            files=self.files,
+            total_bytes=self.total_bytes,
+            folders=self.folders,
+        )
+
+
+def walk(
+    target: DriveTarget,
+    client: GraphClient,
+    *,
+    include: tuple[str, ...] = (),
+    exclude: tuple[str, ...] = (),
+    max_depth: int | None = None,
+    on_progress: ProgressCallback | None = None,
+) -> Iterator[RemoteItem]:
+    """Yield every file beneath ``target``, depth-first, in Graph's own order.
+
+    ``include`` globs are applied first: with any present, a file matching none
+    of them is dropped. ``exclude`` globs are applied second and remove matches.
+    Both match the drive-relative path with :func:`fnmatch.fnmatch`. Neither is
+    applied to a folder, so ``--include '*.pdf'`` still recurses the whole tree.
+
+    ``max_depth`` stops recursion below N levels beneath the addressed folder,
+    and ``0`` means the addressed folder only.
+
+    ``on_progress`` receives an :class:`~sharefetch.events.EnumerationProgress`
+    after each page, a :class:`SkippedItem` for each container the server
+    refuses, and one :class:`~sharefetch.events.EnumerationComplete` when the
+    walk finishes. Counts are cumulative and count what the walk yields, so the
+    final progress and the completion event agree.
+
+    Raises :class:`~sharefetch.errors.ValidationError` for an unusable target or
+    depth limit, and :class:`~sharefetch.errors.EnumerationError` where the
+    addressed folder is refused or a listing is malformed.
+    """
+    drive_id = getattr(target, "drive_id", "")
+    if not isinstance(drive_id, str) or not drive_id.strip():
+        raise ValidationError("the drive target carries no drive id, so nothing can be listed")
+    folder_path = _normalise_folder(getattr(target, "folder_path", ""))
+    state = _WalkState(
+        include=tuple(include),
+        exclude=tuple(exclude),
+        max_depth=check_max_depth(max_depth),
+        on_progress=on_progress,
+    )
+    return _walk(client, drive_id, folder_path, state)
+
+
+def _walk(
+    client: GraphClient,
+    drive_id: str,
+    folder_path: str,
+    state: _WalkState,
+) -> Iterator[RemoteItem]:
+    """Drive the recursion and announce completion once the tree is exhausted."""
+    logger.info(
+        "enumerating %s beneath %s",
+        _shown(folder_path),
+        drive_id,
+    )
+    yield from _walk_folder(
+        client,
+        drive_id,
+        folder_path,
+        _ROOT_DEPTH,
+        state,
+        is_package=False,
+        is_root=True,
+    )
+    logger.info(
+        "enumeration complete: %d files, %d folders, %d bytes",
+        state.files,
+        state.folders,
+        state.total_bytes,
+    )
+    state.report(state.complete())
+
+
+def _walk_folder(
+    client: GraphClient,
+    drive_id: str,
+    folder_path: str,
+    depth: int,
+    state: _WalkState,
+    *,
+    is_package: bool,
+    is_root: bool,
+) -> Iterator[RemoteItem]:
+    """Yield the files in one container and recurse into its own containers.
+
+    Paging terminates when a page carries no ``@odata.nextLink``. A link that
+    has already been followed, or one addressing a host other than Graph, raises
+    :class:`~sharefetch.errors.EnumerationError`: the first is a server loop
+    that would repeat items indefinitely, the second would carry the bearer
+    token off Graph.
+    """
+    url: str | None = _children_url(drive_id, folder_path)
+    seen: set[str] = {url}
+    while url is not None:
+        try:
+            payload = client.get_json(url)
+        except TransferError as exc:
+            reason = str(exc)
+            if is_root:
+                raise EnumerationError(
+                    f"the addressed folder {_shown(folder_path)} could not be listed: {reason}"
+                ) from exc
+            logger.warning(
+                "skipping %s %s, which the server refused: %s",
+                "package" if is_package else "folder",
+                _shown(folder_path),
+                reason,
+            )
+            state.report(
+                SkippedItem(
+                    drive_path=folder_path,
+                    reason=reason,
+                    is_package=is_package,
+                )
+            )
+            return
+
+        children = payload.get("value")
+        if not isinstance(children, list):
+            raise EnumerationError(
+                f"the listing of {_shown(folder_path)} carries no value array, "
+                f"which means the response is not a children collection"
+            )
+
+        for child in children:
+            yield from _walk_child(client, drive_id, folder_path, depth, state, child)
+
+        state.report(state.progress())
+        url = _next_link(payload, folder_path, seen)
+
+
+def _walk_child(
+    client: GraphClient,
+    drive_id: str,
+    folder_path: str,
+    depth: int,
+    state: _WalkState,
+    child: Any,
+) -> Iterator[RemoteItem]:
+    """Yield one child where it is a file, or recurse into it where it is a container."""
+    if not isinstance(child, dict):
+        raise EnumerationError(
+            f"the listing of {_shown(folder_path)} carries a "
+            f"{type(child).__name__} where an item object was expected"
+        )
+    name = child.get("name")
+    if not isinstance(name, str) or not name:
+        raise EnumerationError(f"an item in the listing of {_shown(folder_path)} carries no name")
+    child_path = f"{folder_path}/{name}" if folder_path else name
+
+    is_package = _PACKAGE_FACET in child
+    if _FOLDER_FACET in child or is_package:
+        state.folders += 1
+        child_depth = depth + 1
+        if state.max_depth is not None and child_depth > state.max_depth:
+            logger.debug(
+                "not recursing into %s: depth %d is past the limit of %d",
+                child_path,
+                child_depth,
+                state.max_depth,
+            )
+            return
+        yield from _walk_folder(
+            client,
+            drive_id,
+            child_path,
+            child_depth,
+            state,
+            is_package=is_package,
+            is_root=False,
+        )
+        return
+
+    if _FILE_FACET not in child:
+        # A shortcut or a remote item carries neither facet, so there are no
+        # bytes to fetch and no children to list. It is reported and stepped
+        # over rather than guessed at.
+        reason = "the item carries neither a file, a folder nor a package facet"
+        logger.warning("skipping %s: %s", child_path, reason)
+        state.report(SkippedItem(drive_path=child_path, reason=reason, is_package=False))
+        return
+
+    if not _selected(child_path, state.include, state.exclude):
+        logger.debug("filtered out %s", child_path)
+        return
+
+    item = _to_item(child, child_path, depth, folder_path)
+    state.files += 1
+    state.total_bytes += item.size
+    yield item
+
+
+def _to_item(child: dict[str, Any], drive_path: str, depth: int, folder_path: str) -> RemoteItem:
+    """Build a :class:`RemoteItem` from one Graph driveItem object."""
+    item_id = child.get("id")
+    if not isinstance(item_id, str) or not item_id:
+        raise EnumerationError(
+            f"the item {drive_path} in the listing of {_shown(folder_path)} carries no id"
+        )
+    file_facet = child.get(_FILE_FACET)
+    file_facet = file_facet if isinstance(file_facet, dict) else {}
+    hashes = file_facet.get("hashes")
+    hashes = hashes if isinstance(hashes, dict) else {}
+    return RemoteItem(
+        drive_path=drive_path,
+        item_id=item_id,
+        name=str(child["name"]),
+        size=_size_of(child, drive_path),
+        etag=_text(child.get("eTag")),
+        ctag=_text(child.get("cTag")),
+        mime_type=_text(file_facet.get("mimeType")),
+        quick_xor_hash=_text(hashes.get("quickXorHash")),
+        depth=depth,
+        download_url=_download_url(child.get(_DOWNLOAD_URL), drive_path),
+    )
+
+
+def _download_url(value: Any, drive_path: str) -> str | None:
+    """Return the fallback download URL, or ``None`` where it is unusable.
+
+    The value is server-supplied and is fetched without the bearer token, so
+    the guard the next link needs, that the host is Graph, does not transfer:
+    the token cannot leak to a host this URL names. What remains is a URL whose
+    credential lives in its query string, so the scheme is required to be
+    ``https``. A value that is absent, not a string, not absolute, or on another
+    scheme is dropped and the item transfers from the content endpoint alone.
+
+    Nothing about the rejected value is logged beyond its scheme, because the
+    query string is the credential.
+    """
+    if value is None:
+        return None
+    if not isinstance(value, str) or not value:
+        logger.warning(
+            "%s carries a download URL annotation that is a %s, which is not usable as a URL",
+            drive_path,
+            type(value).__name__,
+        )
+        return None
+    parts = urlsplit(value)
+    if parts.scheme.lower() != _DOWNLOAD_URL_SCHEME or not parts.hostname:
+        logger.warning(
+            "%s carries a download URL on scheme %r, and only %s is accepted, so the "
+            "fallback is dropped",
+            drive_path,
+            parts.scheme,
+            _DOWNLOAD_URL_SCHEME,
+        )
+        return None
+    return value
+
+
+def _size_of(child: dict[str, Any], drive_path: str) -> int:
+    """Return the item's size, treating an absent or unusable value as zero.
+
+    Graph omits ``size`` on some items. A zero-length file is legitimate, and
+    verification compares the bytes written against the recorded size, so a
+    wrong size is caught there rather than guessed at here.
+    """
+    size = child.get("size")
+    if isinstance(size, bool) or not isinstance(size, int) or size < 0:
+        if size is not None:
+            logger.warning("item %s reported an unusable size %r, recording zero", drive_path, size)
+        return 0
+    return size
+
+
+def _text(value: Any) -> str | None:
+    """Return ``value`` where it is a non-empty string, otherwise ``None``."""
+    if isinstance(value, str) and value:
+        return value
+    return None
+
+
+def _selected(drive_path: str, include: tuple[str, ...], exclude: tuple[str, ...]) -> bool:
+    """Apply the include globs, then the exclude globs, to a drive-relative path."""
+    if include and not any(fnmatch.fnmatch(drive_path, pattern) for pattern in include):
+        return False
+    return not any(fnmatch.fnmatch(drive_path, pattern) for pattern in exclude)
+
+
+def _next_link(payload: dict[str, Any], folder_path: str, seen: set[str]) -> str | None:
+    """Return the next page's URL, or ``None`` where the listing is exhausted."""
+    link = payload.get(_NEXT_LINK)
+    if link is None:
+        return None
+    if not isinstance(link, str) or not link:
+        raise EnumerationError(
+            f"the listing of {_shown(folder_path)} carries a "
+            f"{type(link).__name__} where a next link was expected"
+        )
+    host = (urlsplit(link).hostname or "").lower()
+    if host != GRAPH_HOST:
+        raise EnumerationError(
+            f"the listing of {_shown(folder_path)} points its next page at "
+            f"{host or 'no host'}, which is not {GRAPH_HOST}"
+        )
+    if link in seen:
+        raise EnumerationError(
+            f"the listing of {_shown(folder_path)} points its next page at "
+            f"{safe_url(link)}, a page already fetched, so the paging does not terminate"
+        )
+    seen.add(link)
+    return link
+
+
+def _children_url(drive_id: str, folder_path: str) -> str:
+    """Address a folder's children by path."""
+    query = f"?$select={ITEM_SELECT}&$top={PAGE_SIZE}"
+    base = f"{GRAPH_ROOT}/drives/{quote(drive_id, safe='')}"
+    if not folder_path:
+        return f"{base}/root/children{query}"
+    return f"{base}/root:/{quote(folder_path, safe='/')}:/children{query}"
+
+
+def _normalise_folder(folder_path: Any) -> str:
+    """Return the addressed folder as a drive-relative path with no stray separators."""
+    if folder_path is None:
+        return ""
+    if not isinstance(folder_path, str):
+        raise ValidationError(
+            f"the folder path must be a string, received {type(folder_path).__name__}"
+        )
+    return "/".join(segment for segment in folder_path.replace("\\", "/").split("/") if segment)
+
+
+def _shown(folder_path: str) -> str:
+    """Name a folder for a log line or a message, including the drive root."""
+    return folder_path or "the drive root"