graphplug 0.2.0__py3-none-any.whl

graphplug/_http.py

@@ -0,0 +1,198 @@
+"""The transport: Microsoft's middleware pipeline, the bearer token, and the concurrency limit.
+
+Everything that touches the wire is here. The rest of the package sees dictionaries.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from typing import Any, Dict, Mapping, Optional, Sequence
+
+import httpx
+from msgraph_core import GraphClientFactory
+
+from . import _log
+from ._errors import GraphError, as_graph_error, from_response
+from ._request import allowlisted
+
+__all__ = ["Transport", "GRAPH_HOST", "DEFAULT_MAX_CONCURRENCY"]
+
+#: The only host the bearer token may be attached to. Absolute URLs off this host -- a
+#: pre-authenticated download URL, say -- still work; they just travel unauthenticated.
+GRAPH_HOST = "graph.microsoft.com"
+
+#: Deliberately modest. Graph throttles per application and per tenant, and mailbox operations are
+#: limited to a handful of concurrent requests per mailbox, so the ceiling is the service's rather
+#: than Python's. Going wider earns 429s, not throughput.
+DEFAULT_MAX_CONCURRENCY = 12
+
+
+class Transport:
+    """One authenticated HTTP session over Microsoft's middleware pipeline."""
+
+    def __init__(
+        self,
+        credential: Any,
+        scopes: Sequence[str],
+        max_concurrency: int = DEFAULT_MAX_CONCURRENCY,
+        client: Optional[httpx.AsyncClient] = None,
+        owns_credential: bool = True,
+    ) -> None:
+        self._credential = credential
+        # A credential the caller handed in is the caller's to close; it may be shared.
+        self._owns_credential = owns_credential
+        self._scopes = tuple(scopes)
+        self._gate = asyncio.Semaphore(max_concurrency)
+
+        # A caller-supplied client is the test seam: pass one built on httpx.MockTransport and the
+        # middleware still wraps it, so tests exercise the real pipeline.
+        self._client = GraphClientFactory.create_with_default_middleware(client=client)
+        self._closed = False
+
+    # ── the wire ─────────────────────────────────────────────────────────────
+
+    async def send(
+        self,
+        method: str,
+        url: str,
+        headers: Optional[Mapping[str, str]] = None,
+        json_body: Any = None,
+        content: Optional[bytes] = None,
+        stream: bool = False,
+        operation: str = "request",
+    ) -> httpx.Response:
+        """Issue one request through the pipeline, with the token attached if the host allows it.
+
+        Anything that fails before a response exists -- the credential, DNS, TLS, a timeout --
+        is raised as a ``GraphError`` here, so every caller gets the one error shape.
+        """
+        if self._closed:
+            raise GraphError(0, "invalidHandle", "this client has already been closed")
+
+        try:
+            return await self._send(method, url, headers, json_body, content, stream)
+        except GraphError:
+            raise
+        except Exception as exception:
+            raise as_graph_error(exception, operation) from exception
+
+    async def _send(
+        self,
+        method: str,
+        url: str,
+        headers: Optional[Mapping[str, str]],
+        json_body: Any,
+        content: Optional[bytes],
+        stream: bool,
+    ) -> httpx.Response:
+        request = self._client.build_request(
+            method, url, headers=dict(headers or {}), json=json_body, content=content
+        )
+        await self._authorize(request)
+
+        # msgraph-core's transport runs its middleware only when the request carries `options`:
+        #
+        #     if self.pipeline and hasattr(request, 'options'):
+        #
+        # That attribute is normally set by Kiota's RequestAdapter, which this package does not
+        # use. Without it the whole pipeline -- retry, Retry-After, redirect, telemetry -- is
+        # silently skipped and requests go straight to the socket. Setting it here is the one
+        # place that undocumented contract is relied upon, and test_transport.py fails loudly if
+        # it ever stops working.
+        request.options = {}  # type: ignore[attr-defined]
+
+        started = time.monotonic()
+        async with self._gate:
+            if stream:
+                return await self._client.send(request, stream=True)
+            response = await self._client.send(request)
+
+        _log.request(
+            method,
+            url,
+            response.status_code,
+            int((time.monotonic() - started) * 1000),
+            request_id=response.headers.get("request-id"),
+            error_code=None if response.is_success else _peek_error_code(response),
+        )
+        return response
+
+    async def _authorize(self, request: httpx.Request) -> None:
+        """Attach the bearer token, but only for Graph itself.
+
+        Kiota's Python middleware has no authorization handler -- unlike .NET, where
+        AuthorizationHandler both attached the token and enforced allowed hosts. Both are done
+        here, and the host check is what stops a pre-authenticated download URL on some other
+        domain being handed a Graph token.
+        """
+        if request.url.host != GRAPH_HOST:
+            return
+
+        token = await self._credential.get_token(*self._scopes)
+        request.headers["Authorization"] = f"Bearer {token.token}"
+
+    # ── helpers the rest of the package uses ─────────────────────────────────
+
+    async def json(
+        self,
+        method: str,
+        url: str,
+        headers: Optional[Mapping[str, str]] = None,
+        body: Any = None,
+        operation: str = "request",
+    ) -> Dict[str, Any]:
+        """Send, then return the envelope: status, allow-listed headers, body, next link."""
+        response = await self.send(method, url, headers=headers, json_body=body, operation=operation)
+        return self.envelope(response)
+
+    @staticmethod
+    def envelope(response: httpx.Response) -> Dict[str, Any]:
+        """Turn a response into the one shape, raising on failure."""
+        headers = allowlisted(response.headers)
+        body = _decode(response)
+
+        if not response.is_success:
+            raise from_response(response.status_code, headers, body, response.reason_phrase)
+
+        envelope: Dict[str, Any] = {
+            "status": response.status_code,
+            "headers": headers,
+            "body": body,
+        }
+        if isinstance(body, dict) and body.get("@odata.nextLink"):
+            # Promoted so callers never need to know the OData annotation name. Left in the body
+            # too, so the Graph response stays verbatim.
+            envelope["nextLink"] = body["@odata.nextLink"]
+        return envelope
+
+    async def aclose(self) -> None:
+        if not self._closed:
+            self._closed = True
+            await self._client.aclose()
+            closer = getattr(self._credential, "close", None) if self._owns_credential else None
+            if closer is not None:
+                result = closer()
+                if asyncio.iscoroutine(result):
+                    await result
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+
+def _decode(response: httpx.Response) -> Any:
+    if response.status_code == 204 or not response.content:
+        return None
+    try:
+        return response.json()
+    except ValueError:
+        # Not every Graph failure body is JSON -- a gateway can return HTML.
+        return response.text
+
+
+def _peek_error_code(response: httpx.Response) -> Optional[str]:
+    body = _decode(response)
+    if isinstance(body, dict) and isinstance(body.get("error"), dict):
+        return body["error"].get("code")
+    return str(response.status_code)

graphplug/_log.py

@@ -0,0 +1,104 @@
+"""Opt-in structured logging.
+
+One JSON object per line on stderr, off unless ``GRAPHPLUG_LOG_LEVEL`` asks for it. Output goes to
+stderr because a caller may be piping stdout.
+
+Redaction is structural rather than a rule to remember: every function here takes the exact fields
+it may emit, so there is no free-form call through which a header, a token or a request body could
+reach a line. URLs are logged without their query string, because an OData ``$filter`` routinely
+carries email addresses.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import sys
+from typing import Any, Dict, Iterator, Optional, TextIO
+from urllib.parse import urlsplit
+
+__all__ = ["LEVEL_VARIABLE", "capture", "is_enabled", "parse_level", "request", "failure"]
+
+LEVEL_VARIABLE = "GRAPHPLUG_LOG_LEVEL"
+
+OFF, ERROR, INFO = 0, 1, 2
+_NAMES = {ERROR: "error", INFO: "info"}
+
+
+def parse_level(value: Optional[str]) -> int:
+    """Anything unrecognised means off: a typo must not silently enable logging."""
+    return {"error": ERROR, "info": INFO}.get((value or "").strip().lower(), OFF)
+
+
+_configured = parse_level(os.environ.get(LEVEL_VARIABLE))
+
+#: Test-only redirection. Production never sets it, so the production path holds no mutable state.
+_override: Optional[tuple] = None
+
+
+def _level() -> int:
+    return _override[0] if _override else _configured
+
+
+def _writer() -> TextIO:
+    return _override[1] if _override else sys.stderr
+
+
+def is_enabled(level: int) -> bool:
+    return level <= _level()
+
+
+@contextlib.contextmanager
+def capture(level: int, writer: TextIO) -> Iterator[None]:
+    """Capture output at the given level for the duration of the block. Tests only."""
+    global _override
+    previous, _override = _override, (level, writer)
+    try:
+        yield
+    finally:
+        _override = previous
+
+
+def _write(level: int, fields: Dict[str, Any]) -> None:
+    try:
+        line = json.dumps({"level": _NAMES[level], **fields}, separators=(",", ":"))
+        print(line, file=_writer(), flush=True)
+    except Exception:
+        # A diagnostic must never be the thing that takes the process down.
+        pass
+
+
+def _path_only(url: str) -> str:
+    """Scheme, host and path. The query string is never logged."""
+    parts = urlsplit(url)
+    return f"{parts.scheme}://{parts.netloc}{parts.path}"
+
+
+def request(
+    method: str,
+    url: str,
+    status: int,
+    elapsed_ms: int,
+    request_id: Optional[str] = None,
+    error_code: Optional[str] = None,
+) -> None:
+    level = ERROR if error_code else INFO
+    if not is_enabled(level):
+        return
+
+    _write(level, {
+        "event": "request",
+        "method": method,
+        "url": _path_only(url),
+        "status": status,
+        "ms": elapsed_ms,
+        "requestId": request_id,
+        "errorCode": error_code,
+    })
+
+
+def failure(operation: str, code: str, message: str) -> None:
+    """A failure that produced no response. The message is what the caller already receives."""
+    if is_enabled(ERROR):
+        _write(ERROR, {"event": "failure", "operation": operation, "code": code, "message": message})

graphplug/_operations.py

@@ -0,0 +1,310 @@
+"""Paging, batching and file transfer.
+
+Ported from the C# core's `BatchOperation`, `DownloadOperation` and the two upload strategies.
+None of msgraph-core's own helpers can be used for these: `BatchRequestBuilder`, `PageIterator` and
+`LargeFileUploadTask` each require a Kiota `RequestAdapter` and `Parsable` models, which this
+package deliberately does not build.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import uuid
+from pathlib import Path
+from typing import Any, AsyncIterator, Dict, List, Mapping, Optional, Sequence, Tuple, Union
+
+from ._errors import GraphError
+from ._http import Transport
+from ._request import allowlisted, build_url
+
+__all__ = [
+    "paged", "batch", "download", "upload",
+    "MAX_BATCH_SIZE", "CHUNKED_THRESHOLD_BYTES", "CHUNK_SIZE", "CHUNK_ALIGNMENT",
+]
+
+#: Graph's hard limit on requests per batch.
+MAX_BATCH_SIZE = 20
+
+#: Below this a single PUT is fine; at or above it Graph wants an upload session.
+CHUNKED_THRESHOLD_BYTES = 4 * 1024 * 1024
+
+#: Graph requires an upload session's chunk size to be a multiple of 320 KiB.
+CHUNK_ALIGNMENT = 320 * 1024
+CHUNK_SIZE = 10 * 1024 * 1024  # 32 whole alignment units
+
+_COPY_BUFFER = 1 << 16
+_MAX_CHUNK_ATTEMPTS = 3
+
+BatchRequest = Union[Tuple[str, str], Mapping[str, Any]]
+
+
+# ── paging ───────────────────────────────────────────────────────────────────
+
+
+async def paged(
+    transport: Transport,
+    url: str,
+    headers: Optional[Mapping[str, str]] = None,
+) -> AsyncIterator[Dict[str, Any]]:
+    """Walk every page, yielding items.
+
+    Nothing buffers the whole collection, and abandoning the generator leaks nothing: the next
+    link is a complete, self-describing cursor held only by the caller's loop.
+
+    ``headers`` is re-sent on every page, because a header that qualifies the query -- Graph's
+    ``ConsistencyLevel: eventual`` for advanced queries -- has to hold for the whole walk.
+    """
+    while url:
+        envelope = await transport.json("GET", url, headers=headers, operation="paged")
+        body = envelope.get("body") or {}
+
+        for item in body.get("value", []):
+            yield item
+
+        url = envelope.get("nextLink") or ""
+
+
+# ── batching ─────────────────────────────────────────────────────────────────
+
+
+def _prepare(requests: Sequence[BatchRequest]) -> List[Dict[str, Any]]:
+    """Normalise to Graph's sub-request shape, assigning ids so ordering has a key."""
+    prepared: List[Dict[str, Any]] = []
+    for index, request in enumerate(requests):
+        if isinstance(request, tuple):
+            method, url = request
+            item: Dict[str, Any] = {"method": method, "url": url}
+        else:
+            item = dict(request)
+        item.setdefault("id", str(index))
+        prepared.append(item)
+    return prepared
+
+
+async def batch(
+    transport: Transport,
+    requests: Sequence[BatchRequest],
+    version: Optional[str] = None,
+) -> List[Dict[str, Any]]:
+    """Send many requests as one call.
+
+    Two pieces of behaviour belong here rather than to the caller. Graph rejects batches larger
+    than 20, so a bigger set is split; and Graph does not guarantee response order within a batch,
+    so submission order is restored and results align positionally with what was sent.
+
+    Per-request failures are not raised -- each result keeps its own ``status``. One failing
+    sub-request must not discard nineteen successful ones.
+    """
+    prepared = _prepare(requests)
+    if not prepared:
+        return []
+
+    chunks = [prepared[i:i + MAX_BATCH_SIZE] for i in range(0, len(prepared), MAX_BATCH_SIZE)]
+    url = build_url("/$batch", version)
+
+    # Chunks go concurrently; the transport's semaphore is what keeps that within Graph's limits.
+    responses = await asyncio.gather(*(
+        transport.json("POST", url, body={"requests": chunk}, operation="batch")
+        for chunk in chunks
+    ), return_exceptions=True)
+
+    # A chunk that failed outright must not discard the chunks that ran -- with send_many, those
+    # mails have gone, and a caller who only saw an exception would send them twice. Its requests
+    # are reported in place like any failed sub-request. Only when nothing ran is it raised.
+    failures = [r for r in responses if isinstance(r, BaseException)]
+    if len(failures) == len(responses):
+        raise failures[0]
+
+    merged: List[Dict[str, Any]] = []
+    for chunk, envelope in zip(chunks, responses):
+        if isinstance(envelope, GraphError):
+            merged.extend(_failed(chunk, envelope))
+        elif isinstance(envelope, BaseException):
+            raise envelope
+        else:
+            merged.extend(_ordered(chunk, (envelope.get("body") or {}).get("responses", [])))
+    return merged
+
+
+def _failed(sent: Sequence[Mapping[str, Any]], error: GraphError) -> List[Dict[str, Any]]:
+    """One result per request in a chunk that never reached Graph, shaped like a sub-response."""
+    body = {"error": {"code": error.code, "message": error.message}}
+    return [{"id": str(item["id"]), "status": error.status, "body": body} for item in sent]
+
+
+def _ordered(sent: Sequence[Mapping[str, Any]], returned: Sequence[Mapping[str, Any]]) -> List[Dict[str, Any]]:
+    """Restore submission order. Graph returns sub-responses in whatever order they completed."""
+    by_id = {str(item.get("id")): dict(item) for item in returned if item.get("id") is not None}
+    return [by_id[str(item["id"])] for item in sent if str(item["id"]) in by_id]
+
+
+# ── download ─────────────────────────────────────────────────────────────────
+
+
+async def download(transport: Transport, url: str, dest_path: str) -> Dict[str, Any]:
+    """Stream a response straight to disk.
+
+    The bytes never enter a JSON envelope and never fully enter memory, so size is bounded by disk
+    rather than RAM. The destination directory must already exist; this does not create it.
+    """
+    destination = Path(dest_path)
+    if not destination.parent.exists():
+        raise GraphError(
+            0, "invalidRequest", f"the destination directory '{destination.parent}' does not exist"
+        )
+
+    response = await transport.send("GET", url, stream=True, operation="download")
+    try:
+        if not response.is_success:
+            await response.aread()
+            return Transport.envelope(response)  # raises with the Graph error
+
+        # Written to a temporary sibling and renamed on success, so a failed or cancelled
+        # download cannot leave a truncated file at the destination path.
+        partial = destination.with_name(f"{destination.name}.{uuid.uuid4().hex}.partial")
+        written = 0
+        try:
+            with partial.open("wb") as handle:
+                async for block in response.aiter_bytes(_COPY_BUFFER):
+                    handle.write(block)
+                    written += len(block)
+            os.replace(partial, destination)
+        except BaseException:
+            partial.unlink(missing_ok=True)
+            raise
+    finally:
+        await response.aclose()
+
+    return {
+        "status": response.status_code,
+        "headers": allowlisted(response.headers),
+        "bytesWritten": written,
+        "destPath": str(destination),
+    }
+
+
+# ── upload ───────────────────────────────────────────────────────────────────
+
+_CONTENT_SUFFIX = ":/content"
+_SESSION_SUFFIX = ":/createUploadSession"
+
+
+def to_session_path(path: str) -> str:
+    """Turn the content path the caller wrote into the session path Graph expects.
+
+    Both strategies take the same input, so the caller never learns which one ran.
+    """
+    if path.endswith(_CONTENT_SUFFIX):
+        return path[: -len(_CONTENT_SUFFIX)] + _SESSION_SUFFIX
+    if path.endswith("/content"):
+        return path[: -len("/content")] + "/createUploadSession"
+    return path
+
+
+async def upload(
+    transport: Transport,
+    path: str,
+    source_path: str,
+    version: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Send a local file, choosing the strategy by size."""
+    source = Path(source_path)
+    if not source.is_file():
+        raise GraphError(0, "invalidRequest", f"'{source_path}' does not exist")
+
+    size = source.stat().st_size
+    if size < CHUNKED_THRESHOLD_BYTES:
+        envelope = await _upload_simple(transport, path, source, version)
+    else:
+        envelope = await _upload_chunked(transport, path, source, size, version)
+
+    envelope["bytesSent"] = size
+    return envelope
+
+
+async def _upload_simple(
+    transport: Transport, path: str, source: Path, version: Optional[str]
+) -> Dict[str, Any]:
+    """A single PUT of the file. Used below 4 MiB."""
+    response = await transport.send(
+        "PUT",
+        build_url(path, version),
+        headers={"Content-Type": "application/octet-stream"},
+        content=source.read_bytes(),
+        operation="upload",
+    )
+    return Transport.envelope(response)
+
+
+async def _upload_chunked(
+    transport: Transport, path: str, source: Path, size: int, version: Optional[str]
+) -> Dict[str, Any]:
+    """createUploadSession followed by sequential ranged PUTs. Used at 4 MiB and above."""
+    created = await transport.json(
+        "POST", build_url(to_session_path(path), version), operation="upload"
+    )
+    upload_url = (created.get("body") or {}).get("uploadUrl")
+    if not upload_url:
+        raise GraphError(0, "internalError", "the upload session response carried no uploadUrl")
+
+    offset = 0
+    last = None
+    with source.open("rb") as handle:
+        while offset < size:
+            handle.seek(offset)
+            chunk = handle.read(CHUNK_SIZE)
+            if not chunk:
+                break
+
+            last = await _put_chunk(transport, upload_url, chunk, offset, size)
+            if not last.is_success:
+                return Transport.envelope(last)  # raises with the Graph error
+
+            offset += len(chunk)
+
+            # The service's own account of progress wins over ours.
+            resume = _next_expected(last)
+            if resume is not None and resume != offset and resume < size:
+                offset = resume
+
+    if last is None:
+        raise GraphError(0, "invalidRequest", "the file is empty")
+    return Transport.envelope(last)
+
+
+async def _put_chunk(transport: Transport, upload_url: str, chunk: bytes, offset: int, total: int):
+    """Send one chunk, retrying it against a transient failure."""
+    response = None
+    for _ in range(_MAX_CHUNK_ATTEMPTS):
+        response = await transport.send(
+            "PUT",
+            upload_url,
+            headers={
+                "Content-Range": f"bytes {offset}-{offset + len(chunk) - 1}/{total}",
+                "Content-Type": "application/octet-stream",
+            },
+            content=chunk,
+            operation="upload",
+        )
+        if response.is_success or response.status_code not in (408, 429, 500, 502, 503, 504):
+            return response
+    return response
+
+
+def _next_expected(response) -> Optional[int]:
+    """Read the session's nextExpectedRanges, which is what makes a large upload resumable."""
+    try:
+        body = response.json()
+    except ValueError:
+        return None
+    if not isinstance(body, dict):
+        return None
+
+    ranges = body.get("nextExpectedRanges")
+    if not ranges:
+        return None
+    try:
+        return int(str(ranges[0]).split("-", 1)[0])
+    except (ValueError, IndexError):
+        return None