vercel-queue 0.7.0__py3-none-any.whl

vercel/queue/_internal/log.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, cast
+
+import json
+import logging
+import os
+import re
+from collections.abc import Mapping
+from urllib.parse import unquote, urlsplit, urlunsplit
+
+if TYPE_CHECKING:
+    from .types import Message, MessageMetadata
+
+_LOGGER_NAME = "vercel.queue"
+_REDACTED = "<redacted>"
+_REDACTED_RECEIPT_HANDLE = "rh_1_REDACTED"
+_DEBUG_ENV = "VERCEL_QUEUE_DEBUG"
+_SENSITIVE_FIELD_NAMES = frozenset({
+    "authorization",
+    "body",
+    "content",
+    "payload",
+    "raw_body",
+    "receipt_handle",
+    "receipthandle",
+    "token",
+})
+_SENSITIVE_HEADER_NAMES = frozenset({
+    "authorization",
+    "cookie",
+    "set-cookie",
+    "x-vercel-oidc-token",
+})
+_BEARER_RE = re.compile(r"Bearer\s+[^\s,;]+")
+_RECEIPT_RE = re.compile(r"rh_[^\s\"']+")
+
+
+def debug_enabled() -> bool:
+    return os.environ.get(_DEBUG_ENV) in {"1", "true"}
+
+
+def configure_asgi_logger() -> None:
+    logging.getLogger(_LOGGER_NAME).setLevel(logging.INFO if debug_enabled() else logging.WARNING)
+
+
+def debug_log(event: str, **fields: Any) -> None:
+    if not debug_enabled():
+        return
+    payload = {"event": event, **_redact_fields(fields)}
+    logging.getLogger(_LOGGER_NAME).info(
+        json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
+    )
+
+
+def _message_metadata(message: Message[Any] | MessageMetadata) -> MessageMetadata:
+    return cast("MessageMetadata", getattr(message, "metadata", message))
+
+
+def message_debug_fields(message: Message[Any] | MessageMetadata) -> dict[str, object]:
+    metadata = _message_metadata(message)
+    fields: dict[str, object] = {
+        "message_id": metadata.message_id,
+        "topic": metadata.topic,
+        "consumer_group": str(metadata.consumer_group),
+        "delivery_count": metadata.delivery_count,
+    }
+    if metadata.region is not None:
+        fields["region"] = metadata.region
+    return fields
+
+
+def debug_log_for_msg(
+    event: str,
+    message: Message[Any] | MessageMetadata,
+    **fields: Any,
+) -> None:
+    if not debug_enabled():
+        return
+    debug_log(event, **message_debug_fields(message), **fields)
+
+
+def safe_header_names(headers: Mapping[Any, object]) -> list[str]:
+    names = [str(name) for name in headers]
+    return sorted(name for name in names if name.lower() not in _SENSITIVE_HEADER_NAMES)
+
+
+def safe_url(url: str) -> str:
+    parts = urlsplit(url)
+    path_parts = parts.path.split("/")
+    redacted_parts: list[str] = []
+    redact_next = False
+    for part in path_parts:
+        if redact_next and part:
+            redacted_parts.append(_REDACTED_RECEIPT_HANDLE)
+            redact_next = False
+            continue
+        redacted_parts.append(part)
+        redact_next = unquote(part) == "lease"
+    return urlunsplit((parts.scheme, parts.netloc, "/".join(redacted_parts), "", ""))
+
+
+def redact_text(value: str) -> str:
+    return _RECEIPT_RE.sub(
+        _REDACTED_RECEIPT_HANDLE,
+        _BEARER_RE.sub(f"Bearer {_REDACTED}", value),
+    )
+
+
+def content_type(headers: Mapping[str, str]) -> str | None:
+    for name, value in headers.items():
+        if name.lower() == "content-type":
+            return value
+    return None
+
+
+def redact_value(name: str, value: Any) -> Any:
+    lower_name = name.lower()
+    if lower_name in {"receipt_handle", "receipthandle"}:
+        return _REDACTED_RECEIPT_HANDLE
+    if lower_name in _SENSITIVE_FIELD_NAMES:
+        return _REDACTED
+    if isinstance(value, str):
+        return redact_text(value)
+    if isinstance(value, Mapping):
+        return {str(key): redact_value(str(key), item) for key, item in sorted(value.items())}
+    if isinstance(value, tuple | list | set | frozenset):
+        return [redact_value(name, item) for item in value]
+    return _jsonable(value)
+
+
+def _redact_fields(fields: Mapping[str, Any]) -> dict[str, Any]:
+    return {name: redact_value(name, value) for name, value in fields.items() if value is not None}
+
+
+def _jsonable(value: Any) -> Any:
+    if value is None or isinstance(value, str | int | float | bool):
+        return value
+    if isinstance(value, Mapping):
+        return {str(key): _jsonable(item) for key, item in sorted(value.items())}
+    if isinstance(value, tuple | list | set | frozenset):
+        return [_jsonable(item) for item in value]
+    return str(value)

vercel/queue/_internal/messages.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+from typing import TypeVar, cast
+
+from collections.abc import AsyncIterator
+from dataclasses import replace
+from datetime import datetime, timezone
+
+from .constants import (
+    HEADER_CONTENT_TYPE,
+    VQS_HEADER_DELIVERY_COUNT,
+    VQS_HEADER_EXPIRES_AT,
+    VQS_HEADER_MESSAGE_ID,
+    VQS_HEADER_RECEIPT_HANDLE,
+    VQS_HEADER_TIMESTAMP,
+)
+from .errors import MessageCorruptedError
+from .http import headers_from_raw
+from .names import SanitizedName
+from .streams import AsyncStreamPayload, AsyncTextStreamPayload
+from .types import (
+    Message,
+    MessageMetadata,
+    RawHeaders,
+    ReceiptHandle,
+    Transport,
+)
+
+T = TypeVar("T")
+
+
+def _ensure_sanitized_name(value: str | SanitizedName) -> SanitizedName:
+    if isinstance(value, SanitizedName):
+        return value
+    return SanitizedName(value)
+
+
+async def message_from_part_async(
+    topic: str,
+    consumer_group: str | SanitizedName,
+    transport: Transport[T],
+    headers: RawHeaders,
+    payload: AsyncIterator[bytes],
+) -> Message[T]:
+    metadata = _message_metadata_from_part(topic, consumer_group, headers)
+    try:
+        decoded = await transport.deserialize(payload, content_type=metadata.content_type or "")
+    except Exception as exc:
+        raise MessageCorruptedError(
+            metadata.message_id,
+            f"Failed to parse payload: {exc}",
+        ) from exc
+    return Message(payload=decoded, metadata=metadata)
+
+
+def _message_metadata_from_part(
+    topic: str,
+    consumer_group: str | SanitizedName,
+    headers: RawHeaders,
+) -> MessageMetadata:
+    headers = headers_from_raw(headers)
+    message_id = headers.get(VQS_HEADER_MESSAGE_ID)
+    receipt_handle = headers.get(VQS_HEADER_RECEIPT_HANDLE)
+    if not message_id:
+        raise MessageCorruptedError("<unknown>", "Missing Vqs-Message-Id multipart header")
+    if not receipt_handle:
+        raise MessageCorruptedError(message_id, "Missing Vqs-Receipt-Handle multipart header")
+    delivery_count_raw = headers.get(VQS_HEADER_DELIVERY_COUNT) or "0"
+    try:
+        delivery_count = int(delivery_count_raw)
+    except ValueError:
+        delivery_count = 0
+    return MessageMetadata(
+        message_id=message_id,
+        delivery_count=delivery_count,
+        created_at=parse_required_datetime(
+            headers.get(VQS_HEADER_TIMESTAMP),
+            VQS_HEADER_TIMESTAMP,
+        ),
+        expires_at=parse_optional_datetime(
+            headers.get(VQS_HEADER_EXPIRES_AT),
+            VQS_HEADER_EXPIRES_AT,
+        ),
+        topic=topic,
+        consumer_group=_ensure_sanitized_name(consumer_group),
+        receipt_handle=ReceiptHandle(receipt_handle),
+        content_type=headers.get(HEADER_CONTENT_TYPE, ""),
+    )
+
+
+def message_with_region(message: Message[T], region: str | None) -> Message[T]:
+    if region is None:
+        return message
+    return Message(payload=message.payload, metadata=replace(message.metadata, region=region))
+
+
+def sync_message_payload(message: Message[T]) -> Message[T]:
+    if isinstance(message.payload, AsyncStreamPayload):
+        return Message(payload=cast("T", message.payload.to_sync()), metadata=message.metadata)
+    if isinstance(message.payload, AsyncTextStreamPayload):
+        return Message(payload=cast("T", message.payload.to_sync()), metadata=message.metadata)
+    return message
+
+
+def parse_required_datetime(value: str | None, header_name: str) -> datetime:
+    parsed = parse_optional_datetime(value, header_name)
+    if parsed is None:
+        raise ValueError(f"missing required {header_name} header")
+    return parsed
+
+
+def parse_optional_datetime(value: str | None, header_name: str) -> datetime | None:
+    if not value:
+        return None
+    normalized = value[:-1] + "+00:00" if value.endswith("Z") else value
+    try:
+        parsed = datetime.fromisoformat(normalized)
+    except ValueError as exc:
+        raise ValueError(f"invalid {header_name} header: {value!r}") from exc
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    return parsed.astimezone(timezone.utc)

vercel/queue/_internal/multipart.py

@@ -0,0 +1,255 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from typing_extensions import Self
+
+from collections import deque
+from collections.abc import AsyncIterator, Awaitable, Callable
+from dataclasses import dataclass
+
+from python_multipart.exceptions import MultipartParseError
+from python_multipart.multipart import (
+    MultipartParser,
+    parse_options_header,
+)
+
+from .errors import ProtocolError
+from .http import AsyncHttpResponse
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+
+DEFAULT_MAX_HEADER_BUFFER_SIZE = 64 * 1024
+DEFAULT_MAX_BOUNDARY_BUFFER_SIZE = 8 * 1024
+DEFAULT_MAX_HEADER_COUNT = 64
+MULTIPART_MIXED_MEDIA_TYPE = b"multipart/mixed"
+
+
+class _PartBody:
+    """Async iterator over a single multipart part body.
+
+    The parser feeds this object as bytes arrive. If the consumer asks for more
+    bytes before the current part has ended, ``PartBody`` pumps the parent
+    parser by one input chunk.
+    """
+
+    def __init__(self, pump: Callable[[], Awaitable[None]]) -> None:
+        self._pump = pump
+        self._chunks: deque[bytes] = deque()
+        self._done = False
+
+    def __aiter__(self) -> Self:
+        return self
+
+    async def __anext__(self) -> bytes:
+        while not self._chunks and not self._done:
+            await self._pump()
+        if self._chunks:
+            return self._chunks.popleft()
+        raise StopAsyncIteration
+
+    async def drain(self) -> None:
+        """Consume and discard unread bytes for this part."""
+        async for _ in self:
+            pass
+
+    def feed(self, chunk: bytes) -> None:
+        if chunk:
+            self._chunks.append(chunk)
+
+    def feed_slice(self, data: bytes, start: int, end: int) -> None:
+        if start == end:
+            return
+        if start == 0 and end == len(data):
+            self._chunks.append(data)
+            return
+        self._chunks.append(data[start:end])
+
+    def close(self) -> None:
+        self._done = True
+
+
+@dataclass(frozen=True, kw_only=True)
+class _Part:
+    """A streamed multipart part with parsed headers and body iterator."""
+
+    headers: dict[str, str]
+    body: _PartBody
+
+
+class _MultipartParser:
+    """Incremental multipart/mixed parser.
+
+    python-multipart is callback-based and synchronous. This wrapper turns it
+    into an async iterator of parts while preserving streaming behavior: a part
+    is yielded as soon as its headers are complete, and its body continues to be
+    filled as the caller iterates it.
+    """
+
+    def __init__(
+        self,
+        data: AsyncIterator[bytes],
+        /,
+        *,
+        content_type: str,
+    ) -> None:
+        self._input = data
+        self._max_header_buffer_size = DEFAULT_MAX_HEADER_BUFFER_SIZE
+
+        self._parts: deque[_Part] = deque()
+
+        self._headers: dict[str, str] = {}
+        self._field = bytearray()
+        self._value = bytearray()
+        self._header_bytes = 0
+        self._body: _PartBody | None = None
+        self._yielded_body: _PartBody | None = None
+
+        self._ended = False
+        self._exhausted = False
+
+        self._parser = MultipartParser(
+            _multipart_boundary(content_type),
+            max_size=float("inf"),
+            max_header_count=DEFAULT_MAX_HEADER_COUNT,
+            max_header_size=self._max_header_buffer_size,
+            callbacks={
+                "on_header_begin": self._on_header_begin,
+                "on_header_field": self._on_header_field,
+                "on_header_value": self._on_header_value,
+                "on_header_end": self._on_header_end,
+                "on_headers_finished": self._on_headers_finished,
+                "on_part_data": self._on_part_data,
+                "on_part_end": self._on_part_end,
+                "on_end": self._on_end,
+            },
+        )
+
+    def __aiter__(self) -> Self:
+        return self
+
+    async def __anext__(self) -> tuple[Mapping[str, str], AsyncIterator[bytes]]:
+        if self._yielded_body is not None:
+            await self._yielded_body.drain()
+            self._yielded_body = None
+
+        while not self._parts:
+            if self._ended:
+                raise StopAsyncIteration
+            await self._advance()
+        part = self._parts.popleft()
+        self._yielded_body = part.body
+        return part.headers, part.body
+
+    async def _advance(self) -> None:
+        if self._exhausted:
+            if self._ended:
+                return
+            raise ProtocolError("unexpected end of multipart stream")
+        try:
+            chunk = await anext(self._input)
+        except StopAsyncIteration:
+            self._exhausted = True
+            if not self._ended:
+                raise ProtocolError("unexpected end of multipart stream") from None
+            return
+        try:
+            self._parser.write(chunk)
+        except ProtocolError:
+            raise
+        except MultipartParseError as exc:
+            if "Maximum segment count exceeded" in str(exc):
+                raise ProtocolError(
+                    f"multipart header count exceeded {DEFAULT_MAX_HEADER_COUNT}"
+                ) from exc
+            if "Maximum header size exceeded" in str(exc):
+                raise ProtocolError(
+                    f"multipart header buffer exceeded {self._max_header_buffer_size} bytes"
+                ) from exc
+            raise ProtocolError(f"invalid multipart stream: {exc}") from exc
+
+    def _on_header_begin(self) -> None:
+        self._field.clear()
+        self._value.clear()
+
+    def _on_header_field(self, data: bytes, start: int, end: int) -> None:
+        self._add_header_bytes(end - start)
+        self._field.extend(data[start:end])
+
+    def _on_header_value(self, data: bytes, start: int, end: int) -> None:
+        self._add_header_bytes(end - start)
+        self._value.extend(data[start:end])
+
+    def _on_header_end(self) -> None:
+        name = self._field.decode("latin1").strip()
+        if name:
+            self._headers[name] = self._value.decode("latin1").strip()
+
+    def _on_headers_finished(self) -> None:
+        self._body = _PartBody(self._advance)
+        part = _Part(headers=dict(self._headers), body=self._body)
+        self._headers.clear()
+        self._header_bytes = 0
+        self._parts.append(part)
+
+    def _on_part_data(self, data: bytes, start: int, end: int) -> None:
+        if self._body is None:
+            raise RuntimeError("MultipartParser.on_part_data called out of order")
+        self._body.feed_slice(data, start, end)
+
+    def _on_part_end(self) -> None:
+        if self._body is None:
+            raise RuntimeError("MultipartParser.on_part_end called out of order")
+        self._body.close()
+        self._body = None
+
+    def _on_end(self) -> None:
+        self._ended = True
+
+    def _add_header_bytes(self, size: int) -> None:
+        self._header_bytes += size
+        if self._header_bytes > self._max_header_buffer_size:
+            raise ProtocolError(
+                f"multipart header buffer exceeded {self._max_header_buffer_size} bytes"
+            )
+
+
+def parse_multipart_messages(
+    response: AsyncHttpResponse,
+) -> AsyncIterator[tuple[Mapping[str, str], AsyncIterator[bytes]]]:
+    """Stream multipart parts from an HTTP response-like.
+
+    The response must have a ``multipart/mixed`` Content-Type with a boundary.
+    Each yielded part exposes its headers and an async byte iterator for the
+    body. If a caller advances to the next part without consuming the current
+    body, the parser drains the current body first.
+
+    Raises:
+        ProtocolError on malformed headers or other response deformity.
+
+    """
+    content_type = response.headers.get("Content-Type", "")
+    return _MultipartParser(
+        response.aiter_bytes(),
+        content_type=content_type,
+    )
+
+
+def _multipart_boundary(content_type: str) -> bytes:
+    media_type, options = parse_options_header(content_type)
+    if media_type.lower() != MULTIPART_MIXED_MEDIA_TYPE:
+        raise ProtocolError(
+            f"expected a multipart/mixed response, got Content-Type={content_type!r}"
+        )
+    value = options.get(b"boundary")
+    if not value:
+        raise ProtocolError("response did not specify a valid multipart boundary")
+    value = value.strip()
+    if not value:
+        raise ProtocolError("response did not specify a valid multipart boundary")
+    if len(value) > DEFAULT_MAX_BOUNDARY_BUFFER_SIZE:
+        raise ProtocolError(
+            f"multipart boundary buffer exceeded {DEFAULT_MAX_BOUNDARY_BUFFER_SIZE} bytes"
+        )
+    return value

vercel/queue/_internal/names.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import re
+from collections import UserString
+
+VQS_NAME_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$")
+
+
+class SanitizedName(UserString):
+    """A VQS-safe name that should not be sanitized again."""
+
+    __slots__ = ()
+
+    def __init__(self, value: str) -> None:
+        if not value:
+            raise ValueError("name must be a non-empty string")
+        if not _is_valid_vqs_name(value):
+            raise ValueError(f"Invalid queue name: {value!r}; must match {VQS_NAME_PATTERN}")
+        super().__init__(value)
+
+
+def _is_valid_vqs_name(name: str) -> bool:
+    return bool(VQS_NAME_PATTERN.fullmatch(name))
+
+
+# Keep this in sync with @vercel/build-utils:lambda.ts/sanitizeConsumerName
+# https://raw.githubusercontent.com/vercel/vercel/refs/heads/main/packages/build-utils/src/lambda.ts
+def _sanitize_name(name: str) -> str:
+    result = ""
+    for char in name:
+        if char == "_":
+            result += "__"
+        elif char == ".":
+            result += "_D"
+        elif char == "/":
+            result += "_S"
+        elif char.isascii() and (char.isalnum() or char == "-"):
+            result += char
+        else:
+            result += f"_{ord(char):02X}"
+    if not result:
+        raise ValueError("name must be a non-empty string")
+    return result
+
+
+def sanitize_name(name: str | SanitizedName) -> SanitizedName:
+    """Return a VQS-safe name marker for arbitrary input.
+
+    Plain strings are reversibly encoded, including underscores. Use
+    ``SanitizedName`` for values that are already VQS-safe and must not be
+    encoded again.
+    """
+    if isinstance(name, SanitizedName):
+        return name
+    return SanitizedName(_sanitize_name(name))
+
+
+def validate_name(name: object, *, field: str = "name") -> str:
+    name = getattr(name, "name", name)
+    if isinstance(name, SanitizedName):
+        return str(name)
+    if not isinstance(name, str) or not name:
+        raise ValueError(f"{field} must be a non-empty string")
+    if not _is_valid_vqs_name(name):
+        raise ValueError(f"Invalid queue {field}: {name!r}; must match {VQS_NAME_PATTERN}")
+    return name
+
+
+def validate_topic_name(topic: object) -> str:
+    return validate_name(topic, field="topic")
+
+
+def validate_subscription_pattern(topic: str) -> str:
+    if topic == "*":
+        return topic
+    topic_name = topic.removesuffix("*")
+    if not _is_valid_vqs_name(topic_name):
+        raise ValueError(f"Invalid queue topic: {topic!r}; must match {VQS_NAME_PATTERN}")
+    return topic
+
+
+def normalize_name(
+    name: str | SanitizedName,
+    *,
+    field: str = "name",
+) -> SanitizedName:
+    if isinstance(name, SanitizedName):
+        return name
+    if not name:
+        raise ValueError(f"{field} must be a non-empty string")
+    return sanitize_name(name)
+
+
+# Only add public symbols to __all__; internal helpers must stay unexported.
+__all__ = (
+    "SanitizedName",
+    "normalize_name",
+    "sanitize_name",
+    "validate_name",
+    "validate_subscription_pattern",
+)