mfup-core 0.2.0__py3-none-any.whl

mfup_core/__init__.py

@@ -0,0 +1,66 @@
+"""mfup-core — the MFUP/2 upload engine, framework-free.
+
+Protocol codec, per-session SQLite staging, the session state machine,
+Redis expiry index, publish/mapping, and the consumer hook contracts.
+HTTP/WebSocket wiring lives in the companion package ``mfup-fastapi``.
+"""
+
+from .protocol import (
+    CRC32C_IMPL,
+    PROTOCOL_VERSION,
+    FrameReader,
+    SessionState,
+)
+from .hooks import (
+    AuthRequest,
+    AuthResult,
+    AuthorizeHook,
+    CommitEvent,
+    FileMapRequest,
+    MapFileHook,
+    OnCommittedHook,
+    load_authorize_hook,
+    load_hook,
+    load_map_file_hook,
+    load_on_committed_hook,
+    resolve_hook,
+)
+from .session_manager import LiveSession, SessionRegistry
+from .redis_index import SessionIndex
+from .publish import (
+    ConflictError,
+    MappingError,
+    list_payload_files,
+    publish_session,
+    publish_session_mapped,
+)
+from .storage import staging_dir, validate_node_name
+
+__all__ = [
+    "AuthRequest",
+    "AuthResult",
+    "AuthorizeHook",
+    "CommitEvent",
+    "ConflictError",
+    "CRC32C_IMPL",
+    "FileMapRequest",
+    "FrameReader",
+    "list_payload_files",
+    "LiveSession",
+    "load_authorize_hook",
+    "load_hook",
+    "load_map_file_hook",
+    "load_on_committed_hook",
+    "MapFileHook",
+    "MappingError",
+    "OnCommittedHook",
+    "PROTOCOL_VERSION",
+    "publish_session",
+    "publish_session_mapped",
+    "resolve_hook",
+    "SessionIndex",
+    "SessionRegistry",
+    "SessionState",
+    "staging_dir",
+    "validate_node_name",
+]

mfup_core/hooks.py

@@ -0,0 +1,203 @@
+"""MFUP/2 extension hooks — config-driven, no library packaging required.
+
+A consumer plugs their code in via environment variables holding dotted
+paths, e.g.:
+
+    MFUP_AUTHORIZE=myapp.uploads:authorize
+
+The referenced module just has to be importable (on PYTHONPATH / mounted
+into the container). The full contract lives in docs/EXTENDING.md.
+
+Authorize contract
+------------------
+
+    async def authorize(req: AuthRequest) -> AuthResult | None:
+        ...
+
+Called once per HELLO, before the session is created. Return:
+  - ``AuthResult(...)``  — allow, optionally constraining the session;
+  - ``None``             — deny → the client gets SESSION_ABORT(auth_failed).
+
+Raising is treated as a deny (logged server-side, generic reason sent).
+
+If ``MFUP_AUTHORIZE`` is unset the server runs **allow-all** and logs a
+warning at startup — acceptable for development, not for production.
+"""
+
+from __future__ import annotations
+
+import importlib
+import logging
+from dataclasses import dataclass, field
+from typing import Any, Awaitable, Callable, Mapping, Optional
+
+logger = logging.getLogger("mfup.hooks")
+
+
+@dataclass(frozen=True)
+class AuthRequest:
+    """Everything the server knows about an upload attempt at HELLO time."""
+
+    session_id: str
+    target_dir: str
+    #: HTTP headers of the WebSocket handshake (cookies, Authorization, …).
+    headers: Mapping[str, str]
+    #: Client address as reported by the ASGI server ("ip:port" or "").
+    client: str
+    #: Query parameters of the WebSocket URL.
+    query: Mapping[str, str]
+    #: Arbitrary JSON the CONSUMER'S FRONTEND attached to the session
+    #: (MfupSessionConfig.meta → HELLO.meta). Untrusted client input — the
+    #: hook validates it. Typical use: upload scope/purpose ("avatars",
+    #: {"album_id": 123}) that authorization and per-file mapping key on.
+    meta: Any = None
+
+
+@dataclass(frozen=True)
+class AuthResult:
+    """Permission plus per-session constraints. All limits optional."""
+
+    #: Cap on total accepted payload bytes; exceeding aborts the session
+    #: with SESSION_ABORT(quota_exceeded). None = unlimited.
+    max_total_bytes: Optional[int] = None
+    #: Cap on the number of file nodes; exceeding aborts likewise.
+    max_files: Optional[int] = None
+    #: Per-session BASE directory (absolute path) — e.g. the user's home.
+    #: Overrides MFUP_BASE_DIR for this session: the staging dir is created
+    #: inside it (publish stays a same-filesystem rename even when homes live
+    #: on their own mount), relative target_dir resolves against it, and the
+    #: containment check confines the session to it. Created if missing.
+    #: None = the global MFUP_BASE_DIR.
+    base_dir: Optional[str] = None
+    #: Optional override of the client-requested target_dir (e.g. force
+    #: uploads into a fixed subdirectory, or a MAPPING of the client's
+    #: request — the hook receives req.target_dir and may prefix/rewrite it:
+    #:     target_dir=f"incoming/{req.target_dir}"
+    #: Escapes are impossible regardless: the resolved target must stay
+    #: within the session's base_dir or HELLO is refused (bad_target_dir).
+    target_dir: Optional[str] = None
+    #: Free-form bag the consumer may use to correlate sessions with users;
+    #: stored in memory on the LiveSession, never persisted or sent to the
+    #: client.
+    context: dict[str, Any] = field(default_factory=dict)
+
+
+AuthorizeHook = Callable[[AuthRequest], Awaitable[Optional[AuthResult]]]
+
+
+@dataclass(frozen=True)
+class FileMapRequest:
+    """One file about to be published — input to the map_file hook."""
+
+    session_id: str
+    #: Path of the file inside the uploaded tree, "/"-separated, as the
+    #: client sent it (e.g. "photos/2024/img_001.jpg").
+    path: str
+    #: Basename convenience (last segment of `path`).
+    name: str
+    #: Actual size on disk, bytes.
+    size: int
+    #: The session's target_dir (already authorized/mapped at HELLO).
+    target_dir: str
+    #: Client-attached session meta (see AuthRequest.meta).
+    meta: Any
+    #: AuthResult.context from the authorize hook.
+    context: Mapping[str, Any]
+
+
+#: async (FileMapRequest) -> str | None
+#:   str  — new path RELATIVE to target_dir (e.g. "media/img_001.jpg");
+#:   None — keep the client's layout for this file.
+#: Two files mapping to the same destination is a consumer bug → publish
+#: fails with mapping_conflict. Escaping segments ("..", absolute, "\\")
+#: fail publish likewise. The hook runs once per file at PUBLISH time, so
+#: it does not need to be deterministic across retries of the transfer.
+MapFileHook = Callable[[FileMapRequest], Awaitable[Optional[str]]]
+
+
+@dataclass(frozen=True)
+class CommitEvent:
+    """A session just committed — input to the on_committed hook."""
+
+    session_id: str
+    #: The session's (authorized/mapped) target_dir.
+    target_dir: str
+    #: The session's base directory (per-user home or the global base).
+    base_dir: str
+    #: Absolute staging directory holding the committed payload.
+    staging_dir: str
+    #: Committed file count / total payload bytes (as sent in COMMIT_OK).
+    files: int
+    bytes: int
+    #: Client-attached session meta (see AuthRequest.meta). Untrusted.
+    meta: Any
+    #: AuthResult.context from the authorize hook.
+    context: Mapping[str, Any]
+
+
+#: async (CommitEvent) -> str | None
+#:   "publish" — the server publishes immediately (server-side decision:
+#:               scan passed, billing ok, …). The browser's own publish call,
+#:               if any, will find the session gone (404) — harmless.
+#:   None      — do nothing; publish stays client-driven (or the consumer
+#:               backend calls MfupEngine.publish() later).
+#: Raising is logged and treated as None — a broken consumer hook must not
+#: strand committed sessions.
+OnCommittedHook = Callable[[CommitEvent], Awaitable[Optional[str]]]
+
+
+def load_hook(dotted: str) -> Callable[..., Any]:
+    """Import ``pkg.module:attr`` and return the attribute.
+
+    Raises ImportError/AttributeError loudly — a misconfigured hook must
+    fail at startup, not silently run allow-all.
+    """
+    module_path, sep, attr = dotted.partition(":")
+    if not sep or not module_path or not attr:
+        raise ImportError(
+            f"invalid hook path {dotted!r}: expected 'package.module:callable'"
+        )
+    module = importlib.import_module(module_path)
+    return getattr(module, attr)
+
+
+def load_authorize_hook(dotted: str | None) -> AuthorizeHook | None:
+    """Resolve MFUP_AUTHORIZE. None (unset) → allow-all with a warning."""
+    if not dotted:
+        logger.warning(
+            "MFUP_AUTHORIZE is not set — running WITHOUT authorization "
+            "(allow-all). Do not do this in production."
+        )
+        return None
+    hook = load_hook(dotted)
+    logger.info("Authorize hook loaded: %s", dotted)
+    return hook  # type: ignore[return-value]
+
+
+def load_map_file_hook(dotted: str | None) -> MapFileHook | None:
+    """Resolve MFUP_MAP_FILE. None (unset) → identity layout."""
+    if not dotted:
+        return None
+    hook = load_hook(dotted)
+    logger.info("map_file hook loaded: %s", dotted)
+    return hook  # type: ignore[return-value]
+
+
+def load_on_committed_hook(dotted: str | None) -> OnCommittedHook | None:
+    """Resolve MFUP_ON_COMMITTED. None (unset) → client-driven publish."""
+    if not dotted:
+        return None
+    hook = load_hook(dotted)
+    logger.info("on_committed hook loaded: %s", dotted)
+    return hook  # type: ignore[return-value]
+
+
+def resolve_hook(ref: Callable[..., Any] | str | None) -> Callable[..., Any] | None:
+    """Accept a hook given either as the callable itself (library embedding:
+    ``MfupConfig(authorize=my_func)``) or as a dotted path (env-driven:
+    ``"pkg.module:callable"``). None passes through."""
+    if ref is None:
+        return None
+    if callable(ref):
+        return ref
+    return load_hook(ref)

mfup_core/protocol.py

@@ -0,0 +1,347 @@
+"""MFUP/2 protocol types and binary frame decoder (server-side)."""
+
+from __future__ import annotations
+
+import enum
+import struct
+from dataclasses import dataclass
+from typing import Optional
+
+
+PROTOCOL_VERSION = "MFUP/2"
+ROOT_NODE_ID = 0
+
+
+# ---------------------------------------------------------------------------
+# Frame tags
+# ---------------------------------------------------------------------------
+class FrameTag(enum.IntEnum):
+    NODE = 0x01
+    SUMMARY = 0x02
+    FILE_OPEN = 0x03
+    FILE_CHUNK = 0x04
+    FILE_CLOSE = 0x05
+    DIR_CLOSE = 0x06
+    SESSION_END = 0x07
+    CLIENT_ABORT = 0x08
+
+
+class NodeKind(enum.IntEnum):
+    DIR = 0x00
+    FILE = 0x01
+
+
+class ChecksumKind(enum.IntEnum):
+    CRC32C = 0x01
+
+
+# ---------------------------------------------------------------------------
+# Decoded frame dataclasses
+# ---------------------------------------------------------------------------
+@dataclass(slots=True)
+class NodeFrame:
+    tag: FrameTag = FrameTag.NODE
+    node_id: int = 0
+    parent_id: int = 0
+    kind: NodeKind = NodeKind.FILE
+    name: str = ""
+    size_hint: Optional[int] = None
+    mtime_ms: Optional[int] = None
+
+
+@dataclass(slots=True)
+class SummaryFrame:
+    tag: FrameTag = FrameTag.SUMMARY
+    node_id: int = 0
+    scan_done_units: int = 0
+    scan_est_units: int = 0
+    body_done_bytes: int = 0
+    body_est_bytes: int = 0
+    sealed: bool = False
+
+
+@dataclass(slots=True)
+class FileOpenFrame:
+    tag: FrameTag = FrameTag.FILE_OPEN
+    node_id: int = 0
+    size: int = 0
+    mtime_ms: Optional[int] = None
+
+
+@dataclass(slots=True)
+class FileChunkFrame:
+    tag: FrameTag = FrameTag.FILE_CHUNK
+    node_id: int = 0
+    offset: int = 0
+    length: int = 0
+    checksum_kind: ChecksumKind = ChecksumKind.CRC32C
+    checksum: int = 0
+    payload: bytes = b""
+
+
+@dataclass(slots=True)
+class FileCloseFrame:
+    tag: FrameTag = FrameTag.FILE_CLOSE
+    node_id: int = 0
+    size_sent: int = 0
+
+
+@dataclass(slots=True)
+class DirCloseFrame:
+    tag: FrameTag = FrameTag.DIR_CLOSE
+    node_id: int = 0
+
+
+@dataclass(slots=True)
+class SessionEndFrame:
+    tag: FrameTag = FrameTag.SESSION_END
+    scan_done_units: int = 0
+    scan_est_units: int = 0
+    body_done_bytes: int = 0
+    body_est_bytes: int = 0
+    sealed: bool = True
+
+
+@dataclass(slots=True)
+class ClientAbortFrame:
+    tag: FrameTag = FrameTag.CLIENT_ABORT
+    code: str = ""
+    reason: str = ""
+
+
+Frame = (
+    NodeFrame
+    | SummaryFrame
+    | FileOpenFrame
+    | FileChunkFrame
+    | FileCloseFrame
+    | DirCloseFrame
+    | SessionEndFrame
+    | ClientAbortFrame
+)
+
+
+# Server → Client control message: {"t": "PROBE_ACK"}
+
+
+# ---------------------------------------------------------------------------
+# Session states
+# ---------------------------------------------------------------------------
+class SessionState(str, enum.Enum):
+    ACTIVE = "active"
+    PAUSED_BY_SERVER = "paused_by_server"
+    WAITING_RESUME = "waiting_resume"
+    COMMITTING = "committing"
+    COMMITTED = "committed"
+    ABORTED = "aborted"
+    EXPIRED = "expired"
+    FAILED = "failed"
+
+
+# ---------------------------------------------------------------------------
+# Node statuses (in DB)
+# ---------------------------------------------------------------------------
+class NodeStatus(str, enum.Enum):
+    OPEN = "open"
+    CLOSED = "closed"
+    REJECTED = "rejected"
+    PRUNED = "pruned"
+
+
+# ---------------------------------------------------------------------------
+# Binary frame decoder
+#
+# Wire format: [4-byte big-endian length][1-byte tag][payload]
+# The length covers tag + payload.
+# ---------------------------------------------------------------------------
+
+def _read_u8(data: memoryview, off: int) -> tuple[int, int]:
+    return data[off], off + 1
+
+
+def _read_u16(data: memoryview, off: int) -> tuple[int, int]:
+    return struct.unpack_from("!H", data, off)[0], off + 2
+
+
+def _read_u32(data: memoryview, off: int) -> tuple[int, int]:
+    return struct.unpack_from("!I", data, off)[0], off + 4
+
+
+def _read_u64(data: memoryview, off: int) -> tuple[int, int]:
+    return struct.unpack_from("!Q", data, off)[0], off + 8
+
+
+def _read_string(data: memoryview, off: int) -> tuple[str, int]:
+    slen, off = _read_u16(data, off)
+    s = bytes(data[off : off + slen]).decode("utf-8")
+    return s, off + slen
+
+
+def decode_frame_payload(tag: int, payload: memoryview) -> Frame:
+    """Decode the payload bytes (after the tag byte) into a Frame dataclass."""
+    off = 0
+
+    if tag == FrameTag.NODE:
+        node_id, off = _read_u32(payload, off)
+        parent_id, off = _read_u32(payload, off)
+        kind_val, off = _read_u8(payload, off)
+        name, off = _read_string(payload, off)
+        has_size, off = _read_u8(payload, off)
+        size_hint = None
+        if has_size:
+            size_hint, off = _read_u64(payload, off)
+        has_mtime, off = _read_u8(payload, off)
+        mtime_ms = None
+        if has_mtime:
+            mtime_ms, off = _read_u64(payload, off)
+        return NodeFrame(
+            node_id=node_id,
+            parent_id=parent_id,
+            kind=NodeKind(kind_val),
+            name=name,
+            size_hint=size_hint,
+            mtime_ms=mtime_ms,
+        )
+
+    elif tag == FrameTag.SUMMARY:
+        node_id, off = _read_u32(payload, off)
+        scan_done, off = _read_u64(payload, off)
+        scan_est, off = _read_u64(payload, off)
+        body_done, off = _read_u64(payload, off)
+        body_est, off = _read_u64(payload, off)
+        sealed_val, off = _read_u8(payload, off)
+        return SummaryFrame(
+            node_id=node_id,
+            scan_done_units=scan_done,
+            scan_est_units=scan_est,
+            body_done_bytes=body_done,
+            body_est_bytes=body_est,
+            sealed=bool(sealed_val),
+        )
+
+    elif tag == FrameTag.FILE_OPEN:
+        node_id, off = _read_u32(payload, off)
+        size, off = _read_u64(payload, off)
+        has_mtime, off = _read_u8(payload, off)
+        mtime_ms = None
+        if has_mtime:
+            mtime_ms, off = _read_u64(payload, off)
+        return FileOpenFrame(node_id=node_id, size=size, mtime_ms=mtime_ms)
+
+    elif tag == FrameTag.FILE_CHUNK:
+        node_id, off = _read_u32(payload, off)
+        offset_val, off = _read_u64(payload, off)
+        length, off = _read_u32(payload, off)
+        ck_kind, off = _read_u8(payload, off)
+        checksum, off = _read_u32(payload, off)
+        data = bytes(payload[off : off + length])
+        return FileChunkFrame(
+            node_id=node_id,
+            offset=offset_val,
+            length=length,
+            checksum_kind=ChecksumKind(ck_kind),
+            checksum=checksum,
+            payload=data,
+        )
+
+    elif tag == FrameTag.FILE_CLOSE:
+        node_id, off = _read_u32(payload, off)
+        size_sent, off = _read_u64(payload, off)
+        return FileCloseFrame(node_id=node_id, size_sent=size_sent)
+
+    elif tag == FrameTag.DIR_CLOSE:
+        node_id, off = _read_u32(payload, off)
+        return DirCloseFrame(node_id=node_id)
+
+    elif tag == FrameTag.SESSION_END:
+        scan_done, off = _read_u64(payload, off)
+        scan_est, off = _read_u64(payload, off)
+        body_done, off = _read_u64(payload, off)
+        body_est, off = _read_u64(payload, off)
+        sealed_val, off = _read_u8(payload, off)
+        return SessionEndFrame(
+            scan_done_units=scan_done,
+            scan_est_units=scan_est,
+            body_done_bytes=body_done,
+            body_est_bytes=body_est,
+            sealed=bool(sealed_val),
+        )
+
+    elif tag == FrameTag.CLIENT_ABORT:
+        code, off = _read_string(payload, off)
+        reason, off = _read_string(payload, off)
+        return ClientAbortFrame(code=code, reason=reason)
+
+    else:
+        raise ValueError(f"unknown frame tag: {tag:#04x}")
+
+
+class FrameReader:
+    """Incremental frame reader that buffers partial data from a streaming body.
+
+    Feed chunks via `feed(data)` and iterate decoded frames via `drain()`.
+
+    `max_frame_len` bounds the declared frame length: without it a corrupt or
+    malicious 4-byte prefix claiming a multi-gigabyte frame would grow the
+    buffer without limit while the reader waits for it to "complete".
+    """
+
+    # 1 MiB default: comfortably above MAX_CHUNK_BYTES (256 KiB) + headers
+    # and any NODE frame with a long UTF-8 name.
+    DEFAULT_MAX_FRAME_LEN = 1024 * 1024
+
+    def __init__(self, max_frame_len: int = DEFAULT_MAX_FRAME_LEN) -> None:
+        self._buf = bytearray()
+        self._max_frame_len = max_frame_len
+
+    def feed(self, data: bytes) -> None:
+        self._buf.extend(data)
+
+    def drain(self) -> list[Frame]:
+        frames: list[Frame] = []
+        pos = 0
+        total = len(self._buf)
+
+        while pos + 4 <= total:
+            frame_len = struct.unpack_from("!I", self._buf, pos)[0]
+            if frame_len > self._max_frame_len:
+                raise ValueError(
+                    f"frame length {frame_len} exceeds limit {self._max_frame_len}"
+                )
+            if pos + 4 + frame_len > total:
+                break  # incomplete frame
+            tag = self._buf[pos + 4]
+            # Copy payload to a bytes object so no memoryview holds the buffer
+            payload_bytes = bytes(self._buf[pos + 5 : pos + 4 + frame_len])
+            frames.append(decode_frame_payload(tag, memoryview(payload_bytes)))
+            pos += 4 + frame_len
+
+        if pos > 0:
+            del self._buf[:pos]
+
+        return frames
+
+
+# ---------------------------------------------------------------------------
+# CRC-32C (Castagnoli) for checksum verification
+#
+# The C implementation from the `crc32c` package (SSE4.2/ARMv8 accelerated,
+# ~GB/s) is a HARD dependency. A pure-Python fallback used to exist and was
+# removed deliberately: it measured ~6 MB/s and blocked the event loop
+# ~10 ms per 64 KiB chunk, silently throttling the whole server. Failing
+# loudly at import beats degrading quietly in production.
+# ---------------------------------------------------------------------------
+
+try:
+    import crc32c as _crc32c_native  # type: ignore
+except ImportError as exc:  # pragma: no cover
+    raise ImportError(
+        "MFUP/2 requires the C-accelerated 'crc32c' package: pip install crc32c"
+    ) from exc
+
+
+def crc32c(data: bytes | memoryview, initial: int = 0) -> int:
+    return _crc32c_native.crc32c(data, initial)
+
+
+CRC32C_IMPL = "native"