mfup-fastapi 0.2.0__tar.gz → 3.1.0__tar.gz

mfup_fastapi-3.1.0/PKG-INFO

@@ -0,0 +1,62 @@
+Metadata-Version: 2.5
+Name: mfup-fastapi
+Version: 3.1.0
+Summary: MFUP/3 FastAPI adapter with streaming multipart and WebSocket questions
+Project-URL: Homepage, https://github.com/n0isy/mfup
+Project-URL: Repository, https://github.com/n0isy/mfup
+Project-URL: Issues, https://github.com/n0isy/mfup/issues
+Author: n0isy
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: anyio>=4
+Requires-Dist: fastapi>=0.115
+Requires-Dist: mfup-core==3.1.0
+Requires-Dist: python-multipart>=0.0.20
+Requires-Dist: uvicorn[standard]>=0.30
+Provides-Extra: dev
+Requires-Dist: httpx>=0.28; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# mfup-fastapi
+
+```python
+from mfup_fastapi import MfupConfig, create_app
+
+app = create_app(
+    MfupConfig(
+        base_dir="./state",
+        authorize=my_authorize,
+        prefix="/api",
+        client_publish=False,
+        on_committed=my_processor,
+    )
+)
+```
+
+The application supplies identity, scopes and authorize. It may return an
+absolute per-session baseDir, targetDir, quotas and context. The core remains
+independent of application scope names. Ordinary multipart carries files;
+WebSocket carries questions and snapshots.
+
+For embedding, `MfupEngine(config).router` and `.lifespan` (or startup/shutdown)
+manage routing and cleanup. Use one lifecycle per Engine and one process per
+metadata directory. `create_app` applies config.prefix and owns the lifecycle.
+
+map_file runs during manifest reception, before body reads and returns a relative path or None.
+on_committed receives CommitEvent and can return a boolean to override server
+auto_publish (default False). client_publish=False reserves publication for
+the backend. Engine exposes get_session, list_staged, open_staged,
+prepare_publish, publish and retry_committed. Callback failure preserves the
+accepted files and returns a successful commit with processing='failed'.
+
+Standalone: `MFUP_AUTHORIZE=myapp.hooks:authorize python -m mfup_fastapi`.
+MfupConfig.from_env supports the same MFUP_* configuration as the Node package;
+Python hooks can be callables or module:attribute strings.
+See docs/EXTENDING.md and the multiuser-scopes example in the repository.
+
+[Extension API](https://github.com/n0isy/mfup/blob/main/docs/EXTENDING.md) · [HTTP protocol](https://github.com/n0isy/mfup/blob/main/docs/PROTOCOL.md)
+
+[Russian](README_ru.md)

mfup_fastapi-3.1.0/README.md

@@ -0,0 +1,40 @@
+# mfup-fastapi
+
+```python
+from mfup_fastapi import MfupConfig, create_app
+
+app = create_app(
+    MfupConfig(
+        base_dir="./state",
+        authorize=my_authorize,
+        prefix="/api",
+        client_publish=False,
+        on_committed=my_processor,
+    )
+)
+```
+
+The application supplies identity, scopes and authorize. It may return an
+absolute per-session baseDir, targetDir, quotas and context. The core remains
+independent of application scope names. Ordinary multipart carries files;
+WebSocket carries questions and snapshots.
+
+For embedding, `MfupEngine(config).router` and `.lifespan` (or startup/shutdown)
+manage routing and cleanup. Use one lifecycle per Engine and one process per
+metadata directory. `create_app` applies config.prefix and owns the lifecycle.
+
+map_file runs during manifest reception, before body reads and returns a relative path or None.
+on_committed receives CommitEvent and can return a boolean to override server
+auto_publish (default False). client_publish=False reserves publication for
+the backend. Engine exposes get_session, list_staged, open_staged,
+prepare_publish, publish and retry_committed. Callback failure preserves the
+accepted files and returns a successful commit with processing='failed'.
+
+Standalone: `MFUP_AUTHORIZE=myapp.hooks:authorize python -m mfup_fastapi`.
+MfupConfig.from_env supports the same MFUP_* configuration as the Node package;
+Python hooks can be callables or module:attribute strings.
+See docs/EXTENDING.md and the multiuser-scopes example in the repository.
+
+[Extension API](https://github.com/n0isy/mfup/blob/main/docs/EXTENDING.md) · [HTTP protocol](https://github.com/n0isy/mfup/blob/main/docs/PROTOCOL.md)
+
+[Russian](README_ru.md)

mfup_fastapi-3.1.0/README_ru.md

@@ -0,0 +1,27 @@
+# mfup-fastapi
+
+```python
+from mfup_fastapi import MfupConfig, create_app
+
+app = create_app(
+    MfupConfig(
+        base_dir="./state",
+        authorize=my_authorize,
+        prefix="/api",
+        client_publish=False,
+        on_committed=my_processor,
+    )
+)
+```
+
+Приложение предоставляет my_authorize/my_processor, личность и правила доступа. Авторизация может вернуть абсолютный корень сессии, targetDir, квоты и context. Прикладные значения вроде scope передаются через meta. Multipart переносит файлы, WebSocket — вопросы и снимки.
+
+Для встраивания MfupEngine(config).router и lifespan (либо startup/shutdown) управляют маршрутами и очисткой. Один lifecycle на Engine и один процесс на каталог метаданных. Create_app применяет config.prefix и управляет lifecycle.
+
+Map_file вызывается при получении манифеста до чтения тела и возвращает относительный путь либо None. On_committed получает CommitEvent и возвращает boolean для server auto_publish (default False). Client_publish=False оставляет публикацию backend. Engine предоставляет get_session, list_staged, open_staged, prepare_publish, publish, retry_committed. Ошибка callback сохраняет принятые файлы и успешный commit с processing=failed.
+
+Standalone: MFUP_AUTHORIZE=myapp.hooks:authorize python -m mfup_fastapi. MfupConfig.from_env использует те же MFUP_* настройки, что Node; Python hooks — callable или строки module:attribute.
+
+[API расширения](https://github.com/n0isy/mfup/blob/main/docs/ru/EXTENDING.md) · [Протокол](https://github.com/n0isy/mfup/blob/main/docs/ru/PROTOCOL.md)
+
+[English](README.md)

mfup_fastapi-3.1.0/mfup_fastapi/__init__.py

@@ -0,0 +1,4 @@
+from .app import MfupEngine, create_app
+from .config import MfupConfig
+
+__all__ = ["MfupEngine", "MfupConfig", "create_app"]

mfup_fastapi-3.1.0/mfup_fastapi/__main__.py

@@ -0,0 +1,18 @@
+import os
+
+import uvicorn
+
+from . import MfupConfig, create_app
+
+
+def main():
+    app = create_app(MfupConfig.from_env())
+    uvicorn.run(
+        app,
+        host=os.environ.get("MFUP_HOST", "127.0.0.1"),
+        port=int(os.environ.get("MFUP_PORT", "3001")),
+    )
+
+
+if __name__ == "__main__":
+    main()

mfup_fastapi-3.1.0/mfup_fastapi/app.py

@@ -0,0 +1,341 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+from contextlib import asynccontextmanager
+
+from anyio import CancelScope
+from fastapi import APIRouter, FastAPI, Request, WebSocket
+from fastapi.responses import JSONResponse
+from mfup_core import Engine, ProtocolError
+from mfup_core.engine import check, integer, normalize_error
+from python_multipart.multipart import MultipartParser, parse_options_header
+from starlette.requests import ClientDisconnect
+
+from .config import MfupConfig
+
+
+async def json_body(request):
+    chunks = bytearray()
+    async for chunk in request.stream():
+        chunks.extend(chunk)
+        check(len(chunks) <= 262144, "body_too_large", 413)
+    try:
+        data = json.loads(chunks or b"{}")
+        check(isinstance(data, dict), "bad_json")
+        return data
+    except (ValueError, TypeError):
+        raise ProtocolError(400, "bad_json")
+
+
+class MfupEngine:
+    def __init__(self, base_dir, authorize=None, **options):
+        config = (
+            base_dir
+            if isinstance(base_dir, MfupConfig)
+            else MfupConfig(base_dir=base_dir, authorize=authorize, **options)
+        )
+        self.config = config
+        self.engine = Engine(**config.engine_options())
+        self._sweeper = None
+        self.router = APIRouter()
+        engine = self.engine
+
+        @self.router.get("/mfup/health")
+        async def health():
+            return dict(protocol="MFUP/3", backend="python")
+
+        @self.router.post("/mfup/sessions")
+        async def create(request: Request):
+            try:
+                return JSONResponse(
+                    await engine.create(
+                        await json_body(request),
+                        dict(request.headers),
+                        {
+                            "client": f"{request.client.host}:{request.client.port}"
+                            if request.client
+                            else "",
+                            "query": dict(request.query_params),
+                        },
+                    ),
+                    status_code=201,
+                )
+            except Exception as exc:
+                return self.error(engine.record_failure("", exc, "create"))
+
+        @self.router.api_route("/mfup/sessions/{sid}", methods=["GET"])
+        @self.router.api_route("/mfup/sessions/{sid}/{action}", methods=["GET", "POST"])
+        @self.router.api_route("/mfup/sessions/{sid}/{action}/{bid}", methods=["GET", "POST"])
+        async def route(request: Request, sid: str, action: str = "", bid: str = ""):
+            authenticated = False
+            phase = "upload" if action == "batches" else (action or "status")
+            try:
+                engine.authenticate(
+                    sid, request.headers.get("authorization", "").removeprefix("Bearer ")
+                )
+                authenticated = True
+                if request.method == "GET":
+                    if not action:
+                        return engine.snapshot(sid)
+                    if action == "published":
+                        return engine.published_page(sid, request.query_params.get("after", ""))
+                    if action == "files":
+                        raw_limit = request.query_params.get("limit", "256")
+                        check(raw_limit.isdecimal(), "bad_number")
+                        return engine.resume_page(
+                            sid, request.query_params.get("after", ""), int(raw_limit)
+                        )
+                    if action == "batches" and bid:
+                        result = engine.receipt(sid, bid)
+                        check(result is not None, "not_found", 404)
+                        return result
+                if request.method == "POST":
+                    if action == "batches" and bid:
+                        return await self.receive(request, sid, bid)
+                    data = await json_body(request)
+                    if action == "status":
+                        return engine.range_status(sid, data)
+                    if action == "resume":
+                        return await engine.resume(sid)
+                    if action == "commit":
+                        return await engine.commit(sid, data)
+                    if action == "publish":
+                        return await engine.publish_from_client(sid)
+                    if action == "cancel":
+                        return await engine.cancel(sid)
+                    if action == "properties":
+                        return await engine.set_properties(sid, data)
+                    if action == "answers":
+                        return await engine.answer(sid, data.get("id"), data.get("choice"))
+                raise ProtocolError(404, "not_found")
+            except Exception as exc:
+                return self.error(
+                    engine.record_failure(sid, exc, phase)
+                    if authenticated
+                    else normalize_error(exc, phase)
+                )
+
+        @self.router.websocket("/mfup/control")
+        async def control(ws: WebSocket):
+            await ws.accept()
+            unsubscribe = None
+            tasks = []
+            queue = asyncio.Queue(maxsize=1)
+
+            def update(state):
+                if queue.full():
+                    queue.get_nowait()
+                queue.put_nowait(state)
+
+            try:
+                message = await asyncio.wait_for(ws.receive_json(), 5)
+                check(message.get("type") == "subscribe", "bad_message")
+                engine.authenticate(message.get("id"), message.get("token"))
+                unsubscribe = engine.subscribe(message["id"], update)
+
+                async def send():
+                    while True:
+                        await ws.send_json(await queue.get())
+
+                async def receive():
+                    while True:
+                        message = await ws.receive()
+                        if message["type"] == "websocket.disconnect":
+                            return
+
+                tasks = [asyncio.create_task(send()), asyncio.create_task(receive())]
+                await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+            except Exception:
+                with contextlib.suppress(Exception):
+                    await ws.close(code=1008)
+            finally:
+                if unsubscribe:
+                    unsubscribe()
+                for task in tasks:
+                    task.cancel()
+                with CancelScope(shield=True):
+                    await asyncio.gather(*tasks, return_exceptions=True)
+
+    async def startup(self):
+        if self._sweeper is not None or not self.config.sweep_interval_ms:
+            return
+
+        async def sweep():
+            while True:
+                await asyncio.sleep(self.config.sweep_interval_ms / 1000)
+                try:
+                    await self.engine.sweep()
+                except Exception as exc:
+                    self.engine.report("sweep", "", exc)
+
+        self._sweeper = asyncio.create_task(sweep())
+
+    async def shutdown(self):
+        if self._sweeper:
+            self._sweeper.cancel()
+            await asyncio.gather(self._sweeper, return_exceptions=True)
+            self._sweeper = None
+        await self.engine.close()
+
+    @asynccontextmanager
+    async def lifespan(self, app):
+        await self.startup()
+        try:
+            yield
+        finally:
+            await self.shutdown()
+
+    @staticmethod
+    def error(exc):
+        return JSONResponse(
+            dict(error=exc.code, message=str(exc), retryable=exc.retryable, phase=exc.phase),
+            status_code=exc.status,
+        )
+
+    async def receive(self, request, sid, bid):
+        epoch = request.headers.get("x-mfup-epoch", "")
+        check(epoch.isdecimal(), "bad_epoch")
+        task = asyncio.current_task()
+        batch = await self.engine.begin(sid, integer(int(epoch)), bid, task.cancel)
+        output = None
+        failed = False
+        try:
+            content_type, params = parse_options_header(request.headers.get("content-type", ""))
+            check(
+                content_type == b"multipart/form-data"
+                and 0 < len(params.get(b"boundary", b"")) <= 200,
+                "bad_multipart",
+            )
+            events = []
+            callbacks = {}
+            for name in ("part_begin", "header_end", "headers_finished", "part_end", "end"):
+                callbacks["on_" + name] = lambda name=name: events.append((name,))
+            for name in ("header_field", "header_value", "part_data"):
+                callbacks["on_" + name] = lambda data, start, end, name=name: events.append(
+                    (name, data, start, end)
+                )
+            parser = MultipartParser(params[b"boundary"], callbacks)
+            manifest_seen = ended = False
+            cached = None
+            count = total = length = header_size = 0
+            mode = ""
+            headers = {}
+            field, value, manifest = bytearray(), bytearray(), bytearray()
+            expected = 0
+            stream = request.stream().__aiter__()
+            while True:
+                try:
+                    chunk = await asyncio.wait_for(stream.__anext__(), 30)
+                except StopAsyncIteration:
+                    break
+                except asyncio.TimeoutError:
+                    raise ProtocolError(408, "request_timeout") from None
+                total += len(chunk)
+                check(
+                    total <= self.engine.limits["batchBytes"] + 1024 * 1024, "body_too_large", 413
+                )
+                parser.write(chunk)
+                for event in events:
+                    name = event[0]
+                    if name == "part_begin":
+                        headers = {}
+                        field, value = bytearray(), bytearray()
+                        header_size = 0
+                    elif name in ("header_field", "header_value"):
+                        data = memoryview(event[1])[event[2] : event[3]]
+                        header_size += len(data)
+                        check(header_size <= 8192, "headers_too_large", 413)
+                        (field if name == "header_field" else value).extend(data)
+                    elif name == "header_end":
+                        headers[bytes(field).lower()] = bytes(value)
+                        check(len(headers) <= 32, "headers_too_large", 413)
+                        field, value = bytearray(), bytearray()
+                    elif name == "headers_finished":
+                        _, options = parse_options_header(headers.get(b"content-disposition", b""))
+                        part_name = options.get(b"name", b"")
+                        if part_name == b"manifest" and b"filename" not in options:
+                            check(not manifest_seen and count == 0, "bad_manifest")
+                            manifest_seen = True
+                            mode = "manifest"
+                        else:
+                            check(
+                                manifest_seen and "manifest" in batch or cached is not None,
+                                "bad_manifest",
+                            )
+                            check(
+                                part_name == str(count).encode() and b"filename" in options,
+                                "bad_part",
+                            )
+                            check(count < self.engine.limits["maxParts"], "too_many_parts", 413)
+                            mode, length = "file", 0
+                            if cached is None:
+                                check(count < len(batch["entries"]), "unexpected_part")
+                                entry = batch["entries"][count]
+                                expected = entry["item"][4]
+                                if not entry["skip"]:
+                                    output = open(entry["filePath"], "r+b")
+                                    output.seek(entry["item"][3])
+                            count += 1
+                    elif name == "part_data":
+                        data = memoryview(event[1])[event[2] : event[3]]
+                        if mode == "manifest":
+                            manifest.extend(data)
+                            check(len(manifest) <= 262144, "bad_manifest")
+                        else:
+                            length += len(data)
+                            if cached is None:
+                                check(length <= expected, "part_size_mismatch")
+                                if output:
+                                    output.write(data)
+                    elif name == "part_end":
+                        if mode == "manifest":
+                            try:
+                                raw = json.loads(manifest)
+                            except ValueError:
+                                raise ProtocolError(400, "bad_manifest")
+                            cached = await self.engine.prepare(batch, raw)
+                        else:
+                            if output:
+                                output.close()
+                                output = None
+                            if cached is None:
+                                check(length == expected, "part_size_mismatch")
+                    elif name == "end":
+                        ended = True
+                events.clear()
+            parser.finalize()
+            check(ended and manifest_seen, "incomplete_multipart")
+            if cached is not None:
+                return cached
+            check(count == len(batch["manifest"]["files"]), "missing_part")
+            return await self.engine.complete(batch)
+        except (
+            ClientDisconnect,
+            asyncio.CancelledError,
+            ValueError,
+            KeyError,
+        ):
+            failed = True
+            raise ProtocolError(400, "incomplete_multipart")
+        except BaseException:
+            failed = True
+            raise
+        finally:
+            try:
+                if output:
+                    output.close()
+            except OSError:
+                if not failed:
+                    raise
+            finally:
+                self.engine.end(batch)
+
+
+def create_app(base_dir, authorize=None, **options):
+    mfup = MfupEngine(base_dir, authorize, **options)
+    app = FastAPI(lifespan=mfup.lifespan)
+    app.include_router(mfup.router, prefix=mfup.config.prefix)
+    app.state.mfup = mfup
+    return app

mfup_fastapi-3.1.0/mfup_fastapi/config.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import importlib
+import os
+import re
+from dataclasses import dataclass, fields
+from pathlib import Path
+from typing import Any
+
+from mfup_core.contracts import Authorize, MapFile, OnCommitted
+
+
+def resolve_hook(value):
+    if value is None or callable(value):
+        return value
+    module, name = value.rsplit(":", 1)
+    hook = getattr(importlib.import_module(module), name)
+    if not callable(hook):
+        raise ValueError(f"Hook is not callable: {value}")
+    return hook
+
+
+@dataclass
+class MfupConfig:
+    base_dir: str | Path
+    authorize: Authorize | str
+    map_file: MapFile | str | None = None
+    on_committed: OnCommitted | str | None = None
+    limits: dict[str, int] | None = None
+    ttl_ms: int = 86400000
+    sweep_interval_ms: int = 60000
+    max_meta_bytes: int = 16384
+    max_context_bytes: int = 65536
+    auto_publish: bool = False
+    client_publish: bool = True
+    prefix: str = ""
+    on_error: Any = None
+
+    def engine_options(self):
+        self.prefix = self.prefix.rstrip("/")
+        if self.prefix and not re.fullmatch(r"/(?:[A-Za-z0-9_-]+/?)+", self.prefix):
+            raise ValueError("Invalid prefix")
+        result = {
+            f.name: getattr(self, f.name)
+            for f in fields(self)
+            if f.name not in ("prefix", "sweep_interval_ms")
+        }
+        for name in ("authorize", "map_file", "on_committed"):
+            result[name] = resolve_hook(result[name])
+        if not callable(result["authorize"]):
+            raise ValueError("MFUP_AUTHORIZE is required")
+        if type(self.sweep_interval_ms) is not int or self.sweep_interval_ms < 0:
+            raise ValueError("Invalid sweep_interval_ms")
+        return result
+
+    @classmethod
+    def from_env(cls, env=None):
+        env = os.environ if env is None else env
+        values = dict(
+            base_dir=env.get("MFUP_BASE_DIR", "./data"), authorize=env.get("MFUP_AUTHORIZE", "")
+        )
+        for name in ("map_file", "on_committed", "prefix"):
+            if "MFUP_" + name.upper() in env:
+                values[name] = env["MFUP_" + name.upper()]
+        for name in ("ttl_ms", "sweep_interval_ms", "max_meta_bytes", "max_context_bytes"):
+            if "MFUP_" + name.upper() in env:
+                values[name] = int(env["MFUP_" + name.upper()])
+        for name in ("auto_publish", "client_publish"):
+            raw = env.get("MFUP_" + name.upper())
+            if raw is not None:
+                if raw.lower() not in ("true", "false", "1", "0"):
+                    raise ValueError(f"Invalid boolean: {name}")
+                values[name] = raw.lower() in ("true", "1")
+        limits = {}
+        for field, variable in dict(
+            concurrency="CONCURRENCY",
+            maxParts="MAX_PARTS",
+            batchBytes="BATCH_BYTES",
+            partBytes="PART_BYTES",
+        ).items():
+            if "MFUP_" + variable in env:
+                limits[field] = int(env["MFUP_" + variable])
+        if limits:
+            values["limits"] = limits
+        return cls(**values)

mfup_fastapi-3.1.0/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+[project]
+name = "mfup-fastapi"
+version = "3.1.0"
+description = "MFUP/3 FastAPI adapter with streaming multipart and WebSocket questions"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "n0isy" }]
+requires-python = ">=3.10"
+dependencies = ["mfup-core==3.1.0", "fastapi>=0.115", "anyio>=4", "uvicorn[standard]>=0.30", "python-multipart>=0.0.20"]
+[project.scripts]
+mfup-server = "mfup_fastapi.__main__:main"
+
+[project.optional-dependencies]
+dev = ["pytest>=8", "pytest-asyncio>=0.24", "httpx>=0.28"]
+[tool.hatch.build.targets.wheel]
+packages = ["mfup_fastapi"]
+
+[project.urls]
+Homepage = "https://github.com/n0isy/mfup"
+Repository = "https://github.com/n0isy/mfup"
+Issues = "https://github.com/n0isy/mfup/issues"
+
+[tool.hatch.build.targets.sdist]
+include = ["/mfup_fastapi", "/pyproject.toml", "/README.md", "/README_ru.md", "/LICENSE"]
+
+[tool.hatch.build]
+ignore-vcs = true
+exclude = ["**/__pycache__/**", "**/*.py[cod]"]

mfup_fastapi-0.2.0/.gitignore

@@ -1,14 +0,0 @@
-node_modules/
-dist/
-*.egg-info/
-__pycache__/
-.vite/
-uploads/
-repomix-output.md
-*.url
-repomix.sh
-# e2e artifacts
-e2e/.tmp/
-e2e/test-results/
-e2e/playwright-report/
-e2e/webtop/testdata/