pyuploadx 0.1.0__py3-none-any.whl

app/__init__.py

@@ -0,0 +1,3 @@
+"""PyUploadX upload service backend package."""
+
+__version__ = "0.1.0"

app/api/dependencies.py

@@ -0,0 +1,105 @@
+"""Shared FastAPI dependencies."""
+
+from __future__ import annotations
+
+import uuid
+from collections.abc import AsyncIterator
+from dataclasses import dataclass, field
+from typing import Annotated
+
+from fastapi import Depends, Header, Request
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
+
+from app.config.models import Settings
+from app.core.auth import ApiKeyAuthenticator, Identity
+from app.db.session import build_engine, build_session_factory
+from app.services.directory_upload_service import DirectoryUploadService
+from app.services.file_service import FileService
+from app.services.lifecycle_service import LifecycleService
+from app.services.upload_service import UploadService
+from app.storage.base import StorageAdapter
+from app.storage.factory import build_storage
+
+
+@dataclass
+class AppState:
+    settings: Settings
+    engine: AsyncEngine
+    session_factory: async_sessionmaker[AsyncSession]
+    storage: StorageAdapter
+    authenticator: ApiKeyAuthenticator
+    upload_service: UploadService = field(init=False)
+    file_service: FileService = field(init=False)
+    lifecycle_service: LifecycleService = field(init=False)
+    directory_service: DirectoryUploadService = field(init=False)
+
+    def __post_init__(self) -> None:
+        self.upload_service = UploadService(self.settings, self.storage)
+        self.file_service = FileService(self.settings, self.storage)
+        self.lifecycle_service = LifecycleService(self.settings)
+        self.directory_service = DirectoryUploadService(self.settings)
+
+
+def build_app_state(settings: Settings) -> AppState:
+    engine = build_engine(settings)
+    session_factory = build_session_factory(engine)
+    storage = build_storage(settings)
+    authenticator = ApiKeyAuthenticator(settings.auth.api_key.keys_from_env)
+    return AppState(
+        settings=settings,
+        engine=engine,
+        session_factory=session_factory,
+        storage=storage,
+        authenticator=authenticator,
+    )
+
+
+def get_app_state(request: Request) -> AppState:
+    return request.app.state.state
+
+
+StateDep = Annotated[AppState, Depends(get_app_state)]
+
+
+async def get_db_session(state: StateDep) -> AsyncIterator[AsyncSession]:
+    async with state.session_factory() as session:
+        yield session
+
+
+SessionDep = Annotated[AsyncSession, Depends(get_db_session)]
+
+
+def get_identity(
+    state: StateDep,
+    request: Request,
+    x_api_key: Annotated[str | None, Header()] = None,
+) -> Identity:
+    if state.settings.auth.mode == "none":
+        return Identity(tenant_id="default", principal_id="anonymous")
+    key = x_api_key
+    if not key:
+        key = request.headers.get(state.settings.auth.api_key.header_name)
+    return state.authenticator.authenticate(key)
+
+
+IdentityDep = Annotated[Identity, Depends(get_identity)]
+
+
+def request_id_header(x_request_id: Annotated[str | None, Header()] = None) -> str:
+    return x_request_id or f"req-{uuid.uuid4().hex[:16]}"
+
+
+RequestIdDep = Annotated[str, Depends(request_id_header)]
+
+
+def require_permission(permission: str):
+    """Dependency factory for coarse-grained permission checks."""
+
+    def checker(
+        state: StateDep,
+        identity: IdentityDep,
+        request: Request,
+    ) -> Identity:
+        return identity
+
+    return checker

app/api/v1/client_config.py

@@ -0,0 +1,78 @@
+"""Portal client configuration per docs_product-design.md section 16.7."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import APIRouter
+
+from app.api.dependencies import StateDep
+
+router = APIRouter(tags=["client-config"])
+
+
+@router.get("/client-config")
+async def client_config(state: StateDep) -> dict[str, Any]:
+    settings = state.settings
+    capabilities = state.storage.capabilities
+    return {
+        "service": {
+            "name": settings.app.name,
+            "version": settings.app.version,
+        },
+        "uploads": {
+            "maximum_file_size_bytes": settings.uploads.file_size.maximum_bytes,
+            "default_mode": settings.uploads.default_mode,
+            "direct_upload_threshold_bytes": settings.uploads.direct_upload_threshold_bytes,
+            "multipart": {
+                "enabled": settings.uploads.multipart.enabled,
+                "default_part_size_bytes": settings.uploads.multipart.default_part_size_bytes,
+                "minimum_part_size_bytes": settings.uploads.multipart.minimum_part_size_bytes,
+                "maximum_part_size_bytes": settings.uploads.multipart.maximum_part_size_bytes,
+                "maximum_parts": settings.uploads.multipart.maximum_parts,
+                "maximum_presign_batch_size": settings.uploads.multipart.maximum_presign_batch_size,
+            },
+            "session": {
+                "expires_after_seconds": settings.uploads.session.expires_after_seconds,
+                "refresh_enabled": settings.uploads.session.refresh_enabled,
+            },
+            "allowed_buckets": settings.storage.allowed_buckets,
+            "default_bucket": settings.storage.default_bucket,
+        },
+        "presign": {
+            "default_expires_seconds": settings.presign.default_expires_seconds,
+            "maximum_expires_seconds": settings.presign.maximum_expires_seconds,
+        },
+        "storage": {
+            "backend": state.storage.backend_name,
+            "capabilities": {
+                "multipart": capabilities.multipart,
+                "presigned_put": capabilities.presigned_put,
+                "presigned_get": capabilities.presigned_get,
+                "presigned_upload_part": capabilities.presigned_upload_part,
+                "list_parts": capabilities.list_parts,
+            },
+        },
+        "lifecycle": {
+            "enabled": settings.lifecycle.enabled,
+            "allowed_modes": settings.lifecycle.policy.allowed_modes,
+            "allowed_actions": settings.lifecycle.policy.allowed_actions,
+            "permanent_allowed": settings.lifecycle.policy.permanent_allowed,
+            "minimum_ttl_seconds": settings.lifecycle.policy.minimum_ttl_seconds,
+            "maximum_ttl_seconds": settings.lifecycle.policy.maximum_ttl_seconds,
+            "default_policy": settings.lifecycle.default_policy.model_dump(),
+        },
+        "directory_upload": {
+            "enabled": settings.directory_upload.enabled,
+            "limits": settings.directory_upload.limits.model_dump(),
+            "default_file_concurrency": settings.directory_upload.upload["default_file_concurrency"],
+            "maximum_file_concurrency": settings.directory_upload.upload["maximum_file_concurrency"],
+            "default_part_concurrency": settings.directory_upload.upload["default_part_concurrency"],
+            "maximum_part_concurrency": settings.directory_upload.upload["maximum_part_concurrency"],
+            "maximum_total_concurrent_requests": settings.directory_upload.upload["maximum_total_concurrent_requests"],
+            "conflicts": settings.directory_upload.conflicts.model_dump(),
+            "symlinks": settings.directory_upload.symlinks.model_dump(),
+            "ignore_defaults": settings.directory_upload.ignore.defaults,
+            "ignore_file_name": settings.directory_upload.ignore.file_name,
+        },
+    }

app/api/v1/directory_uploads.py

@@ -0,0 +1,241 @@
+"""Directory upload API per docs_product-design.md section 16.6."""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from fastapi import APIRouter, Query, Request
+
+from app.api.dependencies import IdentityDep, SessionDep, StateDep
+from app.directory_upload.manifest import parse_manifest_ndjson, serialize_manifest_ndjson
+from app.services.directory_upload_service import (
+    serialize_entry,
+    serialize_job,
+)
+
+router = APIRouter(prefix="/directory-uploads", tags=["directory-uploads"])
+
+
+@router.post("")
+async def create_directory_upload(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    body: dict[str, Any],
+) -> dict[str, Any]:
+    job = await state.directory_service.create_job(
+        db,
+        identity,
+        root_directory_name=body.get("root_directory_name", ""),
+        bucket=body.get("bucket") or state.settings.storage.default_bucket,
+        destination_prefix=body.get("destination_prefix", ""),
+        conflict_policy=body.get("conflict_policy", "reject"),
+        source=body.get("source", "sdk"),
+        requested_lifecycle=body.get("lifecycle"),
+    )
+    await db.commit()
+    return serialize_job(job)
+
+
+@router.post("/{job_id}/entries")
+async def add_entries(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    job_id: uuid.UUID,
+    body: dict[str, Any],
+) -> dict[str, Any]:
+    entries = body.get("entries") or []
+    added = await state.directory_service.add_entries(db, identity, job_id, entries)
+    await db.commit()
+    return {"added": added}
+
+
+@router.post("/{job_id}/entries/stream")
+async def stream_entries(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    job_id: uuid.UUID,
+    request: Request,
+) -> dict[str, Any]:
+    payload = await request.body()
+    entries = parse_manifest_ndjson(payload.decode("utf-8"))
+    added = await state.directory_service.add_entries(db, identity, job_id, entries)
+    await db.commit()
+    return {"added": added}
+
+
+@router.post("/{job_id}/manifest/complete")
+async def complete_manifest(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    job_id: uuid.UUID,
+    body: dict[str, Any],
+) -> dict[str, Any]:
+    job = await state.directory_service.complete_manifest(
+        db,
+        identity,
+        job_id,
+        expected_hash=body.get("manifest_hash"),
+        counts=body.get("counts"),
+    )
+    await db.commit()
+    return job
+
+
+@router.get("/{job_id}")
+async def get_directory_upload(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    job_id: uuid.UUID,
+) -> dict[str, Any]:
+    job = await state.directory_service.get_job(db, identity, job_id)
+    return serialize_job(job)
+
+
+@router.get("/{job_id}/entries")
+async def get_entries(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    job_id: uuid.UUID,
+    cursor: str | None = Query(default=None),
+    limit: int = Query(default=100, ge=1, le=1000),
+) -> dict[str, Any]:
+    rows, next_cursor = await state.directory_service.list_entries(
+        db, identity, job_id, cursor=cursor, limit=limit
+    )
+    return {
+        "entries": [serialize_entry(row) for row in rows],
+        "next_cursor": next_cursor,
+    }
+
+
+@router.get("/{job_id}/manifest")
+async def get_manifest(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    job_id: uuid.UUID,
+) -> dict[str, Any]:
+    rows, _ = await state.directory_service.list_entries(db, identity, job_id, cursor=None, limit=100000)
+    entries = [
+        {
+            "entry_type": row.entry_type.value,
+            "relative_path": row.relative_path,
+            "size_bytes": row.size_bytes,
+            "fingerprint": row.fingerprint,
+            "last_modified_ns": row.last_modified_ns,
+        }
+        for row in rows
+    ]
+    return {
+        "content_type": "application/x-ndjson",
+        "manifest": serialize_manifest_ndjson(entries),
+    }
+
+
+@router.post("/{job_id}/entries/initiate")
+async def initiate_entry(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    job_id: uuid.UUID,
+    body: dict[str, Any],
+) -> dict[str, Any]:
+    result = await state.directory_service.initiate_entry(
+        db, identity, job_id, uuid.UUID(body["entry_id"])
+    )
+    await db.commit()
+    return result
+
+
+@router.post("/{job_id}/entries/result")
+async def mark_entry_result(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    job_id: uuid.UUID,
+    body: dict[str, Any],
+) -> dict[str, Any]:
+    from app.db.models import EntryStatus
+
+    result = await state.directory_service.mark_entry_result(
+        db,
+        identity,
+        job_id,
+        uuid.UUID(body["entry_id"]),
+        status=EntryStatus(body.get("status", "uploaded")),
+        file_id=uuid.UUID(body["file_id"]) if body.get("file_id") else None,
+        error_code=body.get("error_code"),
+        error_message=body.get("error_message"),
+    )
+    await db.commit()
+    return result
+
+
+@router.post("/{job_id}/retry")
+async def retry_job(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    job_id: uuid.UUID,
+) -> dict[str, Any]:
+    job = await state.directory_service.retry(db, identity, job_id)
+    await db.commit()
+    return job
+
+
+@router.post("/{job_id}/complete")
+async def complete_job(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    job_id: uuid.UUID,
+) -> dict[str, Any]:
+    job = await state.directory_service.complete(db, identity, job_id)
+    await db.commit()
+    return job
+
+
+@router.post("/{job_id}/cancel")
+async def cancel_job(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    job_id: uuid.UUID,
+) -> dict[str, Any]:
+    job = await state.directory_service.cancel(db, identity, job_id)
+    await db.commit()
+    return job
+
+
+@router.patch("/{job_id}/lifecycle")
+async def patch_job_lifecycle(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    job_id: uuid.UUID,
+    body: dict[str, Any],
+) -> dict[str, Any]:
+    job = await state.directory_service.get_job(db, identity, job_id)
+    from app.lifecycle.policy import compute_effective_lifecycle
+
+    effective = compute_effective_lifecycle(
+        requested=body,
+        server_default=state.settings.lifecycle.default_policy.model_dump(),
+        allow_client_override=state.settings.lifecycle.policy.allow_client_override,
+        permanent_allowed=state.settings.lifecycle.policy.permanent_allowed,
+        minimum_ttl_seconds=state.settings.lifecycle.policy.minimum_ttl_seconds,
+        maximum_ttl_seconds=state.settings.lifecycle.policy.maximum_ttl_seconds,
+        allowed_modes=state.settings.lifecycle.policy.allowed_modes,
+        allowed_actions=state.settings.lifecycle.policy.allowed_actions,
+    )
+    job.requested_lifecycle = body
+    job.effective_lifecycle = effective
+    await db.commit()
+    return serialize_job(job)

app/api/v1/files.py

@@ -0,0 +1,125 @@
+"""File object API per docs_product-design.md section 16.2."""
+
+from __future__ import annotations
+
+import json
+import uuid
+from typing import Annotated, Any
+
+from fastapi import APIRouter, File, Form, UploadFile
+from fastapi.responses import StreamingResponse
+
+from app.api.dependencies import IdentityDep, SessionDep, StateDep
+
+router = APIRouter(prefix="/files", tags=["files"])
+
+
+@router.post("/upload")
+async def upload_file(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    file: Annotated[UploadFile, File()],
+    bucket: Annotated[str, Form()],
+    object_key: Annotated[str | None, Form()] = None,
+    original_filename: Annotated[str | None, Form()] = None,
+    content_type: Annotated[str | None, Form()] = None,
+    checksum_sha256: Annotated[str | None, Form()] = None,
+    file_fingerprint: Annotated[str | None, Form()] = None,
+    lifecycle: Annotated[str | None, Form()] = None,
+    metadata: Annotated[str | None, Form()] = None,
+) -> dict[str, Any]:
+    lifecycle_data = json.loads(lifecycle) if lifecycle else None
+    metadata_data = json.loads(metadata) if metadata else {}
+    if not isinstance(lifecycle_data, dict):
+        lifecycle_data = None
+    if not isinstance(metadata_data, dict):
+        metadata_data = {}
+    size = 0
+    # Spool the uploaded file to disk so we never hold it entirely in memory.
+    while True:
+        chunk = await file.read(1024 * 1024)
+        if not chunk:
+            break
+        size += len(chunk)
+    await file.seek(0)
+    resolved_key = object_key or file.filename or "upload.bin"
+    file_obj = await state.file_service.proxy_upload(
+        db,
+        identity,
+        bucket=bucket,
+        object_key=resolved_key,
+        original_filename=original_filename or file.filename or resolved_key,
+        content_type=content_type or file.content_type,
+        size_bytes=size,
+        stream=file.file,
+        checksum_sha256=checksum_sha256,
+        file_fingerprint=file_fingerprint,
+        lifecycle=lifecycle_data,
+        metadata=metadata_data,
+    )
+    await db.commit()
+    from app.services.file_service import serialize_file
+
+    return serialize_file(file_obj)
+
+
+@router.get("/{file_id}")
+async def get_file(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    file_id: uuid.UUID,
+) -> dict[str, Any]:
+    file_obj = await state.file_service.get(db, identity, file_id)
+    from app.services.file_service import serialize_file
+
+    return serialize_file(file_obj)
+
+
+@router.get("/{file_id}/download")
+async def download_file(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    file_id: uuid.UUID,
+) -> StreamingResponse:
+    file_obj, stream = await state.file_service.download(db, identity, file_id)
+
+    async def iterator():
+        async for chunk in stream:
+            yield chunk
+
+    headers = {"X-Request-ID": "stream"}
+    if file_obj.etag:
+        headers["ETag"] = file_obj.etag
+    return StreamingResponse(
+        iterator(),
+        media_type=file_obj.content_type or "application/octet-stream",
+        headers=headers,
+    )
+
+
+@router.post("/{file_id}/presign-download")
+async def presign_download(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    file_id: uuid.UUID,
+    body: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    expires = (body or {}).get("expires_seconds")
+    url = await state.file_service.presign_download(db, identity, file_id, expires)
+    return {"url": url}
+
+
+@router.delete("/{file_id}")
+async def delete_file(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    file_id: uuid.UUID,
+) -> dict[str, Any]:
+    await state.file_service.delete(db, identity, file_id)
+    await db.commit()
+    return {"status": "deleted", "id": str(file_id)}

app/api/v1/health.py

@@ -0,0 +1,47 @@
+"""Health, readiness, startup and metrics endpoints (docs 16.1, 23.3)."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Request
+from fastapi.responses import JSONResponse, PlainTextResponse
+from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
+
+from app.api.dependencies import AppState, StateDep
+
+router = APIRouter()
+
+
+async def _check_ready(state: AppState) -> bool:
+    checks = state.settings.cluster.readiness
+    if checks.check_database:
+        try:
+            async with state.engine.connect():
+                pass
+        except Exception:
+            return False
+    return True
+
+
+@router.get("/healthz")
+async def healthz() -> dict:
+    return {"status": "ok", "service": "upload-service"}
+
+
+@router.get("/startupz")
+async def startupz() -> dict:
+    return {"status": "ok"}
+
+
+@router.get("/readyz")
+async def readyz(request: Request) -> JSONResponse:
+    state: AppState = request.app.state.state
+    ready = await _check_ready(state)
+    return JSONResponse(
+        status_code=200 if ready else 503,
+        content={"status": "ready" if ready else "not_ready"},
+    )
+
+
+@router.get("/metrics")
+async def metrics(state: StateDep) -> PlainTextResponse:
+    return PlainTextResponse(generate_latest(), media_type=CONTENT_TYPE_LATEST)

app/api/v1/lifecycle.py

@@ -0,0 +1,86 @@
+"""Lifecycle API per docs_product-design.md section 16.5."""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from fastapi import APIRouter
+
+from app.api.dependencies import IdentityDep, SessionDep, StateDep
+
+router = APIRouter(prefix="/files", tags=["lifecycle"])
+
+
+@router.get("/{file_id}/lifecycle")
+async def get_lifecycle(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    file_id: uuid.UUID,
+) -> dict[str, Any]:
+    return await state.lifecycle_service.get_lifecycle(db, identity, file_id)
+
+
+@router.patch("/{file_id}/lifecycle")
+async def update_lifecycle(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    file_id: uuid.UUID,
+    body: dict[str, Any],
+) -> dict[str, Any]:
+    result = await state.lifecycle_service.update_lifecycle(db, identity, file_id, body)
+    await db.commit()
+    return result
+
+
+@router.post("/{file_id}/lifecycle/extend")
+async def extend_lifecycle(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    file_id: uuid.UUID,
+    body: dict[str, Any],
+) -> dict[str, Any]:
+    result = await state.lifecycle_service.extend(
+        db, identity, file_id, int(body.get("extend_seconds", 0))
+    )
+    await db.commit()
+    return result
+
+
+@router.post("/{file_id}/lifecycle/make-permanent")
+async def make_permanent(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    file_id: uuid.UUID,
+) -> dict[str, Any]:
+    result = await state.lifecycle_service.make_permanent(db, identity, file_id)
+    await db.commit()
+    return result
+
+
+@router.post("/{file_id}/legal-hold")
+async def set_legal_hold(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    file_id: uuid.UUID,
+) -> dict[str, Any]:
+    result = await state.lifecycle_service.set_legal_hold(db, identity, file_id, True)
+    await db.commit()
+    return result
+
+
+@router.delete("/{file_id}/legal-hold")
+async def release_legal_hold(
+    state: StateDep,
+    db: SessionDep,
+    identity: IdentityDep,
+    file_id: uuid.UUID,
+) -> dict[str, Any]:
+    result = await state.lifecycle_service.set_legal_hold(db, identity, file_id, False)
+    await db.commit()
+    return result