dv-platform 0.1.0__py3-none-any.whl

app/__init__.py

@@ -0,0 +1 @@
+"""Dataset Version Management Platform - backend package."""

app/config.py

@@ -0,0 +1,28 @@
+"""Application settings (env-prefixed with DV_)."""
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix="DV_", env_file=".env", extra="ignore")
+
+    app_name: str = "Dataset Version Platform"
+    database_url: str = "sqlite:///./dv.db"
+    storage_backend: str = "local"          # local | s3
+    storage_root: str = "./storage"         # local backend object root
+    s3_endpoint: str = "http://localhost:9000"
+    s3_bucket: str = "datasets"
+    s3_access_key: str = ""
+    s3_secret_key: str = ""
+    s3_region: str = "us-east-1"
+    presign_ttl: int = 900                  # seconds (15 min)
+    jwt_secret: str = "dev-secret-change-me-0123456789abcdef0123456789"
+    jwt_algorithm: str = "HS256"
+    jwt_expire_minutes: int = 10080         # 7 days
+    public_base_url: str = "http://localhost:8000"
+    web_dist: str = ""                     # optional: absolute path to a built web/dist to serve
+    first_admin_password: str = "admin123"
+    chunk_size: int = 8 * 1024 * 1024       # 8MB reference chunk for docs
+
+
+settings = Settings()
+

app/db.py

@@ -0,0 +1,55 @@
+"""SQLAlchemy engine / session factory.
+
+SQLite notes (development default): we enable WAL journaling plus a busy
+timeout and make every transaction start with ``BEGIN IMMEDIATE``.  When many
+clients upload/download concurrently the metadata writes (blob registration,
+commit creation, ...) are short but frequent; ``BEGIN IMMEDIATE`` serialises
+writers up-front so two concurrent requests never read a stale snapshot and
+then fail with "database is locked" mid-transaction.
+"""
+from sqlalchemy import create_engine, event
+from sqlalchemy.orm import declarative_base, sessionmaker
+
+from .config import settings
+
+connect_args = {}
+if settings.database_url.startswith("sqlite"):
+    connect_args = {"check_same_thread": False}
+
+engine = create_engine(settings.database_url, connect_args=connect_args, future=True)
+
+if settings.database_url.startswith("sqlite") and ":memory:" not in settings.database_url:
+    @event.listens_for(engine, "connect")
+    def _sqlite_pragmas(dbapi_connection, _connection_record):  # pragma: no cover - exercised via app
+        # Disable pysqlite's implicit BEGIN so SQLAlchemy controls transactions
+        # and we can start them with BEGIN IMMEDIATE (see recipe in SQLAlchemy docs).
+        dbapi_connection.isolation_level = None
+        cursor = dbapi_connection.cursor()
+        try:
+            cursor.execute("PRAGMA journal_mode=WAL")
+            cursor.execute("PRAGMA busy_timeout=30000")
+            cursor.execute("PRAGMA foreign_keys=ON")
+        finally:
+            cursor.close()
+
+    @event.listens_for(engine, "begin")
+    def _begin_immediate(conn):
+        # Take the write lock at transaction start instead of upgrading a
+        # read snapshot later (avoids SQLITE_BUSY_SNAPSHOT / lost updates).
+        conn.exec_driver_sql("BEGIN IMMEDIATE")
+
+SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True)
+Base = declarative_base()
+
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+
+
+def init_db() -> None:
+    from . import models  # noqa: F401  (register models)
+    Base.metadata.create_all(bind=engine)

app/engine.py

@@ -0,0 +1,342 @@
+"""Git-like version engine: blob/tree/commit, branches, tags, diff, rollback,
+reference counting and dataset deletion (metadata + physical storage cleanup)."""
+import hashlib
+import json
+from datetime import datetime
+from typing import Iterable
+
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+
+from .models import (
+    AuditLog, Blob, BlobRef, Branch, Commit, Dataset, DatasetAcl, StagedUpload,
+    Tag, Tree, User,
+)
+from .storage import get_storage, object_key
+
+
+# ------------------------------------------------------------------ hashing
+def canonical_json(obj) -> str:
+    return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+
+
+def sha256_hex(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def hash_tree(entries: dict) -> str:
+    return sha256_hex(canonical_json(entries).encode("utf-8"))
+
+
+def hash_commit(tree_id: str, parents: list[str], author: str, message: str, ts: str) -> str:
+    obj = {"tree": tree_id, "parents": parents, "author": author, "message": message, "ts": ts}
+    return sha256_hex(canonical_json(obj).encode("utf-8"))
+
+
+def normalize_files(files: Iterable[dict]) -> dict:
+    """Normalize [{path,hash,size}] -> {path: {"type":"blob","hash","size"}} sorted."""
+    entries = {}
+    for f in files:
+        path = f["path"].replace("\\", "/").lstrip("./")
+        path = "/".join(p for p in path.split("/") if p not in ("", ".", ".."))
+        if not path:
+            continue
+        entries[path] = {"type": "blob", "hash": f["hash"], "size": int(f.get("size", 0))}
+    return dict(sorted(entries.items()))
+
+
+def entries_from_commit(db: Session, commit: Commit) -> dict:
+    tree = db.query(Tree).filter(Tree.content_hash == commit.tree_id).first()
+    return dict(tree.payload) if tree else {}
+
+
+# ------------------------------------------------------------------ refs
+def resolve_ref(db: Session, dataset: Dataset, ref: str) -> Commit:
+    """Resolve a ref to a commit: tag / branch / 'HEAD' / commit hash / seq / vN."""
+    ref = (ref or "HEAD").strip()
+    low = ref.lower()
+    # tag
+    tag = db.query(Tag).filter(Tag.dataset_id == dataset.id, Tag.name == ref).first()
+    if tag:
+        return db.query(Commit).filter(Commit.commit_hash == tag.commit_id).first()
+    # branch
+    if low in ("head", "latest", "main"):
+        branch = db.query(Branch).filter(Branch.dataset_id == dataset.id, Branch.name == "main").first()
+        if branch:
+            return db.query(Commit).filter(Commit.commit_hash == branch.head_commit_id).first()
+        branch = db.query(Branch).filter(Branch.dataset_id == dataset.id).first()
+        if branch:
+            return db.query(Commit).filter(Commit.commit_hash == branch.head_commit_id).first()
+    else:
+        branch = db.query(Branch).filter(Branch.dataset_id == dataset.id, Branch.name == ref).first()
+        if branch:
+            return db.query(Commit).filter(Commit.commit_hash == branch.head_commit_id).first()
+    # vN / seq number
+    if low.startswith("v"):
+        num = low[1:]
+        if num.isdigit():
+            ref = num
+    if ref.isdigit():
+        c = db.query(Commit).filter(Commit.dataset_id == dataset.id, Commit.seq == int(ref)).first()
+        if c:
+            return c
+    # commit hash (full or prefix)
+    q = db.query(Commit).filter(Commit.dataset_id == dataset.id, Commit.commit_hash.startswith(ref.lower()))
+    if ref.lower() and q.count() > 0:
+        return q.first()
+    # latest
+    return db.query(Commit).filter(Commit.dataset_id == dataset.id).order_by(Commit.seq.desc()).first()
+
+
+def commit_or_404(db: Session, dataset: Dataset, ref: str) -> Commit:
+    c = resolve_ref(db, dataset, ref)
+    if c is None:
+        raise HTTPException(404, f"Version/ref '{ref}' not found")
+    return c
+
+
+# ------------------------------------------------------------------ commits
+def create_commit(
+    db: Session,
+    dataset: Dataset,
+    author: User,
+    files: Iterable[dict],
+    message: str,
+    branch_name: str = "main",
+    parent_ref: str | None = None,
+) -> Commit:
+    entries = normalize_files(files)
+    # validate declared blobs are present in object storage; register if missing from DB
+    missing = []
+    for path, meta in entries.items():
+        blob = db.get(Blob, meta["hash"])
+        if blob is None:
+            if get_storage().exists(object_key(meta["hash"])):
+                db.add(Blob(
+                    content_hash=meta["hash"], size=meta["size"],
+                    storage_key=object_key(meta["hash"]), ref_count=0,
+                ))
+            else:
+                missing.append(path)
+    if missing:
+        raise HTTPException(400, f"Blobs not uploaded yet: {missing[:20]}")
+    db.flush()
+
+    tree_id = hash_tree(entries)
+    if db.query(Tree).filter(Tree.content_hash == tree_id).first() is None:
+        db.add(Tree(dataset_id=dataset.id, content_hash=tree_id, payload=entries))
+
+    branch = db.query(Branch).filter(Branch.dataset_id == dataset.id, Branch.name == branch_name).first()
+    parents: list[str] = []
+    if parent_ref:
+        parent = resolve_ref(db, dataset, parent_ref)
+        if parent:
+            parents = [parent.commit_hash]
+    elif branch is not None:
+        parents = [branch.head_commit_id]
+
+    ts = datetime.utcnow().isoformat()
+    commit_hash = hash_commit(tree_id, parents, author.name, message, ts)
+    while db.query(Commit).filter(Commit.commit_hash == commit_hash).first():
+        ts = datetime.utcnow().isoformat() + "-" + str(len(parents))
+        commit_hash = hash_commit(tree_id, parents, author.name, message, ts)
+
+    last = db.query(Commit).filter(Commit.dataset_id == dataset.id).order_by(Commit.seq.desc()).first()
+    commit = Commit(
+        dataset_id=dataset.id,
+        commit_hash=commit_hash,
+        tree_id=tree_id,
+        parent_hashes=parents,
+        author_id=author.id,
+        message=message,
+        seq=(last.seq + 1 if last else 1),
+    )
+    db.add(commit)
+
+    if branch is None:
+        branch = Branch(dataset_id=dataset.id, name=branch_name, head_commit_id=commit_hash)
+        db.add(branch)
+    else:
+        branch.head_commit_id = commit_hash
+
+    # reference counting (dataset-level): +1 only for newly referenced blobs
+    for path, meta in entries.items():
+        existing = (
+            db.query(BlobRef)
+            .filter(BlobRef.dataset_id == dataset.id, BlobRef.blob_hash == meta["hash"])
+            .first()
+        )
+        if existing is None:
+            blob = db.get(Blob, meta["hash"])
+            if blob:
+                blob.ref_count += 1
+            db.add(BlobRef(dataset_id=dataset.id, blob_hash=meta["hash"]))
+    db.commit()
+    db.refresh(commit)
+    return commit
+
+
+def rollback(db: Session, dataset: Dataset, author: User, target_ref: str, message: str) -> Commit:
+    target = commit_or_404(db, dataset, target_ref)
+    entries = entries_from_commit(db, target)
+    files = [
+        {"path": p, "hash": m["hash"], "size": m["size"]}
+        for p, m in entries.items()
+    ]
+    branch = db.query(Branch).filter(Branch.dataset_id == dataset.id, Branch.name == "main").first()
+    parent_ref = branch.head_commit_id if branch else None
+    msg = message or f"rollback to {target_ref} ({target.commit_hash[:8]})"
+    return create_commit(
+        db, dataset, author, files, msg, branch_name=branch.name if branch else "main",
+        parent_ref=parent_ref,
+    )
+
+
+# ------------------------------------------------------------------ diff
+def diff_commits(db: Session, base: Commit, head: Commit) -> list[dict]:
+    base_entries = entries_from_commit(db, base)
+    head_entries = entries_from_commit(db, head)
+    changes = []
+    for path in sorted(set(base_entries) | set(head_entries)):
+        old = base_entries.get(path)
+        new = head_entries.get(path)
+        if old is None:
+            changes.append({"path": path, "status": "added", "old": None, "new": new})
+        elif new is None:
+            changes.append({"path": path, "status": "removed", "old": old, "new": None})
+        elif old["hash"] != new["hash"]:
+            changes.append({"path": path, "status": "modified", "old": old, "new": new})
+    return changes
+
+
+# ------------------------------------------------------------------ delete
+def storage_info(db: Session, dataset: Dataset) -> dict:
+    refs = db.query(BlobRef).filter(BlobRef.dataset_id == dataset.id).all()
+    total_bytes = 0
+    shared = 0
+    shared_bytes = 0
+    for r in refs:
+        blob = db.get(Blob, r.blob_hash)
+        if blob:
+            total_bytes += blob.size
+            if blob.ref_count > 1:
+                shared += 1
+                shared_bytes += blob.size
+    versions = db.query(Commit).filter(Commit.dataset_id == dataset.id).count()
+    latest = db.query(Commit).filter(Commit.dataset_id == dataset.id).order_by(Commit.seq.desc()).first()
+    file_count = len(entries_from_commit(db, latest)) if latest else 0
+    return {
+        "dataset_id": dataset.id,
+        "versions": versions,
+        "file_count": file_count,
+        "storage_bytes": total_bytes,
+        "shared_blobs": shared,
+        "shared_bytes": shared_bytes,
+        "unique_blobs": len(refs),
+    }
+
+
+def delete_dataset(
+    db: Session, dataset: Dataset, user: User, ip: str = ""
+) -> dict:
+    """Metadata deletion + reference counting + physical storage cleanup."""
+    info = storage_info(db, dataset)
+    blob_hashes = [r.blob_hash for r in db.query(BlobRef).filter(BlobRef.dataset_id == dataset.id).all()]
+    zero_ref: list[Blob] = []
+    for h in blob_hashes:
+        blob = db.get(Blob, h)
+        if blob:
+            blob.ref_count -= 1
+            if blob.ref_count <= 0:
+                zero_ref.append(blob)
+
+    # collect tree hashes this dataset's commits use
+    tree_ids = {c.tree_id for c in db.query(Commit).filter(Commit.dataset_id == dataset.id).all()}
+    commit_hashes = {c.commit_hash for c in db.query(Commit).filter(Commit.dataset_id == dataset.id).all()}
+
+    # delete dataset-scoped rows
+    db.query(Commit).filter(Commit.dataset_id == dataset.id).delete()
+    db.query(Branch).filter(Branch.dataset_id == dataset.id).delete()
+    db.query(Tag).filter(Tag.dataset_id == dataset.id).delete()
+    db.query(DatasetAcl).filter(DatasetAcl.dataset_id == dataset.id).delete()
+    db.query(BlobRef).filter(BlobRef.dataset_id == dataset.id).delete()
+    db.query(StagedUpload).filter(StagedUpload.dataset_id == dataset.id).delete()
+    ds_id = dataset.id
+    ds_name = f"{dataset.namespace}/{dataset.name}"
+    db.delete(dataset)
+
+    # remove tree rows no longer referenced by any commit (any dataset)
+    for tree_id in tree_ids:
+        still_used = db.query(Commit).filter(Commit.tree_id == tree_id).count()
+        if still_used == 0:
+            db.query(Tree).filter(Tree.content_hash == tree_id).delete()
+
+    db.commit()
+
+    # physical deletion of unreferenced blobs (after metadata tx)
+    released_bytes = 0
+    deleted_objects = []
+    storage = get_storage()
+    for blob in zero_ref:
+        try:
+            storage.delete(object_key(blob.content_hash))
+            released_bytes += blob.size
+            deleted_objects.append(blob.content_hash)
+        except Exception:
+            pass  # retryable via GC job
+    for h in deleted_objects:
+        db.query(Blob).filter(Blob.content_hash == h).delete()
+    db.commit()
+
+    from .models import GcJob
+    if len(deleted_objects) != len(zero_ref):
+        db.add(GcJob(
+            dataset_id=ds_id,
+            type="dataset_delete",
+            status="pending",
+            total=len(zero_ref),
+            done=len(deleted_objects),
+            detail={"note": "Some objects pending retry"},
+            created_by=user.id,
+        ))
+        db.commit()
+
+    write_audit(
+        db, user, ds_id, "dataset.delete",
+        {
+            "name": ds_name,
+            "versions": info["versions"],
+            "released_bytes": released_bytes,
+            "objects_deleted": len(deleted_objects),
+        },
+        ip,
+    )
+    return {
+        "deleted": True,
+        "released_bytes": released_bytes,
+        "versions_deleted": info["versions"],
+        "objects_deleted": len(deleted_objects),
+        "shared_blobs_kept": info["shared_blobs"],
+    }
+
+
+# ------------------------------------------------------------------ audit
+def write_audit(
+    db: Session, user: User | None, dataset_id: int | None,
+    action: str, detail: dict, ip: str = "",
+) -> None:
+    db.add(AuditLog(
+        user_id=user.id if user else None,
+        dataset_id=dataset_id,
+        action=action,
+        detail=detail or {},
+        ip=ip,
+    ))
+    db.commit()
+
+
+def client_ip(request) -> str:
+    return request.client.host if request.client else ""
+
+
+

app/main.py

@@ -0,0 +1,62 @@
+"""FastAPI application entrypoint."""
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+
+from .config import settings
+from .db import init_db
+from .routers import audit, auth, datasets, storage_objects, tokens, versions
+
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+# DV_WEB_DIST lets a pip-installed deployment serve a pre-built web UI from an
+# arbitrary location; when unset we fall back to the in-repo build (dev).
+WEB_DIST = Path(settings.web_dist).expanduser() if settings.web_dist else REPO_ROOT / "web" / "dist"
+
+app = FastAPI(
+    title=settings.app_name,
+    version="0.1.0",
+    description="Git-like dataset version management platform "
+                "(control plane stores metadata only; data plane is object storage).",
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.middleware("http")
+async def no_store_api(request, call_next):
+    response = await call_next(request)
+    if request.url.path.startswith("/api/"):
+        response.headers["Cache-Control"] = "no-store"
+    return response
+
+app.include_router(auth.router)
+app.include_router(tokens.router)
+app.include_router(datasets.router)
+app.include_router(versions.router)
+app.include_router(storage_objects.router)
+app.include_router(audit.router)
+
+
+@app.on_event("startup")
+def on_startup():
+    init_db()
+
+
+@app.get("/api/v1/health")
+def health():
+    return {"status": "ok", "app": settings.app_name}
+
+
+# Serve the built web UI (if present) - API routes above take precedence.
+if WEB_DIST.is_dir():
+    app.mount("/", StaticFiles(directory=str(WEB_DIST), html=True), name="web")
+
+

app/models.py

@@ -0,0 +1,170 @@
+"""ORM models.  Only metadata is stored here - never file contents."""
+from datetime import datetime
+
+from sqlalchemy import (
+    JSON, Boolean, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint,
+)
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from .db import Base
+
+
+def _now() -> datetime:
+    return datetime.utcnow()
+
+
+class User(Base):
+    __tablename__ = "users"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    name: Mapped[str] = mapped_column(String(128), unique=True, index=True)
+    email: Mapped[str] = mapped_column(String(255), default="")
+    password_hash: Mapped[str] = mapped_column(String(255))
+    role: Mapped[str] = mapped_column(String(16), default="user")  # admin | user
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=_now)
+
+    tokens = relationship("ApiToken", back_populates="user", cascade="all, delete-orphan")
+
+
+class ApiToken(Base):
+    __tablename__ = "api_tokens"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    user_id: Mapped[int] = mapped_column(ForeignKey("users.id"), index=True)
+    name: Mapped[str] = mapped_column(String(128))
+    token_hash: Mapped[str] = mapped_column(String(64), unique=True)
+    scope: Mapped[list] = mapped_column(JSON, default=list)  # ["*"] or [dataset_id,...]
+    expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    last_used_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=_now)
+
+    user = relationship("User", back_populates="tokens")
+
+
+class Dataset(Base):
+    __tablename__ = "datasets"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    name: Mapped[str] = mapped_column(String(128), index=True)
+    namespace: Mapped[str] = mapped_column(String(128), default="default", index=True)
+    description: Mapped[str] = mapped_column(Text, default="")
+    visibility: Mapped[str] = mapped_column(String(16), default="private")  # public|private
+    created_by: Mapped[int] = mapped_column(ForeignKey("users.id"))
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=_now)
+    deleted_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+
+    __table_args__ = (UniqueConstraint("namespace", "name", "deleted_at"),)
+
+    acls = relationship("DatasetAcl", back_populates="dataset", cascade="all, delete-orphan")
+    branches = relationship("Branch", back_populates="dataset", cascade="all, delete-orphan")
+    tags = relationship("Tag", back_populates="dataset", cascade="all, delete-orphan")
+    commits = relationship("Commit", back_populates="dataset", cascade="all, delete-orphan")
+
+
+class DatasetAcl(Base):
+    __tablename__ = "dataset_acls"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
+    user_id: Mapped[int] = mapped_column(ForeignKey("users.id"), index=True)
+    role: Mapped[str] = mapped_column(String(16), default="reader")  # owner|editor|reader
+
+    __table_args__ = (UniqueConstraint("dataset_id", "user_id"),)
+    dataset = relationship("Dataset", back_populates="acls")
+
+
+class Tree(Base):
+    __tablename__ = "trees"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
+    content_hash: Mapped[str] = mapped_column(String(64), unique=True, index=True)
+    payload: Mapped[dict] = mapped_column(JSON)  # {path: {"type","hash","size"}}
+
+
+class Commit(Base):
+    __tablename__ = "commits"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
+    commit_hash: Mapped[str] = mapped_column(String(64), unique=True, index=True)
+    tree_id: Mapped[str] = mapped_column(String(64), index=True)
+    parent_hashes: Mapped[list] = mapped_column(JSON, default=list)
+    author_id: Mapped[int] = mapped_column(ForeignKey("users.id"))
+    message: Mapped[str] = mapped_column(Text, default="")
+    seq: Mapped[int] = mapped_column(Integer, default=0)  # human friendly version number
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=_now)
+
+    dataset = relationship("Dataset", back_populates="commits")
+
+
+class Blob(Base):
+    __tablename__ = "blobs"
+    content_hash: Mapped[str] = mapped_column(String(64), primary_key=True)
+    size: Mapped[int] = mapped_column(Integer, default=0)
+    storage_key: Mapped[str] = mapped_column(String(512))
+    ref_count: Mapped[int] = mapped_column(Integer, default=0)
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=_now)
+
+
+class BlobRef(Base):
+    """Which datasets reference a blob (dataset-level ref counting)."""
+    __tablename__ = "blob_refs"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
+    blob_hash: Mapped[str] = mapped_column(ForeignKey("blobs.content_hash"), index=True)
+
+    __table_args__ = (UniqueConstraint("dataset_id", "blob_hash"),)
+
+
+class Branch(Base):
+    __tablename__ = "branches"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
+    name: Mapped[str] = mapped_column(String(128))
+    head_commit_id: Mapped[str] = mapped_column(String(64))
+
+    __table_args__ = (UniqueConstraint("dataset_id", "name"),)
+    dataset = relationship("Dataset", back_populates="branches")
+
+
+class Tag(Base):
+    __tablename__ = "tags"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
+    name: Mapped[str] = mapped_column(String(128))
+    commit_id: Mapped[str] = mapped_column(String(64))
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=_now)
+
+    __table_args__ = (UniqueConstraint("dataset_id", "name"),)
+    dataset = relationship("Dataset", back_populates="tags")
+
+
+class StagedUpload(Base):
+    __tablename__ = "staged_uploads"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
+    uploader_id: Mapped[int] = mapped_column(ForeignKey("users.id"))
+    blob_hashes: Mapped[list] = mapped_column(JSON, default=list)
+    status: Mapped[str] = mapped_column(String(16), default="pending")
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=_now)
+    expires_at: Mapped[datetime] = mapped_column(DateTime, default=_now)
+
+
+class GcJob(Base):
+    __tablename__ = "gc_jobs"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    dataset_id: Mapped[int] = mapped_column(Integer, nullable=True)
+    type: Mapped[str] = mapped_column(String(32))          # dataset_delete | gc
+    status: Mapped[str] = mapped_column(String(16), default="pending")  # pending|running|done|failed
+    total: Mapped[int] = mapped_column(Integer, default=0)
+    done: Mapped[int] = mapped_column(Integer, default=0)
+    detail: Mapped[dict] = mapped_column(JSON, default=dict)
+    created_by: Mapped[int] = mapped_column(Integer, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=_now)
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+
+
+class AuditLog(Base):
+    __tablename__ = "audit_logs"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    user_id: Mapped[int] = mapped_column(Integer, nullable=True, index=True)
+    dataset_id: Mapped[int] = mapped_column(Integer, nullable=True, index=True)
+    action: Mapped[str] = mapped_column(String(64), index=True)
+    detail: Mapped[dict] = mapped_column(JSON, default=dict)
+    ip: Mapped[str] = mapped_column(String(64), default="")
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=_now, index=True)

app/routers/__init__.py

@@ -0,0 +1 @@
+"""API routers."""