vention-storage 0.0.0__py3-none-any.whl

storage/io_helpers.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+import csv
+import io
+import sqlite3
+import os
+import zipfile
+import tempfile
+from pathlib import Path
+from typing import List
+
+from sqlalchemy import select
+from sqlalchemy.sql.schema import Table
+from sqlmodel import Session, SQLModel
+from storage import database
+from storage.utils import to_primitive
+from fastapi import UploadFile
+
+_SQLITE_INTERNAL_PREFIX = "sqlite_"
+
+__all__ = [
+    "discover_user_tables",
+    "write_table_csv_buffer",
+    "build_export_zip_bytes",
+    "db_file_path",
+    "build_backup_bytes",
+    "validate_sqlite_file",
+    "safe_unlink",
+]
+
+# ---------- Table discovery & CSV ----------
+
+
+def discover_user_tables() -> List[Table]:
+    """Return user tables (excludes SQLite internal)."""
+    user_tables: List[Table] = []
+    for table in SQLModel.metadata.sorted_tables:
+        if not table.name.startswith(_SQLITE_INTERNAL_PREFIX):
+            user_tables.append(table)
+    return user_tables
+
+
+def write_table_csv_buffer(session: Session, table: Table) -> io.StringIO:
+    """SELECT * and return CSV (header + rows) in a StringIO."""
+    buffer = io.StringIO(newline="")
+    columns = list(table.columns)
+    writer = csv.DictWriter(
+        buffer, fieldnames=[column.name for column in columns], extrasaction="ignore"
+    )
+    writer.writeheader()
+    result = session.exec(select(table))
+    for row in result:
+        writer.writerow(
+            {column.name: to_primitive(row._mapping[column]) for column in columns}
+        )
+    return buffer
+
+
+def build_export_zip_bytes(tables: list[Table]) -> bytes:
+    """Return ZIP bytes with one CSV per table (valid empty ZIP if none)."""
+    mem = io.BytesIO()
+    with zipfile.ZipFile(mem, mode="w", compression=zipfile.ZIP_DEFLATED) as zip_file:
+        if tables:
+            with database.use_session() as session:
+                for table in tables:
+                    try:
+                        csv_buffer = write_table_csv_buffer(session, table)
+                    except Exception as inner:  # surface table context upstream
+                        raise RuntimeError(f"table={table.name}: {inner}") from inner
+                    zip_file.writestr(f"{table.name}.csv", csv_buffer.getvalue())
+    return mem.getvalue()
+
+
+# ---------- Backup ----------
+
+
+def db_file_path() -> str:
+    """Return absolute DB file path."""
+    engine = database.get_engine()
+    db_path = engine.url.database or ""
+    return str(Path(db_path).resolve())
+
+
+def build_backup_bytes(db_path: str) -> bytes:
+    """
+    Build a consistent .sqlite backup and return bytes.
+    """
+    db_dir = Path(db_path).resolve().parent
+    with tempfile.NamedTemporaryFile(
+        prefix="backup-", suffix=".sqlite", dir=db_dir, delete=False
+    ) as tmp:
+        tmp_path = Path(tmp.name)
+
+    try:
+        database.get_engine().dispose()
+        with (
+            sqlite3.connect(str(Path(db_path))) as db_connection,
+            sqlite3.connect(str(tmp_path)) as tmp_connection,
+        ):
+            db_connection.backup(tmp_connection)
+
+        return tmp_path.read_bytes()
+    finally:
+        safe_unlink(tmp_path)
+
+
+# ---------- Restore helpers ----------
+
+
+def validate_sqlite_file(path: Path, *, run_integrity_check: bool = True) -> None:
+    """Raise ValueError on validation failure; returns None on success."""
+    # Header check
+    try:
+        with path.open("rb") as file:
+            signature = file.read(16)
+    except Exception as e:
+        raise ValueError(f"Cannot read uploaded file: {e}") from e
+    if signature != b"SQLite format 3\x00":
+        raise ValueError("Invalid SQLite header")
+
+    if not run_integrity_check:
+        return
+
+    # integrity_check
+    try:
+        uri = f"file:{path.as_posix()}?mode=ro"
+        connection = sqlite3.connect(uri, uri=True)
+        try:
+            row = connection.execute("PRAGMA integrity_check;").fetchone()
+            ok = row and str(row[0]).lower() == "ok"
+            if not ok:
+                raise ValueError(
+                    f"Integrity check failed: {row[0] if row else 'unknown'}"
+                )
+        finally:
+            connection.close()
+    except ValueError:
+        raise
+    except Exception as e:
+        raise ValueError(f"Integrity check error: {e}") from e
+
+
+def save_upload_to_temp(file: UploadFile, dest_dir: Path) -> tuple[Path, int]:
+    with tempfile.NamedTemporaryFile(
+        prefix="restore-", suffix=".sqlite", dir=dest_dir, delete=False
+    ) as tmp:
+        tmp_path = Path(tmp.name)
+        total = 0
+        while True:
+            chunk = file.file.read(1024 * 1024)
+            if not chunk:
+                break
+            tmp.write(chunk)
+            total += len(chunk)
+    try:
+        file.file.close()
+    except Exception:
+        pass
+    return tmp_path, total
+
+
+def atomic_replace_db(tmp_path: Path, db_path: Path) -> None:
+    database.get_engine().dispose()
+    os.replace(str(tmp_path), str(db_path))
+
+
+def safe_unlink(path: Path) -> None:
+    try:
+        path.unlink(missing_ok=True)
+    except Exception:
+        pass

storage/router_database.py

@@ -0,0 +1,275 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Dict, List, Optional
+from pathlib import Path
+
+from fastapi import APIRouter, HTTPException, Query, Response, UploadFile, File
+from sqlmodel import SQLModel, select
+from sqlalchemy import desc
+
+from storage import database, io_helpers
+from storage.auditor import AuditLog
+from storage.utils import Operation
+
+from storage.io_helpers import (
+    discover_user_tables,
+    build_export_zip_bytes,
+    db_file_path,
+    build_backup_bytes,
+    validate_sqlite_file,
+    safe_unlink,
+)
+
+__all__ = ["build_db_router"]
+
+
+def build_db_router(
+    *,
+    audit_default_limit: int = 100,
+    audit_max_limit: int = 1000,
+) -> APIRouter:
+    """
+    Build a FastAPI router exposing database-wide utilities.
+
+    Endpoints:
+      - /db/health          : Verify DB engine is available
+      - /db/audit           : Query audit logs (filters + pagination)
+      - /db/diagram.svg     : Schema diagram (requires Graphviz)
+      - /db/export.zip      : CSV export (one CSV per table)
+      - /db/backup.sqlite   : Full SQLite backup file
+      - /db/restore         : Upload and restore a .sqlite backup (atomic replace)
+    """
+    router = APIRouter(prefix="/db", tags=["db"])
+
+    @router.get("/health")
+    def health() -> Dict[str, str]:
+        """
+        Check database connectivity.
+
+        Returns:
+            dict: {"status": "ok"} if the database engine can be initialized.
+        """
+        _ = database.get_engine()
+        return {"status": "ok"}
+
+    @router.get("/audit")
+    def read_audit(
+        component: Optional[str] = Query(None, description="Filter by component name"),
+        record_id: Optional[int] = Query(None, description="Filter by record ID"),
+        actor: Optional[str] = Query(None, description="Filter by actor identifier"),
+        operation: Optional[Operation] = Query(
+            None, description="Filter by operation type"
+        ),
+        since: Optional[datetime] = Query(
+            None, description="Include only logs on/after this timestamp"
+        ),
+        until: Optional[datetime] = Query(
+            None, description="Include only logs before this timestamp"
+        ),
+        limit: int = Query(
+            audit_default_limit,
+            ge=1,
+            le=audit_max_limit,
+            description="Maximum rows to return",
+        ),
+        offset: int = Query(
+            0, ge=0, description="Number of rows to skip (for pagination)"
+        ),
+    ) -> List[AuditLog]:
+        """
+        Query the audit log table.
+
+        Supports filtering by component, record_id, actor, operation,
+        and timestamp range, with pagination.
+
+        Args:
+            component (str, optional): Restrict to a specific component.
+            record_id (int, optional): Restrict to a specific record ID.
+            actor (str, optional): Restrict to a specific actor (user/system).
+            operation (Operation, optional): Restrict to a specific operation.
+            since (datetime, optional): Include only logs since this timestamp.
+            until (datetime, optional): Include only logs before this timestamp.
+            limit (int): Maximum number of logs to return (bounded by audit_max_limit).
+            offset (int): Number of rows to skip for pagination.
+
+        Returns:
+            List[AuditLog]: A list of audit log entries matching the criteria.
+        """
+        with database.use_session() as session:
+            statement = select(AuditLog)
+            if component:
+                statement = statement.where(AuditLog.component == component)
+            if record_id is not None:
+                statement = statement.where(AuditLog.record_id == record_id)
+            if actor:
+                statement = statement.where(AuditLog.actor == actor)
+            if operation:
+                statement = statement.where(AuditLog.operation == operation)
+            if since is not None:
+                statement = statement.where(AuditLog.timestamp >= since)
+            if until is not None:
+                statement = statement.where(AuditLog.timestamp < until)
+            statement = (
+                statement.order_by(desc(AuditLog.timestamp)).offset(offset).limit(limit)
+            )
+            rows: List[AuditLog] = session.exec(statement).all()
+            return rows
+
+    @router.get("/diagram.svg", response_class=Response)
+    def diagram_svg() -> Response:
+        """
+        Generate a database schema diagram in SVG format.
+
+        Requires `sqlalchemy-schemadisplay` and Graphviz to be installed.
+        The diagram reflects the current SQLModel metadata.
+
+        Returns:
+            Response: SVG image of the database schema.
+
+        Raises:
+            HTTPException 503: If required dependencies are missing
+                               or Graphviz is not available.
+        """
+        try:
+            # import here to avoid hard dependency if not used
+            from sqlalchemy_schemadisplay import create_schema_graph
+        except Exception as e:
+            raise HTTPException(
+                status_code=503,
+                detail=(
+                    "sqlalchemy-schemadisplay is required. "
+                    "Install with: pip install sqlalchemy-schemadisplay"
+                ),
+            ) from e
+
+        try:
+            graph = create_schema_graph(
+                engine=database.get_engine(),
+                metadata=SQLModel.metadata,
+                show_datatypes=True,
+                show_indexes=False,
+                concentrate=False,
+            )
+            return Response(content=graph.create_svg(), media_type="image/svg+xml")
+        except Exception as e:
+            msg = str(e).lower()
+            if "executable" in msg or "dot not found" in msg or "graphviz" in msg:
+                raise HTTPException(
+                    status_code=503,
+                    detail=(
+                        "Graphviz is required to render the diagram. "
+                        "Install it (e.g. brew install graphviz / apt-get install graphviz)."
+                    ),
+                ) from e
+            raise
+
+    @router.get("/export.zip")
+    def export_zip() -> Response:
+        """
+        Export the entire database as a ZIP archive.
+
+        The archive contains one CSV file per user-defined table found in SQLModel metadata.
+        SQLite internal tables (e.g., "sqlite_sequence") are excluded.
+
+        Returns:
+            Response: application/zip payload with "{table}.csv" entries.
+        """
+        headers = {"Content-Disposition": 'attachment; filename="export.zip"'}
+        try:
+            zip_bytes = build_export_zip_bytes(discover_user_tables())
+        except Exception as e:
+            raise HTTPException(
+                status_code=503, detail=f"Failed to build export.zip: {e}"
+            ) from e
+        return Response(
+            content=zip_bytes, media_type="application/zip", headers=headers
+        )
+
+    @router.get("/backup.sqlite")
+    def backup_sqlite() -> Response:
+        """
+        Create and return a consistent SQLite backup of the current database file.
+
+        Uses the SQLite Backup API for correctness.
+
+        Returns:
+            Response: application/x-sqlite3 payload with a `.sqlite` file.
+
+        Raises:
+            HTTPException 503: Operational failure creating the backup.
+        """
+        path = db_file_path()
+        headers = {
+            "Content-Disposition": f'attachment; filename="backup-{_backup_timestamp_slug()}.sqlite"'
+        }
+        try:
+            data = build_backup_bytes(path)
+        except Exception as e:
+            raise HTTPException(
+                status_code=503, detail=f"Failed to create backup: {e}"
+            ) from e
+        return Response(
+            content=data, media_type="application/x-sqlite3", headers=headers
+        )
+
+    @router.post("/restore")
+    def restore_sqlite(
+        file: UploadFile = File(..., description="SQLite .sqlite backup to restore"),
+        integrity_check: bool = Query(
+            True, description="Run PRAGMA integrity_check before replacing"
+        ),
+        dry_run: bool = Query(
+            False, description="Validate only; do not modify current database"
+        ),
+    ) -> Dict[str, object]:
+        """
+        Restore the database from an uploaded SQLite file by atomically replacing the current DB file.
+
+        Steps:
+          1. Save upload to a temporary path.
+          2. Validate header and (optionally) PRAGMA integrity_check.
+          3. If dry_run: report validation OK and exit.
+          4. Dispose engine connections and os.replace(temp, db_path).
+
+        Returns:
+            dict: {status: "ok", restored: bool, bytes: int}
+
+        Raises:
+            HTTPException 422: Invalid SQLite file or failed integrity check.
+            HTTPException 503: Operational failure during file I/O.
+        """
+        path = db_file_path()
+        db_dir = Path(path).resolve().parent
+        try:
+            tmp_path, total = io_helpers.save_upload_to_temp(file, db_dir)
+        except Exception as e:
+            raise HTTPException(503, f"Failed to save upload: {e}") from e
+
+        try:
+            validate_sqlite_file(tmp_path, run_integrity_check=integrity_check)
+        except ValueError as ve:
+            safe_unlink(tmp_path)
+            raise HTTPException(422, str(ve)) from ve
+        except Exception as e:
+            safe_unlink(tmp_path)
+            raise HTTPException(422, f"Invalid SQLite file: {e}") from e
+
+        if dry_run:
+            safe_unlink(tmp_path)
+            return {"status": "ok", "restored": False, "bytes": total}
+
+        try:
+            io_helpers.atomic_replace_db(tmp_path, Path(path))
+        except Exception as e:
+            safe_unlink(tmp_path)
+            raise HTTPException(503, f"Failed to replace database file: {e}") from e
+
+        return {"status": "ok", "restored": True, "bytes": total}
+
+    return router
+
+
+def _backup_timestamp_slug() -> str:
+    """Timestamp safe for filenames (UTC, no colons)."""
+    return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")

storage/router_model.py

@@ -0,0 +1,247 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Type
+
+from fastapi import (
+    APIRouter,
+    Depends,
+    HTTPException,
+    Query,
+    Request,
+    Body,
+    Response,
+    status,
+)
+from sqlalchemy.exc import DataError, IntegrityError, StatementError
+
+from storage.accessor import ModelAccessor
+from storage.utils import ModelType
+
+__all__ = ["build_crud_router"]
+
+DEFAULT_MAX_RECORDS_PER_MODEL = 100
+
+
+def get_actor(request: Request) -> str:
+    """
+    Extract the audit actor from the `X-User` header. Eg: 'Operator', 'Admin', etc.
+
+    Notes:
+        - Required for **mutating** endpoints (POST, PUT, DELETE, restore).
+        - Optional for read-only endpoints.
+        - The value is stored verbatim in `AuditLog.actor`.
+        - If missing on a mutating endpoint, a 400 error is returned.
+
+    Returns:
+        The `X-User` header value.
+
+    Raises:
+        HTTPException(400): If the header is missing for a mutating endpoint.
+    """
+    actor = request.headers.get("X-User")
+    if not actor:
+        raise HTTPException(status_code=400, detail="Missing X-User header")
+    return actor
+
+
+def build_crud_router(
+    accessor: ModelAccessor[ModelType],
+    *,
+    max_records: Optional[int] = DEFAULT_MAX_RECORDS_PER_MODEL,
+) -> APIRouter:
+    """
+    Build a FastAPI router exposing CRUD + restore endpoints for a single SQLModel component.
+    """
+    model: Type[ModelType] = accessor.model
+    router = APIRouter(prefix=f"/{accessor.component}", tags=[accessor.component])
+
+    # ---------------- READS ----------------
+
+    @router.get("/", response_model=List[model])  # type: ignore[valid-type]
+    def list_records(
+        include_deleted: bool = Query(False, description="Include soft-deleted rows"),
+    ) -> List[ModelType]:
+        """
+        List all records for this model.
+
+        By default, soft-deleted records are excluded from results.
+
+        Args:
+            include_deleted (bool): Set to true to include soft-deleted records. Defaults to false.
+
+        Returns:
+            List[model]: List of model instances matching the criteria.
+        """
+        return accessor.all(include_deleted=include_deleted)
+
+    @router.get("/{record_id}", response_model=model)
+    def get_record(
+        record_id: int,
+        include_deleted: bool = Query(False, description="Include soft-deleted row"),
+    ) -> ModelType:
+        """
+        Retrieve a single record by its ID.
+
+        Args:
+            record_id (int): ID of the record to fetch.
+            include_deleted (bool): Set to true to allow returning a soft-deleted record. Defaults to false.
+
+        Returns:
+            model: The model instance if found.
+
+        Raises:
+            HTTPException 404: If the record does not exist or is soft-deleted (when include_deleted=false).
+        """
+        obj = accessor.get(record_id, include_deleted=include_deleted)
+        if not obj:
+            raise HTTPException(status_code=404, detail="Not found")
+        return obj
+
+    # ---------------- WRITES ----------------
+
+    @router.post("/", response_model=model)
+    def create_record(
+        payload: Dict[str, Any] = Body(...),
+        actor: str = Depends(get_actor),
+    ) -> ModelType:
+        """
+        Create a new record.
+
+        Args:
+            payload (Dict[str, Any]): JSON body with the record fields.
+            actor (str): User identifier from the `X-User` header.
+
+        Returns:
+            model: The newly created model instance.
+
+        Raises:
+            HTTPException 409: If the maximum number of records has been reached.
+            HTTPException 422: If the payload violates schema or database constraints.
+        """
+        if max_records is not None:
+            total = len(accessor.all(include_deleted=True))
+            if total >= max_records:
+                raise HTTPException(
+                    status_code=409,
+                    detail=f"Max {max_records} records allowed for {accessor.component}",
+                )
+        obj = model(**payload)
+
+        try:
+            return accessor.insert(obj, actor=actor)
+        except (IntegrityError, DataError, StatementError) as e:
+            raise HTTPException(status_code=422, detail=str(e)) from e
+
+    @router.put("/{record_id}", response_model=model)
+    def update_record(
+        record_id: int,
+        response: Response,
+        payload: Dict[str, Any] = Body(...),
+        actor: str = Depends(get_actor),
+    ) -> ModelType:
+        """
+        Upsert a record (PUT semantics).
+
+        Args:
+            record_id (int): ID of the record to update or create.
+            payload (Dict[str, Any]): JSON body with the record fields (the `id` key, if present, is ignored).
+            actor (str): User identifier from the `X-User` header.
+            response (Response): Used to adjust the HTTP status code.
+
+        Returns:
+            model: The updated or newly created model instance.
+
+        If the record exists, it is updated in place (200 OK).
+        If the record does not exist, it is created at this ID (201 Created).
+        """
+        existed = accessor.get(record_id, include_deleted=True) is not None
+
+        # If this PUT will create a new row, enforce the max records.
+        if not existed and max_records is not None:
+            total = len(accessor.all(include_deleted=True))
+            if total >= max_records:
+                raise HTTPException(
+                    status_code=409,
+                    detail=f"Max {max_records} records allowed for {accessor.component}",
+                )
+
+        # Build instance (ignore `id` from body if present)
+        payload_no_id = {key: value for key, value in payload.items() if key != "id"}
+        obj = model(id=record_id, **payload_no_id)
+
+        try:
+            saved = accessor.save(obj, actor=actor)
+        except (IntegrityError, DataError, StatementError) as e:
+            raise HTTPException(status_code=422, detail=str(e)) from e
+
+        if not existed:
+            response.status_code = status.HTTP_201_CREATED
+        return saved
+
+    @router.delete("/{record_id}")
+    def delete_record(
+        record_id: int,
+        actor: str = Depends(get_actor),
+    ) -> Dict[str, str]:
+        """
+        Delete a record by ID.
+
+        If the model supports soft-delete, the record is marked as deleted.
+        Otherwise, the record is permanently removed.
+
+        Args:
+            record_id (int): ID of the record to delete.
+            actor (str): User identifier from the `X-User` header.
+
+        Returns:
+            dict: {"status": "deleted"} on success.
+
+        Raises:
+            HTTPException 404: If the record does not exist.
+        """
+        ok = accessor.delete(record_id, actor=actor)
+        if not ok:
+            raise HTTPException(status_code=404, detail="Not found")
+        return {"status": "deleted"}
+
+    # ---------------- RESTORE (soft-delete only) ----------------
+
+    @router.post("/{record_id}/restore")
+    def restore_record(
+        record_id: int,
+        actor: str = Depends(get_actor),
+    ) -> Dict[str, Any]:
+        """
+        Restore a soft-deleted record.
+
+        This endpoint only applies if the model has a `deleted_at` field.
+
+        Args:
+            record_id (int): ID of the record to restore.
+            actor (str): User identifier from the `X-User` header.
+
+        Returns:
+            dict: {"status": "ok", "restored": True} if the record was restored,
+                  {"status": "ok", "restored": False} if it was not soft-deleted.
+
+        Raises:
+            HTTPException 404: If the record does not exist.
+            HTTPException 409: If the model does not support soft-delete/restore.
+        """
+        if not hasattr(accessor.model, "deleted_at"):
+            raise HTTPException(
+                status_code=409,
+                detail=f"{accessor.component} does not support soft delete/restore",
+            )
+
+        obj = accessor.get(record_id, include_deleted=True)
+        if not obj:
+            raise HTTPException(status_code=404, detail="Not found")
+
+        if getattr(obj, "deleted_at") is None:
+            return {"status": "ok", "restored": False}
+
+        accessor.restore(record_id, actor=actor)
+        return {"status": "ok", "restored": True}
+
+    return router