vention-storage 0.5.4__py3-none-any.whl → 0.6.0__py3-none-any.whl

storage/accessor.py

@@ -29,17 +29,24 @@ class ModelAccessor(Generic[ModelType]):
     """
     Accessor for a single SQLModel type with:
       - strongly-typed lifecycle hooks (before/after insert/update/delete)
-      - atomic writes with auditing
+      - atomic writes with optional auditing
       - optional soft delete (if model defines `deleted_at`)
       - batch helpers
       - implicit session reuse inside hooks (no .bind() needed)
     """
 
-    def __init__(self, model: Type[ModelType], component_name: str) -> None:
+    def __init__(
+        self,
+        model: Type[ModelType],
+        component_name: str,
+        *,
+        enable_auditing: bool = True,
+    ) -> None:
         self.model = model
         self.component = component_name
         self._hooks: HookRegistry[ModelType] = HookRegistry()
         self._has_soft_delete = hasattr(model, "deleted_at")
+        self._enable_auditing = enable_auditing
 
     # ---------- Hook decorators ----------
     def before_insert(self) -> Callable[[HookFn[ModelType]], HookFn[ModelType]]:
@@ -70,6 +77,8 @@ class ModelAccessor(Generic[ModelType]):
         self, *, session: Session, instance: ModelType, actor: str
     ) -> None:
         """Audit a create operation."""
+        if not self._enable_auditing:
+            return
         audit_operation(
             session=session,
             component=self.component,
@@ -150,15 +159,16 @@ class ModelAccessor(Generic[ModelType]):
             self._emit("before_update", session=session, instance=merged)
             session.flush()
             session.refresh(merged)
-            audit_operation(
-                session=session,
-                component=self.component,
-                operation="update",
-                record_id=int(getattr(merged, "id")),
-                actor=actor,
-                before=before,
-                after=merged.model_dump(),
-            )
+            if self._enable_auditing:
+                audit_operation(
+                    session=session,
+                    component=self.component,
+                    operation="update",
+                    record_id=int(getattr(merged, "id")),
+                    actor=actor,
+                    before=before,
+                    after=merged.model_dump(),
+                )
             self._emit("after_update", session=session, instance=merged)
             return cast(ModelType, merged)
 
@@ -173,15 +183,16 @@ class ModelAccessor(Generic[ModelType]):
                 return False
             self._emit("before_delete", session=session, instance=obj)
             op_name, before_payload, after_payload = _soft_or_hard_delete(session, obj)
-            audit_operation(
-                session=session,
-                component=self.component,
-                operation=op_name,
-                record_id=id,
-                actor=actor,
-                before=before_payload,
-                after=after_payload,
-            )
+            if self._enable_auditing:
+                audit_operation(
+                    session=session,
+                    component=self.component,
+                    operation=op_name,
+                    record_id=id,
+                    actor=actor,
+                    before=before_payload,
+                    after=after_payload,
+                )
             self._emit("after_delete", session=session, instance=obj)
             return True
 
@@ -201,15 +212,16 @@ class ModelAccessor(Generic[ModelType]):
             session.add(obj)
             session.flush()
             session.refresh(obj)
-            audit_operation(
-                session=session,
-                component=self.component,
-                operation="restore",
-                record_id=id,
-                actor=actor,
-                before=before,
-                after=obj.model_dump(),
-            )
+            if self._enable_auditing:
+                audit_operation(
+                    session=session,
+                    component=self.component,
+                    operation="restore",
+                    record_id=id,
+                    actor=actor,
+                    before=before,
+                    after=obj.model_dump(),
+                )
             return True
 
         return self._run_write(write_operation)
@@ -248,15 +260,16 @@ class ModelAccessor(Generic[ModelType]):
                 op_name, before_payload, after_payload = _soft_or_hard_delete(
                     session, obj
                 )
-                audit_operation(
-                    session=session,
-                    component=self.component,
-                    operation=op_name,
-                    record_id=id_,
-                    actor=actor,
-                    before=before_payload,
-                    after=after_payload,
-                )
+                if self._enable_auditing:
+                    audit_operation(
+                        session=session,
+                        component=self.component,
+                        operation=op_name,
+                        record_id=id_,
+                        actor=actor,
+                        before=before_payload,
+                        after=after_payload,
+                    )
                 self._emit("after_delete", session=session, instance=obj)
                 count += 1
             return count

storage/bootstrap.py

@@ -1,47 +1,34 @@
 from __future__ import annotations
 
-from typing import Any, Iterable, Optional
-
-from fastapi import FastAPI
+from typing import Optional
 from sqlmodel import SQLModel
 
 from storage.database import get_engine, set_database_url
-from storage.accessor import ModelAccessor
-from storage.router_model import build_crud_router
-from storage.router_database import build_db_router
+
+
+__all__ = ["bootstrap"]
 
 
 def bootstrap(
-    app: FastAPI,
     *,
-    accessors: Iterable[ModelAccessor[Any]],
     database_url: Optional[str] = None,
     create_tables: bool = True,
-    max_records_per_model: Optional[int] = 5,
-    enable_db_router: bool = True,
 ) -> None:
     """
-    Bootstrap the storage system for a FastAPI app.
+    Initialize the database engine and optionally create tables.
+
+    This function performs **environment setup only**:
+      - Applies a database URL override (if provided)
+      - Creates SQLModel tables (if requested)
 
-    This helper wires up:
-      - Database engine initialization (optionally overriding the URL).
-      - Optional table creation via `SQLModel.metadata.create_all`.
-      - One CRUD router per registered `ModelAccessor`.
-      - The global /db router (health, audit, diagram, backup/restore) if enabled.
+    Args:
+        database_url: Optional DB URL override (e.g. sqlite:///foo.sqlite)
+        create_tables: Whether to auto-create tables using metadata.
     """
     if database_url is not None:
         set_database_url(database_url)
 
     engine = get_engine()
+
     if create_tables:
         SQLModel.metadata.create_all(engine)
-
-    # Per-model CRUD routers
-    for accessor in accessors:
-        app.include_router(
-            build_crud_router(accessor, max_records=max_records_per_model)
-        )
-
-    # Global DB router (health, audit, diagram, backup/restore)
-    if enable_db_router:
-        app.include_router(build_db_router())

storage/crud_service.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+from typing import Any, Dict, List, Optional
+
+from sqlalchemy.exc import DataError, IntegrityError, StatementError
+from storage.accessor import ModelAccessor
+from storage.utils import ModelType
+
+
+__all__ = ["CrudService", "CrudError"]
+
+
+# ---------------------------------------------------------
+# Exceptions
+# ---------------------------------------------------------
+
+
+class CrudError(Exception):
+    """Base class for CRUD-related errors."""
+
+    def __init__(self, message: str, code: str = "UNKNOWN"):
+        super().__init__(message)
+        self.code = code
+
+
+class NotFoundError(CrudError):
+    def __init__(self, message: str = "Not found"):
+        super().__init__(message, code="NOT_FOUND")
+
+
+class ConflictError(CrudError):
+    def __init__(self, message: str):
+        super().__init__(message, code="CONFLICT")
+
+
+class ValidationError(CrudError):
+    def __init__(self, message: str):
+        super().__init__(message, code="VALIDATION_ERROR")
+
+
+class UnsupportedError(CrudError):
+    def __init__(self, message: str):
+        super().__init__(message, code="UNSUPPORTED_OPERATION")
+
+
+# ---------------------------------------------------------
+# CRUD Service
+# ---------------------------------------------------------
+
+
+class CrudService:
+    """
+    Transport-agnostic CRUD logic for SQLModel components.
+
+    Provides safe, actor-aware CRUD operations. Intended to be reused
+    by both FastAPI routers (REST) and RPC bundles (ConnectRPC).
+    """
+
+    def __init__(
+        self,
+        accessor: ModelAccessor[ModelType],
+        *,
+        max_records: Optional[int] = None,
+    ) -> None:
+        self.accessor = accessor
+        self.max_records = max_records
+
+    # ---------------- READS ----------------
+
+    def list_records(self, *, include_deleted: bool = False) -> List[ModelType]:
+        """Return all records for this model."""
+        return self.accessor.all(include_deleted=include_deleted)
+
+    def get_record(
+        self,
+        record_id: int,
+        *,
+        include_deleted: bool = False,
+    ) -> ModelType:  # type: ignore[type-var]
+        """Retrieve a single record by ID."""
+        obj = self.accessor.get(record_id, include_deleted=include_deleted)
+        if not obj:
+            raise NotFoundError()
+        return obj
+
+    # ---------------- WRITES ----------------
+
+    def create_record(self, payload: Dict[str, Any], actor: str) -> ModelType:  # type: ignore[type-var]
+        """Insert a new record into the database."""
+        if self.max_records is not None:
+            total = len(self.accessor.all(include_deleted=True))
+            if total >= self.max_records:
+                raise ConflictError(
+                    f"Max {self.max_records} records allowed for {self.accessor.component}"
+                )
+
+        obj = self.accessor.model(**payload)
+        try:
+            result = self.accessor.insert(obj, actor=actor)
+            return result
+        except (IntegrityError, DataError, StatementError) as e:
+            raise ValidationError(str(e)) from e
+
+    def update_record(
+        self,
+        record_id: int,
+        payload: Dict[str, Any],
+        actor: str,
+    ) -> ModelType:  # type: ignore[type-var]
+        """Upsert a record (PUT semantics)."""
+        existed = self.accessor.get(record_id, include_deleted=True) is not None
+
+        if not existed and self.max_records is not None:
+            total = len(self.accessor.all(include_deleted=True))
+            if total >= self.max_records:
+                raise ConflictError(
+                    f"Max {self.max_records} records allowed for {self.accessor.component}"
+                )
+
+        payload_no_id = {k: v for k, v in payload.items() if k != "id"}
+        obj = self.accessor.model(id=record_id, **payload_no_id)
+
+        try:
+            saved = self.accessor.save(obj, actor=actor)
+        except (IntegrityError, DataError, StatementError) as e:
+            raise ValidationError(str(e)) from e
+
+        return saved
+
+    def delete_record(self, record_id: int, actor: str) -> None:
+        """Delete a record by ID (soft or hard)."""
+        ok = self.accessor.delete(record_id, actor=actor)
+        if not ok:
+            raise NotFoundError()
+
+    def restore_record(self, record_id: int, actor: str) -> ModelType:  # type: ignore[type-var]
+        """Restore a soft-deleted record (if supported).
+
+        Returns the restored model instance. If the record was already
+        not deleted, returns it unchanged.
+        """
+        model_cls = self.accessor.model
+        if not hasattr(model_cls, "deleted_at"):
+            raise UnsupportedError(
+                f"{self.accessor.component} does not support soft delete/restore"
+            )
+
+        obj = self.accessor.get(record_id, include_deleted=True)
+        if not obj:
+            raise NotFoundError()
+
+        if getattr(obj, "deleted_at") is None:
+            # already restored, return as-is
+            return obj
+
+        self.accessor.restore(record_id, actor=actor)
+        # Fetch the restored record to return it
+        restored = self.accessor.get(record_id, include_deleted=False)
+        if not restored:
+            raise NotFoundError("Record not found after restore")
+        return restored

storage/database_service.py

@@ -0,0 +1,243 @@
+from __future__ import annotations
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from sqlmodel import SQLModel, select
+from sqlalchemy import desc
+
+from storage import database, io_helpers
+from storage.auditor import AuditLog
+from storage.utils import Operation
+from storage.io_helpers import (
+    discover_user_tables,
+    build_export_zip_bytes,
+    db_file_path,
+    build_backup_bytes,
+    validate_sqlite_file,
+    safe_unlink,
+)
+
+
+__all__ = ["DatabaseService", "DatabaseError"]
+
+
+# ---------------------------------------------------------
+# Exceptions (transport-agnostic)
+# ---------------------------------------------------------
+
+
+class DatabaseError(Exception):
+    """Base class for database-level service errors."""
+
+    def __init__(self, message: str, code: str = "UNKNOWN"):
+        super().__init__(message)
+        self.code = code
+
+
+class DependencyError(DatabaseError):
+    def __init__(self, message: str):
+        super().__init__(message, code="DEPENDENCY_ERROR")
+
+
+class ValidationError(DatabaseError):
+    def __init__(self, message: str):
+        super().__init__(message, code="VALIDATION_ERROR")
+
+
+class OperationalError(DatabaseError):
+    def __init__(self, message: str):
+        super().__init__(message, code="OPERATIONAL_ERROR")
+
+
+# ---------------------------------------------------------
+# Database Service
+# ---------------------------------------------------------
+
+
+class DatabaseService:
+    """
+    Transport-agnostic database utilities.
+
+    Exposes operations like:
+      - health check
+      - audit log queries
+      - schema diagram (SVG)
+      - export.zip / backup.sqlite creation
+      - restore.sqlite upload + validation
+    """
+
+    def __init__(
+        self,
+        *,
+        audit_default_limit: int = 100,
+        audit_max_limit: int = 1000,
+    ) -> None:
+        self.audit_default_limit = audit_default_limit
+        self.audit_max_limit = audit_max_limit
+
+    # ---------------- HEALTH ----------------
+
+    def health(self) -> Dict[str, str]:
+        """Verify that a database engine can be initialized."""
+        _ = database.get_engine()
+        return {"status": "ok"}
+
+    # ---------------- AUDIT LOG ----------------
+
+    def read_audit(
+        self,
+        *,
+        component: Optional[str] = None,
+        record_id: Optional[int] = None,
+        actor: Optional[str] = None,
+        operation: Optional[Operation] = None,
+        since: Optional[datetime] = None,
+        until: Optional[datetime] = None,
+        limit: Optional[int] = None,
+        offset: int = 0,
+    ) -> List[AuditLog]:
+        """
+        Query the audit log table with filtering + pagination.
+        """
+        limit = limit or self.audit_default_limit
+        if limit > self.audit_max_limit:
+            limit = self.audit_max_limit
+
+        with database.use_session() as session:
+            statement = select(AuditLog)
+            if component:
+                statement = statement.where(AuditLog.component == component)
+            if record_id is not None:
+                statement = statement.where(AuditLog.record_id == record_id)
+            if actor:
+                statement = statement.where(AuditLog.actor == actor)
+            if operation:
+                statement = statement.where(AuditLog.operation == operation)
+            if since is not None:
+                statement = statement.where(AuditLog.timestamp >= since)
+            if until is not None:
+                statement = statement.where(AuditLog.timestamp < until)
+
+            statement = (
+                statement.order_by(desc(AuditLog.timestamp)).offset(offset).limit(limit)
+            )
+            results = session.exec(statement).all()
+            return list(results)
+
+    # ---------------- DIAGRAM ----------------
+
+    def schema_diagram_svg(self) -> bytes:
+        """
+        Generate a database schema diagram as SVG bytes.
+        Requires `sqlalchemy-schemadisplay` and Graphviz installed.
+        """
+        try:
+            from sqlalchemy_schemadisplay import create_schema_graph
+        except Exception as e:
+            raise DependencyError(
+                "sqlalchemy-schemadisplay is required. Install with: "
+                "pip install sqlalchemy-schemadisplay"
+            ) from e
+
+        try:
+            graph = create_schema_graph(
+                engine=database.get_engine(),
+                metadata=SQLModel.metadata,
+                show_datatypes=True,
+                show_indexes=False,
+                concentrate=False,
+            )
+            svg_bytes = graph.create_svg()
+            if isinstance(svg_bytes, str):
+                return svg_bytes.encode("utf-8")
+            return bytes(svg_bytes)
+        except Exception as e:
+            msg = str(e).lower()
+            if "executable" in msg or "dot not found" in msg or "graphviz" in msg:
+                raise DependencyError(
+                    "Graphviz is required to render the diagram. "
+                    "Install it via `brew install graphviz` or `apt-get install graphviz`."
+                ) from e
+            raise OperationalError(f"Failed to generate schema diagram: {e}") from e
+
+    # ---------------- EXPORT ZIP ----------------
+
+    def export_zip(self) -> bytes:
+        """
+        Export all user-defined tables as a ZIP archive of CSVs.
+        """
+        try:
+            return build_export_zip_bytes(discover_user_tables())
+        except Exception as e:
+            raise OperationalError(f"Failed to build export.zip: {e}") from e
+
+    # ---------------- BACKUP SQLITE ----------------
+
+    def backup_sqlite(self) -> Dict[str, Any]:
+        """
+        Create a consistent SQLite backup and return metadata + bytes.
+        """
+        path = db_file_path()
+        try:
+            data = build_backup_bytes(path)
+        except Exception as e:
+            raise OperationalError(f"Failed to create backup: {e}") from e
+
+        filename = f"backup-{self._timestamp_slug()}.sqlite"
+        return {"filename": filename, "data": data}
+
+    # ---------------- RESTORE SQLITE ----------------
+
+    def restore_sqlite(
+        self,
+        file_bytes: bytes,
+        *,
+        filename: str = "upload.sqlite",
+        integrity_check: bool = True,
+        dry_run: bool = False,
+    ) -> Dict[str, Any]:
+        """
+        Restore a database from raw SQLite bytes.
+
+        Steps:
+          1. Save to a temp path
+          2. Validate header & integrity
+          3. Optionally replace existing DB atomically
+        """
+        path = db_file_path()
+        db_dir = Path(path).resolve().parent
+
+        try:
+            tmp_path, total = io_helpers.save_bytes_to_temp(
+                file_bytes, db_dir, filename=filename
+            )
+        except Exception as e:
+            raise OperationalError(f"Failed to save upload: {e}") from e
+
+        try:
+            validate_sqlite_file(tmp_path, run_integrity_check=integrity_check)
+        except ValueError as ve:
+            safe_unlink(tmp_path)
+            raise ValidationError(str(ve)) from ve
+        except Exception as e:
+            safe_unlink(tmp_path)
+            raise ValidationError(f"Invalid SQLite file: {e}") from e
+
+        if dry_run:
+            safe_unlink(tmp_path)
+            return {"status": "ok", "restored": False, "bytes": total}
+
+        try:
+            io_helpers.atomic_replace_db(tmp_path, Path(path))
+        except Exception as e:
+            safe_unlink(tmp_path)
+            raise OperationalError(f"Failed to replace database file: {e}") from e
+
+        return {"status": "ok", "restored": True, "bytes": total}
+
+    # ---------------- INTERNAL ----------------
+
+    def _timestamp_slug(self) -> str:
+        """UTC timestamp safe for filenames."""
+        return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")

storage/io_helpers.py

@@ -25,6 +25,7 @@ __all__ = [
     "db_file_path",
     "build_backup_bytes",
     "validate_sqlite_file",
+    "save_bytes_to_temp",
     "safe_unlink",
 ]
 
@@ -159,6 +160,19 @@ def save_upload_to_temp(file: UploadFile, dest_dir: Path) -> tuple[Path, int]:
     return tmp_path, total
 
 
+def save_bytes_to_temp(
+    file_bytes: bytes, dest_dir: Path, *, filename: str = "upload.sqlite"
+) -> tuple[Path, int]:
+    """Save bytes to a temporary file in the destination directory."""
+    with tempfile.NamedTemporaryFile(
+        prefix="restore-", suffix=".sqlite", dir=dest_dir, delete=False
+    ) as tmp:
+        tmp_path = Path(tmp.name)
+        tmp.write(file_bytes)
+        total = len(file_bytes)
+    return tmp_path, total
+
+
 def atomic_replace_db(tmp_path: Path, db_path: Path) -> None:
     database.get_engine().dispose()
     os.replace(str(tmp_path), str(db_path))

storage/utils.py

@@ -1,5 +1,5 @@
 from datetime import datetime, timezone, date, time
-from typing import Literal, TypeVar, Any
+from typing import Literal, TypeVar, Any, Optional, get_args
 from sqlmodel import SQLModel
 
 ModelType = TypeVar("ModelType", bound=SQLModel)
@@ -18,3 +18,28 @@ def to_primitive(value: Any) -> Any:
     if isinstance(value, (datetime, date, time)):
         return value.isoformat()
     return value
+
+
+def parse_audit_operation(operation: Optional[str]) -> Optional[Operation]:
+    """
+    Parse and validate an operation string for audit queries.
+    """
+    if operation is None:
+        return None
+
+    if operation not in get_args(Operation):
+        raise ValueError(
+            f"Invalid operation: {operation}. Must be one of {get_args(Operation)}"
+        )
+
+    return operation  # type: ignore
+
+
+def parse_audit_datetime(dt_str: Optional[str]) -> Optional[datetime]:
+    """
+    Parse an ISO 8601 datetime string for audit queries.
+    Handles both 'Z' and '+00:00' timezone formats.
+    """
+    if dt_str is None:
+        return None
+    return datetime.fromisoformat(dt_str.replace("Z", "+00:00"))