dataman-engine 0.1.0__py3-none-any.whl

dataman/core/middleware.py

@@ -0,0 +1,46 @@
+import time
+
+from .models import APILog
+
+
+class APILoggingMiddleware:
+    """Middleware to log API request metrics for the dashboard."""
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        path = request.path
+        # Only log requests to actual data endpoints,
+        # exclude internal/dashboard/docs/health routes.
+        if (
+            not path.startswith("/api/")
+            or path.startswith("/api/schema")
+            or path.startswith("/api/docs")
+            or path.startswith("/api/redoc")
+            or path.startswith("/api/_internal/")
+            or path.startswith("/api/health")
+            or path.startswith("/health")
+        ):
+            return self.get_response(request)
+
+        start_time = time.time()
+        response = self.get_response(request)
+        duration_ms = int((time.time() - start_time) * 1000)
+
+        # Handle IP Address correctly behind proxies
+        x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR")
+        if x_forwarded_for:
+            ip = x_forwarded_for.split(",")[0]
+        else:
+            ip = request.META.get("REMOTE_ADDR")
+
+        APILog.objects.create(
+            method=request.method,
+            path=path,
+            status_code=response.status_code,
+            duration_ms=duration_ms,
+            ip_address=ip,
+        )
+
+        return response

dataman/core/models.py

@@ -0,0 +1,175 @@
+import contextlib
+import importlib
+import secrets
+import sys
+from pathlib import Path
+
+from django.db import models
+
+# Registry of discovered tables, their databases, and module prefixes
+TABLE_REGISTRY: dict[str, dict] = {}
+
+RESERVED_ROOT_DIRS = {
+    ".agents",
+    ".git",
+    ".pytest_cache",
+    ".ruff_cache",
+    ".system_generated",
+    ".venv",
+    "build",
+    "coverage_hack",
+    "dist",
+    "migrations",
+    "scratch",
+    "src",
+    "tests",
+    "__pycache__",
+}
+
+
+def generate_key():
+    return secrets.token_hex(20)
+
+
+class APIToken(models.Model):
+    """Service Token for DataMan API Access."""
+
+    prefix = models.CharField(max_length=8, unique=True)
+    hashed_secret = models.CharField(max_length=128)
+    name = models.CharField(max_length=255)
+    scopes = models.JSONField(default=list)
+    created_at = models.DateTimeField(auto_now_add=True)
+    expires_at = models.DateTimeField(null=True, blank=True, db_index=True)
+    is_active = models.BooleanField(default=True, db_index=True)
+
+    class Meta:
+        app_label = "dataman"
+        db_table = "dataman_apitoken"
+
+    def __str__(self):
+        return f"{self.name} ({self.prefix}...)"
+
+
+class APILog(models.Model):
+    """Telemetry data for API endpoint analysis."""
+
+    timestamp = models.DateTimeField(auto_now_add=True, db_index=True)
+    method = models.CharField(max_length=10)
+    path = models.CharField(max_length=255, db_index=True)
+    status_code = models.IntegerField()
+    duration_ms = models.IntegerField()
+    ip_address = models.GenericIPAddressField(null=True, blank=True)
+
+    class Meta:
+        app_label = "dataman"
+        db_table = "dataman_apilog"
+        ordering = ["-timestamp"]
+
+    def __str__(self):
+        return f"{self.method} {self.path} - {self.status_code} ({self.duration_ms}ms)"
+
+
+class AuditLog(models.Model):
+    """Enterprise Audit Trail for Security and Compliance."""
+
+    timestamp = models.DateTimeField(auto_now_add=True, db_index=True)
+    event_type = models.CharField(max_length=60, db_index=True)
+    actor = models.CharField(max_length=255, db_index=True, blank=True, default="")
+    ip_address = models.GenericIPAddressField(null=True, blank=True)
+    user_agent = models.TextField(blank=True, default="")
+    status_code = models.IntegerField(null=True, blank=True)
+    severity = models.CharField(max_length=20, default="INFO", db_index=True)
+    details = models.JSONField(default=dict)
+
+    class Meta:
+        app_label = "dataman"
+        db_table = "dataman_auditlog"
+        ordering = ["-timestamp"]
+
+    def __str__(self):
+        return f"[{self.timestamp}] [{self.severity}] {self.event_type} - {self.actor or 'System'}"
+
+
+def _discover_and_load_models():
+    """Discovers and registers models across all configured database directories."""
+    cwd = Path.cwd()
+    if str(cwd) not in sys.path:
+        sys.path.append(str(cwd))
+
+    discovered_candidates: list[tuple[str, str, str]] = []
+    # candidate tuple: (db_name, table_name, module_prefix)
+
+    # 1. Check 'databases/' container directory: databases/<db_name>/<table_name>
+    databases_dir = cwd / "databases"
+    if databases_dir.exists() and databases_dir.is_dir():
+        for db_dir in databases_dir.iterdir():
+            if db_dir.is_dir() and not db_dir.name.startswith((".", "_")):
+                for tbl_dir in db_dir.iterdir():
+                    if tbl_dir.is_dir() and (tbl_dir / "models.py").exists():
+                        discovered_candidates.append(
+                            (
+                                db_dir.name,
+                                tbl_dir.name,
+                                f"databases.{db_dir.name}.{tbl_dir.name}",
+                            )
+                        )
+
+    # 2. Check root-level database directories: <db_name>/<table_name>
+    for item in cwd.iterdir():
+        if (
+            item.is_dir()
+            and not item.name.startswith((".", "_"))
+            and item.name not in RESERVED_ROOT_DIRS
+            and item.name not in ("databases", "tables")
+        ):
+            # Check if this root folder has subdirectories with models.py
+            for sub in item.iterdir():
+                if sub.is_dir() and (sub / "models.py").exists():
+                    discovered_candidates.append(
+                        (item.name, sub.name, f"{item.name}.{sub.name}")
+                    )
+
+    # 3. Check legacy/default 'tables/' directory: tables/<table_name>
+    tables_dir = cwd / "tables"
+    if tables_dir.exists() and tables_dir.is_dir():
+        for tbl_dir in tables_dir.iterdir():
+            if tbl_dir.is_dir() and (tbl_dir / "models.py").exists():
+                discovered_candidates.append(
+                    ("default", tbl_dir.name, f"tables.{tbl_dir.name}")
+                )
+
+    for db_name, tbl_name, module_prefix in discovered_candidates:
+        models_module_name = f"{module_prefix}.models"
+        try:
+            mod = importlib.import_module(models_module_name)
+            # Find models defined in this module
+            for attr_name in dir(mod):
+                attr = getattr(mod, attr_name)
+                if (
+                    isinstance(attr, type)
+                    and issubclass(attr, models.Model)
+                    and attr.__module__ == models_module_name
+                ):
+                    # Check if config overrides database
+                    target_db = db_name
+                    with contextlib.suppress(Exception):
+                        cfg_mod = importlib.import_module(f"{module_prefix}.config")
+                        if hasattr(cfg_mod, "DATABASE") and cfg_mod.DATABASE:
+                            target_db = str(cfg_mod.DATABASE)
+
+                    attr._dataman_db = target_db
+                    entry = {
+                        "database": target_db,
+                        "table_name": tbl_name,
+                        "module_prefix": module_prefix,
+                        "model": attr,
+                    }
+                    TABLE_REGISTRY[attr.__name__] = entry
+                    TABLE_REGISTRY[attr.__name__.lower()] = entry
+                    TABLE_REGISTRY[tbl_name.lower()] = entry
+                    TABLE_REGISTRY[f"{target_db}-{tbl_name.lower()}"] = entry
+        except Exception as e:
+            print(f"Failed to load model from {models_module_name}: {e}")
+
+
+_discover_and_load_models()

dataman/core/permissions.py

@@ -0,0 +1,110 @@
+from rest_framework import permissions
+
+from dataman.core.audit import log_audit_event
+
+
+class HasTableScope(permissions.BasePermission):
+    """
+    Ensures that the incoming request token has the required scope
+    to access this table and action.
+    """
+
+    def has_permission(self, request, view):
+        # Allow superusers (admin panel logins) to do anything
+        if getattr(request.user, "is_superuser", False):
+            return True
+
+        table_name = getattr(view, "basename", "").lower()
+        method = request.method.lower()
+
+        action_map = {
+            "get": "read",
+            "head": "read",
+            "options": "read",
+            "post": "write",
+            "put": "write",
+            "patch": "write",
+            "delete": "write",
+        }
+        action = action_map.get(method, "read")
+        required_scope = f"{table_name}:{action}"
+
+        if not hasattr(request, "auth") or not request.auth:
+            log_audit_event(
+                event_type="PERMISSION_DENIED",
+                request=request,
+                details={
+                    "table": table_name,
+                    "action": action,
+                    "required_scope": required_scope,
+                    "reason": "No authentication token provided",
+                },
+                severity="WARNING",
+                status_code=403,
+            )
+            return False
+
+        token = request.auth
+        if not hasattr(token, "scopes"):
+            log_audit_event(
+                event_type="PERMISSION_DENIED",
+                actor=f"Token:{getattr(token, 'prefix', 'unknown')}",
+                request=request,
+                details={
+                    "table": table_name,
+                    "action": action,
+                    "required_scope": required_scope,
+                    "reason": "Token object missing scopes",
+                },
+                severity="WARNING",
+                status_code=403,
+            )
+            return False
+
+        from dataman.core.models import TABLE_REGISTRY
+
+        clean_table = table_name.split("-")[-1]
+        required_scope = f"{clean_table}:{action}"
+        namespaced_scope = f"{table_name}:{action}"
+
+        possible_scopes = {
+            required_scope,
+            namespaced_scope,
+            f"{clean_table}:*",
+            f"{table_name}:*",
+        }
+
+        reg_entry = TABLE_REGISTRY.get(table_name) or TABLE_REGISTRY.get(clean_table)
+        if reg_entry and "database" in reg_entry:
+            db_name = reg_entry["database"]
+            possible_scopes.update(
+                {
+                    f"{db_name}-{clean_table}:{action}",
+                    f"{db_name}-{clean_table}:*",
+                    f"{db_name}:{clean_table}:{action}",
+                    f"{db_name}:{clean_table}:*",
+                    f"{db_name}:{action}",
+                    f"{db_name}:*",
+                }
+            )
+
+        if "*" in token.scopes:
+            return True
+
+        has_scope = any(s in token.scopes for s in possible_scopes)
+        if not has_scope:
+            log_audit_event(
+                event_type="PERMISSION_DENIED",
+                actor=f"Token:{getattr(token, 'prefix', 'unknown')}",
+                request=request,
+                details={
+                    "table": table_name,
+                    "action": action,
+                    "required_scope": required_scope,
+                    "token_scopes": token.scopes,
+                    "reason": "Missing required scope",
+                },
+                severity="WARNING",
+                status_code=403,
+            )
+        return has_scope

dataman/core/router.py

@@ -0,0 +1,112 @@
+import contextlib
+import logging
+from typing import Any
+
+logger = logging.getLogger("dataman.router")
+
+
+class DataManDatabaseRouter:
+    """
+    Routes database operations to the designated database for each DataMan model.
+    Models can have `_dataman_db` or `_meta.dataman_db` specified, or be registered in TABLE_REGISTRY.
+    """
+
+    DJANGO_INTERNAL_APPS = {
+        "auth",
+        "contenttypes",
+        "sessions",
+        "admin",
+        "messages",
+    }
+
+    def _get_target_db(self, model: Any) -> str:
+        if not model:
+            return "default"
+        if hasattr(model, "_dataman_db") and model._dataman_db:
+            return str(model._dataman_db)
+        if hasattr(model, "_meta") and hasattr(model._meta, "dataman_db"):
+            return str(model._meta.dataman_db)
+
+        # Lookup in TABLE_REGISTRY
+        with contextlib.suppress(Exception):
+            from dataman.core.models import TABLE_REGISTRY
+
+            model_name = getattr(model, "__name__", "") or (
+                getattr(model._meta, "object_name", "")
+                if hasattr(model, "_meta")
+                else ""
+            )
+            if model_name:
+                for reg_name, entry in TABLE_REGISTRY.items():
+                    if reg_name.lower() == model_name.lower():
+                        return entry.get("database", "default")
+
+        return "default"
+
+    def db_for_read(self, model: Any, **hints: Any) -> str:
+        """Determines which database to use for read operations."""
+        if (
+            hasattr(model, "_meta")
+            and model._meta.app_label in self.DJANGO_INTERNAL_APPS
+        ):
+            return "default"
+        return self._get_target_db(model)
+
+    def db_for_write(self, model: Any, **hints: Any) -> str:
+        """Determines which database to use for write operations."""
+        if (
+            hasattr(model, "_meta")
+            and model._meta.app_label in self.DJANGO_INTERNAL_APPS
+        ):
+            return "default"
+        return self._get_target_db(model)
+
+    def allow_relation(self, obj1: Any, obj2: Any, **hints: Any) -> bool | None:
+        """
+        Determines if a relationship between obj1 and obj2 is allowed.
+        Allowed if both objects belong to the same database.
+        """
+        db1 = self._get_target_db(obj1)
+        db2 = self._get_target_db(obj2)
+        if db1 == db2:
+            return True
+        return None
+
+    def allow_migrate(
+        self, db: str, app_label: str, model_name: str | None = None, **hints: Any
+    ) -> bool:
+        """
+        Determines if a migration is allowed to run on the specified database.
+        App-level dependency checks return True to satisfy multi-db migration plans.
+        Model-level table creation is routed strictly to the designated database.
+        """
+        # Internal DataMan models always live on 'default' database
+        if model_name and model_name.lower() in ("apitoken", "apilog", "auditlog"):
+            return db == "default"
+
+        # Lookup model in TABLE_REGISTRY
+        target_name = (model_name or "").lower()
+        model = hints.get("model")
+        if model:
+            target_name = getattr(model, "__name__", "") or getattr(
+                model._meta, "object_name", target_name
+            )
+            target_name = target_name.lower()
+
+        if target_name:
+            with contextlib.suppress(Exception):
+                from dataman.core.models import TABLE_REGISTRY
+
+                for reg_name, entry in TABLE_REGISTRY.items():
+                    if (
+                        reg_name.lower() == target_name
+                        or entry.get("table_name", "").lower() == target_name
+                    ):
+                        target_db = entry.get("database", "default")
+                        return db == target_db
+
+        if model:
+            target_db = self._get_target_db(model)
+            return db == target_db
+
+        return True

dataman/core/signals.py

@@ -0,0 +1,56 @@
+from django.contrib.auth.signals import (
+    user_logged_in,
+    user_logged_out,
+    user_login_failed,
+)
+from django.dispatch import receiver
+
+from .audit import log_audit_event
+
+
+@receiver(user_logged_in)
+def audit_user_logged_in(sender, request, user, **kwargs):
+    """Audit successful user login."""
+    log_audit_event(
+        event_type="ADMIN_LOGIN_SUCCESS",
+        actor=getattr(user, "username", str(user)),
+        request=request,
+        details={
+            "email": getattr(user, "email", ""),
+            "is_superuser": getattr(user, "is_superuser", False),
+        },
+        severity="INFO",
+        status_code=200,
+    )
+
+
+@receiver(user_login_failed)
+def audit_user_login_failed(sender, credentials, request, **kwargs):
+    """Audit failed login attempt."""
+    username = (
+        credentials.get("username", "<unknown>")
+        if isinstance(credentials, dict)
+        else "<unknown>"
+    )
+    log_audit_event(
+        event_type="ADMIN_LOGIN_FAILED",
+        actor=username,
+        request=request,
+        details={"attempted_username": username},
+        severity="WARNING",
+        status_code=401,
+    )
+
+
+@receiver(user_logged_out)
+def audit_user_logged_out(sender, request, user, **kwargs):
+    """Audit user logout."""
+    if user:
+        log_audit_event(
+            event_type="ADMIN_LOGOUT",
+            actor=getattr(user, "username", str(user)),
+            request=request,
+            details={"email": getattr(user, "email", "")},
+            severity="INFO",
+            status_code=200,
+        )