dataman-engine 0.1.0__py3-none-any.whl

dataman/core/apps.py

@@ -0,0 +1,9 @@
+from django.apps import AppConfig
+
+
+class DatamanCoreConfig(AppConfig):
+    name = "dataman.core"
+    label = "dataman"
+
+    def ready(self):
+        import dataman.core.signals  # noqa: F401

dataman/core/audit.py

@@ -0,0 +1,171 @@
+import contextlib
+import json
+import logging
+from typing import Any
+
+audit_logger = logging.getLogger("dataman.audit")
+
+SENSITIVE_EXACT_KEYS = {
+    "password",
+    "passwd",
+    "secret",
+    "token",
+    "auth",
+    "authorization",
+    "credit_card",
+    "api_key",
+    "access_token",
+    "refresh_token",
+    "hash",
+    "hashed_secret",
+    "private_key",
+    "secret_key",
+    "cvv",
+    "cvv2",
+    "pin",
+    "ssn",
+    "social_security",
+    "card_number",
+    "passcode",
+}
+
+SAFE_KEY_SUFFIXES = (
+    "_id",
+    "_name",
+    "_prefix",
+    "_type",
+    "_count",
+    "_scopes",
+    "_status",
+    "_at",
+)
+
+
+def is_sensitive_key(key: str) -> bool:
+    """Checks if a field key corresponds to a secret credential or sensitive PII."""
+    k_lower = key.lower()
+    if any(k_lower.endswith(suffix) for suffix in SAFE_KEY_SUFFIXES):
+        return False
+    if k_lower in SENSITIVE_EXACT_KEYS:
+        return True
+    if any(
+        s_key in k_lower
+        for s_key in ("password", "secret", "credit_card", "api_key", "authorization")
+    ):
+        return True
+    return k_lower.endswith(("_token", "_secret", "_key"))
+
+
+def sanitize_payload(data: Any) -> Any:
+    """Recursively masks sensitive keys in payloads and dictionaries."""
+    if isinstance(data, dict):
+        sanitized = {}
+        for k, v in data.items():
+            if is_sensitive_key(k):
+                sanitized[k] = "********"
+            else:
+                sanitized[k] = sanitize_payload(v)
+        return sanitized
+    elif isinstance(data, list):
+        return [sanitize_payload(item) for item in data]
+    return data
+
+
+def get_client_ip(request: Any) -> str | None:
+    """Resolves client IP address taking proxies and load balancers into account."""
+    if not request:
+        return None
+
+    # Check Cloudflare
+    cf_ip = request.META.get("HTTP_CF_CONNECTING_IP")
+    if cf_ip:
+        return cf_ip.strip()
+
+    # Check X-Forwarded-For
+    x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR")
+    if x_forwarded_for:
+        return x_forwarded_for.split(",")[0].strip()
+
+    # Check X-Real-IP
+    x_real_ip = request.META.get("HTTP_X_REAL_IP")
+    if x_real_ip:
+        return x_real_ip.strip()
+
+    return request.META.get("REMOTE_ADDR")
+
+
+def get_user_agent(request: Any) -> str:
+    """Extracts User-Agent string from request headers."""
+    if not request:
+        return ""
+    return request.META.get("HTTP_USER_AGENT", "")[:500]
+
+
+def log_audit_event(
+    event_type: str,
+    actor: str | None = None,
+    ip_address: str | None = None,
+    user_agent: str | None = None,
+    details: dict | None = None,
+    severity: str = "INFO",
+    status_code: int | None = None,
+    request: Any = None,
+) -> Any | None:
+    """
+    Enterprise Audit Logging function:
+    1. Extracts actor, IP, and User-Agent from request if available.
+    2. Redacts sensitive PII keys.
+    3. Persists record in database AuditLog model.
+    4. Emits structured JSON log via logging.getLogger('dataman.audit').
+    """
+    if request:
+        if not ip_address:
+            ip_address = get_client_ip(request)
+        if not user_agent:
+            user_agent = get_user_agent(request)
+        if not actor:
+            if (
+                hasattr(request, "auth")
+                and request.auth
+                and hasattr(request.auth, "prefix")
+            ):
+                actor = f"Token:{request.auth.prefix}"
+            else:
+                user = getattr(request, "user", None)
+                if user and getattr(user, "is_authenticated", False):
+                    actor = str(user)
+
+    actor = actor or "System"
+    sanitized_details = sanitize_payload(details or {})
+
+    # 1. Structured log stream
+    log_payload = {
+        "event_type": event_type,
+        "actor": actor,
+        "ip_address": ip_address,
+        "user_agent": user_agent,
+        "status_code": status_code,
+        "severity": severity,
+        "details": sanitized_details,
+    }
+
+    log_level = getattr(logging, severity.upper(), logging.INFO)
+    with contextlib.suppress(Exception):
+        audit_logger.log(log_level, json.dumps(log_payload, default=str))
+
+    # 2. Database model persistence
+    try:
+        from dataman.core.models import AuditLog
+
+        return AuditLog.objects.create(
+            event_type=event_type,
+            actor=actor,
+            ip_address=ip_address,
+            user_agent=user_agent or "",
+            status_code=status_code,
+            severity=severity.upper(),
+            details=sanitized_details,
+        )
+    except Exception as e:
+        audit_logger.error(f"Failed to persist AuditLog to DB: {e}")
+        return None

dataman/core/auth.py

@@ -0,0 +1,106 @@
+import hashlib
+
+from django.utils import timezone
+from rest_framework import authentication, exceptions
+
+from dataman.core.audit import log_audit_event
+from dataman.core.models import APIToken
+
+
+class ServiceUser:
+    """A dummy user class to represent a Service Token."""
+
+    is_authenticated = True
+    is_active = True
+    is_staff = False
+    is_superuser = False
+
+    def __str__(self):
+        return "ServiceTokenUser"
+
+
+class ServiceTokenAuthentication(authentication.BaseAuthentication):
+    """Authenticates requests using the dataman APIToken model."""
+
+    def authenticate(self, request):
+        auth = request.headers.get("Authorization", "").split()
+
+        if not auth or auth[0].lower() != "token":
+            return None
+
+        if len(auth) == 1 or len(auth) > 2:
+            raise exceptions.AuthenticationFailed(
+                "Invalid token header. No credentials provided."
+            )
+
+        prefix = ""
+        try:
+            prefix, secret = auth[1].split("_", 1)
+            token = APIToken.objects.get(prefix=prefix)
+
+            hashed_input = hashlib.sha256(secret.encode()).hexdigest()
+            if hashed_input != token.hashed_secret:
+                log_audit_event(
+                    event_type="TOKEN_AUTH_FAILED",
+                    actor=f"Token:{prefix}",
+                    request=request,
+                    details={"prefix": prefix, "reason": "Secret hash mismatch"},
+                    severity="WARNING",
+                    status_code=401,
+                )
+                raise exceptions.AuthenticationFailed("Invalid token.")
+
+            # Check if token is active
+            if not getattr(token, "is_active", True):
+                log_audit_event(
+                    event_type="TOKEN_AUTH_FAILED",
+                    actor=f"Token:{prefix}",
+                    request=request,
+                    details={
+                        "prefix": prefix,
+                        "token_name": token.name,
+                        "reason": "Token is disabled",
+                    },
+                    severity="WARNING",
+                    status_code=401,
+                )
+                raise exceptions.AuthenticationFailed("Token is inactive.")
+
+            # Check if token has expired
+            if token.expires_at and timezone.now() > token.expires_at:
+                log_audit_event(
+                    event_type="TOKEN_EXPIRED",
+                    actor=f"Token:{prefix}",
+                    request=request,
+                    details={
+                        "prefix": prefix,
+                        "token_name": token.name,
+                        "expired_at": token.expires_at.isoformat(),
+                    },
+                    severity="WARNING",
+                    status_code=401,
+                )
+                raise exceptions.AuthenticationFailed("Token has expired.")
+
+        except (ValueError, APIToken.DoesNotExist) as e:
+            log_audit_event(
+                event_type="TOKEN_AUTH_FAILED",
+                actor=f"Token:{prefix}" if prefix else "Unknown",
+                request=request,
+                details={
+                    "prefix": prefix,
+                    "reason": "Token prefix not found or invalid format",
+                },
+                severity="WARNING",
+                status_code=401,
+            )
+            raise exceptions.AuthenticationFailed("Invalid token.") from e
+
+        return (ServiceUser(), token)
+
+
+class CsrfExemptSessionAuthentication(authentication.SessionAuthentication):
+    """SessionAuthentication that does not enforce CSRF checks for internal dashboard APIs."""
+
+    def enforce_csrf(self, request):
+        return  # Skip CSRF check for authenticated dashboard session

dataman/core/crypto.py

@@ -0,0 +1,173 @@
+import abc
+import base64
+import contextlib
+import os
+from typing import Any
+
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+
+
+class CryptoError(Exception):
+    """Base exception for all DataMan cryptographic operations."""
+
+    pass
+
+
+class DecryptionError(CryptoError):
+    """Raised when ciphertext decryption fails due to bad key, corruption, or tampering."""
+
+    pass
+
+
+class BaseCryptoProvider(abc.ABC):
+    """Abstract base class for all DataMan encryption providers."""
+
+    @abc.abstractmethod
+    def encrypt(self, plaintext: Any, context: dict | None = None) -> str:
+        """Encrypts plaintext string/bytes and returns a formatted ciphertext string."""
+        pass
+
+    @abc.abstractmethod
+    def decrypt(self, ciphertext: str, context: dict | None = None) -> str:
+        """Decrypts a formatted ciphertext string and returns plaintext string."""
+        pass
+
+
+class AESGCMCryptoProvider(BaseCryptoProvider):
+    """
+    Production AES-256-GCM authenticated encryption provider.
+
+    Generates 96-bit random nonces for every encryption call and produces
+    versioned, URL-safe base64 encoded ciphertexts prefixed with 'v1:'.
+    """
+
+    CIPHER_PREFIX = "v1:"
+    NONCE_BYTES = 12  # 96-bit standard nonce for AES-GCM
+    KEY_LENGTH_BYTES = 32  # 256-bit AES key
+
+    def __init__(self, key: bytes | str | None = None):
+        self._key = self._resolve_key(key)
+        self._aesgcm = AESGCM(self._key)
+
+    def _resolve_key(self, key: bytes | str | None = None) -> bytes:
+        if isinstance(key, bytes) and len(key) == self.KEY_LENGTH_BYTES:
+            return key
+
+        # Raw secret material
+        secret_source: bytes = b""
+        if isinstance(key, str) and key:
+            secret_source = key.encode("utf-8")
+        elif isinstance(key, bytes) and key:
+            secret_source = key
+        else:
+            # Check environment variables or Django settings
+            env_key = os.getenv("DATAMAN_ENCRYPTION_KEY")
+            if env_key:
+                secret_source = env_key.encode("utf-8")
+            else:
+                secret_key = os.getenv(
+                    "DATAMAN_SECRET_KEY", "default-insecure-key-change-me"
+                )
+                with contextlib.suppress(Exception):
+                    from django.conf import settings
+
+                    if settings.configured and getattr(settings, "SECRET_KEY", None):
+                        secret_key = settings.SECRET_KEY
+                secret_source = secret_key.encode("utf-8")
+
+        # Derive a cryptographically strong 256-bit key using HKDF-SHA256
+        hkdf = HKDF(
+            algorithm=hashes.SHA256(),
+            length=self.KEY_LENGTH_BYTES,
+            salt=b"dataman-field-level-encryption-salt",
+            info=b"dataman-aes256-gcm-key-derivation",
+        )
+        return hkdf.derive(secret_source)
+
+    def encrypt(self, plaintext: Any, context: dict | None = None) -> str:
+        if plaintext is None:
+            return None
+
+        if isinstance(plaintext, bytes):
+            data_bytes = plaintext
+        elif isinstance(plaintext, str):
+            data_bytes = plaintext.encode("utf-8")
+        else:
+            data_bytes = str(plaintext).encode("utf-8")
+
+        nonce = os.urandom(self.NONCE_BYTES)
+        associated_data = None
+        if context:
+            # Sort keys for deterministic bytes representation
+            associated_data = str(sorted(context.items())).encode("utf-8")
+
+        encrypted = self._aesgcm.encrypt(nonce, data_bytes, associated_data)
+        b64_payload = base64.urlsafe_b64encode(nonce + encrypted).decode("ascii")
+        return f"{self.CIPHER_PREFIX}{b64_payload}"
+
+    def decrypt(self, ciphertext: str, context: dict | None = None) -> str:
+        if ciphertext is None:
+            return None
+
+        if not isinstance(ciphertext, str):
+            return str(ciphertext)
+
+        # If not starting with our prefix, return as-is (graceful unencrypted handling)
+        if not ciphertext.startswith(self.CIPHER_PREFIX):
+            return ciphertext
+
+        b64_payload = ciphertext[len(self.CIPHER_PREFIX) :]
+        try:
+            raw = base64.urlsafe_b64decode(b64_payload.encode("ascii"))
+            if len(raw) < self.NONCE_BYTES + 16:  # 12-byte nonce + 16-byte GCM tag
+                raise DecryptionError("Ciphertext payload is truncated or malformed.")
+
+            nonce = raw[: self.NONCE_BYTES]
+            encrypted_payload = raw[self.NONCE_BYTES :]
+
+            associated_data = None
+            if context:
+                associated_data = str(sorted(context.items())).encode("utf-8")
+
+            decrypted_bytes = self._aesgcm.decrypt(
+                nonce, encrypted_payload, associated_data
+            )
+            return decrypted_bytes.decode("utf-8")
+        except DecryptionError:
+            raise
+        except Exception as e:
+            raise DecryptionError(
+                f"Decryption failed: data was tampered with or invalid key used ({e})"
+            ) from e
+
+
+_default_provider: BaseCryptoProvider | None = None
+
+
+def get_crypto_provider() -> BaseCryptoProvider:
+    """Returns the globally configured Crypto Provider instance."""
+    global _default_provider
+    if _default_provider is not None:
+        return _default_provider
+
+    with contextlib.suppress(Exception):
+        import config
+
+        if hasattr(config, "CRYPTO_PROVIDER") and config.CRYPTO_PROVIDER:
+            if isinstance(config.CRYPTO_PROVIDER, BaseCryptoProvider):
+                _default_provider = config.CRYPTO_PROVIDER
+                return _default_provider
+            elif callable(config.CRYPTO_PROVIDER):
+                _default_provider = config.CRYPTO_PROVIDER()
+                return _default_provider
+
+    _default_provider = AESGCMCryptoProvider()
+    return _default_provider
+
+
+def reset_crypto_provider():
+    """Resets the singleton crypto provider (useful for testing)."""
+    global _default_provider
+    _default_provider = None

dataman/core/fields.py

@@ -0,0 +1,120 @@
+import json
+from typing import Any
+
+from django.db import models
+
+from dataman.core.crypto import get_crypto_provider
+
+
+class EncryptedFieldMixin:
+    """Mixin for transparent field-level encryption on Django model fields."""
+
+    def __init__(self, *args, **kwargs):
+        # Encrypted ciphertext strings can exceed typical CharField lengths
+        if "max_length" in kwargs and kwargs["max_length"] is not None:
+            # Ensure sufficient DB column width for base64 nonce + ciphertext + tag
+            kwargs["max_length"] = max(kwargs["max_length"], 512)
+        super().__init__(*args, **kwargs)
+
+    def from_db_value(self, value: str | None, expression, connection) -> Any:
+        if value is None:
+            return None
+        return self._decrypt_value(value)
+
+    def to_python(self, value: Any) -> Any:
+        if value is None:
+            return None
+        if isinstance(value, str) and value.startswith("v1:"):
+            return self._decrypt_value(value)
+        return super().to_python(value)
+
+    def get_prep_value(self, value: Any) -> str | None:
+        value = super().get_prep_value(value)
+        if value is None:
+            return None
+        if isinstance(value, str) and value.startswith("v1:"):
+            return value
+        return self._encrypt_value(value)
+
+    def _encrypt_value(self, raw_value: Any) -> str:
+        provider = get_crypto_provider()
+        val_str = str(raw_value)
+        return provider.encrypt(val_str)
+
+    def _decrypt_value(self, cipher_value: str) -> Any:
+        provider = get_crypto_provider()
+        decrypted = provider.decrypt(cipher_value)
+        return super().to_python(decrypted)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        # Point to dataman.core.fields
+        if path.startswith("dataman.core.fields"):
+            pass
+        return name, path, args, kwargs
+
+
+class EncryptedCharField(EncryptedFieldMixin, models.CharField):
+    """CharField that is encrypted in the database using AES-256-GCM."""
+
+    def __init__(self, *args, **kwargs):
+        if "max_length" not in kwargs or kwargs["max_length"] is None:
+            kwargs["max_length"] = 512
+        super().__init__(*args, **kwargs)
+
+
+class EncryptedTextField(EncryptedFieldMixin, models.TextField):
+    """TextField that is encrypted in the database using AES-256-GCM."""
+
+    pass
+
+
+class EncryptedEmailField(EncryptedFieldMixin, models.EmailField):
+    """EmailField that is encrypted in the database using AES-256-GCM."""
+
+    def __init__(self, *args, **kwargs):
+        if "max_length" not in kwargs or kwargs["max_length"] is None:
+            kwargs["max_length"] = 512
+        super().__init__(*args, **kwargs)
+
+
+class EncryptedJSONField(EncryptedFieldMixin, models.TextField):
+    """JSONField stored as encrypted text in the database."""
+
+    def from_db_value(self, value: str | None, expression, connection) -> Any:
+        if value is None:
+            return None
+        decrypted_str = self._decrypt_value(value)
+        if decrypted_str is None or decrypted_str == "":
+            return None
+        try:
+            return json.loads(decrypted_str)
+        except Exception:
+            return decrypted_str
+
+    def to_python(self, value: Any) -> Any:
+        if value is None:
+            return None
+        if isinstance(value, (dict, list)):
+            return value
+        if isinstance(value, str) and value.startswith("v1:"):
+            decrypted_str = self._decrypt_value(value)
+            try:
+                return json.loads(decrypted_str)
+            except Exception:
+                return decrypted_str
+        if isinstance(value, str):
+            try:
+                return json.loads(value)
+            except Exception:
+                return value
+        return value
+
+    def get_prep_value(self, value: Any) -> str | None:
+        if value is None:
+            return None
+        if isinstance(value, str) and value.startswith("v1:"):
+            return value
+        if not isinstance(value, str):
+            value = json.dumps(value)
+        return self._encrypt_value(value)

dataman/core/masking.py

@@ -0,0 +1,107 @@
+from collections.abc import Callable
+from typing import Any
+
+
+def mask_partial(val: Any, keep_last: int = 4, mask_char: str = "*") -> str:
+    """
+    Masks all alphanumeric characters except the last `keep_last` characters,
+    preserving formatting characters such as hyphens and spaces.
+    Example: '123-45-6789' -> '***-**-6789'
+    """
+    if val is None:
+        return None
+    s = str(val)
+    if not s:
+        return s
+
+    # Find indices of alphanumeric characters
+    alnum_indices = [i for i, c in enumerate(s) if c.isalnum()]
+    if len(alnum_indices) <= keep_last:
+        return mask_char * len(s)
+
+    mask_set = set(alnum_indices[:-keep_last])
+    return "".join(mask_char if i in mask_set else c for i, c in enumerate(s))
+
+
+def mask_last4(val: Any, mask_char: str = "*") -> str:
+    """
+    Masks payment cards / identifiers, retaining only the last 4 digits.
+    Example: '4111-2222-3333-4444' -> '****-****-****-4444'
+    """
+    return mask_partial(val, keep_last=4, mask_char=mask_char)
+
+
+def mask_email(val: Any, mask_char: str = "*") -> str:
+    """
+    Masks email addresses while retaining the domain and outer username boundaries.
+    Example: 'vikash@example.com' -> 'v***h@example.com'
+    """
+    if val is None:
+        return None
+    s = str(val).strip()
+    if "@" not in s:
+        return mask_partial(s, keep_last=2, mask_char=mask_char)
+
+    parts = s.split("@", 1)
+    username, domain = parts[0], parts[1]
+
+    if len(username) <= 1:
+        masked_user = mask_char * 3
+    elif len(username) == 2:
+        masked_user = f"{username[0]}{mask_char * 3}"
+    elif len(username) <= 4:
+        masked_user = f"{username[0]}{mask_char * 3}{username[-1]}"
+    else:
+        masked_user = f"{username[0]}{mask_char * (len(username) - 2)}{username[-1]}"
+
+    return f"{masked_user}@{domain}"
+
+
+def mask_phone(val: Any, mask_char: str = "*") -> str:
+    """
+    Masks telephone numbers, keeping the last 4 digits and country prefix if present.
+    Example: '+1 (555) 234-5678' -> '+1 (***) ***-5678'
+    """
+    if val is None:
+        return None
+    s = str(val)
+    return mask_partial(s, keep_last=4, mask_char=mask_char)
+
+
+def mask_full(val: Any, mask_char: str = "*") -> str:
+    """
+    Completely masks the value with asterisks.
+    Example: 'sensitive_secret' -> '********'
+    """
+    if val is None:
+        return None
+    return mask_char * 8
+
+
+STRATEGY_MAP: dict[str, Callable[[Any], str]] = {
+    "partial": mask_partial,
+    "ssn": mask_partial,
+    "last4": mask_last4,
+    "card": mask_last4,
+    "credit_card": mask_last4,
+    "email": mask_email,
+    "email_mask": mask_email,
+    "phone": mask_phone,
+    "full": mask_full,
+    "redact": mask_full,
+}
+
+
+def mask_value(val: Any, strategy: str | Callable[[Any], str] = "partial") -> Any:
+    """
+    Applies the requested masking strategy to a value.
+    """
+    if val is None or val == "":
+        return val
+
+    if callable(strategy):
+        return strategy(val)
+
+    strategy_key = str(strategy).lower().strip()
+    mask_fn = STRATEGY_MAP.get(strategy_key, mask_partial)
+    return mask_fn(val)