oxutils 0.1.0__py3-none-any.whl

oxutils/__init__.py

@@ -0,0 +1,23 @@
+"""OxUtils - Production-ready utilities for Django applications.
+
+This package provides:
+- JWT authentication with JWKS support
+- S3 storage backends (static, media, private, logs)
+- Structured logging with correlation IDs
+- Audit system with S3 export
+- Celery integration
+- Django model mixins
+- Custom exceptions
+"""
+
+__version__ = "0.1.0"
+
+from oxutils.settings import oxi_settings
+from oxutils.conf import UTILS_APPS, AUDIT_MIDDLEWARE
+
+__all__ = [
+    "oxi_settings",
+    "UTILS_APPS",
+    "AUDIT_MIDDLEWARE",
+    "__version__",
+]

oxutils/apps.py

@@ -0,0 +1,14 @@
+from django.apps import AppConfig
+from django.utils.translation import gettext_lazy as _
+
+
+
+class OxutilsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'oxutils'
+    verbose_name = _("Oxiliere Utilities")
+
+    def ready(self):
+        import oxutils.logger.receivers
+        
+        return super().ready()

oxutils/audit/apps.py

@@ -0,0 +1,12 @@
+from django.apps import AppConfig
+from django.utils.translation import gettext_lazy as _
+
+
+
+class OxutilsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'oxutils_export'
+    verbose_name = _("Oxutils Export")
+
+    def ready(self):
+        return super().ready()

oxutils/audit/export.py

@@ -0,0 +1,229 @@
+"""
+Export utilities for audit logs.
+
+This module provides utilities to export LogEntry records from auditlog,
+compress them into ZIP files, and save them to LogExportState.
+"""
+import io
+import json
+import zipfile
+from datetime import datetime
+from typing import Optional
+
+from django.apps import apps
+from django.core.files.base import ContentFile
+from django.db.models import QuerySet
+from django.utils import timezone
+
+from oxutils.audit.models import LogExportState
+
+
+def get_logentry_model():
+    """Get the LogEntry model from auditlog."""
+    return apps.get_model('auditlog', 'LogEntry')
+
+
+def export_logs_from_date(
+    from_date: datetime,
+    to_date: Optional[datetime] = None,
+    batch_size: int = 5000
+) -> LogExportState:
+    """
+    Export audit logs from a specific date, compress them, and save to LogExportState.
+    Optimized for S3 storage with streaming and minimal memory usage.
+    
+    Args:
+        from_date: Start date for log export (inclusive)
+        to_date: End date for log export (inclusive). If None, uses current time.
+        batch_size: Number of records to process at a time (default: 5000)
+        
+    Returns:
+        LogExportState: The created export state with the compressed data
+        
+    Raises:
+        Exception: If export fails, the LogExportState status will be set to FAILED
+    """
+    if to_date is None:
+        to_date = timezone.now()
+    
+    # Create the export state
+    export_state = LogExportState.create(size=0)
+    
+    try:
+        # Get LogEntry model
+        LogEntry = get_logentry_model()
+        
+        # Query logs within date range - use select_related for optimization
+        logs_queryset = LogEntry.objects.filter(
+            timestamp__gte=from_date,
+            timestamp__lte=to_date
+        ).select_related('content_type', 'actor').order_by('timestamp')
+        
+        # Create ZIP file in memory with optimal compression
+        zip_buffer = io.BytesIO()
+        
+        with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED, compresslevel=6) as zip_file:
+            # Export logs in batches using iterator to avoid loading all in memory
+            total_exported = 0
+            batch_number = 1
+            batch_logs = []
+            
+            # Use iterator() to stream results from database
+            for log in logs_queryset.iterator(chunk_size=batch_size):
+                batch_logs.append(_serialize_log_entry(log))
+                
+                # Write batch when it reaches batch_size
+                if len(batch_logs) >= batch_size:
+                    filename = f'logs_batch_{batch_number:04d}.json'
+                    # Use separators to minimize JSON size
+                    zip_file.writestr(
+                        filename, 
+                        json.dumps(batch_logs, separators=(',', ':'))
+                    )
+                    total_exported += len(batch_logs)
+                    batch_number += 1
+                    batch_logs = []
+            
+            # Write remaining logs
+            if batch_logs:
+                filename = f'logs_batch_{batch_number:04d}.json'
+                zip_file.writestr(
+                    filename, 
+                    json.dumps(batch_logs, separators=(',', ':'))
+                )
+                total_exported += len(batch_logs)
+            
+            # Export metadata at the end (we now have accurate count)
+            metadata = {
+                'export_date': timezone.now().isoformat(),
+                'from_date': from_date.isoformat(),
+                'to_date': to_date.isoformat(),
+                'total_records': total_exported,
+                'batch_size': batch_size,
+                'total_batches': batch_number
+            }
+            zip_file.writestr('metadata.json', json.dumps(metadata, separators=(',', ':')))
+        
+        # Get the ZIP file size and content
+        zip_buffer.seek(0)
+        zip_content = zip_buffer.getvalue()
+        zip_size = len(zip_content)
+        
+        # Save to LogExportState - S3 upload happens here
+        filename = f'audit_logs_{from_date.strftime("%Y%m%d")}_{to_date.strftime("%Y%m%d")}.zip'
+        export_state.data.save(filename, ContentFile(zip_content), save=False)
+        export_state.size = zip_size
+        export_state.set_success()
+        
+        return export_state
+        
+    except Exception as e:
+        export_state.set_failed()
+        raise e
+
+
+def _serialize_log_entry(log) -> dict:
+    """
+    Serialize a single LogEntry to a dictionary (optimized version).
+    
+    Args:
+        log: LogEntry object
+        
+    Returns:
+        dict: Serialized log entry with minimal overhead
+    """
+    return {
+        'id': log.id,
+        'timestamp': log.timestamp.isoformat() if log.timestamp else None,
+        'action': log.action,
+        'content_type': {
+            'app_label': log.content_type.app_label if log.content_type else None,
+            'model': log.content_type.model if log.content_type else None,
+        } if log.content_type else None,
+        'object_pk': log.object_pk,
+        'object_repr': log.object_repr,
+        'changes': log.changes,
+        'actor': {
+            'id': log.actor.id if log.actor else None,
+            'username': str(log.actor) if log.actor else None,
+        } if log.actor else None,
+        'remote_addr': log.remote_addr,
+        'additional_data': log.additional_data if hasattr(log, 'additional_data') else None,
+    }
+
+
+def serialize_log_entries(queryset: QuerySet) -> list:
+    """
+    Serialize LogEntry queryset to a list of dictionaries.
+    
+    Args:
+        queryset: QuerySet of LogEntry objects
+        
+    Returns:
+        list: List of serialized log entries
+    """
+    return [_serialize_log_entry(log) for log in queryset]
+
+
+def export_logs_since_last_export(batch_size: int = 5000) -> Optional[LogExportState]:
+    """
+    Export logs since the last successful export.
+    
+    Args:
+        batch_size: Number of records to process at a time
+        
+    Returns:
+        LogExportState or None: The created export state, or None if no previous export exists
+    """
+    # Get the last successful export
+    last_export = LogExportState.objects.filter(
+        status='success',
+        last_export_date__isnull=False
+    ).order_by('-last_export_date').first()
+    
+    if last_export:
+        from_date = last_export.last_export_date
+    else:
+        # If no previous export, start from the earliest log
+        LogEntry = get_logentry_model()
+        earliest_log = LogEntry.objects.order_by('timestamp').first()
+        
+        if not earliest_log:
+            return None
+        
+        from_date = earliest_log.timestamp
+    
+    return export_logs_from_date(from_date=from_date, batch_size=batch_size)
+
+
+def get_export_statistics() -> dict:
+    """
+    Get statistics about log exports.
+    
+    Returns:
+        dict: Statistics including total exports, successful exports, failed exports, etc.
+    """
+    from oxutils.enums.audit import ExportStatus
+    
+    total_exports = LogExportState.objects.count()
+    successful_exports = LogExportState.objects.filter(status=ExportStatus.SUCCESS).count()
+    failed_exports = LogExportState.objects.filter(status=ExportStatus.FAILED).count()
+    pending_exports = LogExportState.objects.filter(status=ExportStatus.PENDING).count()
+    
+    last_export = LogExportState.objects.filter(
+        status=ExportStatus.SUCCESS
+    ).order_by('-last_export_date').first()
+    
+    total_size = sum(
+        export.size for export in LogExportState.objects.filter(status=ExportStatus.SUCCESS)
+    )
+    
+    return {
+        'total_exports': total_exports,
+        'successful_exports': successful_exports,
+        'failed_exports': failed_exports,
+        'pending_exports': pending_exports,
+        'last_export_date': last_export.last_export_date if last_export else None,
+        'total_size_bytes': total_size,
+        'total_size_mb': round(total_size / (1024 * 1024), 2),
+    }

oxutils/audit/masks.py

@@ -0,0 +1,97 @@
+# Auditlog masks
+
+
+def number_mask(value: str) -> str:
+    """Mask a number showing only the last 4 digits.
+    
+    Args:
+        value: The number string to mask
+        
+    Returns:
+        Masked string with format: ****1234
+    """
+    if not value:
+        return ""
+    
+    value_str = str(value).strip()
+    if len(value_str) <= 4:
+        return "*" * len(value_str)
+    
+    return "****" + value_str[-4:]
+
+
+def phone_number_mask(value: str) -> str:
+    """Mask a phone number showing only the last 4 digits.
+    
+    Args:
+        value: The phone number to mask
+        
+    Returns:
+        Masked phone number with format: ****1234
+    """
+    if not value:
+        return ""
+    
+    # Remove common phone number formatting characters
+    cleaned = str(value).replace(" ", "").replace("-", "").replace("(", "").replace(")", "").replace("+", "")
+    
+    if len(cleaned) <= 4:
+        return "*" * len(cleaned)
+    
+    return "****" + cleaned[-4:]
+
+
+def credit_card_mask(value: str) -> str:
+    """Mask a credit card number showing only the last 4 digits.
+    
+    Args:
+        value: The credit card number to mask
+        
+    Returns:
+        Masked credit card with format: ****1234
+    """
+    if not value:
+        return ""
+    
+    # Remove spaces and dashes from credit card number
+    cleaned = str(value).replace(" ", "").replace("-", "")
+    
+    if len(cleaned) <= 4:
+        return "*" * len(cleaned)
+    
+    return "****" + cleaned[-4:]
+
+def email_mask(value: str) -> str:
+    """Mask an email address showing only the first character and domain.
+    
+    Args:
+        value: The email address to mask
+        
+    Returns:
+        Masked email with format: j***@example.com
+    """
+    if not value:
+        return ""
+    
+    value_str = str(value).strip()
+    
+    if "@" not in value_str:
+        # Not a valid email format, mask most of it
+        if len(value_str) <= 2:
+            return "*" * len(value_str)
+        return value_str[0] + "*" * (len(value_str) - 1)
+    
+    local, domain = value_str.rsplit("@", 1)
+    
+    if not local:
+        return "***@" + domain
+    
+    if len(local) == 1:
+        masked_local = "*"
+    elif len(local) == 2:
+        masked_local = local[0] + "*"
+    else:
+        masked_local = local[0] + "*" * (len(local) - 1)
+    
+    return masked_local + "@" + domain
+

oxutils/audit/models.py

@@ -0,0 +1,75 @@
+from django.utils.translation import gettext_lazy as _
+from django.utils import timezone
+from django.db import models, transaction
+from oxutils.enums.audit import ExportStatus
+from oxutils.models.base import TimestampMixin
+from oxutils.s3.storages import LogStorage
+
+
+
+
+class LogExportHistory(models.Model):
+    state = models.ForeignKey(
+        "LogExportState",
+        related_name='log_histories',
+        on_delete=models.CASCADE
+    )
+    status = models.CharField(
+        default=ExportStatus.PENDING,
+        choices=(
+            (ExportStatus.FAILED, _("Failed")),
+            (ExportStatus.PENDING, _('Pending')),
+            (ExportStatus.SUCCESS, _('Success'))
+        )
+    )
+    created_at = models.DateTimeField(
+        auto_now_add=True,
+        help_text="Date and time when this record was created"
+    )
+
+
+class LogExportState(TimestampMixin):
+    last_export_date = models.DateTimeField(null=True)
+    status = models.CharField(
+        default=ExportStatus.PENDING,
+        choices=(
+            (ExportStatus.FAILED, _("Failed")),
+            (ExportStatus.PENDING, _('Pending')),
+            (ExportStatus.SUCCESS, _('Success'))
+        )
+    )
+    data = models.FileField(storage=LogStorage())
+    size = models.BigIntegerField()
+
+    @classmethod
+    def create(cls, size: int = 0):
+        return cls.objects.create(
+            status=ExportStatus.PENDING,
+            size=size
+        )
+
+    @transaction.atomic
+    def set_success(self):
+        self.status = ExportStatus.SUCCESS
+        self.last_export_date = timezone.now()
+        LogExportHistory.objects.create(
+            state=self,
+            status=ExportStatus.SUCCESS
+        )
+        self.save(update_fields=(
+            'status',
+            'last_export_date',
+            'updated_at'
+        ))
+
+    @transaction.atomic
+    def set_failed(self):
+        self.status = ExportStatus.FAILED
+        self.save(update_fields=(
+            'status',
+            'updated_at'
+        ))
+        LogExportHistory.objects.create(
+            state=self,
+            status=ExportStatus.FAILED
+        )

oxutils/audit/settings.py

@@ -0,0 +1,19 @@
+# Oxiliere Audit settings
+
+AUDITLOG_DISABLE_REMOTE_ADDR = False
+AUDITLOG_MASK_TRACKING_FIELDS = (
+    "password",
+    "api_key",
+    "secret_token",
+    "token",
+)
+
+AUDITLOG_EXCLUDE_TRACKING_FIELDS = (
+    "created_at",
+    "updated_at",
+)
+
+CID_GENERATE = False
+
+AUDITLOG_CID_GETTER = "cid.locals.get_cid"
+AUDITLOG_LOGENTRY_MODEL =  "auditlog.LogEntry"

oxutils/celery/__init__.py

@@ -0,0 +1 @@
+from .base import celery_app

oxutils/celery/base.py

@@ -0,0 +1,98 @@
+import logging
+
+import structlog
+from celery import Celery
+from celery.signals import setup_logging
+from django.conf import settings
+from django_structlog.celery.steps import DjangoStructLogInitStep
+
+
+
+
+
+celery_app = Celery(getattr(settings, "CELERY_APP_NAME", 'oxiliere_celery'))
+
+celery_app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# A step to initialize django-structlog
+celery_app.steps['worker'].add(DjangoStructLogInitStep)
+
+celery_app.autodiscover_tasks()
+
+
+@setup_logging.connect
+def receiver_setup_logging(loglevel, logfile, format, colorize, **kwargs):  # pragma: no cover
+    logging.config.dictConfig(
+        {
+           "version": 1,
+           "disable_existing_loggers": False,
+           "formatters": {
+               "json_formatter": {
+                   "()": structlog.stdlib.ProcessorFormatter,
+                   "processor": structlog.processors.JSONRenderer(),
+               },
+               "plain_console": {
+                   "()": structlog.stdlib.ProcessorFormatter,
+                   "processor": structlog.dev.ConsoleRenderer(),
+               },
+               "key_value": {
+                   "()": structlog.stdlib.ProcessorFormatter,
+                   "processor": structlog.processors.KeyValueRenderer(
+                        key_order=['timestamp', 'level', 'event', 'logger']
+                    ),
+               },
+           },
+           "handlers": {
+               "console": {
+                   "class": "logging.StreamHandler",
+                   "formatter": "plain_console",
+                   'filters': ['correlation'],
+               },
+               "json_file": {
+                   "class": "logging.handlers.WatchedFileHandler",
+                   "filename": "logs/json.log",
+                   "formatter": "json_formatter",
+                   'filters': ['correlation'],
+               },
+               "flat_line_file": {
+                   "class": "logging.handlers.WatchedFileHandler",
+                   "filename": "logs/flat_line.log",
+                   "formatter": "key_value",
+                   'filters': ['correlation'],
+               },
+           },
+            'filters': {
+                'correlation': {
+                    '()': 'cid.log.CidContextFilter'
+                },
+            },
+           "loggers": {
+               "django_structlog": {
+                   "handlers": ["console", "flat_line_file", "json_file"],
+                   "level": "INFO",
+               },
+               "oxiliere_log": {
+                   "handlers": ["console", "flat_line_file", "json_file"],
+                   "level": "INFO",
+               },
+           }
+       }
+    )
+
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            structlog.stdlib.filter_by_level,
+            structlog.processors.TimeStamper(fmt="iso"),
+            structlog.stdlib.add_logger_name,
+            structlog.stdlib.add_log_level,
+            structlog.stdlib.PositionalArgumentsFormatter(),
+            structlog.processors.StackInfoRenderer(),
+            structlog.processors.format_exc_info,
+            structlog.processors.UnicodeDecoder(),
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        cache_logger_on_first_use=True,
+    )
+

oxutils/celery/settings.py

@@ -0,0 +1 @@
+CELERY_CACHE_BACKEND = 'default'

oxutils/conf.py

@@ -0,0 +1,12 @@
+UTILS_APPS = (
+    'django_structlog',
+    'auditlog',
+    'cid.apps.CidAppConfig',
+    'django_celery_results',
+)
+
+AUDIT_MIDDLEWARE = (
+    'cid.middleware.CidMiddleware',
+    'auditlog.middleware.AuditlogMiddleware',
+    'django_structlog.middlewares.RequestMiddleware',
+)

oxutils/enums/__init__.py

@@ -0,0 +1 @@
+from .invoices import InvoiceStatusEnum

oxutils/enums/audit.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+
+class ExportStatus(str, Enum):
+    FAILED = 'failed'
+    SUCCESS = 'success'
+    PENDING = 'pending'

oxutils/enums/invoices.py

@@ -0,0 +1,11 @@
+from enum import Enum
+
+
+
+class InvoiceStatusEnum(str, Enum):
+    DRAFT = "draft"
+    PENDING = "pending"
+    PAID = "paid"
+    OVERDUE = "overdue"
+    CANCELLED = "cancelled"
+    REFUNDED = 'refunded'