django-nativemojo 0.1.10__py3-none-any.whl

mojo/apps/fileman/management/commands/cleanup_expired_uploads.py

@@ -0,0 +1,222 @@
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
+from datetime import datetime, timedelta
+from django.db import transaction
+import logging
+
+from mojo.apps.fileman.models import File, FileManager
+from mojo.apps.fileman.backends import get_backend
+
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = 'Clean up expired file uploads and temporary files'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--days',
+            type=int,
+            default=1,
+            help='Clean up uploads older than this many days (default: 1)'
+        )
+        
+        parser.add_argument(
+            '--hours',
+            type=int,
+            help='Clean up uploads older than this many hours (overrides --days)'
+        )
+        
+        parser.add_argument(
+            '--dry-run',
+            action='store_true',
+            help='Show what would be cleaned up without actually doing it'
+        )
+        
+        parser.add_argument(
+            '--backend-cleanup',
+            action='store_true',
+            default=True,
+            help='Also run backend-specific cleanup (default: True)'
+        )
+        
+        parser.add_argument(
+            '--no-backend-cleanup',
+            action='store_false',
+            dest='backend_cleanup',
+            help='Skip backend-specific cleanup'
+        )
+        
+        parser.add_argument(
+            '--status',
+            choices=['pending', 'uploading', 'failed', 'expired', 'all'],
+            default='all',
+            help='Which upload statuses to clean up (default: all non-completed)'
+        )
+        
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Verbose output'
+        )
+
+    def handle(self, *args, **options):
+        self.dry_run = options['dry_run']
+        self.verbose = options['verbose']
+        
+        # Calculate cutoff date
+        if options['hours']:
+            cutoff_date = timezone.now() - timedelta(hours=options['hours'])
+            cutoff_desc = f"{options['hours']} hours"
+        else:
+            cutoff_date = timezone.now() - timedelta(days=options['days'])
+            cutoff_desc = f"{options['days']} days"
+        
+        if self.dry_run:
+            self.stdout.write(
+                self.style.WARNING(f"DRY RUN: Showing what would be cleaned up")
+            )
+        
+        self.stdout.write(
+            f"Cleaning up uploads older than {cutoff_desc} (before {cutoff_date})"
+        )
+        
+        # Clean up File records
+        files_cleaned = self.cleanup_files(cutoff_date, options['status'])
+        
+        # Clean up backend temporary files
+        if options['backend_cleanup']:
+            backends_cleaned = self.cleanup_backends(cutoff_date)
+        else:
+            backends_cleaned = 0
+        
+        # Summary
+        if self.dry_run:
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"DRY RUN COMPLETE: Would clean up {files_cleaned} file records "
+                    f"and {backends_cleaned} backend temporary files"
+                )
+            )
+        else:
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"CLEANUP COMPLETE: Cleaned up {files_cleaned} file records "
+                    f"and {backends_cleaned} backend temporary files"
+                )
+            )
+
+    def cleanup_files(self, cutoff_date, status_filter):
+        """Clean up File model records"""
+        
+        # Build query based on status filter
+        query = File.objects.filter(created__lt=cutoff_date)
+        
+        if status_filter == 'pending':
+            query = query.filter(upload_status=File.PENDING)
+        elif status_filter == 'uploading':
+            query = query.filter(upload_status=File.UPLOADING)
+        elif status_filter == 'failed':
+            query = query.filter(upload_status=File.FAILED)
+        elif status_filter == 'expired':
+            query = query.filter(upload_status=File.EXPIRED)
+        elif status_filter == 'all':
+            # Clean up all non-completed uploads
+            query = query.exclude(upload_status=File.COMPLETED)
+        
+        # Also include files with expired upload URLs
+        expired_url_query = File.objects.filter(
+            upload_expires_at__lt=timezone.now(),
+            upload_status__in=[File.PENDING, File.UPLOADING]
+        )
+        
+        # Combine queries
+        files_to_cleanup = query.union(expired_url_query).distinct()
+        
+        if self.verbose:
+            self.stdout.write(f"Found {files_to_cleanup.count()} file records to clean up")
+        
+        cleaned_count = 0
+        
+        for file_obj in files_to_cleanup:
+            if self.verbose:
+                self.stdout.write(
+                    f"  File: {file_obj.original_filename} "
+                    f"(status: {file_obj.upload_status}, "
+                    f"created: {file_obj.created})"
+                )
+            
+            if not self.dry_run:
+                try:
+                    with transaction.atomic():
+                        # Try to delete the actual file from storage if it exists
+                        if file_obj.file_path and file_obj.upload_status == File.COMPLETED:
+                            try:
+                                backend = get_backend(file_obj.file_manager)
+                                if backend.exists(file_obj.file_path):
+                                    backend.delete(file_obj.file_path)
+                                    if self.verbose:
+                                        self.stdout.write(f"    Deleted file from storage: {file_obj.file_path}")
+                            except Exception as e:
+                                logger.warning(f"Failed to delete file from storage: {e}")
+                                if self.verbose:
+                                    self.stdout.write(
+                                        self.style.WARNING(f"    Warning: Could not delete from storage: {e}")
+                                    )
+                        
+                        # Mark as expired or delete the record
+                        if file_obj.upload_status in [File.PENDING, File.UPLOADING]:
+                            file_obj.mark_as_expired()
+                            if self.verbose:
+                                self.stdout.write(f"    Marked as expired")
+                        else:
+                            file_obj.delete()
+                            if self.verbose:
+                                self.stdout.write(f"    Deleted record")
+                        
+                        cleaned_count += 1
+                        
+                except Exception as e:
+                    logger.error(f"Error cleaning up file {file_obj.id}: {e}")
+                    if self.verbose:
+                        self.stdout.write(
+                            self.style.ERROR(f"    Error: {e}")
+                        )
+            else:
+                cleaned_count += 1
+        
+        return cleaned_count
+
+    def cleanup_backends(self, cutoff_date):
+        """Run backend-specific cleanup"""
+        cleaned_count = 0
+        
+        # Get all active file managers
+        file_managers = FileManager.objects.filter(is_active=True)
+        
+        if self.verbose:
+            self.stdout.write(f"Running backend cleanup for {file_managers.count()} file managers")
+        
+        for file_manager in file_managers:
+            try:
+                backend = get_backend(file_manager)
+                
+                if self.verbose:
+                    self.stdout.write(
+                        f"  Cleaning up {file_manager.name} ({file_manager.backend_type})"
+                    )
+                
+                if not self.dry_run:
+                    backend.cleanup_expired_uploads(cutoff_date)
+                
+                cleaned_count += 1
+                
+            except Exception as e:
+                logger.error(f"Error running backend cleanup for {file_manager.name}: {e}")
+                if self.verbose:
+                    self.stdout.write(
+                        self.style.ERROR(f"    Error: {e}")
+                    )
+        
+        return cleaned_count

mojo/apps/fileman/models/__init__.py

@@ -0,0 +1,7 @@
+from .manager import FileManager
+from .file import File
+
+__all__ = [
+    'FileManager',
+    'File',
+]

mojo/apps/fileman/models/file.py

@@ -0,0 +1,292 @@
+from django.db import models
+from mojo.models import MojoModel
+import uuid
+import hashlib
+import mimetypes
+from datetime import datetime
+
+
+class File(models.Model, MojoModel):
+    """
+    File model representing uploaded files with metadata and storage information
+    """
+
+    class RestMeta:
+        CAN_SAVE = CAN_CREATE = True
+        CAN_DELETE = True
+        DEFAULT_SORT = "-created"
+        VIEW_PERMS = ["view_fileman"]
+        SEARCH_FIELDS = ["filename", "original_filename", "content_type"]
+        SEARCH_TERMS = [
+            "filename", "original_filename", "content_type",
+            ("group", "group__name"),
+            ("file_manager", "file_manager__name")]
+
+        GRAPHS = {
+            "default": {
+                "graphs": {
+                    "group": "basic",
+                    "file_manager": "basic",
+                    "uploaded_by": "basic"
+                }
+            },
+            "list": {
+                "graphs": {
+                    "group": "basic",
+                    "file_manager": "basic"
+                }
+            }
+        }
+
+    # Upload status choices
+    PENDING = 'pending'
+    UPLOADING = 'uploading'
+    COMPLETED = 'completed'
+    FAILED = 'failed'
+    EXPIRED = 'expired'
+
+    STATUS_CHOICES = [
+        (PENDING, 'Pending Upload'),
+        (UPLOADING, 'Uploading'),
+        (COMPLETED, 'Upload Completed'),
+        (FAILED, 'Upload Failed'),
+        (EXPIRED, 'Upload Expired'),
+    ]
+
+    created = models.DateTimeField(auto_now_add=True, editable=False, db_index=True)
+    modified = models.DateTimeField(auto_now=True)
+
+    group = models.ForeignKey(
+        "account.Group",
+        related_name="files",
+        null=True,
+        blank=True,
+        default=None,
+        on_delete=models.CASCADE,
+        help_text="Group that owns this file"
+    )
+
+    uploaded_by = models.ForeignKey(
+        "account.User",
+        related_name="uploaded_files",
+        null=True,
+        blank=True,
+        default=None,
+        on_delete=models.SET_NULL,
+        help_text="User who uploaded this file"
+    )
+
+    file_manager = models.ForeignKey(
+        "fileman.FileManager",
+        related_name="files",
+        on_delete=models.CASCADE,
+        help_text="File manager configuration used for this file"
+    )
+
+    filename = models.CharField(
+        max_length=255,
+        db_index=True,
+        help_text="Final filename used for storage"
+    )
+
+    original_filename = models.CharField(
+        max_length=255,
+        help_text="Original filename as uploaded by user"
+    )
+
+    file_path = models.TextField(
+        help_text="Full path to file in storage backend"
+    )
+
+    file_size = models.BigIntegerField(
+        null=True,
+        blank=True,
+        help_text="File size in bytes"
+    )
+
+    content_type = models.CharField(
+        max_length=255,
+        db_index=True,
+        help_text="MIME type of the file"
+    )
+
+    checksum = models.CharField(
+        max_length=128,
+        blank=True,
+        default="",
+        help_text="File checksum (MD5, SHA256, etc.)"
+    )
+
+    upload_token = models.CharField(
+        max_length=64,
+        unique=True,
+        db_index=True,
+        help_text="Unique token for tracking direct uploads"
+    )
+
+    upload_status = models.CharField(
+        max_length=32,
+        choices=STATUS_CHOICES,
+        default=PENDING,
+        db_index=True,
+        help_text="Current status of the file upload"
+    )
+
+    upload_url = models.TextField(
+        blank=True,
+        default="",
+        help_text="Pre-signed URL for direct upload (temporary)"
+    )
+
+    upload_expires_at = models.DateTimeField(
+        null=True,
+        blank=True,
+        help_text="When the upload URL expires"
+    )
+
+    metadata = models.JSONField(
+        default=dict,
+        blank=True,
+        help_text="Additional file metadata and custom properties"
+    )
+
+    is_active = models.BooleanField(
+        default=True,
+        help_text="Whether this file is active and accessible"
+    )
+
+    is_public = models.BooleanField(
+        default=False,
+        help_text="Whether this file can be accessed without authentication"
+    )
+
+    class Meta:
+        indexes = [
+            models.Index(fields=['upload_status', 'created']),
+            models.Index(fields=['file_manager', 'upload_status']),
+            models.Index(fields=['group', 'is_active']),
+            models.Index(fields=['content_type', 'is_active']),
+            models.Index(fields=['upload_expires_at']),
+        ]
+
+    def __str__(self):
+        return f"{self.original_filename} ({self.get_upload_status_display()})"
+
+    def save(self, *args, **kwargs):
+        """Custom save to generate upload token and set defaults"""
+        if not self.upload_token:
+            self.upload_token = self.generate_upload_token()
+
+        if not self.filename:
+            self.filename = self.generate_unique_filename()
+
+        if not self.content_type:
+            self.content_type = mimetypes.guess_type(self.filename)[0] or 'application/octet-stream'
+
+        super().save(*args, **kwargs)
+
+    @classmethod
+    def generate_upload_token(cls):
+        """Generate a unique upload token"""
+        return hashlib.sha256(f"{uuid.uuid4()}{datetime.now()}".encode()).hexdigest()[:32]
+
+    def generate_unique_filename(self):
+        """Generate a unique filename for storage"""
+        import os
+        name, ext = os.path.splitext(self.original_filename)
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        unique_id = str(uuid.uuid4())[:8]
+        return f"{name}_{timestamp}_{unique_id}{ext}"
+
+    def get_metadata(self, key, default=None):
+        """Get a specific metadata value"""
+        return self.metadata.get(key, default)
+
+    def set_metadata(self, key, value):
+        """Set a specific metadata value"""
+        self.metadata[key] = value
+
+    @property
+    def is_pending(self):
+        return self.upload_status == self.PENDING
+
+    @property
+    def is_uploading(self):
+        return self.upload_status == self.UPLOADING
+
+    @property
+    def is_completed(self):
+        return self.upload_status == self.COMPLETED
+
+    @property
+    def is_failed(self):
+        return self.upload_status == self.FAILED
+
+    @property
+    def is_expired(self):
+        return self.upload_status == self.EXPIRED
+
+    @property
+    def is_upload_expired(self):
+        """Check if the upload URL has expired"""
+        if not self.upload_expires_at:
+            return False
+        return datetime.now() > self.upload_expires_at
+
+    def mark_as_uploading(self):
+        """Mark file as currently being uploaded"""
+        self.upload_status = self.UPLOADING
+        self.save(update_fields=['upload_status', 'modified'])
+
+    def mark_as_completed(self, file_size=None, checksum=None):
+        """Mark file upload as completed"""
+        self.upload_status = self.COMPLETED
+        if file_size:
+            self.file_size = file_size
+        if checksum:
+            self.checksum = checksum
+        self.save(update_fields=['upload_status', 'file_size', 'checksum', 'modified'])
+
+    def mark_as_failed(self, error_message=None):
+        """Mark file upload as failed"""
+        self.upload_status = self.FAILED
+        if error_message:
+            self.set_metadata('error_message', error_message)
+        self.save(update_fields=['upload_status', 'metadata', 'modified'])
+
+    def mark_as_expired(self):
+        """Mark file upload as expired"""
+        self.upload_status = self.EXPIRED
+        self.save(update_fields=['upload_status', 'modified'])
+
+    def get_file_extension(self):
+        """Get the file extension"""
+        import os
+        return os.path.splitext(self.original_filename)[1].lower()
+
+    def get_human_readable_size(self):
+        """Get human readable file size"""
+        if not self.file_size:
+            return "Unknown"
+
+        for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
+            if self.file_size < 1024.0:
+                return f"{self.file_size:.1f} {unit}"
+            self.file_size /= 1024.0
+        return f"{self.file_size:.1f} PB"
+
+    def can_be_accessed_by(self, user=None, group=None):
+        """Check if file can be accessed by user/group"""
+        if not self.is_active:
+            return False
+
+        if self.is_public:
+            return True
+
+        if user and self.uploaded_by == user:
+            return True
+
+        if group and self.group == group:
+            return True
+
+        return False