django-nativemojo 0.1.10__py3-none-any.whl → 0.1.15__py3-none-any.whl

mojo/apps/fileman/backends/s3.py

@@ -6,6 +6,8 @@ from datetime import datetime, timedelta
 import os
 import uuid
 import hashlib
+from urllib.parse import urlparse
+import json
 
 from .base import StorageBackend
 
@@ -14,33 +16,39 @@ class S3StorageBackend(StorageBackend):
     """
     AWS S3 storage backend implementation
     """
-    
+
     def __init__(self, file_manager, **kwargs):
         super().__init__(file_manager, **kwargs)
-        
+
+        # Parse bucket name and folder path
+        purl = urlparse(file_manager.backend_url)
+        if purl.scheme != "s3":
+            raise ValueError("Invalid scheme for S3 backend")
+        self.bucket_name = purl.netloc
+        self.folder_path = purl.path.lstrip('/')
+
         # S3 configuration
-        self.bucket_name = self.get_setting('bucket_name')
-        self.region_name = self.get_setting('region_name', 'us-east-1')
-        self.access_key_id = self.get_setting('access_key_id')
-        self.secret_access_key = self.get_setting('secret_access_key')
+        self.region_name = self.get_setting('aws_region', 'us-east-1')
+        self.access_key_id = self.get_setting('aws_key')
+        self.secret_access_key = self.get_setting('aws_secret')
         self.endpoint_url = self.get_setting('endpoint_url')  # For S3-compatible services
         self.signature_version = self.get_setting('signature_version', 's3v4')
         self.addressing_style = self.get_setting('addressing_style', 'auto')
-        
+
         # Upload configuration
         self.upload_expires_in = self.get_setting('upload_expires_in', 3600)  # 1 hour
         self.download_expires_in = self.get_setting('download_expires_in', 3600)  # 1 hour
         self.multipart_threshold = self.get_setting('multipart_threshold', 8 * 1024 * 1024)  # 8MB
         self.max_concurrency = self.get_setting('max_concurrency', 10)
-        
+
         # Security settings
         self.server_side_encryption = self.get_setting('server_side_encryption')
         self.kms_key_id = self.get_setting('kms_key_id')
-        
+
         # Initialize S3 client
         self._client = None
         self._resource = None
-    
+
     @property
     def client(self):
         """Lazy initialization of S3 client"""
@@ -50,9 +58,11 @@ class S3StorageBackend(StorageBackend):
                 aws_secret_access_key=self.secret_access_key,
                 region_name=self.region_name
             )
-            
+
             config = Config(
                 signature_version=self.signature_version,
+                connect_timeout=3,
+                read_timeout=3,
                 s3={
                     'addressing_style': self.addressing_style
                 },
@@ -61,15 +71,15 @@ class S3StorageBackend(StorageBackend):
                     'mode': 'adaptive'
                 }
             )
-            
+
             self._client = session.client(
                 's3',
                 endpoint_url=self.endpoint_url,
                 config=config
             )
-        
+
         return self._client
-    
+
     @property
     def resource(self):
         """Lazy initialization of S3 resource"""
@@ -79,51 +89,38 @@ class S3StorageBackend(StorageBackend):
                 aws_secret_access_key=self.secret_access_key,
                 region_name=self.region_name
             )
-            
+
             self._resource = session.resource(
                 's3',
                 endpoint_url=self.endpoint_url
             )
-        
+
         return self._resource
-    
-    def save(self, file_obj, filename: str, **kwargs) -> str:
+
+    def save(self, file_obj, file_path: str, content_type: Optional[str] = None, metadata: Optional[dict] = None) -> str:
         """Save a file to S3"""
-        try:
-            # Generate file path
-            file_path = self.generate_file_path(filename, kwargs.get('group_id'))
-            
-            # Prepare upload parameters
-            upload_params = {
-                'Bucket': self.bucket_name,
-                'Key': file_path,
-                'Body': file_obj
-            }
-            
-            # Add content type if provided
-            content_type = kwargs.get('content_type')
-            if content_type:
-                upload_params['ContentType'] = content_type
-            
-            # Add server-side encryption if configured
-            if self.server_side_encryption:
-                upload_params['ServerSideEncryption'] = self.server_side_encryption
-                if self.kms_key_id:
-                    upload_params['SSEKMSKeyId'] = self.kms_key_id
-            
-            # Add metadata
-            metadata = kwargs.get('metadata', {})
-            if metadata:
-                upload_params['Metadata'] = {k: str(v) for k, v in metadata.items()}
-            
-            # Upload the file
-            self.client.put_object(**upload_params)
-            
-            return file_path
-            
-        except ClientError as e:
-            raise Exception(f"Failed to save file to S3: {e}")
-    
+        # Prepare upload parameters
+        upload_params = {
+            'Bucket': self.bucket_name,
+            'Key': file_path,
+            'ContentType': content_type,
+            'Body': file_obj
+        }
+
+        # Add server-side encryption if configured
+        if self.server_side_encryption:
+            upload_params['ServerSideEncryption'] = self.server_side_encryption
+            if self.kms_key_id:
+                upload_params['SSEKMSKeyId'] = self.kms_key_id
+
+        if metadata:
+            upload_params['Metadata'] = metadata
+
+        # Upload the file
+        self.client.put_object(**upload_params)
+
+        return file_path
+
     def delete(self, file_path: str) -> bool:
         """Delete a file from S3"""
         try:
@@ -134,7 +131,23 @@ class S3StorageBackend(StorageBackend):
             return True
         except ClientError:
             return False
-    
+
+    def delete_folder(self, folder_path: str) -> bool:
+        """Delete a folder from S3"""
+        # List all objects under the prefix
+        response = self.client.list_objects_v2(Bucket=self.bucket_name, Prefix=folder_path)
+        if 'Contents' not in response:
+            return True # Folder is already empty or doesn't exist
+        # Prepare delete batch
+        objects_to_delete = [{'Key': obj['Key']} for obj in response['Contents']]
+
+        # Delete in batch
+        self.client.delete_objects(
+            Bucket=self.bucket_name,
+            Delete={'Objects': objects_to_delete}
+        )
+        return True
+
     def exists(self, file_path: str) -> bool:
         """Check if a file exists in S3"""
         try:
@@ -145,7 +158,7 @@ class S3StorageBackend(StorageBackend):
             return True
         except ClientError:
             return False
-    
+
     def get_file_size(self, file_path: str) -> Optional[int]:
         """Get the size of a file in S3"""
         try:
@@ -156,13 +169,14 @@ class S3StorageBackend(StorageBackend):
             return response['ContentLength']
         except ClientError:
             return None
-    
+
     def get_url(self, file_path: str, expires_in: Optional[int] = None) -> str:
-        """Get a pre-signed URL to access the file"""
+        """Get a URL to access the file, either public or pre-signed based on expiration"""
         if expires_in is None:
-            expires_in = self.download_expires_in
-        
-        try:
+            # Assume the bucket is public and generate a public URL
+            url = f"https://{self.bucket_name}.s3.amazonaws.com/{file_path}"
+        else:
+            # Generate a pre-signed URL
             url = self.client.generate_presigned_url(
                 'get_object',
                 Params={
@@ -171,33 +185,45 @@ class S3StorageBackend(StorageBackend):
                 },
                 ExpiresIn=expires_in
             )
-            return url
-        except ClientError as e:
-            raise Exception(f"Failed to generate download URL: {e}")
-    
-    def generate_upload_url(self, file_path: str, content_type: str, 
-                           file_size: Optional[int] = None, 
+        return url
+
+    def supports_direct_upload(self) -> bool:
+        """
+        Check if this backend supports direct uploads
+
+        Returns:
+            bool: True if direct uploads are supported
+        """
+        return True
+
+    def generate_upload_url(self, file_path: str, content_type: str,
+                           file_size: Optional[int] = None,
                            expires_in: int = 3600) -> Dict[str, Any]:
         """Generate a pre-signed URL for direct upload to S3"""
         try:
             # Conditions for the upload
             conditions = []
-            
+
             # Content type condition
             if content_type:
                 conditions.append({"Content-Type": content_type})
-            
+
             # File size conditions
             if file_size:
                 # Allow some variance in file size (±1KB)
                 conditions.append(["content-length-range", max(0, file_size - 1024), file_size + 1024])
-            
+
             # Server-side encryption conditions
             if self.server_side_encryption:
                 conditions.append({"x-amz-server-side-encryption": self.server_side_encryption})
                 if self.kms_key_id:
                     conditions.append({"x-amz-server-side-encryption-aws-kms-key-id": self.kms_key_id})
-            
+            else:
+                params = dict(Bucket=self.bucket_name, Key=file_path, ContentType=content_type)
+                return self.client.generate_presigned_url(
+                    'put_object',
+                    ExpiresIn=expires_in,
+                    Params=params)
             # Generate the presigned POST
             response = self.client.generate_presigned_post(
                 Bucket=self.bucket_name,
@@ -208,13 +234,13 @@ class S3StorageBackend(StorageBackend):
                 Conditions=conditions,
                 ExpiresIn=expires_in
             )
-            
+
             # Add server-side encryption fields if configured
             if self.server_side_encryption:
                 response['fields']['x-amz-server-side-encryption'] = self.server_side_encryption
                 if self.kms_key_id:
                     response['fields']['x-amz-server-side-encryption-aws-kms-key-id'] = self.kms_key_id
-            
+
             return {
                 'upload_url': response['url'],
                 'method': 'POST',
@@ -223,10 +249,10 @@ class S3StorageBackend(StorageBackend):
                     'Content-Type': content_type
                 }
             }
-            
+
         except ClientError as e:
             raise Exception(f"Failed to generate upload URL: {e}")
-    
+
     def get_file_checksum(self, file_path: str, algorithm: str = 'md5') -> Optional[str]:
         """Get file checksum from S3 metadata or calculate it"""
         try:
@@ -235,52 +261,52 @@ class S3StorageBackend(StorageBackend):
                 Bucket=self.bucket_name,
                 Key=file_path
             )
-            
+
             if algorithm.lower() == 'md5':
                 etag = response.get('ETag', '').strip('"')
                 # ETag is MD5 only for non-multipart uploads (no hyphens)
                 if etag and '-' not in etag:
                     return etag
-            
+
             # If ETag is not usable, download and calculate checksum
             return super().get_file_checksum(file_path, algorithm)
-            
+
         except ClientError:
             return None
-    
+
     def open(self, file_path: str, mode: str = 'rb'):
         """Open a file from S3"""
         if 'w' in mode or 'a' in mode:
             raise ValueError("S3 backend only supports read-only file access")
-        
+
         try:
             obj = self.resource.Object(self.bucket_name, file_path)
             return obj.get()['Body']
         except ClientError as e:
             raise FileNotFoundError(f"File not found in S3: {e}")
-    
+
     def list_files(self, path_prefix: str = "", limit: int = 1000) -> List[str]:
         """List files in S3 with optional path prefix"""
         try:
             paginator = self.client.get_paginator('list_objects_v2')
-            
+
             page_iterator = paginator.paginate(
                 Bucket=self.bucket_name,
                 Prefix=path_prefix,
                 PaginationConfig={'MaxItems': limit}
             )
-            
+
             files = []
             for page in page_iterator:
                 if 'Contents' in page:
                     for obj in page['Contents']:
                         files.append(obj['Key'])
-            
+
             return files
-            
+
         except ClientError:
             return []
-    
+
     def copy_file(self, source_path: str, dest_path: str) -> bool:
         """Copy a file within S3"""
         try:
@@ -288,24 +314,24 @@ class S3StorageBackend(StorageBackend):
                 'Bucket': self.bucket_name,
                 'Key': source_path
             }
-            
+
             self.client.copy_object(
                 CopySource=copy_source,
                 Bucket=self.bucket_name,
                 Key=dest_path
             )
-            
+
             return True
-            
+
         except ClientError:
             return False
-    
+
     def move_file(self, source_path: str, dest_path: str) -> bool:
         """Move a file within S3"""
         if self.copy_file(source_path, dest_path):
             return self.delete(source_path)
         return False
-    
+
     def get_file_metadata(self, file_path: str) -> Dict[str, Any]:
         """Get comprehensive metadata for a file in S3"""
         try:
@@ -313,7 +339,7 @@ class S3StorageBackend(StorageBackend):
                 Bucket=self.bucket_name,
                 Key=file_path
             )
-            
+
             metadata = {
                 'exists': True,
                 'size': response.get('ContentLength'),
@@ -326,22 +352,22 @@ class S3StorageBackend(StorageBackend):
                 'server_side_encryption': response.get('ServerSideEncryption'),
                 'version_id': response.get('VersionId')
             }
-            
+
             return metadata
-            
+
         except ClientError:
             return {'exists': False, 'path': file_path}
-    
+
     def cleanup_expired_uploads(self, before_date: Optional[datetime] = None):
         """Clean up incomplete multipart uploads"""
         if before_date is None:
             before_date = datetime.now() - timedelta(days=1)
-        
+
         try:
             paginator = self.client.get_paginator('list_multipart_uploads')
-            
+
             page_iterator = paginator.paginate(Bucket=self.bucket_name)
-            
+
             for page in page_iterator:
                 if 'Uploads' in page:
                     for upload in page['Uploads']:
@@ -351,35 +377,37 @@ class S3StorageBackend(StorageBackend):
                                 Key=upload['Key'],
                                 UploadId=upload['UploadId']
                             )
-                            
+
         except ClientError:
             pass  # Silently ignore cleanup errors
-    
+
     def get_available_space(self) -> Optional[int]:
         """S3 has virtually unlimited space"""
         return None
-    
+
     def generate_file_path(self, filename: str, group_id: Optional[int] = None) -> str:
         """Generate an S3 key for the file"""
         # Use the base implementation but ensure S3-compatible paths
         path = super().generate_file_path(filename, group_id)
-        
+
         # Ensure no leading slash for S3 keys
         return path.lstrip('/')
-    
+
+
+
     def validate_configuration(self) -> Tuple[bool, List[str]]:
         """Validate S3 configuration"""
         errors = []
-        
+
         if not self.bucket_name:
             errors.append("S3 bucket name is required")
-        
+
         if not self.access_key_id:
             errors.append("AWS access key ID is required")
-        
+
         if not self.secret_access_key:
             errors.append("AWS secret access key is required")
-        
+
         # Test connection if configuration looks valid
         if not errors:
             try:
@@ -394,5 +422,69 @@ class S3StorageBackend(StorageBackend):
                     errors.append(f"Access denied to S3 bucket '{self.bucket_name}'")
                 else:
                     errors.append(f"S3 connection error: {e}")
-        
-        return len(errors) == 0, errors
+
+        return len(errors) == 0, errors
+
+    def make_path_public(self):
+        # Get the current bucket policy (if any)
+        try:
+            current_policy = json.loads(self.client.get_bucket_policy(Bucket=self.bucket_name)["Policy"])
+            statements = current_policy.get("Statement", [])
+        except self.client.exceptions.from_code('NoSuchBucketPolicy'):
+            current_policy = {"Version": "2012-10-17", "Statement": []}
+            statements = []
+
+        # Check if our public-read rule for the prefix already exists
+        public_read_sid = f"AllowPublicReadForPrefix_{self.folder_path.replace('/', '_')}"
+        already_exists = any(stmt.get("Sid") == public_read_sid for stmt in statements)
+
+        if already_exists:
+            return
+
+        # Construct the public read statement for the given prefix
+        new_statement = {
+            "Sid": public_read_sid,
+            "Effect": "Allow",
+            "Principal": "*",
+            "Action": "s3:GetObject",
+            "Resource": f"arn:aws:s3:::{self.bucket_name}/{self.folder_path}*"
+        }
+
+        # Add and apply the new policy
+        current_policy["Statement"].append(new_statement)
+        self.client.put_bucket_policy(
+            Bucket=self.bucket_name,
+            Policy=json.dumps(current_policy))
+
+    def make_path_private(self):
+        # Get the current bucket policy (if any)
+        try:
+            current_policy = json.loads(self.client.get_bucket_policy(Bucket=self.bucket_name)["Policy"])
+            statements = current_policy.get("Statement", [])
+        except self.client.exceptions.from_code('NoSuchBucketPolicy'):
+            current_policy = {"Version": "2012-10-17", "Statement": []}
+            statements = []
+
+        # Check if our public-read rule for the prefix exists
+        public_read_sid = f"AllowPublicReadForPrefix_{self.folder_path.replace('/', '_')}"
+        exists = any(stmt.get("Sid") == public_read_sid for stmt in statements)
+
+        if not exists:
+            return
+
+        # Remove the public read statement for the given prefix
+        statements = [stmt for stmt in statements if stmt.get("Sid") != public_read_sid]
+
+        # Apply the updated policy
+        current_policy["Statement"] = statements
+        self.client.put_bucket_policy(
+            Bucket=self.bucket_name,
+            Policy=json.dumps(current_policy))
+
+    def download(self, file_path: str, local_path: str) -> None:
+        """Download a file from S3 to a local path"""
+        try:
+            with open(local_path, 'wb') as local_file:
+                self.client.download_fileobj(self.bucket_name, file_path, local_file)
+        except ClientError as e:
+            raise Exception(f"Failed to download file from S3: {e}")

mojo/apps/fileman/migrations/0001_initial.py

@@ -0,0 +1,106 @@
+# Generated by Django 4.2.21 on 2025-06-07 13:32
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import mojo.models.rest
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('account', '0003_group_mojo_secrets_user_mojo_secrets'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='FileManager',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('mojo_secrets', models.TextField(blank=True, default=None, null=True)),
+                ('created', models.DateTimeField(auto_now_add=True, db_index=True)),
+                ('modified', models.DateTimeField(auto_now=True)),
+                ('name', models.CharField(db_index=True, help_text='Descriptive name for this file manager configuration', max_length=255)),
+                ('description', models.TextField(blank=True, default='', help_text="Optional description of this file manager's purpose")),
+                ('backend_type', models.CharField(choices=[('file', 'File System'), ('s3', 'AWS S3'), ('azure', 'Azure Blob Storage'), ('gcs', 'Google Cloud Storage'), ('custom', 'Custom Backend')], db_index=True, help_text='Type of storage backend (file, s3, azure, gcs, custom)', max_length=32)),
+                ('backend_url', models.CharField(help_text='Base URL or connection string for the storage backend', max_length=500)),
+                ('supports_direct_upload', models.BooleanField(default=False, help_text='Whether this backend supports direct upload (pre-signed URLs)')),
+                ('max_file_size', models.BigIntegerField(default=104857600, help_text='Maximum file size in bytes (0 for unlimited)')),
+                ('allowed_extensions', models.JSONField(blank=True, default=list, help_text='List of allowed file extensions (empty for all)')),
+                ('allowed_mime_types', models.JSONField(blank=True, default=list, help_text='List of allowed MIME types (empty for all)')),
+                ('is_active', models.BooleanField(default=True, help_text='Whether this file manager is active and can be used')),
+                ('is_default', models.BooleanField(default=False, help_text='Whether this is the default file manager for the group or user')),
+                ('group', models.ForeignKey(blank=True, default=None, help_text='Group that owns this file manager configuration', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='file_managers', to='account.group')),
+                ('user', models.ForeignKey(blank=True, default=None, help_text='User that owns this file manager configuration', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='file_managers', to=settings.AUTH_USER_MODEL)),
+            ],
+            bases=(models.Model, mojo.models.rest.MojoModel),
+        ),
+        migrations.CreateModel(
+            name='File',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True, db_index=True)),
+                ('modified', models.DateTimeField(auto_now=True)),
+                ('filename', models.CharField(db_index=True, help_text='Final filename used for storage', max_length=255)),
+                ('original_filename', models.CharField(help_text='Original filename as uploaded by user', max_length=255)),
+                ('file_path', models.TextField(help_text='Full path to file in storage backend')),
+                ('file_size', models.BigIntegerField(blank=True, help_text='File size in bytes', null=True)),
+                ('content_type', models.CharField(db_index=True, help_text='MIME type of the file', max_length=255)),
+                ('checksum', models.CharField(blank=True, default='', help_text='File checksum (MD5, SHA256, etc.)', max_length=128)),
+                ('upload_token', models.CharField(db_index=True, help_text='Unique token for tracking direct uploads', max_length=64, unique=True)),
+                ('upload_status', models.CharField(choices=[('pending', 'Pending Upload'), ('uploading', 'Uploading'), ('completed', 'Upload Completed'), ('failed', 'Upload Failed'), ('expired', 'Upload Expired')], db_index=True, default='pending', help_text='Current status of the file upload', max_length=32)),
+                ('upload_url', models.TextField(blank=True, default='', help_text='Pre-signed URL for direct upload (temporary)')),
+                ('upload_expires_at', models.DateTimeField(blank=True, help_text='When the upload URL expires', null=True)),
+                ('metadata', models.JSONField(blank=True, default=dict, help_text='Additional file metadata and custom properties')),
+                ('is_active', models.BooleanField(default=True, help_text='Whether this file is active and accessible')),
+                ('is_public', models.BooleanField(default=False, help_text='Whether this file can be accessed without authentication')),
+                ('file_manager', models.ForeignKey(help_text='File manager configuration used for this file', on_delete=django.db.models.deletion.CASCADE, related_name='files', to='fileman.filemanager')),
+                ('group', models.ForeignKey(blank=True, default=None, help_text='Group that owns this file', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='files', to='account.group')),
+                ('uploaded_by', models.ForeignKey(blank=True, default=None, help_text='User who uploaded this file', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='uploaded_files', to=settings.AUTH_USER_MODEL)),
+            ],
+            bases=(models.Model, mojo.models.rest.MojoModel),
+        ),
+        migrations.AddIndex(
+            model_name='filemanager',
+            index=models.Index(fields=['backend_type', 'is_active'], name='fileman_fil_backend_d27c68_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='filemanager',
+            index=models.Index(fields=['group', 'is_default'], name='fileman_fil_group_i_c1c47d_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='filemanager',
+            index=models.Index(fields=['user', 'is_default'], name='fileman_fil_user_id_525e54_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='filemanager',
+            index=models.Index(fields=['group', 'backend_type'], name='fileman_fil_group_i_3f1fc6_idx'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='filemanager',
+            unique_together={('group', 'name')},
+        ),
+        migrations.AddIndex(
+            model_name='file',
+            index=models.Index(fields=['upload_status', 'created'], name='fileman_fil_upload__64e176_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='file',
+            index=models.Index(fields=['file_manager', 'upload_status'], name='fileman_fil_file_ma_765f1a_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='file',
+            index=models.Index(fields=['group', 'is_active'], name='fileman_fil_group_i_4d4a8a_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='file',
+            index=models.Index(fields=['content_type', 'is_active'], name='fileman_fil_content_c5b4a6_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='file',
+            index=models.Index(fields=['upload_expires_at'], name='fileman_fil_upload__c4bc35_idx'),
+        ),
+    ]