django-nativemojo 0.1.10__py3-none-any.whl → 0.1.15__py3-none-any.whl

mojo/serializers/advanced/formats/csv.py

@@ -0,0 +1,416 @@
+import csv
+import io
+from decimal import Decimal
+from datetime import datetime, date
+from django.http import StreamingHttpResponse, HttpResponse
+from django.db.models import QuerySet
+from mojo.helpers import logit
+
+logger = logit.get_logger("csv_formatter", "csv_formatter.log")
+
+
+class CsvFormatter:
+    """
+    Advanced CSV formatter with streaming support and RestMeta.GRAPHS integration.
+    """
+    
+    def __init__(self, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL, 
+                 encoding='utf-8', streaming_threshold=1000):
+        """
+        Initialize CSV formatter.
+        
+        :param delimiter: Field delimiter (default comma)
+        :param quotechar: Quote character for fields containing special chars
+        :param quoting: Quoting behavior (csv.QUOTE_MINIMAL, etc.)
+        :param encoding: Character encoding for output
+        :param streaming_threshold: Minimum rows to trigger streaming response
+        """
+        self.delimiter = delimiter
+        self.quotechar = quotechar
+        self.quoting = quoting
+        self.encoding = encoding
+        self.streaming_threshold = streaming_threshold
+    
+    def serialize_queryset(self, queryset, fields=None, graph=None, filename="export.csv", 
+                          headers=None, localize=None, stream=True):
+        """
+        Serialize a Django QuerySet to CSV format.
+        
+        :param queryset: Django QuerySet to serialize
+        :param fields: List of field names or tuples (field_name, display_name)
+        :param graph: RestMeta graph name to use for field configuration
+        :param filename: Output filename
+        :param headers: Custom header names (overrides field names)
+        :param localize: Localization configuration
+        :param stream: Enable streaming for large datasets
+        :return: HttpResponse or StreamingHttpResponse
+        """
+        # Determine if we should stream based on queryset size
+        should_stream = stream and queryset.count() > self.streaming_threshold
+        
+        # Get fields configuration
+        field_config = self._get_field_config(queryset, fields, graph)
+        
+        if should_stream:
+            return self._create_streaming_response(queryset, field_config, filename, 
+                                                 headers, localize)
+        else:
+            return self._create_standard_response(queryset, field_config, filename, 
+                                                headers, localize)
+    
+    def serialize_data(self, data, fields=None, filename="export.csv", headers=None):
+        """
+        Serialize list of dictionaries or objects to CSV.
+        
+        :param data: List of dictionaries or objects
+        :param fields: Field names to include
+        :param filename: Output filename  
+        :param headers: Custom header names
+        :return: HttpResponse
+        """
+        if not data:
+            return self._create_empty_response(filename)
+        
+        # Auto-detect fields if not provided
+        if not fields:
+            fields = self._auto_detect_fields(data[0])
+        
+        # Prepare field configuration
+        field_config = self._prepare_field_config(fields, headers)
+        
+        # Generate CSV content
+        output = io.StringIO()
+        writer = csv.writer(output, delimiter=self.delimiter, 
+                          quotechar=self.quotechar, quoting=self.quoting)
+        
+        # Write header
+        writer.writerow(field_config['headers'])
+        
+        # Write data rows
+        for item in data:
+            row = self._extract_row_data(item, field_config['field_names'])
+            writer.writerow(row)
+        
+        # Create response
+        response = HttpResponse(output.getvalue(), content_type='text/csv')
+        response['Content-Disposition'] = f'attachment; filename="{filename}"'
+        return response
+    
+    def _get_field_config(self, queryset, fields, graph):
+        """
+        Get field configuration from various sources.
+        """
+        if fields:
+            return self._prepare_field_config(fields)
+        
+        # Try to get from RestMeta.GRAPHS
+        if graph and hasattr(queryset.model, 'RestMeta'):
+            rest_meta = queryset.model.RestMeta
+            if hasattr(rest_meta, 'GRAPHS') and graph in rest_meta.GRAPHS:
+                graph_config = rest_meta.GRAPHS[graph]
+                graph_fields = graph_config.get('fields', [])
+                if graph_fields:
+                    return self._prepare_field_config(graph_fields)
+        
+        # Fallback to model fields
+        model_fields = [f.name for f in queryset.model._meta.fields]
+        return self._prepare_field_config(model_fields)
+    
+    def _prepare_field_config(self, fields, headers=None):
+        """
+        Prepare field configuration for CSV generation.
+        """
+        field_names = []
+        field_headers = []
+        
+        for i, field in enumerate(fields):
+            if isinstance(field, (tuple, list)):
+                field_name, display_name = field
+                field_names.append(field_name)
+                field_headers.append(display_name)
+            else:
+                field_names.append(field)
+                field_headers.append(field.replace('_', ' ').title())
+        
+        # Override with custom headers if provided
+        if headers:
+            field_headers = headers[:len(field_names)]
+        
+        return {
+            'field_names': field_names,
+            'headers': field_headers
+        }
+    
+    def _create_streaming_response(self, queryset, field_config, filename, 
+                                 headers, localize):
+        """
+        Create streaming HTTP response for large datasets.
+        """
+        logger.info(f"Creating streaming CSV response for {queryset.count()} records")
+        
+        def csv_generator():
+            # Create CSV writer with pseudo-buffer
+            pseudo_buffer = EchoWriter()
+            writer = csv.writer(pseudo_buffer, delimiter=self.delimiter,
+                              quotechar=self.quotechar, quoting=self.quoting)
+            
+            # Yield header row
+            yield writer.writerow(field_config['headers'])
+            
+            # Yield data rows
+            for obj in queryset.iterator():  # Use iterator for memory efficiency
+                try:
+                    row = self._extract_row_data(obj, field_config['field_names'], localize)
+                    yield writer.writerow(row)
+                except Exception as e:
+                    logger.error(f"Error processing row for object {obj.pk}: {e}")
+                    # Continue with next row instead of failing completely
+                    continue
+        
+        response = StreamingHttpResponse(csv_generator(), content_type='text/csv')
+        response['Content-Disposition'] = f'attachment; filename="{filename}"'
+        response['Cache-Control'] = 'no-cache'
+        return response
+    
+    def _create_standard_response(self, queryset, field_config, filename, 
+                                headers, localize):
+        """
+        Create standard HTTP response for smaller datasets.
+        """
+        output = io.StringIO()
+        writer = csv.writer(output, delimiter=self.delimiter,
+                          quotechar=self.quotechar, quoting=self.quoting)
+        
+        # Write header
+        writer.writerow(field_config['headers'])
+        
+        # Write data rows
+        for obj in queryset:
+            try:
+                row = self._extract_row_data(obj, field_config['field_names'], localize)
+                writer.writerow(row)
+            except Exception as e:
+                logger.error(f"Error processing row for object {obj.pk}: {e}")
+                continue
+        
+        response = HttpResponse(output.getvalue(), content_type='text/csv')
+        response['Content-Disposition'] = f'attachment; filename="{filename}"'
+        return response
+    
+    def _create_empty_response(self, filename):
+        """
+        Create response for empty dataset.
+        """
+        response = HttpResponse('', content_type='text/csv')
+        response['Content-Disposition'] = f'attachment; filename="{filename}"'
+        return response
+    
+    def _extract_row_data(self, obj, field_names, localize=None):
+        """
+        Extract row data from an object based on field names.
+        """
+        row = []
+        
+        for field_name in field_names:
+            try:
+                value = self._get_field_value(obj, field_name)
+                value = self._process_field_value(value, field_name, localize)
+                row.append(self._format_csv_value(value))
+            except Exception as e:
+                logger.warning(f"Error extracting field '{field_name}': {e}")
+                row.append("N/A")
+        
+        return row
+    
+    def _get_field_value(self, obj, field_name):
+        """
+        Get field value from object, supporting nested field access.
+        """
+        # Handle nested field access (e.g., "user.email", "profile.address.city")
+        if '.' in field_name:
+            return self._get_nested_field_value(obj, field_name)
+        
+        # Handle special metadata fields
+        if field_name.startswith('metadata.') and hasattr(obj, 'getProperty'):
+            parts = field_name.split('.', 2)
+            if len(parts) == 3:
+                return obj.getProperty(parts[2], category=parts[1])
+            elif len(parts) == 2:
+                return obj.getProperty(parts[1])
+        
+        # Standard field access
+        if hasattr(obj, field_name):
+            value = getattr(obj, field_name)
+            return value() if callable(value) else value
+        
+        # Dictionary-style access
+        if isinstance(obj, dict):
+            return obj.get(field_name, None)
+        
+        return None
+    
+    def _get_nested_field_value(self, obj, field_path):
+        """
+        Get value from nested field path like "user.profile.name".
+        """
+        try:
+            current = obj
+            for field_part in field_path.split('.'):
+                if current is None:
+                    return None
+                
+                if hasattr(current, field_part):
+                    current = getattr(current, field_part)
+                elif isinstance(current, dict):
+                    current = current.get(field_part)
+                else:
+                    return None
+                
+                # Handle callable attributes
+                if callable(current):
+                    current = current()
+            
+            return current
+        except Exception as e:
+            logger.warning(f"Error accessing nested field '{field_path}': {e}")
+            return None
+    
+    def _process_field_value(self, value, field_name, localize=None):
+        """
+        Process field value with localization and special handling.
+        """
+        if value is None:
+            return "N/A"
+        
+        # Apply localization if configured
+        if localize and field_name in localize:
+            try:
+                localizer_config = localize[field_name]
+                if '|' in localizer_config:
+                    localizer_name, extra = localizer_config.split('|', 1)
+                else:
+                    localizer_name, extra = localizer_config, None
+                
+                # Import and apply localizer
+                from mojo.serializers.formats.localizers import get_localizer
+                localizer = get_localizer(localizer_name)
+                if localizer:
+                    return localizer(value, extra)
+            except Exception as e:
+                logger.warning(f"Localization failed for field '{field_name}': {e}")
+        
+        return value
+    
+    def _format_csv_value(self, value):
+        """
+        Format value for CSV output.
+        """
+        if value is None:
+            return ""
+        
+        # Handle model instances
+        if hasattr(value, 'pk'):
+            return str(value.pk)
+        
+        # Handle datetime objects
+        elif isinstance(value, datetime):
+            return value.strftime('%Y-%m-%d %H:%M:%S')
+        elif isinstance(value, date):
+            return value.strftime('%Y-%m-%d')
+        
+        # Handle numeric types
+        elif isinstance(value, Decimal):
+            return str(float(value)) if not value.is_nan() else "0"
+        elif isinstance(value, (int, float)):
+            return str(value)
+        
+        # Handle collections
+        elif isinstance(value, (list, tuple)):
+            return '; '.join(str(item) for item in value)
+        elif isinstance(value, dict):
+            return str(value)  # Could be enhanced with better dict formatting
+        
+        # Default string conversion
+        else:
+            return str(value)
+    
+    def _auto_detect_fields(self, sample_item):
+        """
+        Auto-detect fields from a sample data item.
+        """
+        if isinstance(sample_item, dict):
+            return list(sample_item.keys())
+        elif hasattr(sample_item, '_meta'):
+            return [f.name for f in sample_item._meta.fields]
+        elif hasattr(sample_item, '__dict__'):
+            return list(sample_item.__dict__.keys())
+        else:
+            return ['value']  # Fallback for primitive types
+
+
+class EchoWriter:
+    """
+    A writer that implements just the write method for streaming CSV generation.
+    """
+    
+    def writerow(self, row):
+        """Write the row by returning it as a CSV line."""
+        output = io.StringIO()
+        writer = csv.writer(output)
+        writer.writerow(row)
+        return output.getvalue()
+    
+    def write(self, value):
+        """Write the value by returning it directly."""
+        return value
+
+
+# Convenience functions for backwards compatibility
+def generate_csv(queryset, fields, filename, headers=None, localize=None, stream=True):
+    """
+    Generate CSV from queryset.
+    
+    :param queryset: Django QuerySet
+    :param fields: List of field names
+    :param filename: Output filename
+    :param headers: Custom header names
+    :param localize: Localization config
+    :param stream: Enable streaming for large datasets
+    :return: HttpResponse or StreamingHttpResponse
+    """
+    formatter = CsvFormatter()
+    return formatter.serialize_queryset(
+        queryset=queryset,
+        fields=fields,
+        filename=filename,
+        headers=headers,
+        localize=localize,
+        stream=stream
+    )
+
+
+def generate_csv_stream(queryset, fields, filename, localize=None):
+    """
+    Generate streaming CSV response.
+    """
+    formatter = CsvFormatter()
+    return formatter.serialize_queryset(
+        queryset=queryset,
+        fields=fields,
+        filename=filename,
+        localize=localize,
+        stream=True
+    )
+
+
+def serialize_to_csv(data, fields=None, filename="export.csv", headers=None):
+    """
+    Serialize list of data to CSV.
+    """
+    formatter = CsvFormatter()
+    return formatter.serialize_data(
+        data=data,
+        fields=fields,
+        filename=filename,
+        headers=headers
+    )