additory 0.1.0a3__py3-none-any.whl → 0.1.1a1__py3-none-any.whl

additory/core/backends/cudf_bridge.py

@@ -1,355 +0,0 @@
-# cudf_enhanced_bridge.py
-# Enhanced cuDF support for Arrow bridge with proper dtype handling
-
-import gc
-from typing import Any, Dict, Optional, List
-from datetime import datetime
-
-try:
-    import cudf
-    import pyarrow as pa
-    import pandas as pd
-    CUDF_AVAILABLE = True
-except (ImportError, Exception):
-    CUDF_AVAILABLE = False
-    cudf = None
-    pa = None
-    # Create dummy classes to avoid AttributeError
-    class DummyPA:
-        class Table:
-            pass
-    if pa is None:
-        pa = DummyPA()
-
-from ..logging import log_info, log_warning
-
-
-class CuDFBridgeError(Exception):
-    """Raised when cuDF bridge operations fail"""
-    pass
-
-
-class EnhancedCuDFBridge:
-    """Enhanced cuDF support with proper dtype handling and GPU memory management"""
-    
-    def __init__(self):
-        self.cudf_available = CUDF_AVAILABLE
-        self.conversion_stats = {
-            "direct_success": 0,
-            "dtype_conversion_success": 0,
-            "pandas_fallback": 0,
-            "total_failures": 0
-        }
-        
-        if self.cudf_available:
-            log_info("[cudf_bridge] Enhanced cuDF Bridge initialized")
-        else:
-            log_warning("[cudf_bridge] cuDF not available")
-    
-    def detect_cudf(self, df: Any) -> bool:
-        """
-        Enhanced cuDF detection with multiple methods
-        
-        Args:
-            df: Input dataframe
-            
-        Returns:
-            True if dataframe is cuDF
-        """
-        if not self.cudf_available:
-            return False
-        
-        # Method 1: Check module name (most reliable)
-        try:
-            if hasattr(df, '__module__') and 'cudf' in str(df.__module__):
-                return True
-        except Exception:
-            pass
-        
-        # Method 2: Check class name
-        try:
-            if 'cudf' in str(type(df)):
-                return True
-        except Exception:
-            pass
-        
-        # Method 3: isinstance check
-        try:
-            return isinstance(df, cudf.DataFrame)
-        except Exception:
-            pass
-        
-        # Method 4: Check for cuDF-specific attributes
-        try:
-            if (hasattr(df, '_data') and 
-                hasattr(df, '_index') and 
-                hasattr(df, 'to_arrow') and
-                hasattr(df, 'to_pandas')):
-                # Additional check: see if it has cuDF-specific methods
-                if hasattr(df, 'memory_usage') and hasattr(df, 'hash_values'):
-                    return True
-        except Exception:
-            pass
-        
-        return False
-    
-    def cudf_to_arrow(self, df: Any) -> pa.Table:
-        """
-        Enhanced cuDF to Arrow conversion with multiple fallback strategies
-        
-        Args:
-            df: cuDF DataFrame
-            
-        Returns:
-            PyArrow Table
-            
-        Raises:
-            CuDFBridgeError: If all conversion methods fail
-        """
-        if not self.cudf_available:
-            raise CuDFBridgeError("cuDF not available")
-        
-        conversion_errors = []
-        
-        # Strategy 1: Direct conversion (fastest when it works)
-        try:
-            arrow_table = df.to_arrow()
-            self.conversion_stats["direct_success"] += 1
-            log_info("[cudf_bridge] Direct cuDF→Arrow conversion successful")
-            return arrow_table
-        except Exception as e:
-            conversion_errors.append(f"Direct conversion: {e}")
-            log_warning(f"[cudf_bridge] Direct cuDF→Arrow failed: {e}")
-        
-        # Strategy 2: Dtype preprocessing
-        try:
-            arrow_table = self._cudf_to_arrow_with_dtype_conversion(df)
-            self.conversion_stats["dtype_conversion_success"] += 1
-            log_info("[cudf_bridge] cuDF→Arrow with dtype conversion successful")
-            return arrow_table
-        except Exception as e:
-            conversion_errors.append(f"Dtype conversion: {e}")
-            log_warning(f"[cudf_bridge] cuDF dtype conversion failed: {e}")
-        
-        # Strategy 3: Pandas fallback (most compatible)
-        try:
-            pandas_df = df.to_pandas()
-            arrow_table = pa.Table.from_pandas(pandas_df, preserve_index=False)
-            self.conversion_stats["pandas_fallback"] += 1
-            log_info("[cudf_bridge] cuDF→pandas→Arrow fallback successful")
-            return arrow_table
-        except Exception as e:
-            conversion_errors.append(f"Pandas fallback: {e}")
-            log_warning(f"[cudf_bridge] Pandas fallback failed: {e}")
-        
-        # All strategies failed
-        self.conversion_stats["total_failures"] += 1
-        error_summary = "; ".join(conversion_errors)
-        raise CuDFBridgeError(f"All cuDF→Arrow conversion strategies failed: {error_summary}")
-    
-    def _cudf_to_arrow_with_dtype_conversion(self, df: Any) -> pa.Table:
-        """
-        Convert cuDF to Arrow with dtype preprocessing
-        
-        Args:
-            df: cuDF DataFrame
-            
-        Returns:
-            PyArrow Table
-        """
-        # Create a copy to avoid modifying original
-        df_processed = df.copy()
-        
-        # Analyze and convert problematic dtypes
-        dtype_conversions = {}
-        
-        for col in df_processed.columns:
-            col_dtype = df_processed[col].dtype
-            
-            # Handle object dtypes (main source of cupy errors)
-            if col_dtype == 'object':
-                try:
-                    # Try to convert to string
-                    df_processed[col] = df_processed[col].astype('str')
-                    dtype_conversions[col] = 'object→str'
-                except Exception as e:
-                    log_warning(f"[cudf_bridge] Failed to convert column '{col}' from object to string: {e}")
-                    # Try alternative: convert via pandas
-                    try:
-                        pandas_series = df_processed[col].to_pandas().astype('str')
-                        df_processed[col] = cudf.from_pandas(pandas_series)
-                        dtype_conversions[col] = 'object→pandas→str'
-                    except Exception as e2:
-                        log_warning(f"[cudf_bridge] Failed pandas conversion for column '{col}': {e2}")
-                        raise CuDFBridgeError(f"Cannot convert object column '{col}': {e2}")
-            
-            # Handle other problematic dtypes
-            elif 'datetime' in str(col_dtype).lower() and 'ns' not in str(col_dtype):
-                try:
-                    # Ensure datetime has nanosecond precision for Arrow compatibility
-                    df_processed[col] = df_processed[col].astype('datetime64[ns]')
-                    dtype_conversions[col] = f'{col_dtype}→datetime64[ns]'
-                except Exception as e:
-                    log_warning(f"[cudf_bridge] Failed to convert datetime column '{col}': {e}")
-        
-        if dtype_conversions:
-            log_info(f"[cudf_bridge] Applied dtype conversions: {dtype_conversions}")
-        
-        # Try Arrow conversion with processed dtypes
-        return df_processed.to_arrow()
-    
-    def arrow_to_cudf(self, arrow_table: pa.Table) -> Any:
-        """
-        Convert Arrow table to cuDF DataFrame
-        
-        Args:
-            arrow_table: PyArrow Table
-            
-        Returns:
-            cuDF DataFrame
-            
-        Raises:
-            CuDFBridgeError: If conversion fails
-        """
-        if not self.cudf_available:
-            raise CuDFBridgeError("cuDF not available")
-        
-        try:
-            # Direct Arrow to cuDF conversion
-            return cudf.DataFrame.from_arrow(arrow_table)
-        except Exception as e:
-            log_warning(f"[cudf_bridge] Direct Arrow→cuDF failed: {e}")
-            
-            # Fallback: Arrow → pandas → cuDF
-            try:
-                pandas_df = arrow_table.to_pandas()
-                return cudf.from_pandas(pandas_df)
-            except Exception as e2:
-                raise CuDFBridgeError(f"Arrow→cuDF conversion failed. Direct: {e}, Pandas fallback: {e2}")
-    
-    def get_cudf_info(self, df: Any) -> Dict[str, Any]:
-        """
-        Get detailed information about cuDF DataFrame
-        
-        Args:
-            df: cuDF DataFrame
-            
-        Returns:
-            Dictionary with cuDF information
-        """
-        if not self.detect_cudf(df):
-            return {"is_cudf": False}
-        
-        try:
-            info = {
-                "is_cudf": True,
-                "shape": df.shape,
-                "dtypes": {col: str(dtype) for col, dtype in df.dtypes.items()},
-                "memory_usage_bytes": df.memory_usage(deep=True).sum(),
-                "has_object_columns": any(dtype == 'object' for dtype in df.dtypes),
-                "object_columns": [col for col, dtype in df.dtypes.items() if dtype == 'object'],
-                "cudf_version": cudf.__version__ if self.cudf_available else "unknown"
-            }
-            
-            # GPU memory info if available
-            try:
-                import rmm
-                gpu_info = rmm.mr.get_current_device_resource().get_memory_info()
-                info["gpu_memory"] = {
-                    "allocated_bytes": gpu_info[0],
-                    "total_bytes": gpu_info[1],
-                    "usage_percent": (gpu_info[0] / gpu_info[1]) * 100 if gpu_info[1] > 0 else 0
-                }
-            except Exception:
-                info["gpu_memory"] = {"available": False}
-            
-            return info
-            
-        except Exception as e:
-            return {"is_cudf": True, "error": str(e)}
-    
-    def cleanup_gpu_memory(self) -> Dict[str, Any]:
-        """
-        Clean up GPU memory used by cuDF operations
-        
-        Returns:
-            Dictionary with cleanup statistics
-        """
-        if not self.cudf_available:
-            return {"cudf_available": False}
-        
-        try:
-            # Get memory info before cleanup
-            memory_before = 0
-            try:
-                import rmm
-                memory_before = rmm.mr.get_current_device_resource().get_memory_info()[0]
-            except Exception:
-                pass
-            
-            # Force garbage collection
-            gc.collect()
-            
-            # cuDF-specific cleanup
-            try:
-                # Clear any cached data
-                if hasattr(cudf, '_lib') and hasattr(cudf._lib, 'rmm'):
-                    # Note: reinitialize() is aggressive and may affect other GPU operations
-                    # Only use in specific scenarios
-                    pass
-            except Exception:
-                pass
-            
-            # Get memory info after cleanup
-            memory_after = 0
-            try:
-                import rmm
-                memory_after = rmm.mr.get_current_device_resource().get_memory_info()[0]
-            except Exception:
-                pass
-            
-            memory_freed = max(0, memory_before - memory_after)
-            
-            cleanup_stats = {
-                "cudf_available": True,
-                "memory_before_bytes": memory_before,
-                "memory_after_bytes": memory_after,
-                "memory_freed_bytes": memory_freed,
-                "memory_freed_mb": memory_freed / (1024 * 1024)
-            }
-            
-            if memory_freed > 0:
-                log_info(f"[cudf_bridge] GPU memory cleanup: {memory_freed / (1024 * 1024):.1f}MB freed")
-            
-            return cleanup_stats
-            
-        except Exception as e:
-            return {"cudf_available": True, "error": str(e)}
-    
-    def get_conversion_stats(self) -> Dict[str, Any]:
-        """Get conversion statistics"""
-        total_attempts = sum(self.conversion_stats.values())
-        
-        if total_attempts == 0:
-            return {"no_conversions": True}
-        
-        return {
-            "total_attempts": total_attempts,
-            "success_rate": ((total_attempts - self.conversion_stats["total_failures"]) / total_attempts) * 100,
-            "direct_success_rate": (self.conversion_stats["direct_success"] / total_attempts) * 100,
-            "dtype_conversion_rate": (self.conversion_stats["dtype_conversion_success"] / total_attempts) * 100,
-            "pandas_fallback_rate": (self.conversion_stats["pandas_fallback"] / total_attempts) * 100,
-            "failure_rate": (self.conversion_stats["total_failures"] / total_attempts) * 100,
-            "stats": self.conversion_stats.copy()
-        }
-
-
-# Global instance
-_cudf_bridge = None
-
-def get_cudf_bridge() -> EnhancedCuDFBridge:
-    """Get the global cuDF bridge instance"""
-    global _cudf_bridge
-    if _cudf_bridge is None:
-        _cudf_bridge = EnhancedCuDFBridge()
-    return _cudf_bridge