featrixsphere 0.2.3613__tar.gz → 0.2.3737__tar.gz

{featrixsphere-0.2.3613 → featrixsphere-0.2.3737}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featrixsphere
-Version: 0.2.3613
+Version: 0.2.3737
 Summary: Transform any CSV into a production-ready ML model in minutes, not months.
 Home-page: https://github.com/Featrix/sphere
 Author: Featrix

featrixsphere-0.2.3737/VERSION

@@ -0,0 +1 @@
+0.2.3737

{featrixsphere-0.2.3613 → featrixsphere-0.2.3737}/featrixsphere/__init__.py

@@ -38,7 +38,7 @@ Example:
     ...                                labels=['Experiment A', 'Experiment B'])
 """
 
-__version__ = "0.2.3613"
+__version__ = "0.2.3737"
 __author__ = "Featrix"
 __email__ = "support@featrix.com"
 __license__ = "MIT"

{featrixsphere-0.2.3613 → featrixsphere-0.2.3737}/featrixsphere/client.py

@@ -566,6 +566,11 @@ class FeatrixSphereClient:
         response = self._make_request("DELETE", endpoint, max_retries=max_retries, **kwargs)
         return self._unwrap_response(response.json())
     
+    def _post_multipart(self, endpoint: str, data: Dict[str, Any] = None, files: Dict[str, Any] = None, max_retries: int = None, **kwargs) -> Dict[str, Any]:
+        """Make a POST request with multipart/form-data (for file uploads) and return JSON response."""
+        response = self._make_request("POST", endpoint, data=data, files=files, max_retries=max_retries, **kwargs)
+        return self._unwrap_response(response.json())
+    
     # =========================================================================
     # Session Management
     # =========================================================================

{featrixsphere-0.2.3613 → featrixsphere-0.2.3737}/featrixsphere.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featrixsphere
-Version: 0.2.3613
+Version: 0.2.3737
 Summary: Transform any CSV into a production-ready ML model in minutes, not months.
 Home-page: https://github.com/Featrix/sphere
 Author: Featrix

{featrixsphere-0.2.3613 → featrixsphere-0.2.3737}/featrixsphere.egg-info/SOURCES.txt

@@ -82,6 +82,7 @@ src/lib/job_manager.py
 src/lib/json_encoder_cache.py
 src/lib/knn_training.py
 src/lib/meta_learning_client.py
+src/lib/pre_analysis_wrapper.py
 src/lib/queue_manager.py
 src/lib/quick_architecture_search.py
 src/lib/session_chains.py
@@ -109,6 +110,7 @@ src/lib/featrix/neural/domain_codec.py
 src/lib/featrix/neural/dropout_scheduler.py
 src/lib/featrix/neural/embedded_space.py
 src/lib/featrix/neural/embedding_lr_scheduler.py
+src/lib/featrix/neural/embedding_quality.py
 src/lib/featrix/neural/embedding_space_utils.py
 src/lib/featrix/neural/embedding_utils.py
 src/lib/featrix/neural/encoders.py
@@ -134,6 +136,7 @@ src/lib/featrix/neural/integrity.py
 src/lib/featrix/neural/io_utils.py
 src/lib/featrix/neural/json_cache.py
 src/lib/featrix/neural/json_codec.py
+src/lib/featrix/neural/local_string_cache.py
 src/lib/featrix/neural/logging_config.py
 src/lib/featrix/neural/mask_tracker.py
 src/lib/featrix/neural/model_config.py

{featrixsphere-0.2.3613 → featrixsphere-0.2.3737}/src/api.py

@@ -614,24 +614,57 @@ def create_app() -> FastAPI:
         Comprehensive health check endpoint with system metrics.
         Reads cached health data from Redis (populated by system_monitor.py).
         Returns error if cache is stale (>90 seconds old).
+        
+        CRITICAL: Uses timeout on Redis operations to prevent hanging if Redis is slow/unresponsive.
         """
         from lib.job_manager import get_redis_client
         
         current_time = time.time()
         uptime_seconds = current_time - SERVER_STARTUP_TIME
         
-        # Read cached health data from Redis
-        redis_client = get_redis_client()
+        # Read cached health data from Redis with timeout handling
+        # CRITICAL: Run blocking Redis operation in thread pool to avoid blocking event loop
         redis_key = "health:cache"
-        cached_data = redis_client.get(redis_key)
+        cached_data = None
+        redis_error = None
+        
+        def _get_redis_cache():
+            """Blocking Redis operation - runs in thread pool."""
+            import redis
+            try:
+                redis_client = get_redis_client()
+                # Redis client now has socket_timeout=2.0, so this will timeout quickly if Redis is hung
+                return redis_client.get(redis_key), None
+            except redis.TimeoutError as e:
+                return None, f"Redis timeout: {e}"
+            except redis.ConnectionError as e:
+                return None, f"Redis connection error: {e}"
+            except Exception as e:
+                return None, f"Redis error: {e}"
+        
+        try:
+            import asyncio
+            # Run blocking Redis operation in thread pool - prevents blocking event loop
+            cached_data, redis_error = await asyncio.to_thread(_get_redis_cache)
+            if redis_error:
+                logger.warning(f"⚠️  Health endpoint: {redis_error}")
+        except Exception as e:
+            redis_error = f"Unexpected error: {e}"
+            logger.warning(f"⚠️  Health endpoint: {redis_error}")
         
         if not cached_data:
+            # If Redis error occurred, include it in the response
+            error_msg = "Health data not available - system monitor may not be running"
+            if redis_error:
+                error_msg += f" (Redis error: {redis_error})"
+            
             return JSONResponse(
                 status_code=503,
                 content={
                     "status": "unhealthy",
                     "hostname": socket.gethostname(),
-                    "error": "Health data not available - system monitor may not be running",
+                    "error": error_msg,
+                    "redis_error": redis_error,
                     "timestamp": datetime.datetime.now().isoformat(),
                 }
             )

{featrixsphere-0.2.3613 → featrixsphere-0.2.3737}/src/celery_app.py

@@ -144,6 +144,8 @@ def check_duplicate_task_delivery(sender=None, task_id=None, task=None, args=Non
     If a duplicate is detected, we raise Ignore() to gracefully skip the task without
     marking it as failed or acquiring locks that would conflict with the running instance.
     
+    Also logs the full job configuration for all tasks to help debug configuration issues.
+    
     Args:
         sender: The task class
         task_id: Celery task ID
@@ -151,7 +153,59 @@ def check_duplicate_task_delivery(sender=None, task_id=None, task=None, args=Non
         args: Task positional arguments
         kwargs: Task keyword arguments
     """
-    # Only check long-running GPU training tasks
+    # LOG FULL JOB CONFIGURATION FOR ALL TASKS
+    try:
+        import json
+        task_name = sender.name if sender else 'unknown'
+        logger.info(f"\n{'='*80}")
+        logger.info(f"🔵 [TASK START] {task_name}")
+        logger.info(f"   Task ID: {task_id}")
+        logger.info(f"   Timestamp: {datetime.now().isoformat()}")
+        
+        # Extract and log job_spec for all tasks
+        if args and len(args) > 0:
+            job_spec = args[0] if isinstance(args[0], dict) else None
+            if job_spec:
+                logger.info(f"   📋 Job Configuration (job_spec):")
+                # Pretty print the job_spec with indentation
+                try:
+                    job_spec_str = json.dumps(job_spec, indent=4, default=str)
+                    # Log each line with indentation
+                    for line in job_spec_str.split('\n'):
+                        logger.info(f"      {line}")
+                except Exception as json_err:
+                    logger.info(f"      (Could not serialize job_spec: {json_err})")
+                    logger.info(f"      job_spec type: {type(job_spec)}")
+                    logger.info(f"      job_spec keys: {list(job_spec.keys()) if isinstance(job_spec, dict) else 'N/A'}")
+            
+            # Log all args
+            logger.info(f"   📦 Task Arguments:")
+            for i, arg in enumerate(args):
+                if isinstance(arg, dict):
+                    # Skip logging full dicts again (already logged as job_spec)
+                    if i == 0 and job_spec:
+                        logger.info(f"      args[{i}]: <job_spec> (logged above)")
+                    else:
+                        logger.info(f"      args[{i}]: {type(arg).__name__} with {len(arg)} keys" if isinstance(arg, dict) else f"      args[{i}]: {arg}")
+                else:
+                    logger.info(f"      args[{i}]: {arg}")
+        
+        if kwargs:
+            logger.info(f"   📦 Task Keyword Arguments:")
+            for key, value in kwargs.items():
+                if isinstance(value, dict):
+                    logger.info(f"      {key}: <dict with {len(value)} keys>")
+                elif isinstance(value, (list, tuple)):
+                    logger.info(f"      {key}: <{type(value).__name__} with {len(value)} items>")
+                else:
+                    logger.info(f"      {key}: {value}")
+        
+        logger.info(f"{'='*80}\n")
+    except Exception as log_err:
+        # Don't let logging errors break task execution
+        logger.warning(f"⚠️  Error logging job configuration: {log_err}")
+    
+    # Only check long-running GPU training tasks for duplicates
     if sender.name not in ['celery_app.train_es', 'celery_app.train_single_predictor']:
         return
     
@@ -238,7 +292,6 @@ def setup_job_logging(job_id: str, session_id: str, job_type: str):
     import stat
     import socket
     import fcntl
-    import time as time_module
     
     job_dir = get_job_output_path(job_id, session_id, job_type)
     original_cwd = os.getcwd()
@@ -247,6 +300,44 @@ def setup_job_logging(job_id: str, session_id: str, job_type: str):
     job_dir.mkdir(parents=True, exist_ok=True)
     logger.info(f"📁 Job directory: {job_dir} (exists: {job_dir.exists()})")
     
+    # Create JOB_INFO.json with job metadata (queued_at, created_at, etc.)
+    try:
+        from lib.job_manager import load_job
+        job_data = load_job(job_id)
+        
+        job_info = {
+            "job_id": job_id,
+            "session_id": session_id,
+            "job_type": job_type,
+            "created_at": job_data.get("created_at") if job_data else None,
+            "queued_at": job_data.get("created_at") if job_data else None,  # Same as created_at
+            "started_at": datetime.now().isoformat(),
+            "status": job_data.get("status") if job_data else "unknown",
+            "worker_hostname": socket.gethostname(),
+            "celery_task_id": job_id,
+        }
+        
+        # Add any additional job_spec info if available
+        if job_data and "job_spec" in job_data:
+            job_spec = job_data["job_spec"]
+            if isinstance(job_spec, dict):
+                # Include relevant job_spec fields
+                if "target_column" in job_spec:
+                    job_info["target_column"] = job_spec["target_column"]
+                if "target_column_type" in job_spec:
+                    job_info["target_column_type"] = job_spec["target_column_type"]
+        
+        job_info_path = job_dir / "JOB_INFO.json"
+        with open(job_info_path, 'w') as f:
+            json.dump(job_info, f, indent=2, default=str)
+        
+        logger.info(f"📝 Created JOB_INFO.json: {job_info_path}")
+        if job_info.get("queued_at"):
+            logger.info(f"   Queued at: {job_info['queued_at']}")
+    except Exception as job_info_err:
+        logger.warning(f"⚠️  Could not create JOB_INFO.json: {job_info_err}")
+        # Don't fail job setup if this fails
+    
     # NOTE: Duplicate task delivery detection is handled by task_prerun signal (see below)
     # This checks log file mtime BEFORE the task runs to gracefully ignore duplicates
     
@@ -1799,9 +1890,6 @@ def pre_analysis_architecture(self, job_spec: dict, job_id: str, session_id: str
     Returns:
         dict with optimal config
     """
-    from pathlib import Path
-    import sys
-    
     # Use Celery task ID if job_id not provided
     if not job_id:
         job_id = self.request.id
@@ -1827,8 +1915,37 @@ def pre_analysis_architecture(self, job_spec: dict, job_id: str, session_id: str
                 input_data_path = session.get('input_data')
                 
                 if input_data_path:
-                    input_file = FeatrixInputDataFile(str(input_data_path))
-                    df = input_file.df
+                    # Handle relative paths - check if file exists, if not try session directory
+                    input_path = Path(input_data_path)
+                    if not input_path.exists() and not input_path.is_absolute():
+                        # Try to find it in the session directory or data directory
+                        from lib.sphere_config import config
+                        session_dir = Path(config.data_dir) / session_id
+                        possible_paths = [
+                            session_dir / input_path.name,
+                            Path(config.data_dir) / input_path.name,
+                            input_path  # Try original as-is
+                        ]
+                        for possible_path in possible_paths:
+                            if possible_path.exists():
+                                input_data_path = str(possible_path)
+                                logger.info(f"✅ Found input data at: {input_data_path}")
+                                break
+                        else:
+                            # File not found - skip meta-learning query but don't fail
+                            logger.warning(f"⚠️  Input data file not found: {input_data_path}")
+                            logger.warning(f"   Tried paths: {[str(p) for p in possible_paths]}")
+                            logger.warning(f"   Skipping meta-learning query - will use adaptive sampling")
+                            raise FileNotFoundError(f"Input data file not found: {input_data_path}")
+                    
+                    # File exists - try to load it
+                    try:
+                        input_file = FeatrixInputDataFile(str(input_data_path))
+                        df = input_file.df
+                    except Exception as load_err:
+                        logger.warning(f"⚠️  Failed to load input data file: {load_err}")
+                        logger.warning(f"   Skipping meta-learning query - will use adaptive sampling")
+                        raise
                     
                     # Query meta-learning API
                     query_result = query_optimal_parameters(df, target_type=None)
@@ -1893,57 +2010,219 @@ def pre_analysis_architecture(self, job_spec: dict, job_id: str, session_id: str
                 
                 strings_cache = session.get('strings_cache')
             
-            # Import grid search code
-            sys.path.insert(0, str(Path(__file__).parent))
-            from lib.featrix.neural.feature_engineer import FeatureEngineer
-            from lib.quick_architecture_search import run_quick_architecture_search
-            
-            # Run quick search
-            if suggested_configs:
-                logger.info(f"🚀 Running quick architecture search with {len(suggested_configs)} meta-learning suggestions ({quick_epochs} epochs each)")
-            else:
-                logger.info(f"🚀 Running quick architecture search ({n_samples} configs, {quick_epochs} epochs each)")
-            
-            optimal_config = run_quick_architecture_search(
-                data_file=data_file,
-                strings_cache=strings_cache,
-                session_id=session_id,
-                n_samples=n_samples,
-                quick_epochs=quick_epochs,
-                suggested_configs=suggested_configs  # Pass suggested configs
-            )
+            # CRITICAL: Fork/exec pre-analysis in a non-daemon session leader process
+            # Same pattern as ES/SP training for better GPU memory management and process isolation
+            # Serialize args to pass to subprocess
+            args_dict = {
+                'data_file': str(data_file) if data_file else None,
+                'strings_cache': strings_cache or "",
+                'session_id': session_id,
+                'n_samples': n_samples,
+                'quick_epochs': quick_epochs,
+                'suggested_configs': suggested_configs
+            }
             
-            logger.info(f"✅ Quick search complete: {optimal_config}")
+            # Create a temporary file to pass args
+            with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
+                json_file = f.name
+                json_module.dump(args_dict, f)
             
-            # Save results to job directory for debugging
-            results_file = job_dir / "pre_analysis_results.json"
             try:
-                results_output = {
-                    "optimal_config": optimal_config,
-                    "session_id": session_id,
-                    "job_id": job_id,
-                    "timestamp": datetime.now().isoformat(),
-                    "meta": {
-                        "n_samples_requested": job_spec.get('n_samples', 20),
-                        "quick_epochs": job_spec.get('quick_epochs', 25),
-                        "used_suggested_configs": bool(suggested_configs)
-                    }
-                }
+                # Fork/exec pre-analysis in a new session leader process
+                # This provides better process isolation and GPU memory cleanup
+                python_cmd = sys.executable
+                wrapper_script = Path(__file__).parent / "lib" / "pre_analysis_wrapper.py"
                 
-                with open(results_file, 'w') as f:
-                    json.dump(results_output, f, indent=2, default=str)
+                if suggested_configs:
+                    logger.info(f"🚀 Forking pre-analysis process with {len(suggested_configs)} meta-learning suggestions ({quick_epochs} epochs each)")
+                else:
+                    logger.info(f"🚀 Forking pre-analysis process ({n_samples} configs, {quick_epochs} epochs each)")
+                logger.info(f"   Python: {python_cmd}")
+                logger.info(f"   Script: {wrapper_script}")
+                
+                # Fork/exec with setsid to create new session leader (non-daemon)
+                # CRITICAL: Open log file in append mode so subprocess writes to same file
+                log_file_path = job_dir / "logs" / "stdout.log"
+                
+                # Open log file in append mode (line buffered) for subprocess
+                with open(log_file_path, 'a', buffering=1) as log_file:
+                    process = subprocess.Popen(
+                        [python_cmd, str(wrapper_script), json_file, '--job-id', job_id],
+                        cwd=str(job_dir),
+                        stdout=log_file,
+                        stderr=subprocess.STDOUT,  # Redirect stderr to stdout
+                        start_new_session=True,  # setsid - become session leader (non-daemon)
+                        env=os.environ.copy()  # Inherit environment
+                    )
                     
-                logger.info(f"💾 Saved pre-analysis results to {results_file}")
+                    pre_analysis_pid = process.pid
+                    celery_worker_pid = os.getpid()
+                    
+                    # Write Celery worker mapping file for admin monitoring
+                    mapping_file = Path(f"/tmp/featrix-celery-{celery_worker_pid}.json")
+                    try:
+                        mapping_data = {
+                            "timestamp_started": datetime.now().isoformat(),
+                            "celery_worker_pid": celery_worker_pid,
+                            "pre_analysis_pid": pre_analysis_pid,
+                            "job_id": job_id,
+                            "session_id": session_id,
+                            "job_type": "pre_analysis_architecture",
+                        }
+                        with open(mapping_file, 'w') as f:
+                            json_module.dump(mapping_data, f, indent=2)
+                        logger.info(f"📝 Wrote Celery worker mapping: {mapping_file}")
+                    except Exception as e:
+                        logger.warning(f"Failed to write Celery worker mapping: {e}")
+                    
+                    logger.info(f"✅ Pre-analysis process forked with PID {pre_analysis_pid} (session leader)")
+                    logger.info(f"   Celery worker PID: {celery_worker_pid}")
+                    logger.info(f"   Pre-analysis PID: {pre_analysis_pid}")
+                    logger.info(f"   📝 Pre-analysis output in: {log_file_path}")
+                    logger.info(f"   🔍 Monitor with: ffsh tail {session_id}/{job_id}")
+                    logger.info(f"   Monitoring PID {pre_analysis_pid} until it exits...")
+                    
+                    # Wait for process to complete
+                    # CRITICAL: Check for ABORT flag periodically AND update Celery state with progress
+                    abort_flag = job_dir / "ABORT"
+                    last_progress_update = 0
+                    
+                    try:
+                        while True:
+                            # Check if process completed
+                            return_code = process.poll()
+                            if return_code is not None:
+                                # Process finished
+                                if return_code == 0:
+                                    logger.info(f"✅ Pre-analysis process completed successfully (exit code: {return_code})")
+                                else:
+                                    logger.error(f"❌ Pre-analysis process failed with exit code: {return_code}")
+                                break
+                            
+                            # Check for abort flag
+                            if abort_flag.exists():
+                                logger.warning(f"⚠️  ABORT flag detected - terminating pre-analysis process")
+                                process.terminate()
+                                time.sleep(2)
+                                if process.poll() is None:
+                                    logger.warning(f"⚠️  Process didn't terminate, sending SIGKILL")
+                                    process.kill()
+                                break
+                            
+                            # Poll Redis for progress updates (every 2 seconds, like train_single_predictor)
+                            current_time = time.time()
+                            if current_time - last_progress_update >= 2:
+                                try:
+                                    from lib.job_manager import load_job
+                                    job_data = load_job(job_id)
+                                    if job_data:
+                                        progress = job_data.get('progress', 0)
+                                        current_config = job_data.get('current_config', 0)
+                                        total_configs = job_data.get('total_configs', 0)
+                                        current_auc = job_data.get('current_auc')
+                                        best_config_so_far = job_data.get('best_config_so_far')
+                                        best_auc_so_far = job_data.get('best_auc_so_far')
+                                        status_message = job_data.get('status_message', 'Pre-analysis architecture search in progress...')
+                                        
+                                        # Build status message
+                                        if current_config > 0 and total_configs > 0:
+                                            status = f"Testing config {current_config}/{total_configs}"
+                                            if current_auc is not None:
+                                                status += f" (AUC: {current_auc:.4f})"
+                                            if best_config_so_far and best_auc_so_far is not None:
+                                                status += f" | Best so far: d_model={best_config_so_far.get('d_model')}, layers={best_config_so_far.get('n_transformer_layers')}, heads={best_config_so_far.get('n_attention_heads')} (AUC: {best_auc_so_far:.4f})"
+                                        else:
+                                            status = status_message
+                                        
+                                        # Update Celery state so client gets progress
+                                        self.update_state(
+                                            state='PROGRESS',
+                                            meta={
+                                                'status': status,
+                                                'job_id': job_id,
+                                                'session_id': session_id,
+                                                'progress': progress,
+                                                'current_config': current_config,
+                                                'total_configs': total_configs,
+                                                'current_auc': current_auc,
+                                                'best_config_so_far': best_config_so_far,
+                                                'best_auc_so_far': best_auc_so_far
+                                            }
+                                        )
+                                        last_progress_update = current_time
+                                except Exception as e:
+                                    # Don't fail if progress update fails
+                                    logger.debug(f"Failed to update progress: {e}")
+                            
+                            time.sleep(1)  # Check every second
+                        
+                        # Wait for process to fully exit
+                        final_return_code = process.wait()
+                        
+                        if final_return_code != 0:
+                            raise RuntimeError(f"Pre-analysis process failed with exit code: {final_return_code}")
+                    
+                    except KeyboardInterrupt:
+                        logger.warning("⚠️  Received interrupt - terminating pre-analysis process")
+                        process.terminate()
+                        time.sleep(2)
+                        if process.poll() is None:
+                            process.kill()
+                        raise
+                    finally:
+                        # Clean up temp JSON file
+                        try:
+                            os.unlink(json_file)
+                        except Exception:
+                            pass
+                
+                # Load results from session (wrapper saves it there)
+                from lib.session_manager import load_session
+                session = load_session(session_id)
+                optimal_config = session.get('optimal_es_config')
+                
+                if not optimal_config:
+                    raise RuntimeError("Pre-analysis completed but optimal_es_config not found in session")
+                
+                logger.info(f"✅ Quick search complete: {optimal_config}")
+                
+                # Save results to job directory for debugging
+                results_file = job_dir / "pre_analysis_results.json"
+                try:
+                    # Get all tested configs from session if available
+                    session = load_session(session_id)
+                    pre_analysis_results = session.get('pre_analysis_results', {})
+                    all_tested_configs = pre_analysis_results.get('all_tested_configs', [])
+                    
+                    results_output = {
+                        "optimal_config": optimal_config,
+                        "session_id": session_id,
+                        "job_id": job_id,
+                        "timestamp": datetime.now().isoformat(),
+                        "all_tested_configs": all_tested_configs,  # Include all tested configs
+                        "meta": {
+                            "n_samples_requested": job_spec.get('n_samples', 20),
+                            "quick_epochs": job_spec.get('quick_epochs', 25),
+                            "used_suggested_configs": bool(suggested_configs),
+                            "total_configs_tested": len(all_tested_configs),
+                            "successful_configs": len([c for c in all_tested_configs if c.get('tested', False)]),
+                            "failed_configs": len([c for c in all_tested_configs if not c.get('tested', False)])
+                        }
+                    }
+                    
+                    with open(results_file, 'w') as f:
+                        json.dump(results_output, f, indent=2, default=str)
+                        
+                    logger.info(f"💾 Saved pre-analysis results to {results_file} (including {len(all_tested_configs)} tested configs)")
+                except Exception as e:
+                    logger.warning(f"⚠️  Failed to save pre-analysis results: {e}")
+                
+                logger.info(f"💾 Optimal config loaded from session")
+                
             except Exception as e:
-                logger.warning(f"⚠️  Failed to save pre-analysis results: {e}")
-            
-            # Save optimal config to session
-            from lib.session_manager import load_session, save_session
-            session = load_session(session_id)
-            session['optimal_es_config'] = optimal_config
-            save_session(session_id, session, exist_ok=True)
-            
-            logger.info(f"💾 Saved optimal config to session")
+                logger.error(f"❌ Failed to run pre-analysis: {e}")
+                traceback.print_exc()
+                raise
             
             # Dispatch next job (train_es)
             try:
@@ -2511,7 +2790,6 @@ def train_single_predictor(self, *args, **kwargs):
                     
                     # Wait for process to complete
                     # CRITICAL: Check for ABORT flag periodically AND update Celery state with progress
-                    import time
                     abort_flag = job_dir / "ABORT"
                     last_progress_update = 0