featrixsphere 0.2.1439__tar.gz → 0.2.1461__tar.gz

{featrixsphere-0.2.1439 → featrixsphere-0.2.1461}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featrixsphere
-Version: 0.2.1439
+Version: 0.2.1461
 Summary: Transform any CSV into a production-ready ML model in minutes, not months.
 Home-page: https://github.com/Featrix/sphere
 Author: Featrix

featrixsphere-0.2.1461/VERSION

@@ -0,0 +1 @@
+0.2.1461

{featrixsphere-0.2.1439 → featrixsphere-0.2.1461}/featrixsphere/__init__.py

@@ -38,7 +38,7 @@ Example:
     ...                                labels=['Experiment A', 'Experiment B'])
 """
 
-__version__ = "0.2.1439"
+__version__ = "0.2.1461"
 __author__ = "Featrix"
 __email__ = "support@featrix.com"
 __license__ = "MIT"

{featrixsphere-0.2.1439 → featrixsphere-0.2.1461}/featrixsphere/client.py

@@ -1635,7 +1635,7 @@ class FeatrixSphereClient:
             ax.text(0.5, 0.5, f'Error plotting embedding: {e}', 
                    transform=ax.transAxes, ha='center', va='center')
 
-    def create_embedding_space(self, name: str, s3_training_dataset: str, s3_validation_dataset: str, webhooks: Dict[str, str] = None) -> SessionInfo:
+    def create_embedding_space(self, name: str, s3_training_dataset: str, s3_validation_dataset: str, webhooks: Dict[str, str] = None, user_metadata: Dict[str, Any] = None) -> SessionInfo:
         """
         Create a new embedding space from S3 training and validation datasets.
         
@@ -1644,6 +1644,7 @@ class FeatrixSphereClient:
             s3_training_dataset: S3 URL for training dataset (must start with 's3://')
             s3_validation_dataset: S3 URL for validation dataset (must start with 's3://')
             webhooks: Optional dict with webhook configuration keys (webhook_callback_secret, s3_backup_url, model_id_update_url)
+            user_metadata: Optional user metadata for ES/SP identification (max 32KB)
             
         Returns:
             SessionInfo for the newly created embedding space session
@@ -1668,6 +1669,10 @@ class FeatrixSphereClient:
         }
         if webhooks:
             data['webhooks'] = webhooks
+        if user_metadata:
+            import json
+            data['user_metadata'] = json.dumps(user_metadata)
+            print(f"User metadata: {user_metadata}")
         
         response_data = self._post_json("/compute/create-embedding-space", data)
         
@@ -1689,7 +1694,8 @@ class FeatrixSphereClient:
         parent_embedding_space_path: str = None,
         s3_training_dataset: str = None,
         s3_validation_dataset: str = None,
-        webhooks: Dict[str, str] = None
+        webhooks: Dict[str, str] = None,
+        user_metadata: Dict[str, Any] = None
     ) -> SessionInfo:
         """
         Fine-tune an existing embedding space on new data.
@@ -1785,6 +1791,10 @@ class FeatrixSphereClient:
         
         if webhooks:
             data['webhooks'] = webhooks
+        if user_metadata:
+            import json
+            data['user_metadata'] = json.dumps(user_metadata)
+            print(f"User metadata: {user_metadata}")
         
         response_data = self._post_json("/compute/fine-tune-embedding-space", data)
         
@@ -4203,7 +4213,8 @@ class FeatrixSphereClient:
                                     class_imbalance: dict = None,
                                     poll_interval: int = 30, max_poll_time: int = 3600,
                                     verbose: bool = True,
-                                    webhooks: Dict[str, str] = None) -> SessionInfo:
+                                    webhooks: Dict[str, str] = None,
+                                    user_metadata: Dict[str, Any] = None) -> SessionInfo:
         """
         Train a single predictor on a foundation model (existing embedding space).
         
@@ -4227,6 +4238,7 @@ class FeatrixSphereClient:
             max_poll_time: Maximum time to poll in seconds (default: 3600 = 1 hour)
             verbose: Whether to print status updates during polling (default: True)
             webhooks: Optional dict with webhook configuration keys (webhook_callback_secret, s3_backup_url, model_id_update_url)
+            user_metadata: Optional user metadata for ES/SP identification (max 32KB)
             
         Returns:
             SessionInfo for the newly created predictor training session
@@ -4318,6 +4330,11 @@ class FeatrixSphereClient:
             if webhooks:
                 import json
                 data["webhooks"] = json.dumps(webhooks)
+            if user_metadata:
+                import json
+                data["user_metadata"] = json.dumps(user_metadata)
+                if verbose:
+                    print(f"User metadata: {user_metadata}")
             
             # Send request with file if provided
             try:

{featrixsphere-0.2.1439 → featrixsphere-0.2.1461}/featrixsphere.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featrixsphere
-Version: 0.2.1439
+Version: 0.2.1461
 Summary: Transform any CSV into a production-ready ML model in minutes, not months.
 Home-page: https://github.com/Featrix/sphere
 Author: Featrix

{featrixsphere-0.2.1439 → featrixsphere-0.2.1461}/src/api.py

@@ -1031,7 +1031,49 @@ def create_app() -> FastAPI:
                                 
                                 processes.append(process_info)
                     
-                    # Calculate cumulative stats for multiprocess workers of train_es
+                    # Calculate cumulative GPU stats for ALL supervisorctl processes (main + children)
+                    # This ensures every daemon process shows total GPU usage including all child processes
+                    for process in processes:
+                        process_pid = process.get('pid')
+                        if not process_pid:
+                            continue
+                        
+                        try:
+                            main_proc = psutil.Process(process_pid)
+                            # Get all child processes recursively
+                            children = main_proc.children(recursive=True)
+                            
+                            # Start with main process GPU stats from the pre-built map
+                            main_gpu_percent, main_gpu_vram_percent = gpu_stats_map.get(process_pid, (0.0, 0.0))
+                            
+                            # Sum GPU stats from all children using the pre-built map
+                            children_gpu_total = 0.0
+                            children_vram_total = 0.0
+                            
+                            for child in children:
+                                try:
+                                    child_gpu_percent, child_gpu_vram_percent = gpu_stats_map.get(child.pid, (0.0, 0.0))
+                                    children_gpu_total += child_gpu_percent
+                                    children_vram_total += child_gpu_vram_percent
+                                except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+                                    continue
+                            
+                            # Add children totals to main process stats
+                            total_gpu_percent = main_gpu_percent + children_gpu_total
+                            total_gpu_vram_percent = main_gpu_vram_percent + children_vram_total
+                            
+                            # Update the process with total GPU stats (main + all children)
+                            process['gpu_percent'] = round(total_gpu_percent, 1)
+                            process['gpu_vram_percent'] = round(total_gpu_vram_percent, 1)
+                            
+                        except (psutil.NoSuchProcess, psutil.AccessDenied) as e:
+                            logger.debug(f"Could not get children for {process.get('name')} (PID {process_pid}): {e}")
+                            # Keep existing GPU stats (main process only)
+                        except Exception as e:
+                            logger.debug(f"Error calculating GPU stats for {process.get('name')} (PID {process_pid}): {e}")
+                            # Keep existing GPU stats (main process only)
+                    
+                    # Legacy: Calculate cumulative stats for multiprocess workers of train_es (for backward compatibility)
                     for process in processes:
                         if process.get('name') == 'worker-train_es':
                             train_es_pid = process.get('pid')
@@ -1411,7 +1453,49 @@ def create_app() -> FastAPI:
                                 
                                 processes.append(process_info)
                     
-                    # Calculate cumulative stats for multiprocess workers of train_es
+                    # Calculate cumulative GPU stats for ALL supervisorctl processes (main + children)
+                    # This ensures every daemon process shows total GPU usage including all child processes
+                    for process in processes:
+                        process_pid = process.get('pid')
+                        if not process_pid:
+                            continue
+                        
+                        try:
+                            main_proc = psutil.Process(process_pid)
+                            # Get all child processes recursively
+                            children = main_proc.children(recursive=True)
+                            
+                            # Start with main process GPU stats from the pre-built map
+                            main_gpu_percent, main_gpu_vram_percent = gpu_stats_map.get(process_pid, (0.0, 0.0))
+                            
+                            # Sum GPU stats from all children using the pre-built map
+                            children_gpu_total = 0.0
+                            children_vram_total = 0.0
+                            
+                            for child in children:
+                                try:
+                                    child_gpu_percent, child_gpu_vram_percent = gpu_stats_map.get(child.pid, (0.0, 0.0))
+                                    children_gpu_total += child_gpu_percent
+                                    children_vram_total += child_gpu_vram_percent
+                                except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+                                    continue
+                            
+                            # Add children totals to main process stats
+                            total_gpu_percent = main_gpu_percent + children_gpu_total
+                            total_gpu_vram_percent = main_gpu_vram_percent + children_vram_total
+                            
+                            # Update the process with total GPU stats (main + all children)
+                            process['gpu_percent'] = round(total_gpu_percent, 1)
+                            process['gpu_vram_percent'] = round(total_gpu_vram_percent, 1)
+                            
+                        except (psutil.NoSuchProcess, psutil.AccessDenied) as e:
+                            logger.debug(f"Could not get children for {process.get('name')} (PID {process_pid}): {e}")
+                            # Keep existing GPU stats (main process only)
+                        except Exception as e:
+                            logger.debug(f"Error calculating GPU stats for {process.get('name')} (PID {process_pid}): {e}")
+                            # Keep existing GPU stats (main process only)
+                    
+                    # Legacy: Calculate cumulative stats for multiprocess workers of train_es (for backward compatibility)
                     for process in processes:
                         if process.get('name') == 'worker-train_es':
                             train_es_pid = process.get('pid')
@@ -6177,7 +6261,8 @@ def create_app() -> FastAPI:
         class_imbalance: Optional[str] = Form(None),  # JSON string
         use_class_weights: bool = Form(False),
         optimize_for: Optional[str] = Form(None),  # DEPRECATED: Ignored, kept for backward compatibility
-        webhooks: Optional[str] = Form(None)  # JSON string
+        webhooks: Optional[str] = Form(None),  # JSON string
+        user_metadata: Optional[str] = Form(None)  # JSON string
     ) -> JSONResponse:
         """
         Train a single predictor on a foundation model (existing embedding space).
@@ -6213,6 +6298,15 @@ def create_app() -> FastAPI:
             except json.JSONDecodeError:
                 raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail="webhooks must be valid JSON")
         
+        user_metadata_dict = None
+        if user_metadata:
+            try:
+                user_metadata_dict = json.loads(user_metadata)
+                logger.info(f"User metadata received: {len(str(user_metadata_dict))} chars")
+            except json.JSONDecodeError:
+                logger.warning(f"Invalid user_metadata JSON: {user_metadata}")
+                user_metadata_dict = None
+        
         try:
             # Handle file upload if provided
             final_input_filename = input_filename
@@ -6310,7 +6404,8 @@ def create_app() -> FastAPI:
                 foundation_model_id=foundation_model_id,
                 target_spec=target_spec,
                 input_filename=final_input_filename,
-                name=name
+                name=name,
+                user_metadata=user_metadata_dict
             )
             
             # Save the session
@@ -6354,6 +6449,57 @@ def create_app() -> FastAPI:
             post_slack_alert(f"foundation_model={foundation_model_id}; error starting predictor training on foundation model: {error_msg}")
             raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=f"Error starting predictor training on foundation model: {error_msg}")
 
+    @app.post("/session/{id}/dump-to-backplane")
+    async def dump_session_to_backplane(id: str) -> JSONResponse:
+        """
+        Dispatch a Celery task to copy a session's workspace to /backplane/backplane1.
+        
+        This task:
+        1. Verifies /backplane/backplane1 is mounted and different from /backplane
+        2. Rsyncs the session workspace to the backplane
+        3. Creates a LAST_HOST_DUMP.json marker file with metadata
+        
+        Args:
+            id: Session ID to dump
+        
+        Returns:
+            dict with task_id for polling
+        """
+        try:
+            from celery_app import dump_to_backplane
+            
+            # Verify session exists
+            session = load_session(id)
+            
+            # Dispatch Celery task
+            task = dump_to_backplane.delay(id)
+            
+            logger.info(f"✅ Dispatched dump_to_backplane task for session {id} (task_id: {task.id})")
+            
+            return JSONResponse({
+                "task_id": task.id,
+                "status": "submitted",
+                "session_id": id,
+                "message": f"Dump task submitted for session {id}"
+            })
+            
+        except FileNotFoundError:
+            raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=f"Session not found: {id}")
+        except ImportError as e:
+            logger.error(f"❌ Celery not available: {e}")
+            raise HTTPException(
+                status_code=HTTPStatus.SERVICE_UNAVAILABLE,
+                detail="Celery service not available - dump_to_backplane requires Celery"
+            )
+        except Exception as e:
+            error_msg = str(e) if str(e) else repr(e)
+            logger.error(f"❌ DUMP_TO_BACKPLANE - Exception: {error_msg}")
+            logger.exception(e)
+            raise HTTPException(
+                status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+                detail=f"Error dispatching dump task: {error_msg}"
+            )
+
     @app.post("/compute/session/{id}/mark_for_deletion")
     async def mark_session_for_deletion(id: str) -> JSONResponse:
         """
@@ -8701,7 +8847,7 @@ def create_app() -> FastAPI:
                     'create_structured_data': '📊 Data Processing',
                     'train_es': '🧠 Embedding Space',
                     'train_knn': '🔍 Vector Database',
-                    'build_es_projections': '📈 2D Projections',
+                    'run_clustering': '📈 2D Projections',
                     'train_single_predictor': '🎯 Single Predictor'
                 }
                 return display_names.get(job_type, f'⚙️ {job_type}')

{featrixsphere-0.2.1439 → featrixsphere-0.2.1461}/src/auto_upgrade_monitor.py

@@ -119,7 +119,7 @@ CHECK_INTERVAL = 3600  # Check every 60 minutes (3600 seconds)
 TRAINING_QUEUES = [
     "train_es",
     "train_knn", 
-    "build_es_projections",
+    "run_clustering",
     "train_single_predictor",
     "cpu_data_tasks"  # create_structured_data runs here
 ]