featrixsphere 0.2.1462__tar.gz → 0.2.1830__tar.gz

{featrixsphere-0.2.1462 → featrixsphere-0.2.1830}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featrixsphere
-Version: 0.2.1462
+Version: 0.2.1830
 Summary: Transform any CSV into a production-ready ML model in minutes, not months.
 Home-page: https://github.com/Featrix/sphere
 Author: Featrix

featrixsphere-0.2.1830/VERSION

@@ -0,0 +1 @@
+0.2.1830

{featrixsphere-0.2.1462 → featrixsphere-0.2.1830}/featrix-update.py

@@ -409,10 +409,24 @@ def main():
         newest_version = newest['version']
         newest_filename = newest['filename']
         newest_hash = newest.get('hash')
+        newest_date = newest.get('date_modified')
         newest_url = f"{FIRMWARE_SERVER}/{newest_filename}"
         
+        # Format date if available
+        date_str = ""
+        if newest_date:
+            try:
+                # Parse ISO format date and format nicely
+                date_obj = datetime.fromisoformat(newest_date.replace('Z', '+00:00'))
+                date_str = date_obj.strftime("%Y-%m-%d %H:%M:%S")
+            except Exception:
+                # Fallback to raw date if parsing fails
+                date_str = newest_date
+        
         print(f"\n📊 Available versions:")
         print(f"   Newest: {newest_version} ({newest_filename})")
+        if date_str:
+            print(f"   Date: {date_str}")
         if newest_hash:
             print(f"   Hash: {newest_hash[:16]}...")
         

{featrixsphere-0.2.1462 → featrixsphere-0.2.1830}/featrixsphere/__init__.py

@@ -38,7 +38,7 @@ Example:
     ...                                labels=['Experiment A', 'Experiment B'])
 """
 
-__version__ = "0.2.1462"
+__version__ = "0.2.1830"
 __author__ = "Featrix"
 __email__ = "support@featrix.com"
 __license__ = "MIT"

{featrixsphere-0.2.1462 → featrixsphere-0.2.1830}/featrixsphere/client.py

@@ -281,9 +281,20 @@ class FeatrixSphereClient:
             max_retries = self.default_max_retries
         
         # Special handling for session endpoints - longer retry window for 504 errors
-        is_session_endpoint = '/session/' in endpoint
-        if max_retry_time is None and is_session_endpoint:
-            max_retry_time = 120.0  # 120 seconds for session endpoints
+        # Session endpoints include /session/ and /upload_with_new_session/ (creates session)
+        is_session_endpoint = '/session/' in endpoint or '/upload_with_new_session' in endpoint
+        
+        # For upload endpoints, use a much longer timeout (10 minutes for large files)
+        # This MUST happen before setting max_retry_time to ensure uploads get proper timeout
+        if '/upload_with_new_session' in endpoint:
+            # Override timeout if not explicitly set in kwargs
+            if 'timeout' not in kwargs:
+                kwargs['timeout'] = 600  # 10 minutes for file uploads
+            # Also set a longer max_retry_time for upload endpoints (10 minutes)
+            if max_retry_time is None:
+                max_retry_time = 600.0  # 10 minutes for upload endpoints
+        elif max_retry_time is None and is_session_endpoint:
+            max_retry_time = 120.0  # 120 seconds for other session endpoints
             
         # Auto-add /compute prefix for session endpoints
         if endpoint.startswith('/session/') and not endpoint.startswith('/compute/session/'):
@@ -323,7 +334,8 @@ class FeatrixSphereClient:
                         else:
                             # Out of retry time
                             print(f"API request failed: {method} {url}")
-                            print(f"504 Gateway Timeout - exceeded max retry time ({max_retry_time}s)")
+                            max_retry_time_str = f"{max_retry_time}s" if max_retry_time else "None"
+                            print(f"504 Gateway Timeout - exceeded max retry time ({max_retry_time_str})")
                             raise
                     
                     # Check for server restart patterns in 500 errors
@@ -1082,19 +1094,29 @@ class FeatrixSphereClient:
                 """
                 
                 if session_info.jobs:
+                    # Human-readable job type descriptions
+                    JOB_TYPE_NAMES = {
+                        'create_structured_data': 'Creating Structured Data',
+                        'train_es': 'Training Embedding Space',
+                        'train_knn': 'Training KNN Index',
+                        'run_clustering': 'Running Clustering',
+                        'train_single_predictor': 'Training Predictor',
+                    }
+                    
                     html_content += "<h4>Jobs:</h4><ul>"
                     for job_id, job in session_info.jobs.items():
                         job_status = job.get('status', 'unknown')
                         progress = job.get('progress')
                         job_type = job.get('type', job_id.split('_')[0])
+                        job_display_name = JOB_TYPE_NAMES.get(job_type, job_type)
                         
                         if progress is not None:
                             progress_pct = progress * 100
                             progress_bar = "▓" * int(progress_pct / 5) + "░" * (20 - int(progress_pct / 5))
-                            html_content += f"<li><strong>{job_type}:</strong> {job_status} [{progress_bar}] {progress_pct:.1f}%</li>"
+                            html_content += f"<li><strong>{job_display_name}:</strong> {job_status} [{progress_bar}] {progress_pct:.1f}%</li>"
                         else:
                             status_emoji = "✅" if job_status == "done" else "🔄" if job_status == "running" else "❌"
-                            html_content += f"<li>{status_emoji} <strong>{job_type}:</strong> {job_status}</li>"
+                            html_content += f"<li>{status_emoji} <strong>{job_display_name}:</strong> {job_status}</li>"
                     html_content += "</ul>"
                 
                 display(HTML(html_content))
@@ -1149,6 +1171,15 @@ class FeatrixSphereClient:
             from rich.text import Text
             import time
             
+            # Human-readable job type descriptions
+            JOB_TYPE_NAMES = {
+                'create_structured_data': 'Creating Structured Data',
+                'train_es': 'Training Embedding Space',
+                'train_knn': 'Training KNN Index',
+                'run_clustering': 'Running Clustering',
+                'train_single_predictor': 'Training Predictor',
+            }
+            
             start_time = time.time()
             job_tasks = {}  # Track progress tasks for each job
             
@@ -1180,15 +1211,18 @@ class FeatrixSphereClient:
                     for job_id, job in session_info.jobs.items():
                         if job_id not in job_tasks:
                             job_type = job.get('type', job_id.split('_')[0])
-                            job_tasks[job_id] = progress.add_task(f"[cyan]{job_type}", total=100)
+                            job_display_name = JOB_TYPE_NAMES.get(job_type, job_type)
+                            job_tasks[job_id] = progress.add_task(f"[cyan]{job_display_name}", total=100)
                         
                         # Update job progress
                         job_status = job.get('status', 'unknown')
                         raw_progress = job.get('progress', 0)
                         job_progress = 100 if job_status == 'done' else (raw_progress * 100 if raw_progress else 0)
                         
+                        job_type = job.get('type', job_id.split('_')[0])
+                        job_display_name = JOB_TYPE_NAMES.get(job_type, job_type)
                         progress.update(job_tasks[job_id], completed=job_progress,
-                                      description=f"[cyan]{job.get('type', job_id.split('_')[0])} ({job_status})")
+                                      description=f"[cyan]{job_display_name} ({job_status})")
                     
                     # Check completion
                     if session_info.status in ['done', 'failed', 'cancelled']:
@@ -1232,8 +1266,20 @@ class FeatrixSphereClient:
             status_callback(session_info, elapsed)
             
             # Check if completed
+            # CRITICAL: Only return done if we actually have jobs that completed
+            # If no jobs exist, session shouldn't be marked as done
             if session_info.status in ['completed', 'done', 'DONE']:
-                return session_info
+                # Verify that jobs actually exist and completed
+                if session_info.jobs:
+                    # Check if all jobs are in terminal states
+                    terminal_states = {'done', 'failed', 'cancelled'}
+                    all_terminal = all(job.get('status') in terminal_states for job in session_info.jobs.values())
+                    if all_terminal:
+                        return session_info
+                else:
+                    # No jobs but status is "done" - this is a bug, keep waiting
+                    logger.warning(f"⚠️  Session {session_id} status is 'done' but no jobs exist - this shouldn't happen, continuing to wait...")
+                    # Don't return - keep waiting for jobs to appear
             
             time.sleep(check_interval)
         
@@ -2145,8 +2191,38 @@ class FeatrixSphereClient:
             data['quick_run'] = 'true'
             data['epochs'] = str(epochs)
             print(f"Training epochs: {epochs} (quick_run mode enabled)")
-            
-        response = self._make_request("POST", "/compute/upload_with_new_session/", files=files, data=data)
+        
+        # Check file size - warn if very large
+        file_size_mb = len(file_content) / (1024 * 1024)
+        CHUNK_SIZE_MB = 512  # 512 MB chunk size
+        CHUNK_SIZE_BYTES = CHUNK_SIZE_MB * 1024 * 1024
+        
+        if file_size_mb > CHUNK_SIZE_MB:
+            print(f"⚠️  Warning: File size ({file_size_mb:.1f} MB) exceeds {CHUNK_SIZE_MB} MB threshold")
+            print(f"   Large uploads may timeout. Consider splitting the data or using smaller batches.")
+        
+        # Try upload with retry on 504
+        import time
+        upload_trace_id = f"UPLOAD-{int(time.time()*1000)}"
+        print(f"🔵 [CLIENT] {upload_trace_id} Starting upload request")
+        print(f"   Endpoint: /compute/upload_with_new_session/")
+        print(f"   File: {filename}, Size: {file_size_mb:.2f} MB")
+        print(f"   Form data keys: {list(data.keys())}")
+        try:
+            response = self._make_request("POST", "/compute/upload_with_new_session/", files=files, data=data)
+            print(f"🔵 [CLIENT] {upload_trace_id} Received response: HTTP {response.status_code}")
+        except requests.exceptions.HTTPError as e:
+            # If we get a 504 and file is large, suggest chunking
+            if e.response and e.response.status_code == 504 and file_size_mb > CHUNK_SIZE_MB:
+                print(f"\n❌ 504 Gateway Timeout on large file upload ({file_size_mb:.1f} MB)")
+                print(f"   File exceeds {CHUNK_SIZE_MB} MB - chunking not yet implemented for regular uploads")
+                print(f"   Consider:")
+                print(f"   1. Splitting your data into smaller files (< {CHUNK_SIZE_MB} MB each)")
+                print(f"   2. Using the chunked upload endpoint (if available)")
+                print(f"   3. Retrying the upload (server may have been temporarily busy)")
+                raise
+            else:
+                raise
         
         response_data = response.json()
         session_id = response_data.get('session_id')
@@ -5296,6 +5372,219 @@ class FeatrixSphereClient:
             if verbose:
                 print(f"❌ Error starting predictor continuation: {e}")
             raise
+    
+    def train_on_partial_foundation(
+        self,
+        foundation_session_id: str,
+        local_data_file: str,
+        target_column: str,
+        target_column_type: str,
+        checkpoint_epoch: int = None,
+        epochs: int = 0,
+        rare_label_value: str = None,
+        use_class_weights: bool = True,
+        class_imbalance: dict = None,
+        cost_false_positive: float = None,
+        cost_false_negative: float = None,
+        name: str = None,
+        session_name_prefix: str = None,
+        webhooks: Dict[str, str] = None,
+        poll_interval: int = 30,
+        max_poll_time: int = 3600,
+        verbose: bool = True
+    ) -> Dict[str, Any]:
+        """
+        Train a predictor on a partial/in-progress foundation model.
+        
+        Designed for use while the foundation model is STILL TRAINING. Test predictor
+        performance on intermediate checkpoints without waiting for full ES training
+        to complete. Also works with completed foundation training.
+        
+        Creates a NEW session using:
+        - Checkpoint from foundation's ES training (specific epoch or best/latest)
+        - Your uploaded data file for predictor training
+        - Metadata about foundation training progress at checkpoint time
+        
+        Perfect for:
+        - Testing predictor performance while ES is still training
+        - Training on partially-trained foundation models
+        - Running parallel experiments on different checkpoint epochs
+        
+        Args:
+            foundation_session_id: Session ID with ES training (in-progress or completed)
+            local_data_file: Path to CSV file on YOUR MACHINE (will be uploaded)
+            target_column: Column to predict
+            target_column_type: "set" for classification, "scalar" for regression
+            checkpoint_epoch: Which epoch checkpoint to use (None = best/latest available)
+            epochs: Predictor training epochs (0 = auto-calculate based on data size)
+            rare_label_value: Minority class label for binary classification
+            use_class_weights: Enable class weighting for imbalanced data
+            class_imbalance: Expected class distribution from real world
+            cost_false_positive: Cost of false positive (classification only)
+            cost_false_negative: Cost of false negative (classification only)
+            name: Optional name for the new session
+            session_name_prefix: Optional prefix for session ID
+            webhooks: Webhook configuration dict
+            poll_interval: Seconds between status checks (default: 30)
+            max_poll_time: Maximum polling time in seconds (default: 3600 = 1 hour)
+            verbose: Print status updates
+            
+        Returns:
+            dict with:
+                - new_session_id: Created session ID
+                - foundation_session_id: Original foundation session
+                - checkpoint_epoch: Epoch used for checkpoint
+                - foundation_training_metadata: Progress info when checkpoint was created
+                - target_column: Column being predicted
+                - predictor_epochs: Training epochs
+                
+        Example:
+            ```python
+            # Train on epoch 50 checkpoint while ES is still training
+            result = client.train_on_partial_foundation(
+                foundation_session_id="abc-def-ghi",
+                local_data_file="my_data.csv",
+                target_column="price",
+                target_column_type="scalar",
+                checkpoint_epoch=50,
+                epochs=100
+            )
+            
+            print(f"New session: {result['new_session_id']}")
+            print(f"Foundation was {result['foundation_training_metadata']['training_progress_percent']}% trained")
+            ```
+        """
+        import os
+        from pathlib import Path
+        
+        # Validate inputs
+        if not os.path.exists(local_data_file):
+            raise FileNotFoundError(f"Local data file not found: {local_data_file}")
+        
+        if target_column_type not in ["set", "scalar"]:
+            raise ValueError("target_column_type must be 'set' or 'scalar'")
+        
+        # Prepare form data
+        form_data = {
+            'target_column': target_column,
+            'target_column_type': target_column_type,
+            'epochs': str(epochs),
+            'use_class_weights': str(use_class_weights).lower(),
+        }
+        
+        if checkpoint_epoch is not None:
+            form_data['checkpoint_epoch'] = str(checkpoint_epoch)
+        if rare_label_value:
+            form_data['rare_label_value'] = rare_label_value
+        if class_imbalance:
+            import json
+            form_data['class_imbalance'] = json.dumps(class_imbalance)
+        if cost_false_positive is not None:
+            form_data['cost_false_positive'] = str(cost_false_positive)
+        if cost_false_negative is not None:
+            form_data['cost_false_negative'] = str(cost_false_negative)
+        if name:
+            form_data['name'] = name
+        if session_name_prefix:
+            form_data['session_name_prefix'] = session_name_prefix
+        if webhooks:
+            import json
+            form_data['webhooks'] = json.dumps(webhooks)
+        
+        # Upload file
+        try:
+            with open(local_data_file, 'rb') as f:
+                files = {'file': (Path(local_data_file).name, f, 'text/csv')}
+                response_data = self._post_multipart(
+                    f"/compute/session/{foundation_session_id}/train_on_partial_foundation",
+                    data=form_data,
+                    files=files
+                )
+            
+            new_session_id = response_data.get('new_session_id')
+            checkpoint_epoch_used = response_data.get('checkpoint_epoch')
+            training_metadata = response_data.get('foundation_training_metadata', {})
+            
+            if verbose:
+                print(f"✅ {response_data.get('message')}")
+                print(f"   New session ID: {new_session_id}")
+                print(f"   Checkpoint epoch: {checkpoint_epoch_used}")
+                if training_metadata.get('training_progress_percent'):
+                    print(f"   Foundation training progress: {training_metadata['training_progress_percent']}%")
+                if training_metadata.get('validation_loss_at_checkpoint'):
+                    print(f"   Val loss at checkpoint: {training_metadata['validation_loss_at_checkpoint']:.4f}")
+            
+            # Poll for completion if requested
+            if poll_interval > 0 and max_poll_time > 0 and new_session_id:
+                import time
+                start_time = time.time()
+                last_message = ""
+                
+                while time.time() - start_time < max_poll_time:
+                    try:
+                        session_info = self.get_session_status(new_session_id)
+                        jobs = session_info.jobs if hasattr(session_info, 'jobs') else {}
+                        
+                        # Find predictor training jobs
+                        sp_jobs = {j_id: j for j_id, j in jobs.items() 
+                                 if 'train_single_predictor' in j.get('type', '')}
+                        
+                        if not sp_jobs:
+                            if verbose:
+                                print("⏳ Waiting for predictor job to start...")
+                            time.sleep(poll_interval)
+                            continue
+                        
+                        # Check statuses
+                        running = [j_id for j_id, j in sp_jobs.items() if j.get('status') == 'running']
+                        done = [j_id for j_id, j in sp_jobs.items() if j.get('status') == 'done']
+                        failed = [j_id for j_id, j in sp_jobs.items() if j.get('status') == 'failed']
+                        
+                        status_msg = f"Running: {len(running)}, Done: {len(done)}, Failed: {len(failed)}"
+                        if status_msg != last_message and verbose:
+                            print(f"📊 {status_msg}")
+                            last_message = status_msg
+                        
+                        if not running and (done or failed):
+                            if done:
+                                if verbose:
+                                    print(f"✅ Predictor training completed!")
+                                return {
+                                    **response_data,
+                                    "status": "completed",
+                                    "poll_time": int(time.time() - start_time)
+                                }
+                            elif failed:
+                                if verbose:
+                                    print(f"❌ Predictor training failed")
+                                return {
+                                    **response_data,
+                                    "status": "failed",
+                                    "failed_jobs": failed,
+                                    "poll_time": int(time.time() - start_time)
+                                }
+                        
+                        time.sleep(poll_interval)
+                    except Exception as poll_error:
+                        if verbose:
+                            print(f"⚠️  Polling error: {poll_error}")
+                        time.sleep(poll_interval)
+                
+                if verbose:
+                    print(f"⏰ Polling timeout ({max_poll_time}s). Training may still be in progress.")
+                
+                return {
+                    **response_data,
+                    "status": "timeout",
+                    "poll_time": max_poll_time
+                }
+            
+            return response_data
+            
+        except Exception as e:
+            if verbose:
+                print(f"❌ Error training on partial foundation: {e}")
+            raise
 
     def foundation_model_train_more(self, session_id: str, es_id: str = None, data_passes: int = None,
                                    epochs: int = None, poll_interval: int = 30, max_poll_time: int = 3600,

{featrixsphere-0.2.1462 → featrixsphere-0.2.1830}/featrixsphere.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featrixsphere
-Version: 0.2.1462
+Version: 0.2.1830
 Summary: Transform any CSV into a production-ready ML model in minutes, not months.
 Home-page: https://github.com/Featrix/sphere
 Author: Featrix

{featrixsphere-0.2.1462 → featrixsphere-0.2.1830}/featrixsphere.egg-info/SOURCES.txt

@@ -7,7 +7,6 @@ requirements.txt
 setup.py
 system_monitor.py
 featrixsphere/__init__.py
-featrixsphere/cli.py
 featrixsphere/client.py
 featrixsphere/test_client.py
 featrixsphere.egg-info/PKG-INFO
@@ -21,7 +20,6 @@ src/api.py
 src/auto_upgrade_monitor.py
 src/build_version.py
 src/celery_app.py
-src/cli.py
 src/config.py
 src/demo_existing_model.py
 src/demo_label_updates.py
@@ -33,17 +31,18 @@ src/event_log.py
 src/example_api_usage.py
 src/example_prediction_feedback.py
 src/example_train_predictor.py
-src/featrix_queue.py
 src/featrix_watchdog.py
 src/gc_cleanup.py
 src/llm_client.py
 src/manage_churro.sh
+src/migrate_string_cache_naming.py
 src/neural.py
 src/node-install.sh
 src/prediction_drift_monitor.py
 src/prediction_persistence_worker.py
 src/quick_test_deployment.sh
 src/recreate_session.py
+src/redis_job_progress.py
 src/redis_prediction_cli.py
 src/redis_prediction_store.py
 src/regenerate_training_movie.py
@@ -53,6 +52,8 @@ src/run_api_server.sh
 src/send_email.py
 src/slack.py
 src/standalone_prediction.py
+src/start_celery_cpu_worker.sh
+src/start_celery_gpu_worker.sh
 src/start_celery_worker.sh
 src/start_churro_server.sh
 src/tail-watch.py
@@ -67,17 +68,27 @@ src/test_upload_endpoint.py
 src/tree.py
 src/utils.py
 src/version.py
+src/lib/api_event_retry.py
+src/lib/celery_job_recovery.py
 src/lib/convergence_monitor.py
+src/lib/crash_tracker.py
+src/lib/distribution_shift_detector.py
 src/lib/epoch_projections.py
 src/lib/es_projections.py
 src/lib/es_training.py
+src/lib/es_training_wrapper.py
 src/lib/featrix_debug.py
+src/lib/job_manager.py
 src/lib/json_encoder_cache.py
 src/lib/knn_training.py
+src/lib/queue_manager.py
+src/lib/session_chains.py
+src/lib/session_manager.py
 src/lib/single_predictor_cv.py
 src/lib/single_predictor_training.py
 src/lib/sphere_config.py
 src/lib/structureddata.py
+src/lib/system_health_monitor.py
 src/lib/training_monitor.py
 src/lib/utils.py
 src/lib/vector_db.py
@@ -112,6 +123,7 @@ src/lib/featrix/neural/hubspot_free_domains_list_may_2025.py
 src/lib/featrix/neural/input_data_file.py
 src/lib/featrix/neural/input_data_set.py
 src/lib/featrix/neural/integrity.py
+src/lib/featrix/neural/io_utils.py
 src/lib/featrix/neural/json_cache.py
 src/lib/featrix/neural/json_codec.py
 src/lib/featrix/neural/logging_config.py
@@ -151,6 +163,7 @@ src/lib/featrix/neural/qa/generate_focal_report.py
 src/lib/featrix/neural/qa/model_advisor.py
 src/lib/featrix/neural/qa/show_results.py
 src/lib/featrix/neural/qa/test_adaptive_training.py
+src/lib/featrix/neural/qa/test_checkpoint_dict_reconstruction.py
 src/lib/featrix/neural/qa/test_confusion_matrix_metadata.py
 src/lib/featrix/neural/qa/test_embedding_quality.py
 src/lib/featrix/neural/qa/test_embedding_space.py