featrixsphere 0.2.1461__tar.gz → 0.2.1637__tar.gz

{featrixsphere-0.2.1461 → featrixsphere-0.2.1637}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featrixsphere
-Version: 0.2.1461
+Version: 0.2.1637
 Summary: Transform any CSV into a production-ready ML model in minutes, not months.
 Home-page: https://github.com/Featrix/sphere
 Author: Featrix

featrixsphere-0.2.1637/VERSION

@@ -0,0 +1 @@
+0.2.1637

{featrixsphere-0.2.1461 → featrixsphere-0.2.1637}/featrixsphere/__init__.py

@@ -38,7 +38,7 @@ Example:
     ...                                labels=['Experiment A', 'Experiment B'])
 """
 
-__version__ = "0.2.1461"
+__version__ = "0.2.1637"
 __author__ = "Featrix"
 __email__ = "support@featrix.com"
 __license__ = "MIT"

{featrixsphere-0.2.1461 → featrixsphere-0.2.1637}/featrixsphere/client.py

@@ -281,9 +281,20 @@ class FeatrixSphereClient:
             max_retries = self.default_max_retries
         
         # Special handling for session endpoints - longer retry window for 504 errors
-        is_session_endpoint = '/session/' in endpoint
-        if max_retry_time is None and is_session_endpoint:
-            max_retry_time = 120.0  # 120 seconds for session endpoints
+        # Session endpoints include /session/ and /upload_with_new_session/ (creates session)
+        is_session_endpoint = '/session/' in endpoint or '/upload_with_new_session' in endpoint
+        
+        # For upload endpoints, use a much longer timeout (10 minutes for large files)
+        # This MUST happen before setting max_retry_time to ensure uploads get proper timeout
+        if '/upload_with_new_session' in endpoint:
+            # Override timeout if not explicitly set in kwargs
+            if 'timeout' not in kwargs:
+                kwargs['timeout'] = 600  # 10 minutes for file uploads
+            # Also set a longer max_retry_time for upload endpoints (10 minutes)
+            if max_retry_time is None:
+                max_retry_time = 600.0  # 10 minutes for upload endpoints
+        elif max_retry_time is None and is_session_endpoint:
+            max_retry_time = 120.0  # 120 seconds for other session endpoints
             
         # Auto-add /compute prefix for session endpoints
         if endpoint.startswith('/session/') and not endpoint.startswith('/compute/session/'):
@@ -323,7 +334,8 @@ class FeatrixSphereClient:
                         else:
                             # Out of retry time
                             print(f"API request failed: {method} {url}")
-                            print(f"504 Gateway Timeout - exceeded max retry time ({max_retry_time}s)")
+                            max_retry_time_str = f"{max_retry_time}s" if max_retry_time else "None"
+                            print(f"504 Gateway Timeout - exceeded max retry time ({max_retry_time_str})")
                             raise
                     
                     # Check for server restart patterns in 500 errors
@@ -1082,19 +1094,29 @@ class FeatrixSphereClient:
                 """
                 
                 if session_info.jobs:
+                    # Human-readable job type descriptions
+                    JOB_TYPE_NAMES = {
+                        'create_structured_data': 'Creating Structured Data',
+                        'train_es': 'Training Embedding Space',
+                        'train_knn': 'Training KNN Index',
+                        'run_clustering': 'Running Clustering',
+                        'train_single_predictor': 'Training Predictor',
+                    }
+                    
                     html_content += "<h4>Jobs:</h4><ul>"
                     for job_id, job in session_info.jobs.items():
                         job_status = job.get('status', 'unknown')
                         progress = job.get('progress')
                         job_type = job.get('type', job_id.split('_')[0])
+                        job_display_name = JOB_TYPE_NAMES.get(job_type, job_type)
                         
                         if progress is not None:
                             progress_pct = progress * 100
                             progress_bar = "▓" * int(progress_pct / 5) + "░" * (20 - int(progress_pct / 5))
-                            html_content += f"<li><strong>{job_type}:</strong> {job_status} [{progress_bar}] {progress_pct:.1f}%</li>"
+                            html_content += f"<li><strong>{job_display_name}:</strong> {job_status} [{progress_bar}] {progress_pct:.1f}%</li>"
                         else:
                             status_emoji = "✅" if job_status == "done" else "🔄" if job_status == "running" else "❌"
-                            html_content += f"<li>{status_emoji} <strong>{job_type}:</strong> {job_status}</li>"
+                            html_content += f"<li>{status_emoji} <strong>{job_display_name}:</strong> {job_status}</li>"
                     html_content += "</ul>"
                 
                 display(HTML(html_content))
@@ -1149,6 +1171,15 @@ class FeatrixSphereClient:
             from rich.text import Text
             import time
             
+            # Human-readable job type descriptions
+            JOB_TYPE_NAMES = {
+                'create_structured_data': 'Creating Structured Data',
+                'train_es': 'Training Embedding Space',
+                'train_knn': 'Training KNN Index',
+                'run_clustering': 'Running Clustering',
+                'train_single_predictor': 'Training Predictor',
+            }
+            
             start_time = time.time()
             job_tasks = {}  # Track progress tasks for each job
             
@@ -1180,15 +1211,18 @@ class FeatrixSphereClient:
                     for job_id, job in session_info.jobs.items():
                         if job_id not in job_tasks:
                             job_type = job.get('type', job_id.split('_')[0])
-                            job_tasks[job_id] = progress.add_task(f"[cyan]{job_type}", total=100)
+                            job_display_name = JOB_TYPE_NAMES.get(job_type, job_type)
+                            job_tasks[job_id] = progress.add_task(f"[cyan]{job_display_name}", total=100)
                         
                         # Update job progress
                         job_status = job.get('status', 'unknown')
                         raw_progress = job.get('progress', 0)
                         job_progress = 100 if job_status == 'done' else (raw_progress * 100 if raw_progress else 0)
                         
+                        job_type = job.get('type', job_id.split('_')[0])
+                        job_display_name = JOB_TYPE_NAMES.get(job_type, job_type)
                         progress.update(job_tasks[job_id], completed=job_progress,
-                                      description=f"[cyan]{job.get('type', job_id.split('_')[0])} ({job_status})")
+                                      description=f"[cyan]{job_display_name} ({job_status})")
                     
                     # Check completion
                     if session_info.status in ['done', 'failed', 'cancelled']:
@@ -1232,8 +1266,20 @@ class FeatrixSphereClient:
             status_callback(session_info, elapsed)
             
             # Check if completed
+            # CRITICAL: Only return done if we actually have jobs that completed
+            # If no jobs exist, session shouldn't be marked as done
             if session_info.status in ['completed', 'done', 'DONE']:
-                return session_info
+                # Verify that jobs actually exist and completed
+                if session_info.jobs:
+                    # Check if all jobs are in terminal states
+                    terminal_states = {'done', 'failed', 'cancelled'}
+                    all_terminal = all(job.get('status') in terminal_states for job in session_info.jobs.values())
+                    if all_terminal:
+                        return session_info
+                else:
+                    # No jobs but status is "done" - this is a bug, keep waiting
+                    logger.warning(f"⚠️  Session {session_id} status is 'done' but no jobs exist - this shouldn't happen, continuing to wait...")
+                    # Don't return - keep waiting for jobs to appear
             
             time.sleep(check_interval)
         
@@ -2145,8 +2191,38 @@ class FeatrixSphereClient:
             data['quick_run'] = 'true'
             data['epochs'] = str(epochs)
             print(f"Training epochs: {epochs} (quick_run mode enabled)")
-            
-        response = self._make_request("POST", "/compute/upload_with_new_session/", files=files, data=data)
+        
+        # Check file size - warn if very large
+        file_size_mb = len(file_content) / (1024 * 1024)
+        CHUNK_SIZE_MB = 512  # 512 MB chunk size
+        CHUNK_SIZE_BYTES = CHUNK_SIZE_MB * 1024 * 1024
+        
+        if file_size_mb > CHUNK_SIZE_MB:
+            print(f"⚠️  Warning: File size ({file_size_mb:.1f} MB) exceeds {CHUNK_SIZE_MB} MB threshold")
+            print(f"   Large uploads may timeout. Consider splitting the data or using smaller batches.")
+        
+        # Try upload with retry on 504
+        import time
+        upload_trace_id = f"UPLOAD-{int(time.time()*1000)}"
+        print(f"🔵 [CLIENT] {upload_trace_id} Starting upload request")
+        print(f"   Endpoint: /compute/upload_with_new_session/")
+        print(f"   File: {filename}, Size: {file_size_mb:.2f} MB")
+        print(f"   Form data keys: {list(data.keys())}")
+        try:
+            response = self._make_request("POST", "/compute/upload_with_new_session/", files=files, data=data)
+            print(f"🔵 [CLIENT] {upload_trace_id} Received response: HTTP {response.status_code}")
+        except requests.exceptions.HTTPError as e:
+            # If we get a 504 and file is large, suggest chunking
+            if e.response and e.response.status_code == 504 and file_size_mb > CHUNK_SIZE_MB:
+                print(f"\n❌ 504 Gateway Timeout on large file upload ({file_size_mb:.1f} MB)")
+                print(f"   File exceeds {CHUNK_SIZE_MB} MB - chunking not yet implemented for regular uploads")
+                print(f"   Consider:")
+                print(f"   1. Splitting your data into smaller files (< {CHUNK_SIZE_MB} MB each)")
+                print(f"   2. Using the chunked upload endpoint (if available)")
+                print(f"   3. Retrying the upload (server may have been temporarily busy)")
+                raise
+            else:
+                raise
         
         response_data = response.json()
         session_id = response_data.get('session_id')
@@ -4248,9 +4324,28 @@ class FeatrixSphereClient:
         
         # Get the compute cluster from the foundation model session
         # This ensures we upload files to the same node where the foundation model lives
-        foundation_session = self.get_session_status(foundation_model_id)
-        foundation_compute_cluster = self.get_last_server_metadata()
-        foundation_compute_cluster = foundation_compute_cluster.get('compute_cluster') if foundation_compute_cluster else None
+        # If the foundation session doesn't exist (404), we'll proceed with current compute cluster
+        foundation_compute_cluster = None
+        try:
+            foundation_session = self.get_session_status(foundation_model_id)
+            foundation_compute_cluster = self.get_last_server_metadata()
+            foundation_compute_cluster = foundation_compute_cluster.get('compute_cluster') if foundation_compute_cluster else None
+        except Exception as e:
+            # Foundation session might not exist or be accessible - that's okay
+            # The server will validate it when we submit the training request
+            if verbose:
+                # Check if it's a 404 HTTP error
+                is_404 = False
+                if isinstance(e, requests.exceptions.HTTPError):
+                    if hasattr(e, 'response') and e.response.status_code == 404:
+                        is_404 = True
+                
+                if is_404:
+                    print(f"  ⚠️  Foundation session not found (404) - will use current compute cluster")
+                    print(f"     Server will validate foundation model when training starts")
+                else:
+                    print(f"  ⚠️  Could not fetch foundation session: {e}")
+                    print(f"     Will proceed with current compute cluster")
         
         # Temporarily set compute cluster for file uploads if we found one
         original_compute_cluster = self.compute_cluster
@@ -4259,6 +4354,8 @@ class FeatrixSphereClient:
             self.set_compute_cluster(foundation_compute_cluster)
             if verbose:
                 print(f"  Using compute cluster: {foundation_compute_cluster}")
+        elif verbose and self.compute_cluster:
+            print(f"  Using current compute cluster: {self.compute_cluster}")
         
         try:
             # Validate that only one data source is provided

{featrixsphere-0.2.1461 → featrixsphere-0.2.1637}/featrixsphere.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featrixsphere
-Version: 0.2.1461
+Version: 0.2.1637
 Summary: Transform any CSV into a production-ready ML model in minutes, not months.
 Home-page: https://github.com/Featrix/sphere
 Author: Featrix

{featrixsphere-0.2.1461 → featrixsphere-0.2.1637}/featrixsphere.egg-info/SOURCES.txt

@@ -7,7 +7,6 @@ requirements.txt
 setup.py
 system_monitor.py
 featrixsphere/__init__.py
-featrixsphere/cli.py
 featrixsphere/client.py
 featrixsphere/test_client.py
 featrixsphere.egg-info/PKG-INFO
@@ -21,7 +20,6 @@ src/api.py
 src/auto_upgrade_monitor.py
 src/build_version.py
 src/celery_app.py
-src/cli.py
 src/config.py
 src/demo_existing_model.py
 src/demo_label_updates.py
@@ -33,17 +31,18 @@ src/event_log.py
 src/example_api_usage.py
 src/example_prediction_feedback.py
 src/example_train_predictor.py
-src/featrix_queue.py
 src/featrix_watchdog.py
 src/gc_cleanup.py
 src/llm_client.py
 src/manage_churro.sh
+src/migrate_string_cache_naming.py
 src/neural.py
 src/node-install.sh
 src/prediction_drift_monitor.py
 src/prediction_persistence_worker.py
 src/quick_test_deployment.sh
 src/recreate_session.py
+src/redis_job_progress.py
 src/redis_prediction_cli.py
 src/redis_prediction_store.py
 src/regenerate_training_movie.py
@@ -53,6 +52,8 @@ src/run_api_server.sh
 src/send_email.py
 src/slack.py
 src/standalone_prediction.py
+src/start_celery_cpu_worker.sh
+src/start_celery_gpu_worker.sh
 src/start_celery_worker.sh
 src/start_churro_server.sh
 src/tail-watch.py
@@ -67,13 +68,20 @@ src/test_upload_endpoint.py
 src/tree.py
 src/utils.py
 src/version.py
+src/lib/api_event_retry.py
+src/lib/celery_job_recovery.py
 src/lib/convergence_monitor.py
 src/lib/epoch_projections.py
 src/lib/es_projections.py
 src/lib/es_training.py
+src/lib/es_training_wrapper.py
 src/lib/featrix_debug.py
+src/lib/job_manager.py
 src/lib/json_encoder_cache.py
 src/lib/knn_training.py
+src/lib/queue_manager.py
+src/lib/session_chains.py
+src/lib/session_manager.py
 src/lib/single_predictor_cv.py
 src/lib/single_predictor_training.py
 src/lib/sphere_config.py
@@ -112,6 +120,7 @@ src/lib/featrix/neural/hubspot_free_domains_list_may_2025.py
 src/lib/featrix/neural/input_data_file.py
 src/lib/featrix/neural/input_data_set.py
 src/lib/featrix/neural/integrity.py
+src/lib/featrix/neural/io_utils.py
 src/lib/featrix/neural/json_cache.py
 src/lib/featrix/neural/json_codec.py
 src/lib/featrix/neural/logging_config.py
@@ -151,6 +160,7 @@ src/lib/featrix/neural/qa/generate_focal_report.py
 src/lib/featrix/neural/qa/model_advisor.py
 src/lib/featrix/neural/qa/show_results.py
 src/lib/featrix/neural/qa/test_adaptive_training.py
+src/lib/featrix/neural/qa/test_checkpoint_dict_reconstruction.py
 src/lib/featrix/neural/qa/test_confusion_matrix_metadata.py
 src/lib/featrix/neural/qa/test_embedding_quality.py
 src/lib/featrix/neural/qa/test_embedding_space.py