featrixsphere 2.0.12346__tar.gz → 2.0.12349__tar.gz

{featrixsphere-2.0.12346/featrixsphere.egg-info → featrixsphere-2.0.12349}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featrixsphere
-Version: 2.0.12346
+Version: 2.0.12349
 Summary: Transform any CSV into a production-ready ML model in minutes, not months.
 Home-page: https://github.com/Featrix/sphere
 Author: Featrix

featrixsphere-2.0.12349/VERSION

@@ -0,0 +1 @@
+2.0.12349

{featrixsphere-2.0.12346 → featrixsphere-2.0.12349}/featrixsphere/__init__.py

@@ -44,7 +44,7 @@ Usage:
 For more examples, see the documentation at https://docs.featrix.com
 """
 
-__version__ = "2.0.12346"
+__version__ = "2.0.12349"
 __author__ = "Featrix"
 __email__ = "support@featrix.com"
 __license__ = "MIT"

{featrixsphere-2.0.12346 → featrixsphere-2.0.12349/featrixsphere.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featrixsphere
-Version: 2.0.12346
+Version: 2.0.12349
 Summary: Transform any CSV into a production-ready ML model in minutes, not months.
 Home-page: https://github.com/Featrix/sphere
 Author: Featrix

{featrixsphere-2.0.12346 → featrixsphere-2.0.12349}/src/celery_app.py

@@ -3872,12 +3872,17 @@ def train_single_predictor(self, *args, **kwargs):
                     break
             
             if create_structured_data_job:
-                job_id = create_structured_data_job.get('job_id')
-                if job_id and job_id != 'None':
+                # NOTE: local var name kept distinct from the outer `job_id` (this
+                # task's own job_id, used later for crash-log / status-update calls).
+                # Reusing `job_id` here previously clobbered it for the rest of the
+                # function, so a sqlite_db-not-found failure got attributed to the
+                # create_structured_data job instead of this train_single_predictor job.
+                csd_job_id = create_structured_data_job.get('job_id')
+                if csd_job_id and csd_job_id != 'None':
                     try:
                         from lib.job_manager import get_job_output_path
-                        job_output_dir = get_job_output_path(job_id, session_id, 'create_structured_data')
-                        
+                        job_output_dir = get_job_output_path(csd_job_id, session_id, 'create_structured_data')
+
                         # Look for sqlite_db files in the job output directory
                         possible_db_names = ['embedding_space.db', 'data.db']
                         for db_name in possible_db_names:
@@ -3888,7 +3893,7 @@ def train_single_predictor(self, *args, **kwargs):
                                 if emergency_log:
                                     emergency_log.write(f"✅ Found sqlite_db in job output: {sqlite_db_path}\n")
                                     emergency_log.flush()
-                                
+
                                 # Update session with the found path
                                 session['sqlite_db'] = sqlite_db_path
                                 save_session(session_id, session, exist_ok=True)
@@ -3904,7 +3909,7 @@ def train_single_predictor(self, *args, **kwargs):
                     error_msg = (
                         f"SQLite database not found and create_structured_data job has not been dispatched yet. "
                         f"train_single_predictor cannot run until create_structured_data completes. "
-                        f"Job plan shows create_structured_data but job_id is {job_id!r}"
+                        f"Job plan shows create_structured_data but job_id is {csd_job_id!r}"
                     )
                     if emergency_log:
                         emergency_log.write(f"❌ {error_msg}\n")

{featrixsphere-2.0.12346 → featrixsphere-2.0.12349}/src/lib/featrix/neural/embedded_space.py

@@ -20435,7 +20435,10 @@ class EmbeddingSpace(EmbeddingSpaceLoggingMixin, EmbeddingSpaceCheckpointMixin,
                 
                 # CRITICAL: Check for existing workers before creating new DataLoader
                 # This helps detect worker accumulation issues
-                existing_workers_before = _check_and_cleanup_existing_workers(context=" before creating training DataLoader")
+                existing_workers_before = _check_and_cleanup_existing_workers(
+                    context=" before creating training DataLoader",
+                    min_create_time=_phase_wall_t0,
+                )
                 if existing_workers_before > 0:
                     logger.warning(
                         f"⚠️  Found {existing_workers_before} existing worker(s) before creating training DataLoader. "
@@ -20471,7 +20474,10 @@ class EmbeddingSpace(EmbeddingSpaceLoggingMixin, EmbeddingSpaceCheckpointMixin,
                 # CRITICAL: Check for existing training workers before creating validation workers
                 # This prevents having both training and validation workers active simultaneously,
                 # which doubles memory usage and can cause OOM
-                existing_workers = _check_and_cleanup_existing_workers(context=" before creating validation DataLoader")
+                existing_workers = _check_and_cleanup_existing_workers(
+                    context=" before creating validation DataLoader",
+                    min_create_time=_phase_wall_t0,
+                )
                 if existing_workers > 0:
                     logger.warning(
                         f"⚠️  Found and terminated {existing_workers} stale training DataLoader worker(s) "

{featrixsphere-2.0.12346 → featrixsphere-2.0.12349}/src/lib/featrix/neural/encoders/core.py

@@ -2499,6 +2499,30 @@ class ColumnEncoders(nn.Module):
                     for bname, buf in encoder.named_buffers():
                         logger.error(f"💥 [DEVICE_DIAG] MISMATCH detail: {col_or_group_name} buffer {bname}.device = {buf.device}")
 
+            # DEVICE SELF-HEAL: force this column's returned activations onto
+            # expected_device. The straggler fix above only moves the ENCODER
+            # MODULE's own parameters/buffers before calling forward() — it does
+            # not guarantee the returned tensor lands on expected_device, e.g. an
+            # encoder whose output is built from an input TokenBatch that is
+            # itself still resident on the wrong device. Previously this was only
+            # ever *detected* (DEVICE_DIAG table above), and only for the first 2
+            # batches — the mismatched tensor still sailed into the final
+            # torch.stack() unmodified and crashed with "Expected all tensors to
+            # be on the same device" on whichever batch actually hit it, with NO
+            # diagnostic output once the 2-batch diag window closed (fingerprint
+            # 6c081d82644fd833: 33 crashes on churro/burrito/taco since 2026-08-19,
+            # inside encode_records_batch / warm_start.py multi_query probing).
+            # Checked and logged every batch (not gated by _do_device_diag) since
+            # the whole point is visibility past batch 2.
+            if full_col_encoding.device != expected_device:
+                logger.warning(
+                    f"[DEVICE_STRAGGLER] '{col_or_group_name}' output on "
+                    f"{full_col_encoding.device}, expected {expected_device} — moving in place"
+                )
+                full_col_encoding = full_col_encoding.to(expected_device)
+            if short_col_encoding.device != expected_device:
+                short_col_encoding = short_col_encoding.to(expected_device)
+
             # CRITICAL: Ensure encodings are 2D [batch_size, d_model]
             # Some encoders might accidentally return 3D tensors [batch_size, seq_len, d_model]
             if len(full_col_encoding.shape) == 3:

{featrixsphere-2.0.12346 → featrixsphere-2.0.12349}/src/lib/featrix/neural/es_job_control.py

@@ -119,7 +119,8 @@ def _check_for_runaway_processes(context: str = "", max_expected: int = None):
 _MIN_WORKER_AGE_SECONDS = 3.0
 
 
-def _check_and_cleanup_existing_workers(parent_pid: int = None, context: str = ""):
+def _check_and_cleanup_existing_workers(parent_pid: int = None, context: str = "",
+                                         min_create_time: float = None):
     """
     Check for existing DataLoader worker processes and terminate any found.
 
@@ -142,9 +143,30 @@ def _check_and_cleanup_existing_workers(parent_pid: int = None, context: str = "
     a worker off mid-handshake is what destabilizes the tracker for every
     spawn after it, not the worker count itself.)
 
+    That fixed-age guard alone isn't enough on larger datasets: worker
+    startup (fork + first-batch collation) routinely takes longer than
+    _MIN_WORKER_AGE_SECONDS, so a brand-new, legitimate worker belonging to
+    the DataLoader this very call is about to hand off to still reads as
+    "old enough" and gets killed anyway — corrupting the resource_tracker
+    handshake for the next spawn, which then fails with "Connection
+    refused" (2026-09-01, Filipino_Family_Income_and_Expenditure on churro:
+    the "1 existing worker, RSS=0.11GB" found here was the just-created
+    training DataLoader's own worker, not a leftover). `min_create_time`
+    closes that gap: pass the wall-clock time this training attempt began
+    (before its own DataLoader(s) were created) and any worker spawned AT OR
+    AFTER that point is by definition part of THIS attempt, not a leftover
+    from a previous one — regardless of how long its startup took.
+
     Args:
         parent_pid: Parent process ID (None = current process)
         context: Context string for logging
+        min_create_time: Unix epoch seconds marking when the current
+            training attempt began. Workers created at or after this time
+            belong to this attempt and are never treated as stale, no
+            matter their age. None (default) disables this check — used by
+            callers doing an unconditional full teardown (e.g.
+            reset_training_state before an OOM retry) where anything found
+            really is being abandoned.
 
     Returns:
         Number of existing workers found (and terminated)
@@ -161,6 +183,13 @@ def _check_and_cleanup_existing_workers(parent_pid: int = None, context: str = "
                 f"   Skipping worker PID={w['pid']}{context}: age={age:.1f}s "
                 f"< {_MIN_WORKER_AGE_SECONDS}s minimum — too young to be a stale leftover"
             )
+        elif (min_create_time is not None and w.get('create_time')
+              and w['create_time'] >= min_create_time):
+            logger.debug(
+                f"   Skipping worker PID={w['pid']}{context}: created "
+                f"{w['create_time']:.1f} >= attempt start {min_create_time:.1f} — "
+                f"belongs to this attempt, not a leftover from a previous one"
+            )
         else:
             workers.append(w)
 

{featrixsphere-2.0.12346 → featrixsphere-2.0.12349}/src/lib/featrix/neural/loss/loss_components/halfspace_coverage.py

@@ -40,23 +40,30 @@ class HalfspaceCoverageLoss(LossComponent):
         self.resample_every = config.params.get("resample_every", 50)
         # Split directions will be lazily initialized on first call
         self._split_directions = None
-        self._splits_device = None
-        self._splits_dtype = None
 
     def _get_split_directions(self, D: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor:
         """Get or resample random split directions."""
+        # Check the cached tensor's OWN .device, not the self._splits_device
+        # bookkeeping variable. This component is a plain object (LossComponent
+        # is an ABC, not an nn.Module), so _split_directions is never touched by
+        # the encoder's .to(device)/.cpu() calls, and torch.load(map_location=...)
+        # remaps the tensor's storage without touching a separately-pickled
+        # torch.device attribute — the bookkeeping var can silently say "cuda:0"
+        # while the tensor it describes is physically on cpu. Comparing the real
+        # tensor's device closes that drift. Root cause of fingerprint
+        # d50f5ccc00db40e1 ("mat2 is on cpu, different from other tensors on
+        # cuda:0" in emb @ splits.T) — 2 crashes on churro since 2026-08-19,
+        # only after a checkpoint-restore mid-training.
         needs_resample = (
             self._split_directions is None
             or self._call_count % self.resample_every == 0
             or self._split_directions.shape[1] != D
-            or self._splits_device != device
+            or self._split_directions.device != device
         )
         if needs_resample:
             # Sample random unit vectors as hyperplane normals
             directions = torch.randn(self.n_splits, D, device=device, dtype=dtype)
             self._split_directions = F.normalize(directions, dim=1, eps=1e-8)
-            self._splits_device = device
-            self._splits_dtype = dtype
         return self._split_directions
 
     def compute(

{featrixsphere-2.0.12346 → featrixsphere-2.0.12349}/src/lib/featrixevents/__init__.py

@@ -86,7 +86,7 @@ _key_resolution.get_api_key() (env var -> ~/.featrix -> /etc/.featrix_key,
 the file present on compute nodes running as root).
 """
 
-__version__ = "2.0.12346"
+__version__ = "2.0.12349"
 __author__ = "Featrix"
 __email__ = "support@featrix.com"
 __license__ = "MIT"

{featrixsphere-2.0.12346 → featrixsphere-2.0.12349}/src/lib/single_predictor_training.py

@@ -5977,7 +5977,7 @@ def train_single_predictor(args: LightSinglePredictorArgs):
         raise Exception("Single predictor training failed with all attempted batch sizes")
     
     logger.info(f"training_worked = {training_worked}")
-    
+
     if not training_worked:
         _set_job_failed(args)
         _send_failure_webhooks(args, error_message="Training failed")
@@ -5990,7 +5990,23 @@ def train_single_predictor(args: LightSinglePredictorArgs):
             throttle=False,
         )
         raise Exception("Single predictor training failed")
-    
+
+    # Tell job control directly, now, while we're still inside the training
+    # process — do not rely on the celery supervisor's process.wait()-gated
+    # write, which only fires after this process's OS exit (and can be
+    # delayed arbitrarily long, e.g. by the interpreter-shutdown resource_tracker
+    # hang documented in
+    # docs/internal/plans/2026-09-resource-tracker-shutdown-hang-after-dataloader-sigkill.md).
+    # Mirrors es_training.py's success-path update_job_status(..., DONE) calls;
+    # single_predictor_training.py previously only called update_job_status on
+    # the failure path (_set_job_failed), leaving Beagle/predictabar showing
+    # "running" long after training actually finished.
+    if args.job_id:
+        try:
+            update_job_status(job_id=args.job_id, status=JobStatus.DONE)
+        except Exception as e:
+            logger.warning(f"Could not push DONE status to job control: {e}")
+
     return 0
 
 

{featrixsphere-2.0.12346 → featrixsphere-2.0.12349}/src/lib/sp_training_wrapper.py

@@ -288,7 +288,16 @@ if __name__ == "__main__":
     try:
         train_single_predictor(args)
         logger.info("✅ Training completed successfully")
-        sys.exit(0)
+        # os._exit() skips CPython's normal interpreter shutdown (resource_tracker
+        # finalizers etc.), which can hang for many minutes waiting on
+        # multiprocessing.resource_tracker after a SIGKILLed DataLoader worker
+        # left orphaned semaphore registrations behind — the celery supervisor's
+        # process.wait() (and therefore the job's terminal status) blocks for the
+        # whole hang. See docs/internal/plans/2026-09-resource-tracker-shutdown-hang-after-dataloader-sigkill.md.
+        # atexit handlers do NOT run with os._exit(), so clean up GPU memory
+        # explicitly first.
+        cleanup_gpu_memory()
+        os._exit(0)
     except Exception as e:
         logger.error(f"❌ Training failed: {e}")
         traceback.print_exc()

featrixsphere-2.0.12346/VERSION

@@ -1 +0,0 @@
-2.0.12346