shareddata 6.83.21__tar.gz → 6.83.22__tar.gz

{shareddata-6.83.21/src/shareddata.egg-info → shareddata-6.83.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.83.21
+Version: 6.83.22
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License

{shareddata-6.83.21 → shareddata-6.83.22}/setup.py

@@ -34,7 +34,7 @@ install_requires = [
 
 setup(
     name='shareddata',
-    version='6.83.21',
+    version='6.83.22',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     long_description=open('README.md').read(),
     long_description_content_type='text/markdown',

{shareddata-6.83.21 → shareddata-6.83.22}/src/SharedData/Defaults.py

@@ -221,6 +221,27 @@ def _classify_on_mounted_device(resolved_path, current_mounts, declared_mounts):
     return True
 
 
+def _windows_folder_is_live(folder: str) -> bool:
+    """Windows has no fstab/mountinfo stub semantics; an existing dir is live."""
+    try:
+        return bool(folder) and Path(folder).is_dir()
+    except (OSError, TypeError):
+        return False
+
+
+def _is_filesystem_root(current) -> bool:
+    """True for POSIX ``/``, Windows drive roots (``C:\\``), and UNC share roots.
+
+    ``Path('C:\\\\').parent`` is itself and ``os.path.ismount('C:\\\\')`` is
+    True, so the st_dev==parent_dev stub heuristic must not treat a drive
+    letter as an unmounted mountpoint.
+    """
+    try:
+        return current == current.parent or current == type(current)(current.anchor)
+    except (OSError, TypeError, ValueError):
+        return False
+
+
 def _is_on_mounted_device_legacy(folder: str) -> bool:
     """Original ``os.path.ismount()`` st_dev heuristic — fallback for platforms
     without ``/proc/self/mountinfo``.  Cannot detect an unmounted mountpoint
@@ -232,20 +253,20 @@ def _is_on_mounted_device_legacy(folder: str) -> bool:
         while True:
             if os.path.ismount(current):
                 try:
+                    # Drive / UNC / POSIX root is a live filesystem, not a stub.
+                    if _is_filesystem_root(current):
+                        return True
                     dev = current.stat().st_dev
                     parent_dev = current.parent.stat().st_dev
                     if dev != parent_dev:
                         return True  # device is mounted
                     else:
                         # Mountpoint stub — device not mounted.
-                        # But if this is /, it's the root fs (always live).
-                        if current == Path('/'):
-                            return True
                         return False
                 except OSError:
                     return False
             if current == current.parent:
-                # Reached / without finding a mountpoint — root fs.
+                # Reached filesystem root without finding a mountpoint — live.
                 return True
             current = current.parent
     except (OSError, TypeError):
@@ -265,11 +286,15 @@ def _is_on_mounted_device(folder: str) -> bool:
     detection (writes silently landing on ``/`` instead of the intended disk).
 
     Falls back to the legacy ``os.path.ismount()`` heuristic when mountinfo is
-    unavailable (non-Linux).
+    unavailable (non-Linux).  On Windows there is no fstab/mountinfo stub
+    semantics — an existing directory is live (drive roots are not unmounted
+    mountpoint stubs).
     """
     try:
         if not folder:
             return False
+        if os.name == 'nt':
+            return _windows_folder_is_live(folder)
         resolved = str(Path(folder).resolve())
         current = _current_mount_points()
         if current is None:

{shareddata-6.83.21 → shareddata-6.83.22}/src/SharedData/IO/MongoDBClient.py

@@ -260,4 +260,4 @@ class MongoDBClient:
             if pymongo.helpers._index_list(index_data['key']) == target_index:
                 return  # Index already exists
 
-        coll.create_index(index_fields, **kwargs)
+        coll.create_index(index_fields, **kwargs)

{shareddata-6.83.21 → shareddata-6.83.22}/src/SharedData/Logger.py

@@ -340,14 +340,18 @@ class Logger:
         Returns:
             pandas.DataFrame: The updated DataFrame containing processed log entries.
         """
-        dfnewlines_sorted = Logger.readLogs()
-        if dfnewlines_sorted is None or len(dfnewlines_sorted) == 0:
+        # Serialize cursor + dflogs mutation: gunicorn gthread workers run
+        # concurrent request threads that would otherwise race _logfileposition
+        # / dflogs and serve inconsistent tails to UI polls.
+        with Logger.lock:
+            dfnewlines_sorted = Logger.readLogs()
+            if dfnewlines_sorted is None or len(dfnewlines_sorted) == 0:
+                return Logger.dflogs
+
+            Logger.getLastLog(dfnewlines_sorted)
+            Logger.getStatus(dfnewlines_sorted)
+            Logger.handleHeartbeats(keep_latest_heartbeat)
             return Logger.dflogs
-        
-        Logger.getLastLog(dfnewlines_sorted)
-        Logger.getStatus(dfnewlines_sorted)
-        Logger.handleHeartbeats(keep_latest_heartbeat)
-        return Logger.dflogs
     
     @staticmethod
     def readLogs():

shareddata-6.83.22/src/SharedData/Routines/UpdateActiveJobs.py

@@ -0,0 +1,75 @@
+"""Scheduler-supervised realtime WorkerPool job-state reconciler."""
+
+import faulthandler
+import time
+
+from SharedData.Logger import Logger
+from SharedData.Routines.WorkerPool import WorkerPool
+
+faulthandler.enable()
+
+
+def _wait_with_heartbeats(delay, heartbeat_interval, error_count, sleep):
+    """Wait without letting Scheduler supervision go stale."""
+    remaining = delay
+    while remaining > 0:
+        wait_for = min(heartbeat_interval, remaining)
+        sleep(wait_for)
+        remaining -= wait_for
+        if remaining > 0:
+            Logger.log.debug(f'#heartbeat#errors={error_count}')
+
+
+def run(
+    shdata,
+    interval=15,
+    retry_interval=60,
+    max_errors=10,
+    sleep=time.sleep,
+):
+    """Reconcile active jobs forever while publishing scheduler heartbeats."""
+    if interval <= 0 or retry_interval <= 0:
+        raise ValueError('intervals must be greater than zero')
+    if max_errors < 1:
+        raise ValueError('max_errors must be at least one')
+
+    Logger.log.info('ROUTINE STARTED!')
+    error_count = 0
+
+    while True:
+        delay = interval
+        try:
+            counts = WorkerPool.update_active_jobs_once(shdata)
+            error_count = 0
+            if counts['pending'] or counts['timed_out'] or counts['removed']:
+                Logger.log.info(
+                    'WorkerPool jobs status: '
+                    f"scanned={counts['scanned']},pending={counts['pending']},"
+                    f"timed_out={counts['timed_out']},removed={counts['removed']}"
+                )
+        except Exception as e:
+            error_count += 1
+            delay = retry_interval
+            Logger.log.error(f'Error in update_active_jobs_once: {e}')
+            if error_count >= max_errors:
+                Logger.log.error(
+                    f'Max errors ({max_errors}) reached, exiting'
+                )
+                raise SystemExit(1)
+
+        Logger.log.debug(f'#heartbeat#errors={error_count}')
+        _wait_with_heartbeats(delay, interval, error_count, sleep)
+
+
+def main():
+    from SharedData.SharedData import SharedData
+
+    shdata = SharedData(
+        'SharedData.Routines.UpdateActiveJobs',
+        user='master',
+    )
+    run(shdata)
+
+
+if __name__ == '__main__':
+    main()

{shareddata-6.83.21 → shareddata-6.83.22}/src/SharedData/Routines/Worker.py

@@ -330,4 +330,4 @@ while True:
         )
         # Logger.log.debug(f'#heartbeat# {nroutines}routines,{running_batch_jobs}/{MAX_BATCH_JOBS}jobs,{completed_batch_jobs}completed,{error_batch_jobs}errors,{SHAREDDATA_VERSION}')
     
-    time.sleep(SLEEP_TIME * np.random.rand())
+    time.sleep(SLEEP_TIME * np.random.rand())

{shareddata-6.83.21 → shareddata-6.83.22}/src/SharedData/Routines/WorkerPool.py

@@ -406,139 +406,134 @@ class WorkerPool:
         return fetched_jobs
 
     @staticmethod
-    def update_active_jobs_once(shdata) -> None:
-        """
-        Runs a single pass of the NEW/WAITING -> PENDING promotion and
-        FETCHED/RUNNING -> ERROR timeout sweep over the WORKERPOOL/JOBS
-        active-jobs cache. Raises on failure; callers own the retry/backoff
-        policy. Always releases the JOBS mutex before returning, even on
-        error, so a failed pass never blocks other JOBS readers/writers.
-
-        Extracted from the former update_active_jobs() so it can run as the
-        body of a standalone routine (SharedData.Routines.UpdateActiveJobs)
-        instead of an in-process Worker thread — see that module for the
-        owning loop, heartbeat, and MAX_ERRORS handling.
+    def update_active_jobs_once(shdata, now=None):
+        """Run one atomic WorkerPool job-state reconciliation pass.
+
+        NEW/WAITING jobs become PENDING once due and all dependencies are
+        COMPLETED. FETCHED/RUNNING jobs become ERROR after one hour. An absent
+        ACTIVE_JOBS cache is a normal empty-queue condition.
         """
-        isacquired = False
+        tnow = (
+            pd.Timestamp.utcnow().tz_localize(None)
+            if now is None
+            else pd.Timestamp(now)
+        )
+        one_hour_ago = tnow - pd.Timedelta(hours=1)
+        counts = {
+            'scanned': 0,
+            'pending': 0,
+            'timed_out': 0,
+            'removed': 0,
+        }
+
         job_table = WorkerPool.get_job_table(shdata)
+        job_table.acquire()
         try:
-            tnow = pd.Timestamp.utcnow().tz_localize(None)
-            one_hour_ago = tnow - pd.Timedelta(hours=1)
-
-            job_table.acquire()
-            isacquired = True
-
-            # Get active jobs cache
             buff = np.full((1,), dtype=job_table.dtype, fill_value=np.nan)
             buff['hash'][0] = 'ACTIVE_JOBS'
             loc = job_table.get_loc(buff, acquire=False)
             if loc[0] == -1:
-                return  # No active jobs yet — nothing to do this pass
+                return counts
 
-            active_jobs = job_table.get_dict_list(job_table[loc[0]], acquire=False)[0]
+            active_jobs = job_table.get_dict_list(
+                job_table[loc[0]], acquire=False
+            )[0]
             modified = False
-            timeout_count = 0
-            pending_count = 0
+            if isinstance(active_jobs.get('jobs'), list):
+                active_jobs['jobs'] = {}
+                modified = True
+
+            started_at = time.monotonic()
+            active_statuses = {'NEW', 'WAITING', 'PENDING', 'FETCHED', 'RUNNING'}
 
-            ti = time.time()
-            # Process each active job
             for job_hash, job in list(active_jobs['jobs'].items()):
-                # Check if job status is active
-                if job['status'] not in ['NEW', 'WAITING', 'PENDING', 'FETCHED', 'RUNNING']:
-                    del active_jobs['jobs'][job_hash] # remove from active jobs
+                counts['scanned'] += 1
+
+                if job['status'] not in active_statuses:
+                    del active_jobs['jobs'][job_hash]
                     modified = True
+                    counts['removed'] += 1
                     continue
 
-                # 1. Check timeout for FETCHED/RUNNING jobs
-                if job['status'] in ['FETCHED', 'RUNNING'] and job['mtime'] < one_hour_ago:
+                if (
+                    job['status'] in {'FETCHED', 'RUNNING'}
+                    and job['mtime'] < one_hour_ago
+                ):
                     job['status'] = 'ERROR'
                     job['stderr'] = 'timeout'
                     job['mtime'] = tnow
                     job_table.upsert(job, acquire=False)
-                    del active_jobs['jobs'][job_hash] # remove from active jobs
+                    del active_jobs['jobs'][job_hash]
                     modified = True
-                    timeout_count += 1
+                    counts['timed_out'] += 1
+                    counts['removed'] += 1
                     continue
 
-                # 2. Update NEW/WAITING → PENDING if due and dependencies met
-                if job['status'] not in ['NEW', 'WAITING']:
+                if job['status'] not in {'NEW', 'WAITING'} or job['date'] > tnow:
                     continue
 
-                # Check if job is due
-                if job['date'] > tnow:
-                    continue
-
-                # Check dependencies
+                dependencies = job.get('dependencies') or []
                 all_deps_completed = True
-                if job.get('dependencies'):
-                    ndep = len(job['dependencies'])
-                    buff = np.full((ndep,), dtype=job_table.dtype, fill_value=np.nan)
-
-                    for i, dep_hash in enumerate(job['dependencies']):
+                if dependencies:
+                    buff = np.full(
+                        (len(dependencies),),
+                        dtype=job_table.dtype,
+                        fill_value=np.nan,
+                    )
+                    for i, dep_hash in enumerate(dependencies):
                         buff['hash'][i] = dep_hash
 
                     locs = job_table.get_loc(buff, acquire=False)
-
-                    for i in range(ndep):
-                        if locs[i] != -1:
-                            dep_job = job_table.get_dict_list(job_table[locs[i]], acquire=False)[0]
-                            if dep_job['status'] != 'COMPLETED':
-                                all_deps_completed = False
-                                break
-                        else:
-                            # Dependency not found - cannot proceed
+                    for dep_loc in locs:
+                        if dep_loc == -1:
+                            all_deps_completed = False
+                            break
+                        dep_job = job_table.get_dict_list(
+                            job_table[dep_loc], acquire=False
+                        )[0]
+                        if dep_job['status'] != 'COMPLETED':
                             all_deps_completed = False
                             break
 
-                # Update to PENDING if all conditions met
                 if all_deps_completed:
                     job['status'] = 'PENDING'
                     job['mtime'] = tnow
                     job_table.upsert(job, acquire=False)
                     active_jobs['jobs'][job_hash] = job
                     modified = True
-                    pending_count += 1
+                    counts['pending'] += 1
 
-            te = time.time() - ti
-            if te > 5:
-                Logger.log.warning(f"update_active_jobs processing time high: {te:.2f} seconds")
+            elapsed = time.monotonic() - started_at
+            if elapsed > 5:
+                Logger.log.warning(
+                    f"update_active_jobs processing time high: {elapsed:.2f} seconds"
+                )
 
-            # Save updated active_jobs cache if modified
             if modified:
                 job_table.upsert(active_jobs, acquire=False)
 
-            # Log warnings (matches MongoDB logging behavior)
-            if timeout_count > 0:
-                Logger.log.warning(f"Marked {timeout_count} jobs as ERROR due to timeout (>1 hour)")
+            if counts['timed_out']:
+                Logger.log.warning(
+                    f"Marked {counts['timed_out']} jobs as ERROR due to timeout (>1 hour)"
+                )
 
+            return counts
         finally:
-            if isacquired:
-                job_table.release()
+            job_table.release()
 
     @staticmethod
-    def update_active_jobs(shdata) -> None:
-        """
-        Legacy in-process loop kept for backward compatibility with any
-        external caller still spawning this as a thread. New deployments
-        should run SharedData.Routines.UpdateActiveJobs as a standalone
-        routine instead — it owns this same retry/backoff policy plus a
-        heartbeat and MAX_ERRORS exit, and is independently
-        restartable/killable via the normal workerpool commands.
-        """
+    def update_active_jobs(shdata, interval=15, retry_interval=60) -> None:
+        """Compatibility loop; scheduled deployments should use UpdateActiveJobs."""
         while True:
-            retry_delay = 15
             try:
                 WorkerPool.update_active_jobs_once(shdata)
+                time.sleep(interval)
             except Exception as e:
                 try:
                     Logger.log.error(f"Error in update_active_jobs: {e}")
-                except:
+                except Exception:
                     pass
-                retry_delay = 60
-            # Never hold the JOBS mutex during the retry backoff —
-            # update_active_jobs_once() always releases it before raising
-            # or returning.
-            time.sleep(retry_delay)
+                time.sleep(retry_interval)
         
     @staticmethod
     def get_active_jobs(shdata, acquire=True):

{shareddata-6.83.21 → shareddata-6.83.22/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.83.21
+Version: 6.83.22
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License

shareddata-6.83.21/src/SharedData/Routines/UpdateActiveJobs.py

@@ -1,53 +0,0 @@
-import faulthandler
-import time
-
-# Capture a Python-level traceback if a C extension (pandas_parser, numpy,
-# pyarrow, ...) crashes the interpreter with SIGSEGV/SIGABRT/SIGFPE/SIGBUS.
-# Without this, a crash leaves only a kernel segfault line with a raw IP.
-faulthandler.enable()
-
-# PROPRIETARY LIBS
-from SharedData.Routines.WorkerPool import WorkerPool
-from SharedData.Logger import Logger
-from SharedData.SharedData import SharedData
-shdata = SharedData('SharedData.Routines.UpdateActiveJobs', user='master')
-
-Logger.log.info('ROUTINE STARTED!')
-
-# Standalone routine form of the promotion sweep formerly run as an
-# in-process Worker thread (WorkerPool.update_active_jobs). Splitting it out
-# lets it be killed/restarted independently via the normal workerpool
-# commands without tearing down the Worker's API server and every other
-# child routine on the host — see docs/INCIDENT_WORKERPOOL_JOBS_INDEX_20260724.md.
-MAX_ERRORS = 10
-error_count = 0
-lastheartbeat = time.time()
-
-while True:
-    try:
-        if time.time() - lastheartbeat > 15:
-            lastheartbeat = time.time()
-            Logger.log.debug('#heartbeat#errors=%d' % error_count)
-
-        retry_delay = 15
-        try:
-            WorkerPool.update_active_jobs_once(shdata)
-            error_count = 0
-        except Exception as e:
-            Logger.log.error('Error in update_active_jobs_once: %s' % (str(e)))
-            error_count += 1
-            retry_delay = 60
-            if error_count >= MAX_ERRORS:
-                Logger.log.error(
-                    'Max errors (%d) reached, exiting' % (MAX_ERRORS))
-                raise SystemExit(1)
-
-        # Never hold the JOBS mutex during the retry backoff —
-        # update_active_jobs_once() always releases it before raising or
-        # returning.
-        time.sleep(retry_delay)
-    except SystemExit:
-        raise
-    except Exception as e:
-        Logger.log.error('UpdateActiveJobs Loop Exception: %s' % (str(e)))
-        time.sleep(60)