shareddata 6.83.16__tar.gz → 6.83.17__tar.gz

{shareddata-6.83.16 → shareddata-6.83.17}/MANIFEST.in

@@ -1,2 +1 @@
 include src/SharedData/sharedmutexwin.pyd
-include src/SharedData/IO/apidocs.yml

{shareddata-6.83.16/src/shareddata.egg-info → shareddata-6.83.17}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.83.16
+Version: 6.83.17
 Summary: Memory Mapped / Shared Memory Database with S3 repository
+License: MIT
 Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9, <3.13
 Description-Content-Type: text/markdown
@@ -39,6 +39,7 @@ Requires-Dist: redis==6.2.0
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
+Dynamic: license
 Dynamic: license-file
 Dynamic: requires-dist
 Dynamic: requires-python

{shareddata-6.83.16 → shareddata-6.83.17}/setup.py

@@ -34,13 +34,13 @@ install_requires = [
 
 setup(
     name='shareddata',
-    version='6.83.16',
+    version='6.83.17',
     description='Memory Mapped / Shared Memory Database with S3 repository',
+    license='MIT',
     long_description=open('README.md').read(),
     long_description_content_type='text/markdown',
     classifiers=[
         'Programming Language :: Python :: 3',
-        'License :: OSI Approved :: MIT License',
         'Operating System :: OS Independent',
     ],
     packages=find_packages(where='src'),

{shareddata-6.83.16 → shareddata-6.83.17}/src/SharedData/IO/MongoDBClient.py

@@ -62,7 +62,7 @@ class MongoDBClient:
                 f'&serverSelectionTimeoutMS=10000'
             )
         elif 'MONGODB_REPLICA_SET' not in os.environ:
-            # Single server connection (MONGODB_HOST includes port: '10.0.0.50:27017')
+            # Single server connection (MONGODB_HOST includes port: '192.0.2.10:27017')
             self.mongodb_conn_str = (
                 f'mongodb://{os.environ["MONGODB_USER"]}:'
                 f'{os.environ["MONGODB_PWD"]}@'
@@ -260,4 +260,4 @@ class MongoDBClient:
             if pymongo.helpers._index_list(index_data['key']) == target_index:
                 return  # Index already exists
 
-        coll.create_index(index_fields, **kwargs)
+        coll.create_index(index_fields, **kwargs)

{shareddata-6.83.16 → shareddata-6.83.17}/src/SharedData/Logger.py

@@ -340,14 +340,18 @@ class Logger:
         Returns:
             pandas.DataFrame: The updated DataFrame containing processed log entries.
         """
-        dfnewlines_sorted = Logger.readLogs()
-        if dfnewlines_sorted is None or len(dfnewlines_sorted) == 0:
+        # Serialize cursor + dflogs mutation: gunicorn gthread workers run
+        # concurrent request threads that would otherwise race _logfileposition
+        # / dflogs and serve inconsistent tails to UI polls.
+        with Logger.lock:
+            dfnewlines_sorted = Logger.readLogs()
+            if dfnewlines_sorted is None or len(dfnewlines_sorted) == 0:
+                return Logger.dflogs
+
+            Logger.getLastLog(dfnewlines_sorted)
+            Logger.getStatus(dfnewlines_sorted)
+            Logger.handleHeartbeats(keep_latest_heartbeat)
             return Logger.dflogs
-        
-        Logger.getLastLog(dfnewlines_sorted)
-        Logger.getStatus(dfnewlines_sorted)
-        Logger.handleHeartbeats(keep_latest_heartbeat)
-        return Logger.dflogs
     
     @staticmethod
     def readLogs():

shareddata-6.83.17/src/SharedData/Routines/UpdateActiveJobs.py

@@ -0,0 +1,75 @@
+"""Scheduler-supervised realtime WorkerPool job-state reconciler."""
+
+import faulthandler
+import time
+
+from SharedData.Logger import Logger
+from SharedData.Routines.WorkerPool import WorkerPool
+
+faulthandler.enable()
+
+
+def _wait_with_heartbeats(delay, heartbeat_interval, error_count, sleep):
+    """Wait without letting Scheduler supervision go stale."""
+    remaining = delay
+    while remaining > 0:
+        wait_for = min(heartbeat_interval, remaining)
+        sleep(wait_for)
+        remaining -= wait_for
+        if remaining > 0:
+            Logger.log.debug(f'#heartbeat#errors={error_count}')
+
+
+def run(
+    shdata,
+    interval=15,
+    retry_interval=60,
+    max_errors=10,
+    sleep=time.sleep,
+):
+    """Reconcile active jobs forever while publishing scheduler heartbeats."""
+    if interval <= 0 or retry_interval <= 0:
+        raise ValueError('intervals must be greater than zero')
+    if max_errors < 1:
+        raise ValueError('max_errors must be at least one')
+
+    Logger.log.info('ROUTINE STARTED!')
+    error_count = 0
+
+    while True:
+        delay = interval
+        try:
+            counts = WorkerPool.update_active_jobs_once(shdata)
+            error_count = 0
+            if counts['pending'] or counts['timed_out'] or counts['removed']:
+                Logger.log.info(
+                    'WorkerPool jobs status: '
+                    f"scanned={counts['scanned']},pending={counts['pending']},"
+                    f"timed_out={counts['timed_out']},removed={counts['removed']}"
+                )
+        except Exception as e:
+            error_count += 1
+            delay = retry_interval
+            Logger.log.error(f'Error in update_active_jobs_once: {e}')
+            if error_count >= max_errors:
+                Logger.log.error(
+                    f'Max errors ({max_errors}) reached, exiting'
+                )
+                raise SystemExit(1)
+
+        Logger.log.debug(f'#heartbeat#errors={error_count}')
+        _wait_with_heartbeats(delay, interval, error_count, sleep)
+
+
+def main():
+    from SharedData.SharedData import SharedData
+
+    shdata = SharedData(
+        'SharedData.Routines.UpdateActiveJobs',
+        user='master',
+    )
+    run(shdata)
+
+
+if __name__ == '__main__':
+    main()

{shareddata-6.83.16 → shareddata-6.83.17}/src/SharedData/Routines/Worker.py

@@ -226,8 +226,8 @@ while True:
 
     # Scheduler lifecycle — watchdog works in both election and non-election
     # modes. In election mode, only the current master owns the scheduler and
-    # runs 'MASTER'. In non-election mode (static deploy, e.g. TB08-WS with
-    # --schedules MASTER,TRADEBOT08-VM1), this worker always owns the scheduler
+    # runs 'MASTER'. In non-election mode (static deploy, e.g. WORKER01 with
+    # --schedules MASTER,WORKER02), this worker always owns the scheduler
     # and runs its configured SCHEDULE_NAMES.
     election_mode = str(os.environ.get('RUN_MASTER_ELECTION', '')).upper() == 'TRUE'
     if election_mode:
@@ -330,4 +330,4 @@ while True:
         )
         # Logger.log.debug(f'#heartbeat# {nroutines}routines,{running_batch_jobs}/{MAX_BATCH_JOBS}jobs,{completed_batch_jobs}completed,{error_batch_jobs}errors,{SHAREDDATA_VERSION}')
     
-    time.sleep(SLEEP_TIME * np.random.rand())
+    time.sleep(SLEEP_TIME * np.random.rand())

{shareddata-6.83.16 → shareddata-6.83.17}/src/SharedData/Routines/WorkerPool.py

@@ -406,139 +406,134 @@ class WorkerPool:
         return fetched_jobs
 
     @staticmethod
-    def update_active_jobs_once(shdata) -> None:
-        """
-        Runs a single pass of the NEW/WAITING -> PENDING promotion and
-        FETCHED/RUNNING -> ERROR timeout sweep over the WORKERPOOL/JOBS
-        active-jobs cache. Raises on failure; callers own the retry/backoff
-        policy. Always releases the JOBS mutex before returning, even on
-        error, so a failed pass never blocks other JOBS readers/writers.
-
-        Extracted from the former update_active_jobs() so it can run as the
-        body of a standalone routine (SharedData.Routines.UpdateActiveJobs)
-        instead of an in-process Worker thread — see that module for the
-        owning loop, heartbeat, and MAX_ERRORS handling.
+    def update_active_jobs_once(shdata, now=None):
+        """Run one atomic WorkerPool job-state reconciliation pass.
+
+        NEW/WAITING jobs become PENDING once due and all dependencies are
+        COMPLETED. FETCHED/RUNNING jobs become ERROR after one hour. An absent
+        ACTIVE_JOBS cache is a normal empty-queue condition.
         """
-        isacquired = False
+        tnow = (
+            pd.Timestamp.utcnow().tz_localize(None)
+            if now is None
+            else pd.Timestamp(now)
+        )
+        one_hour_ago = tnow - pd.Timedelta(hours=1)
+        counts = {
+            'scanned': 0,
+            'pending': 0,
+            'timed_out': 0,
+            'removed': 0,
+        }
+
         job_table = WorkerPool.get_job_table(shdata)
+        job_table.acquire()
         try:
-            tnow = pd.Timestamp.utcnow().tz_localize(None)
-            one_hour_ago = tnow - pd.Timedelta(hours=1)
-
-            job_table.acquire()
-            isacquired = True
-
-            # Get active jobs cache
             buff = np.full((1,), dtype=job_table.dtype, fill_value=np.nan)
             buff['hash'][0] = 'ACTIVE_JOBS'
             loc = job_table.get_loc(buff, acquire=False)
             if loc[0] == -1:
-                return  # No active jobs yet — nothing to do this pass
+                return counts
 
-            active_jobs = job_table.get_dict_list(job_table[loc[0]], acquire=False)[0]
+            active_jobs = job_table.get_dict_list(
+                job_table[loc[0]], acquire=False
+            )[0]
             modified = False
-            timeout_count = 0
-            pending_count = 0
+            if isinstance(active_jobs.get('jobs'), list):
+                active_jobs['jobs'] = {}
+                modified = True
+
+            started_at = time.monotonic()
+            active_statuses = {'NEW', 'WAITING', 'PENDING', 'FETCHED', 'RUNNING'}
 
-            ti = time.time()
-            # Process each active job
             for job_hash, job in list(active_jobs['jobs'].items()):
-                # Check if job status is active
-                if job['status'] not in ['NEW', 'WAITING', 'PENDING', 'FETCHED', 'RUNNING']:
-                    del active_jobs['jobs'][job_hash] # remove from active jobs
+                counts['scanned'] += 1
+
+                if job['status'] not in active_statuses:
+                    del active_jobs['jobs'][job_hash]
                     modified = True
+                    counts['removed'] += 1
                     continue
 
-                # 1. Check timeout for FETCHED/RUNNING jobs
-                if job['status'] in ['FETCHED', 'RUNNING'] and job['mtime'] < one_hour_ago:
+                if (
+                    job['status'] in {'FETCHED', 'RUNNING'}
+                    and job['mtime'] < one_hour_ago
+                ):
                     job['status'] = 'ERROR'
                     job['stderr'] = 'timeout'
                     job['mtime'] = tnow
                     job_table.upsert(job, acquire=False)
-                    del active_jobs['jobs'][job_hash] # remove from active jobs
+                    del active_jobs['jobs'][job_hash]
                     modified = True
-                    timeout_count += 1
+                    counts['timed_out'] += 1
+                    counts['removed'] += 1
                     continue
 
-                # 2. Update NEW/WAITING → PENDING if due and dependencies met
-                if job['status'] not in ['NEW', 'WAITING']:
+                if job['status'] not in {'NEW', 'WAITING'} or job['date'] > tnow:
                     continue
 
-                # Check if job is due
-                if job['date'] > tnow:
-                    continue
-
-                # Check dependencies
+                dependencies = job.get('dependencies') or []
                 all_deps_completed = True
-                if job.get('dependencies'):
-                    ndep = len(job['dependencies'])
-                    buff = np.full((ndep,), dtype=job_table.dtype, fill_value=np.nan)
-
-                    for i, dep_hash in enumerate(job['dependencies']):
+                if dependencies:
+                    buff = np.full(
+                        (len(dependencies),),
+                        dtype=job_table.dtype,
+                        fill_value=np.nan,
+                    )
+                    for i, dep_hash in enumerate(dependencies):
                         buff['hash'][i] = dep_hash
 
                     locs = job_table.get_loc(buff, acquire=False)
-
-                    for i in range(ndep):
-                        if locs[i] != -1:
-                            dep_job = job_table.get_dict_list(job_table[locs[i]], acquire=False)[0]
-                            if dep_job['status'] != 'COMPLETED':
-                                all_deps_completed = False
-                                break
-                        else:
-                            # Dependency not found - cannot proceed
+                    for dep_loc in locs:
+                        if dep_loc == -1:
+                            all_deps_completed = False
+                            break
+                        dep_job = job_table.get_dict_list(
+                            job_table[dep_loc], acquire=False
+                        )[0]
+                        if dep_job['status'] != 'COMPLETED':
                             all_deps_completed = False
                             break
 
-                # Update to PENDING if all conditions met
                 if all_deps_completed:
                     job['status'] = 'PENDING'
                     job['mtime'] = tnow
                     job_table.upsert(job, acquire=False)
                     active_jobs['jobs'][job_hash] = job
                     modified = True
-                    pending_count += 1
+                    counts['pending'] += 1
 
-            te = time.time() - ti
-            if te > 5:
-                Logger.log.warning(f"update_active_jobs processing time high: {te:.2f} seconds")
+            elapsed = time.monotonic() - started_at
+            if elapsed > 5:
+                Logger.log.warning(
+                    f"update_active_jobs processing time high: {elapsed:.2f} seconds"
+                )
 
-            # Save updated active_jobs cache if modified
             if modified:
                 job_table.upsert(active_jobs, acquire=False)
 
-            # Log warnings (matches MongoDB logging behavior)
-            if timeout_count > 0:
-                Logger.log.warning(f"Marked {timeout_count} jobs as ERROR due to timeout (>1 hour)")
+            if counts['timed_out']:
+                Logger.log.warning(
+                    f"Marked {counts['timed_out']} jobs as ERROR due to timeout (>1 hour)"
+                )
 
+            return counts
         finally:
-            if isacquired:
-                job_table.release()
+            job_table.release()
 
     @staticmethod
-    def update_active_jobs(shdata) -> None:
-        """
-        Legacy in-process loop kept for backward compatibility with any
-        external caller still spawning this as a thread. New deployments
-        should run SharedData.Routines.UpdateActiveJobs as a standalone
-        routine instead — it owns this same retry/backoff policy plus a
-        heartbeat and MAX_ERRORS exit, and is independently
-        restartable/killable via the normal workerpool commands.
-        """
+    def update_active_jobs(shdata, interval=15, retry_interval=60) -> None:
+        """Compatibility loop; scheduled deployments should use UpdateActiveJobs."""
         while True:
-            retry_delay = 15
             try:
                 WorkerPool.update_active_jobs_once(shdata)
+                time.sleep(interval)
             except Exception as e:
                 try:
                     Logger.log.error(f"Error in update_active_jobs: {e}")
-                except:
+                except Exception:
                     pass
-                retry_delay = 60
-            # Never hold the JOBS mutex during the retry backoff —
-            # update_active_jobs_once() always releases it before raising
-            # or returning.
-            time.sleep(retry_delay)
+                time.sleep(retry_interval)
         
     @staticmethod
     def get_active_jobs(shdata, acquire=True):

{shareddata-6.83.16 → shareddata-6.83.17/src/shareddata.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.83.16
+Version: 6.83.17
 Summary: Memory Mapped / Shared Memory Database with S3 repository
+License: MIT
 Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9, <3.13
 Description-Content-Type: text/markdown
@@ -39,6 +39,7 @@ Requires-Dist: redis==6.2.0
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
+Dynamic: license
 Dynamic: license-file
 Dynamic: requires-dist
 Dynamic: requires-python

{shareddata-6.83.16 → shareddata-6.83.17}/src/shareddata.egg-info/SOURCES.txt

@@ -86,6 +86,7 @@ tests/test_cache_redis.py
 tests/test_extend_rt.py
 tests/test_listcache.py
 tests/test_loc.py
+tests/test_logger_thread_safety.py
 tests/test_metadata.py
 tests/test_mount_unmount_detection.py
 tests/test_read_write_tail.py
@@ -98,5 +99,6 @@ tests/test_upsert_unordered_get_date_loc_d1.py
 tests/test_upsert_unordered_get_date_loc_m1.py
 tests/test_upsert_unordered_get_date_loc_m15.py
 tests/test_upsert_unordered_positions_m1.py
+tests/test_workerpool_jobs_status.py
 tests/test_workerpool_mutex_release.py
 tests/test_workerpool_post_commands.py

{shareddata-6.83.16 → shareddata-6.83.17}/tests/test_bson_last_pos_reuse.py

@@ -117,7 +117,7 @@ def test_bson_last_pos_reuse():
                 'routine': f'Scripts/Routine_{j}.py',
                 'status': np.random.choice(['running', 'pending', 'completed', 'error']),
                 'started': time.time() - np.random.randint(0, 3600),
-                'worker': f'TRADEBOT{j % 10:02d}-WS',
+                'worker': f'WORKER{j % 10:02d}',
                 'iteration': i,
             }
         
@@ -363,11 +363,11 @@ def test_bson_backwards_compatibility():
     # --- Case 1: Insert multiple records to simulate old-format data ---
     print(f"\n[1] Inserting records to simulate pre-existing BSON file...")
     test_data = [
-        {'hash': 'WORKER_01', 'status': 'ONLINE', 'info': {'host': 'TB01', 'cpu': 12}},
-        {'hash': 'WORKER_02', 'status': 'ONLINE', 'info': {'host': 'TB02', 'cpu': 28}},
-        {'hash': 'WORKER_03', 'status': 'OFFLINE', 'info': {'host': 'TB03', 'cpu': 16}},
+        {'hash': 'WORKER_01', 'status': 'ONLINE', 'info': {'host': 'HOST01', 'cpu': 12}},
+        {'hash': 'WORKER_02', 'status': 'ONLINE', 'info': {'host': 'HOST02', 'cpu': 28}},
+        {'hash': 'WORKER_03', 'status': 'OFFLINE', 'info': {'host': 'HOST03', 'cpu': 16}},
         {'hash': 'ACTIVE_JOBS', 'status': 'ACTIVE', 'jobs': {'j1': {'repo': 'R1'}}},
-        {'hash': 'WORKER_04', 'status': 'ONLINE', 'info': {'host': 'TB04', 'cpu': 16}},
+        {'hash': 'WORKER_04', 'status': 'ONLINE', 'info': {'host': 'HOST04', 'cpu': 16}},
     ]
     for d in test_data:
         records = records.upsert(d)

shareddata-6.83.17/tests/test_logger_thread_safety.py

@@ -0,0 +1,47 @@
+from concurrent.futures import ThreadPoolExecutor
+import threading
+import time
+
+import pandas as pd
+
+from SharedData.Logger import Logger
+
+
+def test_getlogs_serializes_the_full_mutation_pipeline(monkeypatch):
+    call_count = 8
+    start = threading.Barrier(call_count)
+    state_guard = threading.Lock()
+    state = {"active": 0, "max_active": 0}
+    new_lines = pd.DataFrame({"message": ["new log line"]})
+
+    def read_logs():
+        with state_guard:
+            state["active"] += 1
+            state["max_active"] = max(state["max_active"], state["active"])
+        time.sleep(0.005)
+        return new_lines
+
+    def process_logs(_):
+        time.sleep(0.005)
+
+    def handle_heartbeats(_):
+        time.sleep(0.005)
+        with state_guard:
+            state["active"] -= 1
+
+    monkeypatch.setattr(Logger, "lock", threading.Lock())
+    monkeypatch.setattr(Logger, "dflogs", pd.DataFrame())
+    monkeypatch.setattr(Logger, "readLogs", staticmethod(read_logs))
+    monkeypatch.setattr(Logger, "getLastLog", staticmethod(process_logs))
+    monkeypatch.setattr(Logger, "getStatus", staticmethod(process_logs))
+    monkeypatch.setattr(Logger, "handleHeartbeats", staticmethod(handle_heartbeats))
+
+    def poll_logs():
+        start.wait()
+        return Logger.getLogs()
+
+    with ThreadPoolExecutor(max_workers=call_count) as executor:
+        results = list(executor.map(lambda _: poll_logs(), range(call_count)))
+
+    assert state == {"active": 0, "max_active": 1}
+    assert all(result is Logger.dflogs for result in results)

{shareddata-6.83.16 → shareddata-6.83.17}/tests/test_mount_unmount_detection.py

@@ -13,14 +13,14 @@ ancestor is a stub -> not accessible.
 """
 from SharedData.Defaults import _classify_on_mounted_device
 
-# Model of TB08-WS: /home is a real mount; /nvme1../nvme6 are declared fstab
-# mount targets. DATABASE_FOLDERS='/home/jcooloj/db,/nvme4/db,/nvme6/db'.
+# Example host: /home is a real mount; /nvme1../nvme6 are declared fstab
+# mount targets. DATABASE_FOLDERS='/home/example/db,/nvme4/db,/nvme6/db'.
 DECLARED = {'/', '/home', '/nvme1', '/nvme2', '/nvme3', '/nvme4', '/nvme5', '/nvme6'}
 
 
 def test_root_fs_dbfolder_on_home_mount_is_accessible():
     current = {'/', '/home', '/nvme4', '/nvme6'}
-    assert _classify_on_mounted_device('/home/jcooloj/db', current, DECLARED) is True
+    assert _classify_on_mounted_device('/home/example/db', current, DECLARED) is True
 
 
 def test_mounted_nvme_dbfolder_is_accessible():

{shareddata-6.83.16 → shareddata-6.83.17}/tests/test_schema_acquire_timeout.py

@@ -14,13 +14,13 @@ from SharedData.SharedData import SharedData
 
 def test_schema_relpath_forward_slashes_matches():
     assert SharedData._is_schema_relpath(
-        'master/Symbols/D1/SCHEMA/table/TRADEBOT08-WS') is True
+        'master/Symbols/D1/SCHEMA/table/WORKER01') is True
 
 
 def test_schema_relpath_backslashes_matches():
     # POSIX Table converts shm_name to backslashes before acquire().
     assert SharedData._is_schema_relpath(
-        'master\\Symbols\\D1\\SCHEMA\\table\\TRADEBOT08-WS') is True
+        'master\\Symbols\\D1\\SCHEMA\\table\\WORKER01') is True
 
 
 def test_non_schema_relpath_does_not_match_either_separator():
@@ -42,6 +42,6 @@ def test_schema_predicate_rejects_non_singleton_schema_source():
 
 def test_schema_predicate_accepts_singleton_for_any_host():
     assert SharedData._is_schema_relpath(
-        'master/Symbols/D1/SCHEMA/table/TRADEBOT08-WS') is True
+        'master/Symbols/D1/SCHEMA/table/WORKER01') is True
     assert SharedData._is_schema_relpath(
         'master/Symbols/D1/SCHEMA/table/OTHERHOST') is True