shareddata 6.83.7__tar.gz → 6.83.8__tar.gz

{shareddata-6.83.7/src/shareddata.egg-info → shareddata-6.83.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.83.7
+Version: 6.83.8
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License

{shareddata-6.83.7 → shareddata-6.83.8}/setup.py

@@ -34,7 +34,7 @@ install_requires = [
 
 setup(
     name='shareddata',
-    version='6.83.7',
+    version='6.83.8',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     long_description=open('README.md').read(),
     long_description_content_type='text/markdown',

{shareddata-6.83.7 → shareddata-6.83.8}/src/SharedData/Logger.py

@@ -390,10 +390,10 @@ class Logger:
                 dfnewlines_sorted = dfnewlines.sort_values(['asctime', 'sequence_number'])
                 if need_full_sort:
                     Logger.dflogs = pd.concat([Logger.dflogs, dfnewlines_sorted], ignore_index=True)
-                    Logger.dflogs['asctime'] = pd.to_datetime(Logger.dflogs['asctime'])
+                    Logger.dflogs['asctime'] = pd.to_datetime(Logger.dflogs['asctime'], format='ISO8601', errors='coerce')
                     Logger.dflogs = Logger.dflogs.sort_values(['asctime', 'sequence_number'], ignore_index=True)
-                    Logger._sorted_until = len(Logger.dflogs)                                            
-                    Logger._max_asctime = pd.to_datetime(Logger.dflogs['asctime']).max()
+                    Logger._sorted_until = len(Logger.dflogs)
+                    Logger._max_asctime = Logger.dflogs['asctime'].max()
                 else:
                     # Append new lines and sort only the new part
                     Logger.dflogs = pd.concat([Logger.dflogs, dfnewlines_sorted], ignore_index=True)
@@ -452,7 +452,7 @@ class Logger:
             pd.DataFrame: DataFrame containing the updated last log entries for the affected (user_name, logger_name) groups.
         """
         _dflast = df.groupby(['user_name', 'logger_name']).last()
-        _dflast['asctime'] = pd.to_datetime(_dflast['asctime'])
+        _dflast['asctime'] = pd.to_datetime(_dflast['asctime'], format='ISO8601', errors='coerce')
 
         notinidx = _dflast.index.difference(Logger.dflast.index)
         Logger.dflast = Logger.dflast.reindex(_dflast.index.union(Logger.dflast.index))

{shareddata-6.83.7 → shareddata-6.83.8}/src/SharedData/Routines/Schedule.py

@@ -222,11 +222,26 @@ class Schedule:
     def update(self):
         """
         Updates the object's logs, batch status, and realtime status, then merges the realtime and batch DataFrames into a single schedule DataFrame with a reset index.
+
+        Each stage is isolated: a failure in UpdateLogs must not prevent
+        UpdateBatchStatus from running, and vice versa, so a single malformed
+        log row (or a pandas parser hiccup) cannot stall the whole scheduler.
         """
-        self.UpdateLogs()
-        self.UpdateBatchStatus()
-        self.UpdateRealtimeStatus()
-        self.schedule = pd.concat([self.realtime,self.batch]).reset_index()
+        for stage_name, stage_fn in (
+            ('UpdateLogs', self.UpdateLogs),
+            ('UpdateBatchStatus', self.UpdateBatchStatus),
+            ('UpdateRealtimeStatus', self.UpdateRealtimeStatus),
+        ):
+            try:
+                stage_fn()
+            except Exception as e:
+                Logger.log.error(
+                    'Schedule.%s[%s] failed: %s' % (stage_name, self.schedule_name, e))
+        try:
+            self.schedule = pd.concat([self.realtime, self.batch]).reset_index()
+        except Exception as e:
+            Logger.log.error(
+                'Schedule.update merge[%s] failed: %s' % (self.schedule_name, e))
 
     def UpdateLogs(self):
         """
@@ -252,7 +267,7 @@ class Schedule:
         if not Logger.dfrun.empty:
             df = Logger.dfrun.copy()
             df['routine'] = df['user_name']+':'+df['logger_name']
-            df['asctime'] = pd.to_datetime(df['asctime'])
+            df['asctime'] = pd.to_datetime(df['asctime'], format='ISO8601', errors='coerce')
             df = df.sort_values('asctime')
             df = df.groupby('routine').last()
             idisin = dfsched.index[dfsched.index.isin(df.index)]
@@ -267,7 +282,7 @@ class Schedule:
         if not Logger.dfstarted.empty:
             df = Logger.dfstarted.copy()
             df['routine'] = df['user_name']+':'+df['logger_name']
-            df['asctime'] = pd.to_datetime(df['asctime'])
+            df['asctime'] = pd.to_datetime(df['asctime'], format='ISO8601', errors='coerce')
             df = df.sort_values('asctime')
             df = df.groupby('routine').last()        
             idisin = dfsched.index[dfsched.index.isin(df.index)]
@@ -281,7 +296,7 @@ class Schedule:
         if not Logger.dferror.empty:
             df = Logger.dferror.copy()
             df['routine'] = df['user_name']+':'+df['logger_name']
-            df['asctime'] = pd.to_datetime(df['asctime'])
+            df['asctime'] = pd.to_datetime(df['asctime'], format='ISO8601', errors='coerce')
             df = df.sort_values('asctime')
             df = df.groupby('routine').last()
             idisin = dfsched.index[dfsched.index.isin(df.index)]
@@ -293,7 +308,7 @@ class Schedule:
         if not Logger.dfcompleted.empty:
             df = Logger.dfcompleted.copy()
             df['routine'] = df['user_name']+':'+df['logger_name']
-            df['asctime'] = pd.to_datetime(df['asctime'])
+            df['asctime'] = pd.to_datetime(df['asctime'], format='ISO8601', errors='coerce')
             df = df.sort_values('asctime')
             df = df.groupby('routine').last()
             idisin = dfsched.index[dfsched.index.isin(df.index)]
@@ -305,7 +320,7 @@ class Schedule:
         if not Logger.dflast.empty:
             df = Logger.dflast.copy().reset_index()
             df['routine'] = df['user_name']+':'+df['logger_name']
-            df['asctime'] = pd.to_datetime(df['asctime'])
+            df['asctime'] = pd.to_datetime(df['asctime'], format='ISO8601', errors='coerce')
             df = df.sort_values('asctime')
             df = df.groupby('routine').last()
             idisin = dfsched.index[dfsched.index.isin(df.index)]
@@ -320,7 +335,7 @@ class Schedule:
         if not Logger.dfheartbeats.empty:
             df = Logger.dfheartbeats.copy().reset_index()
             df['routine'] = df['user_name']+':'+df['logger_name']
-            df['asctime'] = pd.to_datetime(df['asctime'])
+            df['asctime'] = pd.to_datetime(df['asctime'], format='ISO8601', errors='coerce')
             df = df.sort_values('asctime')
             df = df.groupby('routine').last()
             idisin = dfsched.index[dfsched.index.isin(df.index)]            

shareddata-6.83.8/src/SharedData/Routines/Scheduler.py

@@ -0,0 +1,109 @@
+import faulthandler
+import os
+import sys
+import time
+from datetime import datetime
+import pytz
+from tzlocal import get_localzone
+local_tz = pytz.timezone(str(get_localzone()))
+
+# Capture a Python-level traceback if a C extension (pandas_parser, numpy,
+# pyarrow, ...) crashes the interpreter with SIGSEGV/SIGABRT/SIGFPE/SIGBUS.
+# Without this, a crash leaves only a kernel segfault line with a raw IP.
+faulthandler.enable()
+
+# PROPRIETARY LIBS
+from SharedData.Routines.Schedule import Schedule
+from SharedData.Logger import Logger
+from SharedData.SharedData import SharedData
+shdata = SharedData('SharedData.Routines.Scheduler', user='master')
+
+
+def _safe(desc, fn, *args, **kwargs):
+    """Run fn(*args, **kwargs), log any exception, never propagate."""
+    try:
+        return fn(*args, **kwargs)
+    except Exception as e:
+        Logger.log.error('Scheduler %s failed: %s' % (desc, e))
+        return None
+
+
+if len(sys.argv) >= 2:
+    ARGS = str(sys.argv[1])
+else:
+    Logger.log.error('Schedules not provided, please specify!')
+    raise SystemExit(2)
+
+Logger.log.info(
+    'SharedData Routines Scheduler starting for %s...' % (ARGS))
+
+schedule_names = ARGS.split(',')
+
+lastheartbeat = time.time()
+Logger.log.info('ROUTINE STARTED!')
+
+# Boot each schedule independently — one bad SCHEDULES/<name> must not
+# prevent the others from running.
+schedules = {}
+for schedule_name in schedule_names:
+    try:
+        sched = Schedule(schedule_name)
+        sched.update()
+        sched.save()
+        schedules[schedule_name] = sched
+    except Exception as e:
+        Logger.log.error(
+            'Scheduler failed to load schedule %s: %s' % (schedule_name, e))
+
+if not schedules:
+    Logger.log.error('Scheduler has no loadable schedules — retrying in main loop')
+
+while True:
+    try:
+        if time.time() - lastheartbeat > 15:
+            lastheartbeat = time.time()
+            Logger.log.debug('#heartbeat#schedule:%s' % (ARGS))
+
+        # Retry failed schedule loads every tick so a transient MongoDB / metadata
+        # outage doesn't leave a schedule permanently dark.
+        for schedule_name in schedule_names:
+            if schedule_name not in schedules:
+                try:
+                    sched = Schedule(schedule_name)
+                    sched.update()
+                    sched.save()
+                    schedules[schedule_name] = sched
+                    Logger.log.info(
+                        'Scheduler recovered schedule %s' % (schedule_name))
+                except Exception as e:
+                    Logger.log.error(
+                        'Scheduler still cannot load %s: %s' % (schedule_name, e))
+
+        now = datetime.now().astimezone(tz=local_tz)
+        for s in list(schedules.keys()):
+            sched = schedules[s]
+            try:
+                if (
+                    sched.schedule is not None
+                    and len(sched.schedule) > 0
+                    and 'runtimes' in sched.schedule.columns
+                    and len(sched.schedule['runtimes']) > 0
+                    and now.date() > sched.schedule['runtimes'][0].date()
+                ):
+                    Logger.log.info(
+                        'Reloading Schedule %s %s' % (s, str(datetime.now())))
+                    _safe('%s.load' % s, sched.load)
+
+                _safe('%s.update' % s, sched.update)
+                _safe('%s.run' % s, sched.run)
+                _safe('%s.save' % s, sched.save)
+            except Exception as e:
+                # Belt-and-suspenders: even with per-stage _safe above, one
+                # bad schedule cannot stop others from running this tick.
+                Logger.log.error(
+                    'Scheduler schedule %s tick failed: %s' % (s, e))
+
+        time.sleep(5)
+    except Exception as e:
+        Logger.log.error('Scheduler Loop Exception: %s' % (str(e)))
+        time.sleep(60)

{shareddata-6.83.7 → shareddata-6.83.8/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.83.7
+Version: 6.83.8
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License

shareddata-6.83.7/src/SharedData/Routines/Scheduler.py

@@ -1,60 +0,0 @@
-import os
-import sys
-import time
-from datetime import datetime
-import pytz
-from tzlocal import get_localzone
-local_tz = pytz.timezone(str(get_localzone()))
-
-# PROPRIETARY LIBS
-from SharedData.Routines.Schedule import Schedule
-from SharedData.Logger import Logger
-from SharedData.SharedData import SharedData
-shdata = SharedData('SharedData.Routines.Scheduler', user='master')
-
-try:
-    if len(sys.argv) >= 2:
-        ARGS = str(sys.argv[1])
-    else:
-        Logger.log.error('Schedules not provided, please specify!')
-        raise Exception('Schedules not provided, please specify!')
-
-    Logger.log.info(
-        'SharedData Routines Scheduler starting for %s...' % (ARGS))
-
-    schedule_names = ARGS.split(',')
-
-    lastheartbeat = time.time()
-    Logger.log.info('ROUTINE STARTED!')
-
-    schedules = {}
-    for schedule_name in schedule_names:
-        schedules[schedule_name] = Schedule(schedule_name)
-        schedules[schedule_name].update()
-        schedules[schedule_name].save()
-
-    while (True):
-        try:
-            if time.time()-lastheartbeat>15:
-                lastheartbeat=time.time()
-                Logger.log.debug('#heartbeat#schedule:%s' % (ARGS))
-
-            now = datetime.now().astimezone(tz=local_tz)
-            for s in schedules:
-                sched = schedules[s]
-                if now.date() > sched.schedule['runtimes'][0].date():            
-                    Logger.log.info('Reloading Schedule %s' % (str(datetime.now())))            
-                    sched.load()
-                    
-                sched.update()
-                sched.run()
-                sched.save()
-
-            time.sleep(5)
-        except Exception as e:
-            Logger.log.error('Scheduler Loop Exception: %s' % (str(e)))
-            time.sleep(60)
-
-except Exception as e:
-    Logger.log.error('Scheduler Routine Exception: %s' % (str(e)))
-    time.sleep(60)