ciocore 5.1.1__py2.py3-none-any.whl → 10.0.0b3__py2.py3-none-any.whl

ciocore/worker.py

@@ -1,19 +1,21 @@
 import logging
+import queue
 import sys
 import threading
+import time
 import traceback
 
-try:
-    import Queue as queue
-except ImportError: 
-    import queue
-
 import ciocore
+import ciocore.loggeria
+import ciocore.uploader.thread_queue_job
 
-logger = logging.getLogger("{}.worker".format(ciocore.common.CONDUCTOR_LOGGER_NAME))
+logger = logging.getLogger("{}.uploader.worker".format(
+    ciocore.loggeria.CONDUCTOR_LOGGER_NAME))
 
-# This is used to signal to workers if work should continue or not
+# This is used to signal to workers i workf should continue or not
 WORKING = True
+EMPTY_JOB = "empty_job"
+
 
 class Reporter():
     def __init__(self, metric_store=None):
@@ -42,13 +44,14 @@ class Reporter():
             return self.thread
 
         logger.debug('starting reporter thread')
-        thd = threading.Thread(target=self.target, name=self.__class__.__name__)
+        thd = threading.Thread(
+            target=self.target, name=self.__class__.__name__)
         thd.daemon = True
         thd.start()
         self.thread = thd
         return self.thread
-    
-    
+
+
 class ThreadWorker(object):
     '''
     Abstract worker class.
@@ -57,7 +60,7 @@ class ThreadWorker(object):
 
     TODO: move this into it's own lib
     '''
-    
+
     def __init__(self, **kwargs):
 
         # the in_queue provides work for us to do
@@ -77,10 +80,10 @@ class ThreadWorker(object):
 
         # create a list to hold the threads that we create
         self.threads = []
-        
+
         self.thread_complete_counter = Counter()
-        self._worker_started = Marker()
         self._job_counter = Counter()
+        self.task_count = 0
 
     def do_work(self, job):
         '''
@@ -102,7 +105,8 @@ class ThreadWorker(object):
             exit()
 
     def kill(self, block=False):
-        logger.debug('killing workers %s (%s threads)', self.__class__.__name__, len(self.threads))
+        logger.debug('killing workers %s (%s threads)',
+                     self.__class__.__name__, len(self.threads))
         for _ in self.threads:
             self.in_queue.put(self.PoisonPill())
 
@@ -115,52 +119,62 @@ class ThreadWorker(object):
         return True
 
     def join(self):
-        logger.debug("Waiting for in_queue to join. ({}-{}). {} items left.".format(self.__class__.__name__, self, self.in_queue.qsize()))
-        
+        logger.debug("Waiting for in_queue to join. (%s-%s). %s items left.",
+                     self.__class__.__name__, self, self.in_queue.qsize())
+
         while True:
             try:
-                logger.debug("Getting remaining task from in_queue ({}-{}). {} items left.".format(self.__class__.__name__, self, self.in_queue.qsize()))
+                logger.debug("Getting remaining task from in_queue (%s-%s). %s items left.",
+                             self.__class__.__name__, self, self.in_queue.qsize())
                 job = self.in_queue.get(block=False)
-                logger.debug("Dropping task {} from in_queue ({}-{}). {} items left.".format(job, self.__class__.__name__, self, self.in_queue.qsize()))
+                logger.debug("Dropping task %s from in_queue (%s-%s). %s items left.",
+                             job, self.__class__.__name__, self, self.in_queue.qsize())
                 self.in_queue.task_done()
             except queue.Empty:
-                logger.debug("in_queue is empty ({}-{}). {} items left.".format(self.__class__.__name__, self, self.in_queue.qsize()))
+                logger.debug("in_queue is empty (%s-%s). %s items left.",
+                             self.__class__.__name__, self, self.in_queue.qsize())
                 break
 
         self.kill(True)
 
     # Basic thread target loop.
-    @ciocore.common.dec_catch_exception(raise_=True)
+    # @ciocore.common.dec_catch_exception(raise_=True)
     def target(self, thread_int):
 
         while not ciocore.common.SIGINT_EXIT:
             try:
-                
+
                 job = None
 
                 try:
                     logger.debug("Worker querying for job")
                     job = self.in_queue.get(timeout=2)
-                    queue_size = self.in_queue.qsize()                     
+                    queue_size = self.in_queue.qsize()
 
                 except queue.Empty:
-                    
-                    if self._job_counter.value > 0:
-                        logger.debug("Worker has completed all of its tasks.".format(job))
+
+                    logger.debug("Worker in queue raised Empty (_job_counter=%s, task_count=%s",
+                                 self._job_counter.value, self.task_count)
+
+                    if self._job_counter.value >= self.task_count:
+                        logger.debug(
+                            "Worker has completed all of its tasks (%s)", job)
                         self.thread_complete_counter.decrement()
                         break
-                    
-                    else:
+
+                    elif self._job_counter.value == 0:
                         logger.debug("Worker waiting for first job")
-                        continue
-                    
-                logger.debug("Worker got job {}".format(job))
+
+                    time.sleep(1)
+                    continue
+
+                logger.debug("Worker got job %s", job)
                 self._job_counter.increment()
-                logger.debug("Processing Job '{}' #{} on {}. {} tasks remaining in queue".format( job, 
-                                                                                                  self._job_counter.value, 
-                                                                                                  self, 
-                                                                                                  queue_size))
-                
+                logger.debug("Processing Job '%s' #%s on %s. %s tasks remaining in queue", job,
+                             self._job_counter.value,
+                             self,
+                             queue_size)
+
                 # exit if we were passed 'PoisonPill'
                 self.check_for_poison_pill(job)
 
@@ -168,11 +182,15 @@ class ThreadWorker(object):
                 try:
                     output = None
                     output = self.do_work(job, thread_int)
+                    logger.debug("Output from do_work(): '%s'", output)
                 except Exception as exception:
-                    logger.exception('CAUGHT EXCEPTION on job "{}" [{}]":\n'.format(job, self))
+                    logger.error(
+                        'CAUGHT EXCEPTION on job "%s" [%s]":\n', job, self)
+                    logger.error(traceback.format_exc())
 
                     # if there is no error queue to dump data into, then simply raise the exception
-                    if not self.error_queue:
+                    if self.error_queue is None:
+                        logger.debug("Re-raising exception")
                         raise
 
                     # Otherwise put the exception in the error queue
@@ -189,12 +207,13 @@ class ThreadWorker(object):
                 self.mark_done()
 
             except Exception:
-                logger.error('[thread %s]+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=', thread_int)
+                logger.error(
+                    '[thread %s]+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=', thread_int)
                 logger.error(traceback.print_exc())
                 logger.error(traceback.format_exc())
                 raise
-        
-        logger.info("Worker {} completed".format(self))
+
+        logger.info("Worker %s completed", self)
 
     def start(self):
         '''
@@ -207,9 +226,10 @@ class ThreadWorker(object):
         for thread_int in range(self.thread_count):
             name = "%s-%s" % (self.__class__.__name__, thread_int)
             logger.debug('starting thread %s', name)
-            
+
             # thread will begin execution on self.target()
-            thd = threading.Thread(target=self.target, args=(thread_int,), name=name)
+            thd = threading.Thread(
+                target=self.target, args=(thread_int,), name=name)
 
             # make sure threads don't stop the program from exiting
             thd.daemon = True
@@ -217,7 +237,7 @@ class ThreadWorker(object):
             # start thread
             thd.start()
             self.threads.append(thd)
-            
+
         self.thread_complete_counter.value = self.thread_count
 
         return self.threads
@@ -225,11 +245,11 @@ class ThreadWorker(object):
     def mark_done(self):
         try:
             self.in_queue.task_done()
-            
+
         except ValueError:
             # this will happen if we are draining queues
             logger.debug('WORKING: %s', WORKING)
-            
+
             if WORKING:
                 logger.error('error hit when marking task_done')
                 # this should not happen if we are still working
@@ -237,25 +257,32 @@ class ThreadWorker(object):
         return
 
     def put_job(self, job):
+
+        if job is None:
+            logger.debug(
+                "Attempting to put job 'None' in out_queue (%s -> %s). Skipping.", self, self.out_queue)
+            return
+
         # don't to anything if we were not provided an out_queue
-        logger.debug("Attempting to put job to out_queue ({} -> {})".format(job,self, self.out_queue))
-        
+        logger.debug(
+            "Attempting to put job to out_queue (%s -> %s)", job, self.out_queue)
+
         if not self.out_queue:
             return
 
         # if were not supposed to be working, don't create new jobs
         if not WORKING:
             return
-        
-        logger.debug("Adding job to out_queue ({} -> {})".format(job,self, self.out_queue))
-        
+
+        logger.debug("Adding job to out_queue (%s -> %s)", job, self.out_queue)
+
         # add item to job
         self.out_queue.put(job)
         return True
-    
-    def is_complete(self):        
+
+    def is_complete(self):
         return self.thread_complete_counter.value == 0
-            
+
 
 class MetricStore():
     '''
@@ -282,15 +309,16 @@ class MetricStore():
         if self.started:
             logger.debug('metric_store already started')
             return None
-        
+
         logger.debug('starting metric_store')
-        
+
         self.stop = False
-        self.thread = threading.Thread(target=self.target, name=self.__class__.__name__)
+        self.thread = threading.Thread(
+            target=self.target, name=self.__class__.__name__)
         self.thread.daemon = True
         self.thread.start()
         self.started = True
-        
+
         return self.thread
 
     def set(self, key, value):
@@ -315,19 +343,20 @@ class MetricStore():
         if filename:
             if 'files' not in self.metric_store:
                 self.metric_store['files'] = {}
-            
+
             if filename not in self.metric_store['files']:
-                self.metric_store['files'][filename] = {'bytes_to_upload': 0, 'bytes_uploaded': 0, 'already_uploaded': False}
-            
+                self.metric_store['files'][filename] = {
+                    'bytes_to_upload': 0, 'bytes_uploaded': 0, 'already_uploaded': False}
+
             if variable == 'bytes_uploaded':
                 self.metric_store['files'][filename]['bytes_uploaded'] += step_size
-            
+
             elif variable == 'bytes_to_upload':
                 self.metric_store['files'][filename]['bytes_to_upload'] = step_size
-                
+
             elif variable == 'already_uploaded':
-                self.metric_store['files'][filename]['already_uploaded'] = step_size # True/False            
-                
+                # True/False
+                self.metric_store['files'][filename]['already_uploaded'] = step_size
 
     def set_dict(self, dict_name, key, value):
         self.update_queue.put(('set_dict', dict_name, key, value))
@@ -371,17 +400,17 @@ class MetricStore():
     @ciocore.common.dec_catch_exception(raise_=True)
     def target(self):
         logger.debug('created metric_store target thread')
-        
+
         while not self.stop:
-            
-            logger.debug("Metric store self.stop={}".format(self.stop))
-            
+
+            logger.debug("Metric store self.stop=%s", self.stop)
+
             try:
                 # block until update given
                 update_tuple = self.update_queue.get(True, 2)
-                
+
             except queue.Empty:
-                continue                
+                continue
 
             method = update_tuple[0]
             method_args = update_tuple[1:]
@@ -413,9 +442,10 @@ class JobManager():
         self._queue_started = False
         self.error_handler_stop = False
         self.error_handler_thread = None
+        self.task_count = None
 
     def drain_queues(self):
-        logger.error('draining queues')
+        logger.debug('draining queues')
         # http://stackoverflow.com/questions/6517953/clear-all-items-from-the-queue
         for the_queue in self.work_queues:
             the_queue.mutex.acquire()
@@ -424,7 +454,7 @@ class JobManager():
         return True
 
     def mark_all_tasks_complete(self):
-        logger.error('clearing out all tasks')
+        logger.debug('clearing out all tasks')
         # http://stackoverflow.com/questions/6517953/clear-all-items-from-the-queue
         for the_queue in self.work_queues:
             the_queue.mutex.acquire()
@@ -438,7 +468,7 @@ class JobManager():
         WORKING = False
         for worker in self.workers:
             logger.debug("Killing worker: %s", worker)
-            worker.kill(block=True) # Wait to ensure the worker was killed
+            worker.kill(block=True)  # Wait to ensure the worker was killed
 
     def kill_reporters(self):
         for reporter in self.reporters:
@@ -446,67 +476,89 @@ class JobManager():
             reporter.kill()
 
     def stop_work(self, force=False):
-        
+
         global WORKING
-        
+
         if WORKING:
-            
+
             logger.info("Stopping Worker Manager")
-            
+
             WORKING = False  # stop any new jobs from being created
             self.drain_queues()  # clear out any jobs in queue
             self.kill_workers()  # kill all threads
             self.kill_reporters()
             self.mark_all_tasks_complete()  # reset task counts
-            
+
             self.metric_store.stop = True
-            self.metric_store.join()
-            
+            # self.metric_store.join()
+
             self.error_handler_stop = True
-            self.error_handler_thread.join()
-            
+            # self.error_handler_thread.join()
+
         else:
             logger.info("Worker Manager has already been stopped.")
-        
+
         return self.error
 
     @ciocore.common.dec_catch_exception(raise_=True)
     def error_handler_target(self):
 
         while not self.error_handler_stop:
-            
+
             try:
-                error = self.error_queue.get(True, 2)
-                
+                error = self.error_queue.get(True, 0.5)
+
             except queue.Empty:
-                continue
-            
-            logger.error('Got something from the error queue: {}'.format(error))
+
+                if self.error:
+                    break
+
+                else:
+                    continue
+
+            logger.error('Got something from the error queue: %s', error)
             self.error.append(error)
             try:
                 self.error_queue.task_done()
             except ValueError:
                 pass
 
+        self.stop_work(force=True)
+
     def start_error_handler(self):
-        
+
         logger.debug('Creating error handler thread')
-        self.error_handler_thread = threading.Thread(target=self.error_handler_target, name="ErrorThread")
+        self.error_handler_thread = threading.Thread(
+            target=self.error_handler_target, name="ErrorThread")
         self.error_handler_thread.daemon = True
         self.error_handler_thread.start()
-        
+
         return None
 
-    def add_task(self, task):
-        
+    def add_task(self, task, project=None):
+
         # This allows us to keep track of the difference between an empty queue
         # because no tasks have been added or an empty queue because all the tasks
         # have been completed
-        if not self._queue_started:            
+        if not self._queue_started:
+            logger.debug("Initializing Queue - queue started")
             self._queue_started = True
-        
-        self.work_queues[0].put(task)
-        
+            self.task_count = 0
+
+        job = ciocore.uploader.thread_queue_job.ThreadQueueJob(path=task[0],
+                                                               md5=task[1],
+                                                               project=project)
+        self.work_queues[0].put(job)
+
+        for worker in self.workers:
+            worker.task_count += 1
+            logger.debug("Incremented task count on worker %s to %s",
+                         worker, worker.task_count)
+
+        self.task_count += 1
+        logger.debug("Incremented task count on manager to %s",
+                     self.task_count)
+
         return True
 
     def start(self):
@@ -523,7 +575,6 @@ class JobManager():
         next_queue = None
         last_queue = self.work_queues[0]
         last_worker = next(reversed(self.job_description))
-
         for worker_description in self.job_description:
             worker_class = worker_description[0]
             args = []
@@ -537,13 +588,9 @@ class JobManager():
 
             kwargs['in_queue'] = last_queue
 
-            if last_worker == worker_class:
-                # the last worker does not need an output queue
-                kwargs['out_queue'] = None
-            else:
-                next_queue = queue.Queue()
-                self.work_queues.append(next_queue)
-                kwargs['out_queue'] = next_queue
+            next_queue = queue.Queue()
+            self.work_queues.append(next_queue)
+            kwargs['out_queue'] = next_queue
 
             kwargs['error_queue'] = self.error_queue
             kwargs['metric_store'] = self.metric_store
@@ -576,79 +623,55 @@ class JobManager():
         self.metric_store.stop = True
         self.metric_store.join()
         logger.debug('metric store in sync')
-        
+
         self.error_handler_stop = True
         self.error_handler_thread.join()
-        
+
         if self.error:
             return self.error
-        
+
         self.kill_workers()
         self.kill_reporters()
-        
+
         return None
 
     def worker_queue_status_text(self):
-        msg = '\n' + '#' * 80 + '\n'
+        msg = "\n{:#^80}\n".format('QUEUE STATUS')
         for index, worker_info in enumerate(self.job_description):
             worker_class = worker_info[0]
             q_size = self.work_queues[index].qsize()
             worker_threads = self.workers[index].threads
 
-            # thread.isAlive() was renamed to is_alive() in Python 3.9 
-            try:
-                num_active_threads = len([thd for thd in worker_threads if thd.isAlive()])
-            except AttributeError:
-                num_active_threads = len([thd for thd in worker_threads if thd.is_alive()])
+            num_active_threads = len(
+                [thd for thd in worker_threads if thd.is_alive()])
 
             msg += '%s \titems in queue: %s' % (q_size, worker_class.__name__)
             msg += '\t\t%s threads' % num_active_threads
             msg += '\n'
         return msg
-    
+
     def is_complete(self):
 
-        is_complete = True
-        
-        # Manager has completed if all workers have completed
         for worker in self.workers:
-            logger.debug("Worker {} is complete: {}".format(worker.__class__.__name__, worker.is_complete()))
-            is_complete &= worker.is_complete()
-            
-        logger.debug("All Workers have completed: {}".format(is_complete))
-            
-        return is_complete
-    
+            logger.debug("Worker %s is complete: %s",
+                         worker.__class__.__name__, worker.is_complete())
+
+        return self.get_last_worker().is_complete()
+
+    def get_last_worker(self):
+        return self.workers[-1]
+
+
 class Counter(object):
-    
+
     def __init__(self):
         self.value = 0
         self._lock = threading.Lock()
-        
+
     def increment(self):
         with self._lock:
             self.value += 1
-            
+
     def decrement(self):
         with self._lock:
-            self.value -= 1            
-            
-class Marker(object):
-    
-    def __init__(self):
-        self.marker = False
-        self._lock = threading.Lock()
-        
-    def mark(self):
-        
-        with self._lock:
-            
-            logger.debug("Lock aquired (marker={})".format(self.marker))
-                    
-            if self.marker:
-                return True
-            
-            else:            
-            
-                self.marker = True
-                return False        
+            self.value -= 1