python-microscopy 25.11.20__cp313-cp313-win_amd64.whl

PYME/cluster/ruleserver.py

@@ -0,0 +1,1106 @@
+"""
+This module defines a RESTful HTTP server, the PYME Rule Server, which manages task distribution across a cluster through the use of rules.
+Rules are essentially a json template which defines how tasks may be generated (by substitution into the template on the
+client side). Generation of individual tasks on the client has the dual benefits of a) reducing the CPU load and memory
+requirements on the server and b) dramatically decreasing the network bandwidth used for task distribution.
+ 
+The exposed REST API is as follows:
+
+============================================================= ====== ========================================
+Endpoint                                                      Verb   Brief Description
+============================================================= ====== ========================================
+:meth:`/add_integer_id_rule <RuleServer.add_integer_id_rule>` POST   Add a new rule
+:meth:`/release_rule_tasks <RuleServer.release_rule_tasks>`   POST   Release tasks IDs associated with a rule
+:meth:`/task_advertisments <RuleServer.task_advertisements>`  GET    Retrieve a list of advertisements
+:meth:`/bid_on_tasks <RuleServer.bid_on_tasks>`               POST   Submit bids on advertised tasks
+:meth:`/handin <RuleServer.handin>`                           POST   Advise rule server of task completion
+:meth:`/distributor/queues <Ruleserver.get_queues>`           GET    Get status information
+============================================================= ====== ========================================
+
+Implementation details follow. The json format and parameters for the REST calls are defined in the docstrings of the
+python functions defining that method (linked from the table above). The server is launched by the ``PYMERuleServer``
+command (see :mod:`PYME.cluster.PYMERuleServer`) which loads this in a separate thread and takes care of logging and zeroconf
+registration, rather than by directly running this file.
+"""
+#import cherrypy
+import threading
+import requests
+import queue as Queue
+from six.moves import xrange
+
+import logging
+logging.basicConfig(level=logging.WARN)
+logger = logging.getLogger('ruleserver')
+logger.setLevel(logging.DEBUG)
+
+import time
+import sys
+import ujson as json
+#import json
+import os
+
+from PYME.misc import computerName
+from PYME import config
+from PYME import resources
+#from PYME.IO import clusterIO
+from PYME.util import webframework
+import collections
+
+import uuid
+
+import numpy as np
+
+class Rule(object):
+    pass
+
+STATUS_UNAVAILABLE, STATUS_AVAILABLE, STATUS_ASSIGNED, STATUS_COMPLETE, STATUS_FAILED = range(5)
+
+Bid = collections.namedtuple('Bid', 'bidder_id, cost')
+
+DEFAULT_BID = Bid(0, sys.maxsize)
+
+class IntegerIDRule(Rule):
+    """
+    A rule which generates tasks based on a template.
+    
+    Parameters
+    ----------
+    ruleID : str
+        A unique ID for this rule
+    task_template : str
+        A json template for generating tasks (see templates section below)
+    inputs_by_task : list of dict
+        A list of dictionaries mapping recipe variable names to URIs to treat as inputs (recipes only)
+    max_task_ID : int
+        The maximum number of tasks that can be generated by this rule. If the final number of tasks is not known at
+        creation, set this to a value which is garuanteed to be higher and then call `mark_release_complete()` after all tasks
+        have been released / made available.
+    task_timeout: float
+        A timeout in seconds for each task. If a task takes longer than task_timeout seconds it is assumed that
+        the node executing it has fallen over and we retry on a different node (up to a maximum number of times
+        set by the 'ruleserver-retries` config option).
+    rule_timeout : float
+        A timeout in seconds from the last task we processed after which we assume that no more tasks are coming and
+        we can delete the rule (to keep memory down in a long-term usage scenario).
+    
+        
+    Notes
+    -----
+    
+    *Templates*
+    
+    Templates are a string which can be substituted to generate tasks. There are currently two supported formats for
+    templates, those for localization tasks, and those for recipes. Each take the form of a json dictionary:
+    
+    **Localization**
+    
+    .. code-block:: json
+        
+        {
+         "id" : "{{ruleID}}~{{taskID}}",
+         "type" : "localization",
+         "taskdef" : {"frameIndex" : {{taskID}}, "metadata" : "PYME-CLUSTER://path/to/series/metadata.json"},
+         "inputs" : {"frames" : "PYME-CLUSTER://path/to/series.pcs"},
+         "outputs" : {"fitResults" : "PYME-CLUSTER://__aggregate_h5r/path/to/analysis.h5r/FitResults",
+                      "driftResults" : "PYME-CLUSTER://__aggregate_h5r/path/to/analysis.h5r/DriftResults"}
+        }
+    
+    
+    **Recipes**
+    The recipe can either be specified inline:
+    
+    .. code-block:: json
+        
+        {
+         "id" : "{{ruleID}}~{{taskID}}",
+         "type" : "recipe",
+         "taskdef" : {"recipe" : "<recipe as a YAML string>"},
+         "inputs" : {{taskInputs}},
+         "output_dir" : "PYME-CLUSTER://path/to/output/dir",
+        }
+    
+        
+    or using a cluster URI:
+    
+    .. code-block:: json
+        
+        {
+         "id" : "{{ruleID}}~{{taskID}}",
+         "type" : "recipe",
+         "taskdefRef" : "PYME-CLUSTER://path/to/recipe.yaml",
+         "inputs" : {{taskInputs}},
+         "output_dir" : "PYME-CLUSTER://path/to/output/dir",
+        }
+     
+        
+    The rule will substitute ``{{taskInputs}}`` with a dictionary mapping integer task IDs to recipe input files, e.g.
+    
+    .. code-block:: json
+    
+        {0 : {"recipe_input_0" : "input_0_URI_0","recipe_input_1" : "input_1_URI_0"},
+         1 : {"recipe_input_0" : "input_0_URI_1","recipe_input_1" : "input_1_URI_1"},
+         2 : {"recipe_input_0" : "input_0_URI_2","recipe_input_1" : "input_1_URI_2"},
+         }
+       
+         
+    Alternatively the inputs dictionary can be supplied directly (without relying on the taskInputs substitution).
+    
+    *Rule Chaining*
+    
+    Rules may also define a chained rule, to be run on completion of the original rule. This is accomplished by
+    supplying an "on_completion" dictionary. This is a dictionary, ``{'template' : <template>, 'max_tasks' : max_tasks,
+    'rule_timeout' : timeout, 'on_completion' : {...}}``, with the template following the format above and everything
+    but the template being optional (max_tasks defaults to 1). Follow on / chained rules are slightly more restricted
+    than standard rules in that recipe "inputs" must be hardcoded (no ``{{taskInputs}}`` substitution).
+    
+    Chained rules are created once the original rule is finished (see 
+    `IntegerIDRule.finished`). All tasks for the chained rule will be released 
+    immediately.
+        
+    """
+    TASK_INFO_DTYPE = np.dtype([('status', 'uint8'), ('nRetries', 'uint8'), ('expiry', 'f4'), ('cost', 'f4')])
+    
+    # task cost threshold below which tasks are deemed 'local' and directly awarded to the first bidder
+    # rather than going to "auction" - ie. the "Buy Now" price
+    # TODO - Revisit - should this be a settable rule parameter rather than a constant?
+    COST_THRESHOLD = 0.2
+    
+    
+    def __init__(self, ruleID, task_template, inputs_by_task = None,
+                 max_task_ID=100000, task_timeout=600, rule_timeout=3600, 
+                 on_completion=None):
+        self.ruleID = ruleID
+        
+        if not inputs_by_task is None:
+            self._inputs_by_task = {int(k):v for k, v in inputs_by_task.items()}
+        else:
+            self._inputs_by_task = None
+            
+        self._template = task_template
+        self._task_info = np.zeros(max_task_ID, self.TASK_INFO_DTYPE)
+        
+        # Number of times to re-queue a task if it times out is set by the 'ruleserver-retries' config option
+        # Setting a value of 0 effectively disables re-trying and makes analysis less robust.
+        # Note that a timeout is different to a failure - failing tasks will be marked as having failed and will not be re-tried. Timeouts will
+        # occur in one of 4 scenarios:
+        # - a worker falls over completely or is disconnected from the network
+        # - an unhandled exception - e.g. due to an IO error connecting to the ruleserver in `taskWorkerHTTP.taskWorker._return_task_results()`
+        # - analysis is getting massively bogged down and nothing is keeping up
+        # - the task timeout is unrealistically short for a given processing task
+        # 
+        # In scenarios 1 & 2 it's reasonable to expect that retrying will result in success. In scenarios 3 & 4 it's a bit muddier, but seeing as a) these kind of
+        # failures tend to be a bit stochastic and b) the retries get punted to the back of the queue, when the load might have let up a bit, the odds are
+        # reasonably good.
+        self._n_retries = config.get('ruleserver-retries', 1)
+        self._timeout = task_timeout
+        
+        self._rule_timeout = rule_timeout
+        self._cached_advert = None
+        self._active = True # making this rule inactive will cause it not to generate adverts (this is the closest we  get to aborting)
+        self._seen_as_finished = False # flag to make sure completion logic only gets triggered once.
+        
+        self.nTotal = 0
+        self.nAssigned = 0
+        self.nAvailable = 0
+        self.nCompleted = 0
+        self.nFailed = 0
+        self.n_returned_after_timeout = 0
+        self.n_repeats = 0
+        self.n_timed_out = 0
+        
+        self._n_max = max_task_ID
+        
+        self.on_completion = on_completion
+        
+        self.avCost = 0
+        
+        self.expiry = time.time() + self._rule_timeout
+        
+        # store pending bids
+        self._pending_bids = {}
+        self._current_bidder_id = 0
+              
+        self._info_lock = threading.Lock()
+        self._advert_lock = threading.Lock()
+    
+    def mark_release_complete(self, n_tasks=None):
+        """
+        Signal that all tasks which are going to be released for this rule have been, and that the rule should evaluate as finished once these tasks have been
+        evaluated. Used when the number of tasks is not known in advance. (See also __init__ docs) 
+        
+        FIXME - What happens when task release doesn't start at 0?
+
+        Parameters
+        ----------
+        n_tasks : int, optional
+            mark complete at a given number of tasks. The default is to mark complete at the highest task number which has been released prior to making this call. 
+            The use case for this optional parameter is to avoid the need for locking when this gets called from another thread than the `make_range_available()`
+            and implies a promise by the calling code that it will not release any more tasks above the given number.
+
+        Raises
+        ------
+        ValueError
+            If `n_tasks` is supplied and is either smaller than the largest task already released or larger than larger than `max_task_ID`
+        """
+        
+        with self._info_lock: #TODO - is this lock necessary (I suspect it's not)
+            if (n_tasks is not None): 
+                if self.nTotal > n_tasks:
+                        raise ValueError('n_tasks cannot be less than nTotal')
+
+                # extend _task_info array if necessary
+                if n_tasks > self._n_max:
+                   raise ValueError('n_tasks cannot be greater than max_task_ID')
+
+                self._n_max = n_tasks
+            else:
+                self._n_max = self.nTotal
+       
+        
+    
+        
+    def _update_nums(self):
+        self.nAvailable = int((self._task_info['status'] == STATUS_AVAILABLE).sum())
+        self.nAssigned = int((self._task_info['status'] == STATUS_ASSIGNED).sum())
+        
+    def _update_cost(self):
+        av_cost = np.mean(self._task_info['cost'][self._task_info['status'] > STATUS_AVAILABLE])
+        if np.isnan(av_cost):
+            av_cost = 0
+        else:
+            av_cost = float(av_cost)
+            
+        self.avCost = av_cost
+        
+    def make_range_available(self, start, end):
+        """
+        Make a range of tasks available (to be called once the underlying data is available)
+        [start, end)
+        
+        Parameters
+        ----------
+        start : int
+            first task number to release (inclusive)
+        end : int
+            last task number to release (exclusive)
+        
+        Raises
+        ------
+        RuntimeError
+            if asked to release a range which is invalid for the max tasks we can create from this rule
+        """
+
+        if start < 0 or start > self._task_info.size or end < 0 or end > self._task_info.size:
+            raise RuntimeError('Range (%d, %d) invalid with maxTasks=%d' % (start, end, self._task_info.size))
+        
+        #TODO - check existing status - it probably makes sense to only apply this to tasks which have STATUS_UNAVAILABLE
+        with self._info_lock:
+            self._task_info['status'][start:end] = STATUS_AVAILABLE
+            
+            self.nTotal = int((self._task_info['status'] >0).sum())
+        
+            self._update_nums()
+
+        self.expiry = time.time() + self._rule_timeout
+        
+        with self._advert_lock:
+            self._cached_advert = None
+            
+    def bid(self, bid):
+        """Bid on tasks (and return any that match). Note the current implementation is very naive and doesn't
+        check bid cost - i.e. the first bid gets the task. This works if (and only if) the clients are well behaved
+        and preferentially bid on tasks which have a lowest cost for them.
+        
+        Parameters
+        ----------
+        
+        bid : dict
+            A dictionary containing the ruleID, the IDs of the tasks to bid on, and their costs
+            ``{"ruleID" : str ,"taskIDs" : [list of int],"taskCosts" : [list of float]}``
+        
+        Returns
+        -------
+        
+        successful_bids: dict
+            A dictionary containing the ruleID, the IDs of the tasks awarded, and the rule template
+            ``{"ruleID" : ruleID, "taskIDs": [list of int], "template" : "<rule template>"}``
+        
+        """
+        if not self._active:
+            # don't accept any bids
+            return {'ruleID': bid['ruleID'], 'taskIDs':[], 'template' : ''}
+        
+        taskIDs = np.array(bid['taskIDs'], 'i')
+        costs = np.array(bid['costs'], 'f4')
+        
+        if np.all(costs < self.COST_THRESHOLD):
+            # all tasks are below the "Buy Now" threshold, take a shortcut and directly award them to the bidder bypassing
+            # the rest of the biddng process. This enables high performance on localisation tasks
+            pass
+        else:
+            # NOTE: this bidding code is not high-performance. This is probably OK as code is bypassed in normal
+            # localisation scenario but might need to be revisited as it will be triggered in mixed cases where
+            # some bids are local and some are not.
+            with self._info_lock:
+                # get a unique bidder ID
+                self._current_bidder_id += 1
+                bidder_id = self._current_bidder_id
+                
+                # record bids
+                for taskID, cost in zip(taskIDs, costs):
+                    # get existing winning bid
+                    _, winning_cost = self._pending_bids.get(taskID, DEFAULT_BID)
+                    if cost < winning_cost:
+                        # if we have a lower cost, over-write the winning bid
+                        self._pending_bids[taskID] = Bid(bidder_id, cost)
+            
+            #wait for other bids
+            time.sleep(0.2) #TODO - make this time configurable??
+            
+            with self._info_lock:
+                # filter our tasks to only keep the wining ones
+                winners = [self._pending_bids[taskID].bidder_id==bidder_id for taskID in taskIDs]
+                
+                taskIDs = taskIDs[winners]
+                costs = costs[winners]
+        
+        
+        with self._info_lock:
+            successful_bid_mask = self._task_info['status'][taskIDs] == STATUS_AVAILABLE
+            successful_bid_ids = taskIDs[successful_bid_mask]
+            self._task_info['status'][successful_bid_ids] = STATUS_ASSIGNED
+            self._task_info['cost'][successful_bid_ids] = costs[successful_bid_mask]
+            self._task_info['expiry'][successful_bid_ids] = time.time() + self._timeout
+            
+            nTasks = len(successful_bid_ids)
+            self.nAvailable -= nTasks
+            self.nAssigned += nTasks
+            
+            self._update_cost()
+
+        self.expiry = time.time() + self._rule_timeout
+        
+        with self._advert_lock:
+            self._cached_advert = None
+            
+        return {'ruleID': bid['ruleID'], 'taskIDs':successful_bid_ids.tolist(), 'template' : self._template}
+    
+    def mark_complete(self, info):
+        """
+        Mark a set of tasks as completed and/or failed
+        
+        Parameters
+        ----------
+        info : dict
+            A dictionary of the form: ``{"ruleID": str, "taskIDs" : [list of int], "status" : [list of int]}``
+            
+            There should be an entry in status for each entry in taskIDs, the valid values of status being
+            ``STATUS_COMPLETE=3`` or ``STATUS_FAILED=4``.
+
+        Returns
+        -------
+
+        """
+        taskIDs = np.array(info['taskIDs'], 'i')
+        status = np.array(info['status'], 'uint8')
+        
+        with self._info_lock:
+            old_status = np.copy(self._task_info['status'][taskIDs])
+            self._task_info['status'][taskIDs] = status
+            
+            # if tasks have timed out (or results have already been recieved), they will register as not assigned
+            n_not_assigned = int((old_status != STATUS_ASSIGNED).sum())
+            
+            # if we re-queue tasks after timeout we might receive answers from the re-queued tasks twice
+            n_already_complete = int((old_status == STATUS_COMPLETE).sum())
+            n_already_failed = int((old_status == STATUS_FAILED).sum())
+            
+            
+            self.nCompleted += (int((status == STATUS_COMPLETE).sum()) - n_already_complete)
+            self.nFailed += (int((status == STATUS_FAILED).sum()) - n_already_failed)
+            
+            self.n_repeats += (n_already_complete + n_already_failed)
+            self.n_returned_after_timeout += n_not_assigned
+            self.nAvailable -= (n_not_assigned - (n_already_complete + n_already_failed))
+            
+            nTasks = len(taskIDs) - n_not_assigned#(n_already_complete + n_already_failed)
+            self.nAssigned -= nTasks
+
+        self.expiry = time.time() + self._rule_timeout
+            
+    @property
+    def advert(self):
+        """ The task advertisment.
+        
+        If the rule has tasks available, a dictionary of the form:
+        ``{"ruleID" : str, "taskTemplate" : str, "availableTaskIDs" : [list of int], "inputsByTask" : [optional] dict mapping task IDs to inputs}``
+        
+        "inputsByTask" is only provided for some recipe tasks.
+        """
+        if not self._active:
+            return None
+        
+        with self._advert_lock:
+            if not self._cached_advert:
+                availableTasks = np.where(self._task_info['status'] == STATUS_AVAILABLE)[0].tolist()
+                
+                if len(availableTasks) == 0:
+                    self._cached_advert = None
+                else:
+                    self._cached_advert = {'ruleID' : self.ruleID,
+                        'taskTemplate': self._template,
+                        'availableTaskIDs': availableTasks}
+                    
+                    #print self._inputs_by_task
+                    
+                    if not self._inputs_by_task is None:
+                        self._cached_advert['inputsByTask'] = {taskID: self._inputs_by_task[taskID] for taskID in availableTasks}
+                
+            return self._cached_advert
+    
+    # @property
+    # def nAvailable(self):
+    #     return (self._task_info['status'] == STATUS_AVAILABLE).sum()
+    #
+    # @property
+    # def nAssigned(self):
+    #     return (self._task_info['status'] == STATUS_ASSIGNED).sum()
+
+    # @property
+    # def nCompleted(self):
+    #     return (self._task_info['status'] == STATUS_COMPLETE).sum()
+
+    # @property
+    # def nFailed(self):
+    #     return (self._task_info['status'] == STATUS_FAILED).sum()
+
+    
+    @property
+    def expired(self):
+        """ Whether the rule has expired (no available tasks, no tasks assigned, and time > expiry) and can be removed"""
+        return (self.nAvailable == 0) and (self.nAssigned == 0) and (time.time() > self.expiry)
+    
+    @property
+    def finished(self):
+        """ 
+        Whether the rule has finished (the maximum number of tasks which can be created have been completed).
+        """
+        
+        # TODO - change this so that we can release starting at task_ID > 0???? 
+        # TODO - make rules finish when some tasks fail.
+        
+        # To fix: Potentially replace with `np.all(self._task_info['status']>=STATUS_COMPLETE)` (although this would need to be cached and refreshed - property access should be cheap). 
+        # combined with a new enum value STATUS_INVALID==6 -  `self.mark_release_complete()` could be re-written as `self._task_info['status'][self._task_info['status'] == 0] = STATUS_INVALID`
+        
+        return (self.nAvailable == 0) and ((self.nCompleted + self.nFailed) >= self._n_max)
+    
+    def inactivate(self):
+        """
+        Mark rule as inactive (generates no adverts) to facilitate aborting / pausing long-running rules.
+        """
+        self._active = False
+    
+    def info(self):
+        """
+        Get information / status about this rule
+        
+        Returns
+        -------
+        
+        status : dict
+            A status dictionary of the form:
+            ``{'tasksPosted': int, 'tasksRunning': int, 'tasksCompleted': int, 'tasksFailed' : int, 'averageExecutionCost' : float}``
+
+        """
+        return {'tasksPosted': self.nTotal,
+                  'tasksRunning': self.nAssigned,
+                  'tasksCompleted': self.nCompleted,
+                  'tasksFailed' : self.nFailed,
+                  'averageExecutionCost' : self.avCost,
+                  'active' : self._active,
+                  'tasksTimedOut' : self.n_timed_out,
+                  'tasksCompleteAfterTimeout' : self.n_returned_after_timeout,
+                  'finished' : self.finished,
+                  'expired' : self.expired,
+                }
+    
+    def poll_timeouts(self):
+        t = time.time()
+        
+        with self._info_lock:
+            timed_out = np.where((self._task_info['status'] == STATUS_ASSIGNED)*(self._task_info['expiry'] < t))[0]
+            
+            
+            nTimedOut = len(timed_out)
+            if nTimedOut > 0:
+                self._task_info['status'][timed_out] = STATUS_AVAILABLE
+                self._task_info['nRetries'][timed_out] += 1
+                
+                self.nAssigned -= nTimedOut
+                self.nAvailable += nTimedOut
+                
+                self.n_timed_out += nTimedOut
+    
+                retry_failed = self._task_info['nRetries'][timed_out] > self._n_retries
+                self._task_info['status'][timed_out[retry_failed]] = STATUS_FAILED
+                self.nAvailable -= int(retry_failed.sum())
+                self.nFailed += int(retry_failed.sum())
+                
+                #self._update_nums()
+            
+        with self._advert_lock:
+            self._cached_advert = None
+        
+        
+    
+        
+
+
+class RuleServer(object):
+    MAX_ADVERTISEMENTS = 5 * 10 * 50 * 12 #only advertise enough for 100 tasks on each core of each cluster node
+    def __init__(self):
+        self._rules = collections.OrderedDict()
+        
+        #cherrypy.engine.subscribe('stop', self.stop)
+        
+        self._do_poll = True
+        
+        self._queueLock = threading.Lock()
+        
+        self._advert_lock = threading.Lock()
+        
+        #set up threads to poll the distributor and announce ourselves and get and return tasks
+        #self.pollThread = threading.Thread(target=self._poll)
+        #self.pollThread.start()
+        
+        self._cached_advert = None
+        self._cached_advert_expiry = 0
+        
+        
+        self._info_lock = threading.Lock()
+        self._cached_info = None
+        self._cached_info_expiry = 0
+        self._cached_info_timeout = 5
+        self._finished_rule_timeout = 60 # keep finished rules around for a minute (so we can see them in the GUI)
+        self._failed_finished_rule_timeout = 5*60 # keep rules with failures around for 5 mins to give us a chance to look at errors
+        
+        self._rule_n = 0
+        
+        self._rule_lock = threading.Lock() # lock for when we modify the dictionary of rules
+        
+        self.rulePollThread = threading.Thread(target=self._poll_rules)
+        self.rulePollThread.start()
+        
+        with open(os.path.join(resources.get_web_dir(),  'ruleserver.html'), 'r') as f:
+            self._status_page = f.read()
+    
+    
+    def _poll(self):
+        while self._do_poll:
+            time.sleep(1)
+    
+    def _poll_rules(self):
+        while self._do_poll:
+            for qn in list(self._rules.keys()):
+                self._rules[qn].poll_timeouts()
+                
+                # look for rules that have processed all tasks
+                if self._rules[qn].finished:
+                    #with self._rule_lock:
+                    #    r = self._rules.pop(qn)
+                    
+                    r = self._rules[qn]
+                    if not r._seen_as_finished:
+                        r._seen_as_finished = True # prevent following logic from executing twice
+
+                        # shorten rule expiry (no need to keep lots of finished rules in memory)
+                        if r.nFailed > 0:
+                            # Allow a little more time if we have errors so we can click through to the diagnostics (TODO)
+                            r.expiry = time.time() + self._failed_finished_rule_timeout
+                        else:
+                            # Allow some time so that rules can be seen as complete in the GUI
+                            r.expiry = time.time() + self._finished_rule_timeout
+                        
+                        follow_on = r.on_completion
+                        if follow_on is not None:
+                            # if a follow on rule is defined, add it
+                            template = follow_on['template']
+                            n_tasks = follow_on.get('max_tasks', 1)
+                            timeout = follow_on.get('rule_timeout', 3600.)
+                            ruleID = '%06d-%s' % (self._rule_n, uuid.uuid4().hex)
+        
+                            rule = IntegerIDRule(ruleID, template, max_task_ID=int(n_tasks),
+                                                rule_timeout=float(timeout), on_completion=follow_on.get('on_completion', None))
+        
+                            rule.make_range_available(0, int(n_tasks))
+        
+                            with self._rule_lock:
+                                self._rules[ruleID] = rule
+        
+                            self._rule_n += 1
+                    
+                
+                #remore queue if expired (no activity for an hour) to free up memory
+                if self._rules[qn].expired:
+                    logger.debug('removing expired rule: %s' % self._rules[qn].ruleID)
+                    with self._rule_lock:
+                        r = self._rules.pop(qn)
+                        
+            
+            time.sleep(5)
+    
+    def stop(self):
+        self._do_poll = False
+        
+        #for queue in self._queues.values():
+        #    queue.stop()
+            
+    @webframework.register_endpoint('/task_advertisements')
+    def task_advertisements(self):
+        """
+        HTTP endpoint (GET) for retrieving task advertisements.
+        
+        Note - by default, a limited number of advertisements are posted at any given time (to limit bandwidth). This
+        should be enough to keep all the workers busy, with new adverts being posted once tasks are assigned to workers.
+        
+        Returns
+        -------
+        
+        adverts : json str
+            List of advertisements. See :attr:`IntegerIDRule.advert`
+
+        """
+        with self._advert_lock:
+            t = time.time()
+            if (t > self._cached_advert_expiry):
+                adverts = []
+                nTasks = 0
+                ruleN = 0
+                
+                rules = list(self._rules.values())
+                while ruleN < len(rules) and nTasks < self.MAX_ADVERTISEMENTS:
+                    advert = rules[ruleN].advert
+                    
+                    if not advert is None:
+                        adverts.append(advert)
+                        nTasks += len(advert['availableTaskIDs'])
+                        
+                    ruleN += 1
+                    
+                self._cached_advert = json.dumps(adverts)
+                self._cached_advert_expiry = time.time() + 1 #regenerate advert once every second
+            
+        return self._cached_advert
+        
+        
+    @webframework.register_endpoint('/bid_on_tasks')
+    def bid_on_tasks(self, body=''):
+        """
+        HTTP endpoint (POST) for bidding on tasks.
+        
+        Parameters
+        ----------
+        body : json list of bids
+            A list of bids, each of which is a dictionary of the form
+            ``{"ruleID" : str ,"taskIDs" : [list of int],"taskCosts" : [list of float]}``
+            see :meth:`IntegerIDRule.Bid` for details.
+
+        Returns
+        -------
+        assignments: json str
+            A json formatted list of task assignments, each of which has the form:
+            ``{"ruleID" : ruleID, "taskIDs": [list of int], "template" : "<rule template>"}``
+            See :meth:`IntegerIDTask.bid`
+
+        """
+        bids = json.loads(body)
+        
+        succesfull_bids = []
+        
+        for bid in bids:
+            rule = self._rules[bid['ruleID']]
+            
+            succesfull_bids.append(rule.bid(bid))
+            #task_ids = bid['taskIDs']
+            #costs = bid['taskCosts']
+            
+        #print(succesfull_bids)
+        return json.dumps(succesfull_bids)
+        
+            
+        
+    @webframework.register_endpoint('/add_integer_id_rule')
+    def add_integer_id_rule(self, max_tasks= 1e6, release_start=None, release_end = None, ruleID = None, timeout=3600, body=''):
+        """
+        HTTP endpoint (POST) for adding a new rule.
+        
+        Add a rule that generates tasks based on an integer ID, such as a frame number (localization) or an index into
+        a list of inputs (recipes). By default, tasks are not released on creation, but later with
+        :func:`release_rule_tasks()`. This allows for the creation of a rule before a series has finished spooling.
+        
+        Parameters
+        ----------
+        max_tasks : int
+            The maximum number of tasks that could match this rule. Generally the number of frames in a localization
+            series (if known in advance) or the number of inputs on which to run a recipe. Allows us to pre-allocate a
+            task array of the correct size (passing this rather than relying on the default reduces memory usage).
+        release_start, release_end : int
+            Release a range of tasks for computation after creating the rule (avoids a
+            call to `release_rule_tasks()`
+        ruleID : str
+            A unique identifier for the rule. If not provided, one will be automatically generated
+        timeout : int
+            How long this rule should live for, in seconds. Defaults to an hour.
+        body : str
+            A json dictionary  ``{'template' : "<rule template>", 'inputsByTask' : [list of URIs]}``. See :class:`IntegerIDRule`
+            for the format of the rule template. The ``inputsByTask`` parameter is only used for recipes, and can be omitted
+            for localisation analysis, or for recipe tasks using a hard coded ``"inputs"`` dictionary. An optional additional
+            parameter, "on_completion" parameter may be given, itself consisting of a new
+            ``{'template': <template>, 'on_completion': {...}}`` dictionary.
+
+        Returns
+        -------
+        
+        result : json str
+            The result of adding the rule. A dict of the form:
+            ``{"ok" : "True", 'ruleID' : str}``
+
+        """
+        with self._rule_lock:
+            # lock ~entire call so we don't hit KeyErrors if clients immediately
+            # try to abort/mark datasource complete, etc. after posting
+            rule_info = json.loads(body)
+            
+            if ruleID is None:
+                ruleID = '%06d-%s' % (self._rule_n, uuid.uuid4().hex)
+            
+            rule = IntegerIDRule(ruleID, rule_info['template'], max_task_ID=int(max_tasks), rule_timeout=float(timeout),
+                                inputs_by_task=rule_info.get('inputsByTask', None), on_completion=rule_info.get('on_completion', None))
+            
+            #print rule._inputs_by_task
+            if not release_start is None:
+                rule.make_range_available(int(release_start), int(release_end))
+            
+
+            self._rules[ruleID] = rule
+        
+        self._rule_n += 1
+        
+        return json.dumps({'ok': 'True', 'ruleID' : ruleID})
+
+    @webframework.register_endpoint('/release_rule_tasks')
+    def release_rule_tasks(self, ruleID, release_start, release_end, body=''):
+        """
+        HTTP Endpoint (POST or GET ) for releasing tasks associated with a rule.
+        
+        When performing spooling data analysis we typically have one rule per series and one task per frame.
+        This method allows for the tasks associated with a particular frame (or range of frames) to be released as they
+        become available.
+        
+        Parameters
+        ----------
+        ruleID : str
+            The rule ID
+        release_start, release_end: int
+            The range of tasks IDs to release
+        body : str, empty
+            Needed for interface compatibility, ignored
+
+        Returns
+        -------
+        
+        success : json str
+            ``{"ok" : "True"}`` if successful.
+
+        """
+        with self._rule_lock:
+            # take out the lock in case the rule is still being added in another
+            # request and we are already trying to release tasks
+            rule = self._rules[ruleID]
+            
+        logger.debug('release_rule_tasks(ruleID = %s, release_start=%d, release_end=%d)' % (ruleID, int(release_start), int(release_end )))
+        
+        rule.make_range_available(int(release_start), int(release_end))
+    
+    
+        return json.dumps({'ok': 'True'})
+    
+    @webframework.register_endpoint('/inactivate_rule')
+    def inactivate_rule(self, ruleID):
+        with self._rule_lock:
+            # take out the lock in case the rule is still being added in another
+            # request and we are already trying to abort
+            self._rules[ruleID].inactivate()
+
+        return json.dumps({'ok': 'True'})
+    
+    @webframework.register_endpoint('/handin')
+    def handin(self, body):
+        """
+        HTTP Endpoint (POST) to mark tasks as having been completed
+        
+        Parameters
+        ----------
+        body : json list
+            A list of entries of the form ``{"ruleID": str, "taskIDs" : [list of int], "status" : [list of int]}``
+            where status has the same length as taskIDs with each entry being either ``STATUS_COMPLETE`` or
+            ``STATUS_FAILED``. See :meth:`IntegerIDRule.mark_complete`.
+
+        Returns
+        -------
+        
+        success : json str
+            ``{"ok" : "True"}`` if successful.
+        
+
+        """
+        
+        #logger.debug('Handing in tasks...')
+        expired_rules = set()
+        for handin in json.loads(body):
+            ruleID = handin['ruleID']
+            try:
+                rule = self._rules[ruleID]
+                rule.mark_complete(handin)
+            except KeyError:  # rule may have expired
+                expired_rules.add(ruleID)
+        
+        if len(expired_rules) == 0:
+            return json.dumps({'ok': 'True'})
+        else:
+            # rulenodeserver currently ignores what we say here other than 'ok'
+            return json.dumps({'ok': 'False', 'error': str(expired_rules)})
+    
+    @webframework.register_endpoint('/mark_release_complete')
+    def mark_release_complete(self, ruleID, n_tasks=None):
+        """
+        
+        HTTP Endpoint (POST) to signal that no more tasks will be released for a rule and the rule can be regarded as finished once the previously released
+        tasks are complete. 
+        
+        Parameters
+        ----------
+        rule_id : str
+            ID of the rule to update
+        n_tasks : int, [optional, discouraged]
+            a fixed number of tasks to truncate the rule at. Used for avoiding locks/race conditions in a multi-threaded client process. NOTE - this parameter 
+            may disappear in a future version to enable rules with incomplete ranges to be marked as complete (see comments in `IntegerIDRule.finished`). Locking or 
+            otherwise structuring the client such that no `release_rule_tasks()` calls can occur after a call to `mark_release_complete()` is therefore preferred.
+        
+        Returns
+        -------
+        success : str
+            ``{"ok" : "True"}`` if successful.
+        
+        Raises
+        ------
+        ValueError
+            If `value` is less than number of tasks already assigned.
+            
+        FIXME - HTTP endpoints should not raise - they should handle the error and return an error response to the client.
+        """
+        if n_tasks is not None:
+            n_tasks = int(n_tasks)
+        
+        # take out the rule lock in case we are still creating the rule and the
+        # client POSTs this (e.g. if a series is started/stopped quickly)
+        with self._rule_lock:
+            self._rules[ruleID].mark_release_complete(n_tasks)
+        return json.dumps({'ok': 'True'})
+    
+    @webframework.register_endpoint('/distributor/queues')
+    def get_queues(self):
+        """
+        HTTP Endpoint (GET) - visible at "distributor/queues" - for querying ruleserver status.
+        
+        Returns
+        -------
+        
+        status: json str
+            A dictionary of the form ``{"ok" : True, "result" : {ruleID0 : rule0.info(), ruleID1 : rule1.info()}}``
+            See :meth:`IntegerIDRule.info`.
+        """
+        with self._info_lock:
+            t = time.time()
+            if (t > self._cached_info_expiry):
+                with self._rule_lock:
+                    self._cached_info = json.dumps({'ok': True, 'result': {qn: self._rules[qn].info() for qn in self._rules.keys()}})
+                self._cached_info_expiry = time.time() + self._cached_info_timeout
+                
+        return self._cached_info
+    
+    @webframework.register_endpoint('/queue_info_longpoll')
+    def get_queue_info(self):
+        """
+        a throttled version of queue info
+        
+        Returns
+        -------
+
+        """
+        time.sleep(0.5)
+        return self.get_queues()
+    
+    @webframework.register_endpoint('/', mimetype='text/html')
+    def status(self):
+        return self._status_page
+
+
+# class CPRuleServer(RuleServer):
+#     @cherrypy.expose
+#     def add_integer_rule(self, queue=None, nodeID=None, numWant=50, timeout=5):
+#         cherrypy.response.headers['Content-Type'] = 'application/json'
+#
+#         body = ''
+#         #if cherrypy.request.method == 'GET':
+#
+#         if cherrypy.request.method == 'POST':
+#             body = cherrypy.request.body.read()
+#
+#         return self._tasks(queue, nodeID, numWant, timeout, body)
+#
+#     @cherrypy.expose
+#     def handin(self, nodeID):
+#         cherrypy.response.headers['Content-Type'] = 'application/json'
+#
+#         body = cherrypy.request.body.read()
+#
+#         return self._handin(nodeID, body)
+#
+#     @cherrypy.expose
+#     def announce(self, nodeID, ip, port):
+#         cherrypy.response.headers['Content-Type'] = 'application/json'
+#
+#         self._announce(nodeID, ip, port)
+#
+#     @cherrypy.expose
+#     def queues(self):
+#         cherrypy.response.headers['Content-Type'] = 'application/json'
+#
+#         return self._get_queues()
+
+
+class WFRuleServer(webframework.APIHTTPServer, RuleServer):
+    """
+    Combines the RuleServer with it's web framework.
+    
+    Largely an artifact of initial experiments using cherrypy (allowed quickly switching between cherrypy
+    and our internal webframework).
+    """
+    def __init__(self, port, bind_addr=''):
+        RuleServer.__init__(self)
+        
+        server_address = (bind_addr, port)
+        webframework.APIHTTPServer.__init__(self, server_address)
+        self.daemon_threads = True
+
+
+# def runCP(port):
+#     import socket
+#     cherrypy.config.update({'server.socket_port': port,
+#                             'server.socket_host': '0.0.0.0',
+#                             'log.screen': False,
+#                             'log.access_file': '',
+#                             'log.error_file': '',
+#                             'server.thread_pool': 50,
+#                             })
+#
+#     logging.getLogger('cherrypy.access').setLevel(logging.ERROR)
+#
+#     #externalAddr = socket.gethostbyname(socket.gethostname())
+#
+#     distributor = CPRuleServer()
+#
+#     app = cherrypy.tree.mount(distributor, '/distributor/')
+#     app.log.access_log.setLevel(logging.ERROR)
+#
+#     try:
+#
+#         cherrypy.quickstart()
+#     finally:
+#         distributor._do_poll = False
+
+
+import threading
+
+class ServerThread(threading.Thread):
+    """"""
+    def __init__(self, port, bind_addr='', profile=False):
+        self.port = int(port)
+        self._profile = profile
+        self.bind_addr = bind_addr
+        threading.Thread.__init__(self)
+        
+    def run(self):
+        """"""
+        
+        if self._profile:
+            from PYME.util import mProfile
+        
+            mProfile.profileOn(['ruleserver.py', ])
+            profileOutDir = config.get('dataserver-root', os.curdir) + '/LOGS/%s/mProf' % computerName.GetComputerName()
+
+        if self.bind_addr == '':
+            import socket
+            self.externalAddr = socket.gethostbyname(socket.gethostname())
+        else:
+            self.externalAddr = self.bind_addr
+            
+        self.distributor = WFRuleServer(self.port, bind_addr=self.bind_addr)
+
+        logger.info('Starting ruleserver on %s:%d' % (self.externalAddr, self.port))
+        try:
+            self.distributor.serve_forever()
+        finally:
+            self.distributor._do_poll = False
+            #logger.info('Shutting down ...')
+            #self.distributor.shutdown()
+            logger.info('Closing server ...')
+            self.distributor.server_close()
+
+            if self._profile:
+                mProfile.report(False, profiledir=profileOutDir)
+            
+    
+    def shutdown(self):
+        self.distributor._do_poll = False
+        logger.info('Shutting down ...')
+        self.distributor.shutdown()
+        logger.info('Closing server ...')
+        self.distributor.server_close()
+
+
+def on_SIGHUP(signum, frame):
+    """"""
+    from PYME.util import mProfile
+    mProfile.report(False, profiledir=profileOutDir)
+    raise RuntimeError('Recieved SIGHUP')
+
+
+if __name__ == '__main__':
+    import signal
+    
+    port = sys.argv[1]
+    
+    if (len(sys.argv) == 3) and (sys.argv[2] == '-k'):
+        profile = True
+        from PYME.util import mProfile
+        
+        mProfile.profileOn(['ruleserver.py', ])
+        profileOutDir = config.get('dataserver-root', os.curdir) + '/LOGS/%s/mProf' % computerName.GetComputerName()
+    else:
+        profile = False
+        profileOutDir = None
+    
+    if not sys.platform == 'win32':
+        #windows doesn't support handling signals ... don't catch and hope for the best.
+        #Note: This will make it hard to cleanly shutdown the distributor on Windows, but should be OK for testing and
+        #development
+        signal.signal(signal.SIGHUP, on_SIGHUP)
+    
+    try:
+        run(int(port))
+    finally:
+        if profile:
+            mProfile.report(False, profiledir=profileOutDir)
+