basepair 1.5__0-py3.8.egg

basepair/utils/colors.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+from builtins import object
+
+
+class color(object):
+    HEADER = '\033[95m'
+    BLUE = '\033[34m'
+    CYAN = '\033[36;1m'
+    GREEN = '\033[38;5;28m'
+    BROWN = '\033[33m'
+    RED = '\033[31m'
+    ENDC = '\033[0m'
+
+    @staticmethod
+    def primary(text):
+        return color.BLUE + text + color.ENDC
+
+    @staticmethod
+    def info(text):
+        return color.CYAN + text + color.ENDC
+
+    @staticmethod
+    def success(text):
+        return color.GREEN + text + color.ENDC
+
+    @staticmethod
+    def warning(text):
+        return color.BROWN + text + color.ENDC
+
+    @staticmethod
+    def error(text):
+        return color.RED + text + color.ENDC
+
+
+if __name__ == '__main__':
+    print(color.primary('primary'))
+    print(color.primary('info'))
+    print(color.success('success'))
+    print(color.warning('warning'))
+    print(color.error('error'))
+    print('ok')

basepair/utils/monitor.py

@@ -0,0 +1,38 @@
+# monitors multiple files for modification
+# can check for modification of one or all of them
+#
+# usage:
+# monitor = Monitor()
+# monitor.monitor('decider.py')
+
+# if monitor.is_modified('decider.py'):     # check one specific file
+# if monitor.is_modified():                 # check all monitored files
+
+from builtins import object
+import os
+
+class Monitor(object):
+    def __init__(self, filename=None):
+        self.last_modified = {}
+        if filename:
+            self.monitor(filename)
+
+    def monitor(self, filename):
+        self.last_modified[filename] = os.path.getmtime(filename)
+    
+    def is_modified(self, filename=None):
+        if filename:
+            return self.is_modified_filename(filename)
+        else:
+            # if any single file is modified, return True
+            for filename in list(self.last_modified.keys()):
+                if self.is_modified_filename(filename):
+                    return True
+            return False
+
+    def is_modified_filename(self, filename):
+        if os.path.getmtime(filename) > self.last_modified[filename]:
+            self.last_modified[filename] = os.path.getmtime(filename)
+            return True
+        else:
+            return False

basepair/utils/s3.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+
+from builtins import object
+import argparse
+import json
+
+import boto
+from boto.s3.key import Key
+
+
+class BpS3(object):
+    def __init__(self, conf):
+        self.conf = conf
+
+        region = None
+        try:
+            for reg in boto.sqs.regions():
+                if reg.name == self.conf['aws']['s3']['region']:
+                    region = reg
+                    break
+        except KeyError:
+            region = None
+
+        self.conn = boto.connect_s3(
+            aws_access_key_id=self.conf['aws']['aws_id'],
+            aws_secret_access_key=self.conf['aws']['aws_secret'],
+            region=region,
+        )
+
+    def save_data(self, data, bucket_name, key_name):
+        bucket = self.conn.create_bucket(
+            bucket_name, location=boto.s3.connection.Location.DEFAULT)
+        k = Key(bucket)
+        k.key = key_name
+        k.set_contents_from_string(json.dumps(data, indent=2))
+
+    # TODO: URL valid for 1 year
+    # generate s3 path
+    def get_s3_url(self, path, expires_in=None):
+        if not expires_in:
+            expires_in = 3600 * 24 * 365
+
+        bucket = self.conn.get_bucket(self.conf['aws']['s3']['bucket'])
+        key = bucket.get_key(path)
+        url = key.generate_url(expires_in) if key else ''
+
+        return url
+
+
+def main():
+    args = read_args()
+    conf = json.load(open(args.config_file))
+    s3 = BpS3(conf)
+
+    data = {
+        'abc': 'def',
+        'rst': 'xyz',
+    }
+    bucket_name = 'bpdev0'
+    key_name = 'analysis/1890/__private__/workflow.json'
+    s3.save_data(data, bucket_name, key_name)
+
+
+def read_args():
+    parser = argparse.ArgumentParser(description='cmd line instance control')
+    parser.add_argument('-c', '--config-file')
+    parser.set_defaults(
+        config_file='setup/config.dev.json'
+    )
+
+    args = parser.parse_args()
+
+    return args
+
+
+if __name__ == '__main__':
+    main()

basepair/utils/sqs.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python
+'''wrapper for boto AWS SQS'''
+from __future__ import print_function
+
+from builtins import object
+import sys
+# import os
+import json
+import argparse
+import random
+# import time
+
+import boto
+from boto.sqs.jsonmessage import JSONMessage
+import requests
+
+
+class BpSQS(object):
+    '''wrapper for boto AWS SQS'''
+    def __init__(self, conf, config_file=None, queue='decider', timeout=120, wait_time=20):
+        if config_file:
+            self.conf = json.load(open(config_file))
+        else:
+            self.conf = conf
+        self.wait_time = wait_time
+        self.q = None
+
+        internet = True
+        try:
+            requests.get('https://www.google.com/', timeout=1)
+        except requests.exceptions.ConnectionError:
+            print('warning: no internet', file=sys.stderr)
+            internet = False
+
+        region = None
+        try:
+            for reg in boto.sqs.regions():
+                if reg.name == self.conf['aws']['sqs']['region']:
+                    region = reg
+                    break
+        except KeyError:
+            region = None
+
+        if internet:
+            self.conn = boto.connect_sqs(
+                aws_access_key_id=self.conf['aws']['aws_id'],
+                aws_secret_access_key=self.conf['aws']['aws_secret'],
+                region=region
+                )
+
+            self.q = self.conn.create_queue(queue, timeout)
+            self.q.set_message_class(JSONMessage)
+
+    def send_message(self, _msg):
+        '''send message'''
+        delay = _msg.get('delay', 0)
+
+        message = JSONMessage()
+        message.set_body(_msg)
+        status = self.q.write(message, delay_seconds=delay)
+        return status
+
+    # {u'ApproximateFirstReceiveTimestamp' '1535158416671',
+    # u'SenderId': u'',
+    # u'ApproximateReceiveCount': u'1
+    # u'SentTimestamp': u'1535158416657'}
+    def get_messages(self):
+        '''get messages'''
+        return self.q.get_messages(
+            attributes=['All'],
+            num_messages=10,
+            wait_time_seconds=self.wait_time)
+
+    def get_message(self):
+        '''get a message'''
+        return self.q.read(wait_time_seconds=self.wait_time)
+        # except socket.gaierror:
+        #     print >>sys.stderr, 'no connection, sleeping for a minute'
+        #     time.sleep(60)
+        #     return None
+
+    def clear(self, wetrun=False):
+        '''clear the queue, delete messages'''
+        msgs = self.get_messages()
+        print(msgs, file=sys.stderr)
+        for msg in msgs:
+            print(msg, file=sys.stderr)
+            if wetrun:
+                self.q.delete_message(msg)
+
+
+def main():
+    '''script'''
+    args = read_args()
+    # test_decider_queue(args)
+    # test_instance_queue(args)
+    if args.clear:
+        sqs = BpSQS(conf=None, config_file=args.config_file)
+        sqs.clear()
+
+
+def test_decider_queue(args):
+    '''test decider queue'''
+    sqs = BpSQS(conf=None, config_file=args.config_file)
+    msg = {
+        "message": "completed",
+        "analysis-id": 1481,
+        "data": {},
+        "sender": "decider"
+    }
+    sqs.send_message(msg)
+    print(json.dumps(msg, indent=2), file=sys.stderr)
+
+
+def test_instance_queue(args):
+    '''test instance queue'''
+    analysis_id = int(1000000 * random.random())
+    sqs = BpSQS(conf=None, config_file=args.config_file, queue='instance', timeout=600)
+    msg = {
+        'action': 'start-instance',
+        'name': 'bp-%s' % analysis_id,
+        'kind': 'spot',
+        'mode': 'dev',
+        'task_list': 'worker',
+        'analysis_id': analysis_id,
+    }
+    sqs.send_message(msg)
+    print(json.dumps(msg, indent=2), file=sys.stderr)
+
+
+def read_args():
+    '''user args'''
+    parser = argparse.ArgumentParser(description='cmd line instance control')
+    parser.add_argument('-a', '--action')
+    parser.add_argument('-c', '--config-file')
+    parser.add_argument('-C', '--clear')
+    parser.set_defaults(clear=False)
+    parser.set_defaults(
+        config_file='setup/config.dev.json'
+    )
+    args = parser.parse_args()
+
+    return args
+
+
+if __name__ == '__main__':
+    main()

basepair/utils/swf.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+from builtins import object
+import sys
+import json
+import argparse
+import time
+
+import boto.swf
+from boto.swf.exceptions import SWFDomainAlreadyExistsError, \
+    SWFTypeAlreadyExistsError
+
+
+class SWF(object):
+    def __init__(self, config_file):
+        self.conf = json.load(open(config_file))
+        region = None
+        try:
+            for reg in boto.swf.regions():
+                if reg.name == self.conf['aws']['swf']['region']:
+                    region = reg
+                    break
+
+        except KeyError:
+            region = None
+
+        self.layer = boto.swf.layer1.Layer1(
+            aws_access_key_id=self.conf['aws']['aws_id'],
+            aws_secret_access_key=self.conf['aws']['aws_secret'],
+            region=region
+            )
+
+        # self.s3conn = boto.connect_s3(
+        #         aws_access_key_id=self.conf['aws']['aws_id'],
+        #         aws_secret_access_key=self.conf['aws']['aws_secret'],
+        #         )
+
+    def terminate(self):
+        workflows = self.layer.list_open_workflow_executions(
+            self.conf['aws']['swf']['domain'], oldest_date=1
+        )
+
+        for workflow in workflows['executionInfos']:
+            workflow_id = workflow['execution']['workflowId']
+            self.layer.terminate_workflow_execution(
+                self.conf['aws']['swf']['domain'], workflow_id)
+            print(workflow_id, file=sys.stderr)
+
+    def register(self):
+        self.register_domain()
+        self.register_workflow_type()
+        self.register_activity_type()
+
+    def register_domain(self):
+        domain = self.conf['aws']['swf']['domain']
+
+        try:
+            self.layer.register_domain(domain, '7')
+        except SWFDomainAlreadyExistsError:
+            print('domain', domain, 'exists', file=sys.stderr)
+
+        # print layer.list_domains('REGISTERED')
+
+    def register_workflow_type(self):
+        domain = self.conf['aws']['swf']['domain']
+        workflow = self.conf['aws']['swf']['workflow_type']
+        version = '1.0'
+
+        try:
+            self.layer.register_workflow_type(
+                domain, workflow, version,
+                task_list=self.conf['aws']['swf']['decider_task_list'],
+                default_child_policy='REQUEST_CANCEL',
+                default_execution_start_to_close_timeout='31536000',
+                default_task_start_to_close_timeout='31536000',
+            )
+        except SWFTypeAlreadyExistsError:
+            print('workflow', workflow, 'exists', file=sys.stderr)
+
+    def register_activity_type(self):
+        domain = self.conf['aws']['swf']['domain']
+        activity = self.conf['aws']['swf']['activity_type']
+        version = '1.0'
+
+        try:
+            self.layer.register_activity_type(
+                domain, activity, version,
+                default_task_heartbeat_timeout='31536000',
+                default_task_schedule_to_close_timeout='31536000',
+                default_task_schedule_to_start_timeout='31536000',
+                default_task_start_to_close_timeout='31536000',
+                task_list=self.conf['aws']['swf']['worker_task_list'],
+            )
+        except SWFTypeAlreadyExistsError:
+            print('activity', activity, 'exists', file=sys.stderr)
+
+
+def main():
+    args = read_args()
+    swf = SWF(args.config_file)
+
+    if args.register:
+        swf.register()
+
+    if args.terminate:
+        print('terminating ...', file=sys.stderr)
+        swf.terminate()
+        time.sleep(3)
+        swf.terminate()
+
+
+def read_args():
+    parser = argparse.ArgumentParser(description='cmd line instance control')
+    parser.add_argument('-t', '--terminate', action='store_true',
+                        help='terminate open workflows')
+    parser.add_argument('-r', '--register', action='store_true')
+    parser.add_argument('-c', '--config-file')
+    parser.set_defaults(
+        config_file='setup/config.dev.json'
+    )
+
+    args = parser.parse_args()
+
+    return args
+
+
+if __name__ == '__main__':
+    main()