syncloud-lib 333__py3-none-any.whl

syncloudlib/integration/loop.py

@@ -0,0 +1,65 @@
+from syncloudlib.integration.ssh import run_ssh
+import re
+
+
+class LosetupEntry:
+    def __init__(self, device, file):
+        self.file = file
+        self.device = device
+
+    def is_deleted(self):
+        return '(deleted)' in self.file
+
+
+class MountEntry:
+    def __init__(self, device):
+        self.device = device
+
+
+def loop_device_cleanup(host, dev_file, password):
+    print('cleanup')
+    for mount in parse_mount(run_ssh(host, 'mount', password=password)):
+        if dev_file == mount.device:
+            run_ssh(host, 'umount {0}'.format(mount.device), throw=False, password=password)
+
+    for loop in parse_losetup(run_ssh(host, 'losetup', password=password)):
+        if loop.file == dev_file or loop.is_deleted():
+            run_ssh(host, 'losetup -d {0}'.format(loop.device), throw=False, password=password)
+
+    run_ssh(host, 'losetup', password=password)
+
+    for loop in run_ssh(host, 'dmsetup ls', password=password).splitlines():
+        if 'loop0p1' in loop:
+            run_ssh(host, 'sudo dmsetup remove loop0p1', password=password)
+        if 'loop0p2' in loop:
+            run_ssh(host, 'sudo dmsetup remove loop0p2', password=password)
+
+    run_ssh(host, 'rm -rf {0}'.format(dev_file), throw=False, password=password)
+
+
+def parse_losetup(output):
+    entries = []
+    for line in output.splitlines():
+        match = re.match(r'(.*[0-9]+)\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)\s+(.*)\s+([0-9]+)\s+([0-9]+)', line.strip())
+        if match:
+            entry = LosetupEntry(match.group(1).strip(), match.group(6).strip())
+            entries.append(entry)
+    return entries
+
+
+def parse_mount(output):
+    entries = []
+    for line in output.splitlines():
+        match = re.match(r'(.*)\son\s.*', line.strip())
+        if match:
+            entry = MountEntry(match.group(1).strip())
+            entries.append(entry)
+    return entries
+
+
+def loop_device_add(host, fs, dev_file, password):
+    print('adding loop device')
+    run_ssh(host, 'dd if=/dev/zero bs=1M count=10 of={0}'.format(dev_file), password=password)
+    loop = run_ssh(host, 'losetup -f --show {0}'.format(dev_file), password=password)
+    run_ssh(host, 'mkfs.{0} {1}'.format(fs, loop), password=password)
+    return loop 

syncloudlib/integration/screenshots.py

@@ -0,0 +1,7 @@
+from os.path import join
+
+
+def screenshots(driver, screenshot_dir, name):
+    driver.get_screenshot_as_file(join(screenshot_dir, '{0}.png'.format(name)))
+    with open(join(screenshot_dir, '{0}.html.log'.format(name)), "w") as f:
+        f.write(str(driver.page_source.encode("utf-8")))

syncloudlib/integration/selenium_wrapper.py

@@ -0,0 +1,127 @@
+import time
+
+from retry import retry
+
+from syncloudlib.integration.screenshots import screenshots
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions
+
+
+class SeleniumWrapper:
+    def __init__(self, driver, ui_mode, screenshot_dir, app_domain, timeout, browser):
+        self.app_domain = app_domain
+        self.screenshot_dir = screenshot_dir
+        self.ui_mode = ui_mode
+        self.driver = driver
+        self.browser = browser
+        self.wait_driver = WebDriverWait(self.driver, timeout)
+
+    def find_by_xpath(self, xpath):
+        return self.find_by(By.XPATH, xpath)
+
+    def find_by_name(self, name):
+        return self.find_by(By.NAME, name)
+
+    def find_by_id(self, field_id):
+        return self.find_by(By.ID, field_id)
+
+    def find_by_css(self, css):
+        return self.find_by(By.CSS_SELECTOR, css)
+
+    def find_by(self, by, value, parent=None):
+        if not parent:
+            parent = self.driver
+        #self.wait_or_screenshot(expected_conditions.visibility_of_element_located((by, value)))
+        self.wait_or_screenshot(expected_conditions.presence_of_element_located((by, value)))
+        try:
+            return parent.find_element(by, value)
+        except Exception as e:
+            self.screenshot('exception', True)
+
+    def invisible_by(self, by, value):
+        self.wait_or_screenshot(expected_conditions.invisibility_of_element_located((by, value)))
+
+    def click_by(self, by, value):
+        #self.wait_or_screenshot(expected_conditions.element_to_be_clickable((by, value)))
+        self.wait_or_screenshot(expected_conditions.presence_of_element_located((by, value)))
+        try:
+            self.driver.find_element(by, value).click()
+        except Exception as e:
+            self.screenshot('exception', True)
+
+    def clickable_by(self, by, value):
+        self.wait_or_screenshot(expected_conditions.element_to_be_clickable((by, value)))
+        try:
+            return self.driver.find_element(by, value)
+        except Exception as e:
+            self.screenshot('exception', True)
+
+    def present_by(self, by, value):
+        self.wait_or_screenshot(expected_conditions.presence_of_element_located((by, value)))
+        try:
+            return self.driver.find_element(by, value)
+        except Exception as e:
+            self.screenshot('exception', True)
+
+    def exists_by(self, by, value, timeout=10):
+        driver = WebDriverWait(self.driver, timeout)
+        cond = expected_conditions.visibility_of_element_located((by, value))
+        try:
+            driver.until(cond)
+            return True
+        except Exception as _:
+            self.screenshot('exception', False)
+            return False
+
+    @retry(exceptions=Exception, tries=3, delay=1, backoff=2)
+    def wait_or_screenshot(self, method, throw=True):
+        try:
+            self.wait_driver.until(method)
+            return True
+        except Exception as e:
+            self.screenshot('exception', throw)
+            if throw:
+                raise e
+            else:
+                return False
+
+    def screenshot(self, name, throw=True):
+        retries = 5
+        retry = 0
+        while True:
+            try:
+                screenshots(self.driver, self.screenshot_dir, '{}-{}'.format(name, self.ui_mode))
+                break
+            except Exception as e:
+                if retry >= retries:
+                    if throw:
+                        raise
+                    else:
+                        return
+                retry += 1
+                time.sleep(1)
+                print('retrying screenshot {0}'.format(retry))
+
+    def open_app(self, path=''):
+        self.driver.get("https://{0}{1}".format(self.app_domain, path))
+
+    def log(self):
+        if self.browser != "chrome":
+            print("browser logs are only supported in chrome")
+            return
+
+        print("browser log")
+        for entry in self.driver.get_log('browser'):
+            print(entry)
+
+    @retry(exceptions=Exception, tries=10, delay=1, backoff=2)
+    def element_by_js(self, js):
+        try:
+            elem = self.driver.execute_script('return ' + js)
+            self.driver.execute_script("arguments[0].scrollIntoView();", elem)
+            return elem
+        except Exception:
+            self.screenshot('exception')
+            raise
+

syncloudlib/integration/ssh.py

@@ -0,0 +1,51 @@
+from subprocess import check_output, STDOUT, CalledProcessError
+
+import time
+
+
+def run_scp(command, throw=True, debug=True, password='syncloud', retries=0, sleep=1, port=22):
+    retry = 0
+    while True:
+        try:
+            return _run_command('scp -P {0} -o StrictHostKeyChecking=no {1}'.format(port, command), throw, debug, password)
+        except Exception as e:
+            if retry >= retries:
+                raise
+            retry += 1
+            time.sleep(sleep)
+            sleep = sleep * 2
+            print('retrying {0}'.format(retry))
+
+
+def run_ssh(host, command, throw=True, debug=True, password='syncloud', retries=0, sleep=1, env_vars='', port=22):
+    ssh_command='{0} {1}'.format(env_vars, command)
+    retry = 0
+    while True:
+        try:
+            return _run_command('ssh -p {0} -o StrictHostKeyChecking=no root@{1} "{2}"'.format(port, host, ssh_command), throw, debug, password)
+        except Exception as e:
+            if retry >= retries:
+                raise
+            retry += 1
+            time.sleep(sleep)
+            sleep = sleep * 2
+            print('retrying {0}'.format(retry))
+
+
+def ssh_command(password, command):
+    return 'sshpass -p {0} {1}'.format(password, command)
+
+
+def _run_command(command, throw, debug, password):
+    try:
+        print('ssh command: {0}'.format(command))
+        output = str(check_output(ssh_command(password, command), shell=True, stderr=STDOUT, encoding='UTF-8')).strip()
+        if debug:
+            print("ssh output: " + output)
+            print
+        return output
+    except CalledProcessError as e:
+        print("ssh error: " + str(e.output))
+        if throw:
+            raise
+        return str(e.output)

syncloudlib/json/convertible.py

@@ -0,0 +1,114 @@
+import sys
+
+import jsonpickle
+import os
+import datetime
+import inspect
+
+jsonpickle.set_preferred_backend('json')
+if sys.version_info.major < 3:
+    types = (bool, str, unicode, int, long, float, datetime.datetime)
+else:
+    types = (bool, str, int, float, datetime.datetime)
+
+
+def pretty_print():
+    jsonpickle.set_encoder_options('json', indent=2)
+
+def ugly_print():
+    jsonpickle.set_encoder_options('json', indent=None)
+
+class List:
+    def __init__(self, item_type):
+        self.item_type = item_type
+
+class Field:
+    def __init__(self, field_type=None, default=None):
+        self.field_type = field_type
+        self.default = default
+
+class Struct(object):
+    def __init__(self, adict):
+        self.__dict__.update(adict)
+        for k, v in adict.items():
+            if isinstance(v, (dict, list)):
+                self.__dict__[k] = to_object(v)
+
+
+def get_field_value(field, field_meta, structure):
+    if field in structure.keys():
+        value = structure[field]
+        if field_meta.field_type:
+            return to_object(value, field_meta.field_type)
+        return value
+    else:
+        if field_meta.default is not None:
+            return field_meta.default
+    return None
+
+
+def to_object(structure, obj_type=None):
+    if obj_type:
+        if isinstance(obj_type, List):
+            return [to_object(item, obj_type.item_type) for item in structure]
+        else:
+            obj = obj_type()
+            for field, field_meta in inspect.getmembers(obj):
+                if isinstance(field_meta, Field):
+                    value = get_field_value(field, field_meta, structure)
+                    setattr(obj, field, value)
+            return obj
+    else:
+        if structure is None or isinstance(structure, types):
+            return structure
+        if isinstance(structure, list):
+            return [to_object(item) for item in structure]
+        return Struct(structure)
+
+def limit(dictionary, keys):
+    filtered = [key for key in dictionary.keys() if key not in keys]
+    for key in filtered:
+        dictionary.pop(key, None)
+
+def to_dict(value):
+    if isinstance(value, list):
+        return [to_dict(item) for item in value]
+    if isinstance(value, dict):
+        cloned = value.copy()
+        for key, value in cloned.items():
+            cloned[key] = to_dict(value)
+        return cloned
+    if value is None or isinstance(value, types):
+        return value
+    result = value.__dict__.copy()
+    if hasattr(value, '__public__'):
+        limit(result, value.__public__)
+    for member, mvalue in result.items():
+        result[member] = to_dict(mvalue)
+    return result
+
+def from_json(json_text, obj_type=None):
+    return to_object(jsonpickle.decode(json_text), obj_type)
+
+def to_json(value):
+    return jsonpickle.encode(value, unpicklable=False)
+
+def reformat(json_text):
+    return to_json(from_json(json_text))
+
+def read_json(filename, obj_type=None):
+    if not os.path.isfile(filename):
+        return None
+    with open(filename, 'r') as f:
+        json = f.read()
+    if json.strip(' \t\n\r') == '':
+        return None
+    return from_json(json, obj_type)
+
+def write_json(filename, obj):
+    if not obj:
+        json = ''
+    else:
+        json = to_json(obj)
+    with open(filename, 'w') as file:
+        file.write(json)

syncloudlib/linux.py

@@ -0,0 +1,15 @@
+from subprocess import check_output
+import pwd
+
+
+def useradd(user, home_folder=None, shell='/bin/false'):
+    try:
+        pwd.getpwnam(user)
+        return 'user {0} exists'.format(user)
+    except KeyError:
+        options = '-r -s {0}'.format(shell)
+        if home_folder:
+            home_folder_options = '-m -d {0}'.format(home_folder)
+            options = home_folder_options + ' ' + options
+        command_line = '/usr/sbin/useradd {0} {1}'.format(options, user)
+        return check_output(command_line, shell=True).decode()

syncloudlib/logger.py

@@ -0,0 +1,54 @@
+import logging
+import logging.handlers
+import sys
+
+factory_instance = None
+
+
+class WhitespaceRemovingFormatter(logging.Formatter):
+    def format(self, record):
+        record.msg = clean(record.msg)
+        return super(WhitespaceRemovingFormatter, self).format(record)
+
+
+def clean(message):
+        return str(message).strip()
+
+
+class LoggerFactory:
+    def __init__(self, level, console, filename, line_format):
+        self.level = level
+        self.console = console
+        self.filename = filename
+        self.formatter = WhitespaceRemovingFormatter(line_format)
+        if filename:
+            self.file_handler = logging.handlers.RotatingFileHandler(self.filename, maxBytes=1000000, backupCount=5)
+            self.file_handler.setFormatter(self.formatter)
+        if console:
+            self.console_handler = logging.StreamHandler(sys.stdout)
+            self.console_handler.setFormatter(self.formatter)
+
+    def get_logger(self, name):
+        logger = logging.getLogger(name)
+        logger.setLevel(self.level)
+        if self.filename:
+            if self.file_handler not in logger.handlers:
+                logger.addHandler(self.file_handler)
+        if self.console:
+            if self.console_handler not in logger.handlers:
+                logger.addHandler(self.console_handler)
+        return logger
+
+default_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+
+
+def init(level=logging.INFO, console=False, filename=None, line_format=default_format):
+    global factory_instance
+    factory_instance = LoggerFactory(level, console, filename, line_format)
+
+
+def get_logger(name):
+    global factory_instance
+    if not factory_instance:
+        raise Exception('Logging is not initialized')
+    return factory_instance.get_logger(name)