RLTest 0.7.14__py3-none-any.whl

RLTest/Enterprise/CcsMock.py

@@ -0,0 +1,183 @@
+from __future__ import print_function
+import subprocess
+import redis
+import os
+from RLTest.utils import wait_for_conn, Colors
+
+
+class CcsMock():
+    CCS_UNIX_SOCKET_DEFAULT_PATH = '/tmp/ccs.sock'
+    CCS_DB_RDB_FILE_NAME = 'syncer-pixie-ccs.rdb'
+    CCS_LOG_FILE_NAME = 'ccs.log'
+
+    def __init__(self, redisBinaryPath, proxyPort, directory=None, useSlaves=False, password=None, libPath=None):
+        self.redisBinaryPath = os.path.expanduser(redisBinaryPath) if redisBinaryPath.startswith('~/') else redisBinaryPath
+        self.useSlaves = useSlaves
+        self.directory = directory
+        self.generateArgs()
+        self.libPath = os.path.expanduser(libPath) if libPath.startswith('~/') else libPath
+        self.env = {}
+        if self.libPath:
+            self.env['LD_LIBRARY_PATH'] = self.libPath
+        self.password = password
+        self.proxyPort = proxyPort
+
+    def generateArgs(self):
+        self.args = self.redisBinaryPath.split()
+        if self.directory:
+            self.args += ['--dir', self.directory]
+        self.args += ['--port', '0',
+                      '--unixsocket', self.CCS_UNIX_SOCKET_DEFAULT_PATH,
+                      '--dbfilename', self.CCS_DB_RDB_FILE_NAME,
+                      '--logfile', self.CCS_LOG_FILE_NAME]
+
+    def Start(self, shards, bdb_fields=None, endpoint_ccs_params=None, legacy_hash_slots=True, extra_keys=None):
+        self.setup(shards, bdb_fields, endpoint_ccs_params, legacy_hash_slots, extra_keys)
+
+    def Stop(self):
+        if self.process:
+            try:
+                self.process.terminate()
+                self.process.wait()
+            except OSError:
+                pass
+            self.process = None
+
+    def PrintEnvData(self, prefix=''):
+        print(Colors.Yellow(prefix + 'pid: %d' % self.process.pid))
+        print(Colors.Yellow(prefix + 'unix socket: %s' % self.CCS_UNIX_SOCKET_DEFAULT_PATH))
+        print(Colors.Yellow(prefix + 'binary path: %s' % self.redisBinaryPath))
+        if self.directory:
+            print(Colors.Yellow(prefix + 'db dir path: %s' % (self.directory)))
+        print(Colors.Yellow(prefix + 'rdb file name: %s' % (self.CCS_DB_RDB_FILE_NAME)))
+        print(Colors.Yellow(prefix + 'log file name: %s' % (self.CCS_LOG_FILE_NAME)))
+        if self.libPath:
+            print(Colors.Yellow(prefix + 'lib path: %s' % (self.libPath)))
+
+    def setup(self, shards, bdb_fields=None, endpoint_ccs_params=None, legacy_hash_slots=True, extra_keys=None):
+
+        self.process = subprocess.Popen(stdout=subprocess.PIPE, stderr=subprocess.STDOUT, args=self.args, env=self.env)
+
+        # Create fake cluster, node and dmc objects
+        self.conn = redis.Redis(unix_socket_path=self.CCS_UNIX_SOCKET_DEFAULT_PATH)
+        wait_for_conn(self.conn)
+
+        self.conn.hmset('cluster', {'name': 'fakecluster'})
+        self.conn.hmset('node:1', {'addr': '127.0.0.1',
+                                   'dmc_uid': '1'})
+        self.conn.sadd('node:all', '1')
+        self.conn.hmset('dmc:1', {'threads': '1',
+                                  'max_threads': '1',
+                                  'log_level': 'info'})
+        self.conn.sadd('dmc:all', '1')
+        # self.conn.hmset('cluster_cert', {"syncer_cert": "syncer_cert", "syncer_key": "syncer_key"}) todo: check if needed
+        self.disperse_shards = False
+        self.next_node_uid = 1
+
+        self.create_bdb(shards, bdb_fields, endpoint_ccs_params, legacy_hash_slots, extra_keys)
+
+    def set_server_ccs_values(self, serverId, port, assigned_slots, bdb_uid, role):
+        """
+        Set all the neccesary values of a server in the ccs
+        """
+        redis_uid = serverId
+        self.conn.hmset('redis:%s' % str(redis_uid),
+                        {'port': port,
+                         'node_uid': str(self.next_node_uid),
+                         'bdb_uid': bdb_uid,
+                         'role': role,
+                         'assigned_slots': assigned_slots})
+        self.conn.sadd('redis:all', redis_uid)
+        # we promote the node uid just in case of master shard because we want disperion of master shards
+        if role == 'master' and self.disperse_shards:
+            n_nodes = len(self.conn.smembers('node:all'))
+            self.next_node_uid = self.next_node_uid % n_nodes + 1
+
+    def update_bdb_redis_list(self, bdb_uid, shards):
+        """
+        Update the redis list in the ccs of the given bdb
+        """
+        bdb_uid = str(bdb_uid)
+        redis_uids = []
+        for shard in shards:
+            redis_uids.append(str(shard.masterServerId))
+            if self.useSlaves:
+                redis_uids.append(str(shard.slaveServerId))
+        self.conn.hset('bdb:%s' % bdb_uid, 'redis_list', ','.join(redis_uids))
+
+    def create_bdb(self, shards, bdb_fields=None, endpoint_ccs_params=None, legacy_hash_slots=True, extra_keys=None):
+        bdb_uid = 1
+        port = self.proxyPort
+
+        if legacy_hash_slots:
+            number_of_slots_in_bdb = 4096
+            translation_offset = 1
+        else:
+            number_of_slots_in_bdb = 16384
+            translation_offset = 0
+        slots_per_shard = float(number_of_slots_in_bdb) / len(shards)
+        first_slot = translation_offset
+        last_slot = number_of_slots_in_bdb - 1 + translation_offset
+
+        start_slots = [int(round(first_slot + i * slots_per_shard)) for i in range(len(shards))]
+        end_slots = [next_start_slot - 1 for next_start_slot in start_slots[1:]] + [last_slot]
+        slots = ["%s-%s" % (pair[0], pair[1]) for pair in zip(start_slots, end_slots)]
+
+        for index, shard in enumerate(shards):
+            self.set_server_ccs_values(shard.masterServerId, shard.port, str(slots[index]), bdb_uid, 'master')
+
+            if self.useSlaves:
+                self.set_server_ccs_values(shard.slaveServerId, shard.GetSlavePort(), str(slots[index]), bdb_uid, 'slave')
+
+        self.conn.hmset('bdb:%s' % bdb_uid,
+                        {'shards_count': len(shards),
+                         'type': 'redis',
+                         'endpoint_list': '%s:1' % bdb_uid,
+                         'replication': ('enabled' if self.useSlaves else 'disabled'),
+                         'authentication_admin_pass': 'secret',
+                         'internal_pass': self.password,
+                         'redis_version': '4.0',
+                         'implicit_shard_key': 'enabled'})
+
+        if legacy_hash_slots:
+            self.conn.hmset('bdb:%s' % bdb_uid,
+                            {'hash_slots_policy': 'legacy',
+                             'shard_function': 'crc12'})
+        else:
+            self.conn.hmset('bdb:%s' % bdb_uid,
+                            {'hash_slots_policy': '16k'})
+
+        if bdb_fields:
+            self.conn.hmset('bdb:%s' % bdb_uid, bdb_fields)
+            # if 'ssl' in bdb_fields and bdb_fields['ssl'] in ["enabled", "replica_ssl"]:
+            #     with open(CLIENT_CERT_FILE, 'r') as cert_file:
+            #         cert = cert_file.readlines()
+            #     self.conn.hmset('bdb:%s' % bdb_uid,
+            #                     {'authentication_ssl_client_certs': "".join(cert)})
+
+        if extra_keys:
+            for key, val in extra_keys.items():
+                self.conn.hmset(key, val)
+
+        # Set up endpoint
+        proxy_policy = self.conn.hget('bdb:%s' % bdb_uid, 'proxy_policy')
+        if not proxy_policy:
+            proxy_policy = 'all-nodes'
+
+        self.conn.hmset('endpoint:%s:1' % bdb_uid,
+                        {'bdb_uid': bdb_uid,
+                         'port': port,
+                         'proxy_policy': proxy_policy})
+        self.conn.hmset('endpoint:%s:1' % bdb_uid,
+                        {'bdb_uid': bdb_uid,
+                         'port': port,
+                         'proxy_policy': proxy_policy,
+                         'proxy_uids': '1'})
+
+        if endpoint_ccs_params:
+            self.conn.hmset('endpoint:%s:1' % bdb_uid, endpoint_ccs_params)
+
+        self.conn.sadd('endpoint:all', '%s:1' % bdb_uid)
+
+        self.conn.sadd('bdb:all', bdb_uid)
+        self.update_bdb_redis_list(bdb_uid, shards)

RLTest/Enterprise/Dmc.py

@@ -0,0 +1,47 @@
+from __future__ import print_function
+
+import os
+import psutil
+import subprocess
+from RLTest.utils import Colors
+
+
+class Dmc():
+    DMC_LOG_FILE_NAME = 'dmc.log'
+
+    def __init__(self, dmcBinaryPath, libPath, directory=None):
+        self.dmcBinaryPath = os.path.expanduser(dmcBinaryPath) if dmcBinaryPath.startswith('~/') else dmcBinaryPath
+        self.env = {'MEMTIER_NODE_ID': '1'}
+        self.libPath = libPath
+        if 'LD_LIBRARY_PATH' in os.environ:
+            self.env['LD_LIBRARY_PATH'] = os.environ['LD_LIBRARY_PATH']
+        else:
+            self.env['LD_LIBRARY_PATH'] = os.path.expanduser(self.libPath) if dmcBinaryPath.startswith('~/') else self.libPath
+        self.directory = directory
+
+        logFile = self.DMC_LOG_FILE_NAME if self.directory is None else os.path.join(self.directory, self.DMC_LOG_FILE_NAME)
+        self.proxyArgs = [
+            self.dmcBinaryPath,
+            '-c', '500',
+            '-O', logFile
+        ]
+
+    def Start(self):
+        self.process = psutil.Popen(executable=self.dmcBinaryPath,
+                                    stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                                    env=self.env,
+                                    args=self.proxyArgs)
+
+    def Stop(self):
+        if self.process:
+            self.process.kill()
+            self.process.wait()
+        self.process = None
+
+    def PrintEnvData(self, prefix=''):
+        print(Colors.Yellow(prefix + 'dmc binary path: %s' % self.dmcBinaryPath))
+        print(Colors.Yellow(prefix + 'dmc env: %s' % str(self.env)))
+        print(Colors.Yellow(prefix + 'dir path: %s' % str(self.directory)))
+        print(Colors.Yellow(prefix + 'log file name: %s' % str(self.DMC_LOG_FILE_NAME)))
+        if self.libPath:
+            print(Colors.Yellow(prefix + 'lib path: %s' % str(self.libPath)))

RLTest/Enterprise/EnterpriseClusterEnv.py

@@ -0,0 +1,158 @@
+from __future__ import print_function
+
+from RLTest.redis_std import StandardEnv
+from RLTest.utils import Colors, wait_for_conn
+from .CcsMock import CcsMock
+from .Dmc import Dmc
+import redis
+import os
+import json
+from zipfile import ZipFile
+
+
+SHARD_PASSWORD = 'password'
+
+
+class EnterpriseClusterEnv():
+    DMC_PORT = 10000
+    MODULE_WORKING_DIR = '/tmp/'
+
+    def __init__(self, **kwargs):
+
+        self.shards = []
+        self.envIsUp = False
+        self.envIsHealthy = False
+        self.modulePath = kwargs.pop('modulePath')
+        self.moduleArgs = kwargs['moduleArgs']
+        self.shardsCount = kwargs.pop('shardsCount')
+        self.dmcBinaryPath = kwargs.pop('dmcBinaryPath')
+        useSlaves = kwargs.get('useSlaves', False)
+
+        self.preperModule()
+        startPort = 20000
+        totalRedises = self.shardsCount * (2 if useSlaves else 1)
+        for i in range(0, totalRedises, (2 if useSlaves else 1)):
+            shard = StandardEnv(port=startPort, serverId=(i + 1), password=SHARD_PASSWORD, modulePath=self.moduleSoFilePath, **kwargs)
+            self.shards.append(shard)
+            startPort += 2
+
+        self.ccs = CcsMock(redisBinaryPath=kwargs['redisBinaryPath'], directory=kwargs['dbDirPath'], useSlaves=kwargs['useSlaves'],
+                           password=SHARD_PASSWORD, proxyPort=self.DMC_PORT, libPath=kwargs['libPath'])
+        self.dmc = Dmc(directory=kwargs['dbDirPath'], dmcBinaryPath=self.dmcBinaryPath, libPath=kwargs['libPath'])
+        self.envIsUp = False
+
+    def preperModule(self):
+        if self.modulePath is None:
+            self.moduleSoFilePath = None
+            self.moduleConfig = None
+            return
+        if not self.modulePath.endswith('zip'):
+            raise Exception('module on enterprise cluster must be a zip file')
+        with ZipFile(self.modulePath, 'r') as myzip:
+            moduleJson = [a.filename for a in myzip.infolist() if a.filename.endswith('.json')]
+            if len(moduleJson) != 1:
+                self.moduleSoFilePath = None
+                self.moduleConfig = None
+                return
+            moduleJson = moduleJson[0]
+            myzip.extractall(self.MODULE_WORKING_DIR)
+            with open(os.path.join(self.MODULE_WORKING_DIR, moduleJson), 'rt') as f:
+                self.moduleConfig = json.load(f)
+            self.moduleSoFilePath = os.path.join(self.MODULE_WORKING_DIR, self.moduleConfig['module_file'])
+            if self.moduleArgs is None:
+                self.moduleArgs = self.moduleConfig['command_line_args']
+
+    def printEnvData(self, prefix=''):
+        print(Colors.Yellow(prefix + 'bdb info:'))
+        print(Colors.Yellow(prefix + '\tlistening port:%d' % self.DMC_PORT))
+        print(Colors.Yellow(prefix + '\tshards count:%d' % len(self.shards)))
+        if self.modulePath:
+            print(Colors.Yellow(prefix + '\tzip module path:%s' % self.modulePath))
+        if self.moduleSoFilePath:
+            print(Colors.Yellow(prefix + '\tso module path:%s' % self.moduleSoFilePath))
+        if self.moduleArgs:
+            print(Colors.Yellow(prefix + '\tmodule args:%s' % self.moduleArgs))
+        for i, shard in enumerate(self.shards):
+            print(Colors.Yellow(prefix + 'shard: %d' % (i + 1)))
+            shard.printEnvData(prefix + '\t')
+        print(Colors.Yellow(prefix + 'ccs:'))
+        self.ccs.PrintEnvData(prefix + '\t')
+        print(Colors.Yellow(prefix + 'dmc:'))
+        self.dmc.PrintEnvData(prefix + '\t')
+
+    def startEnv(self, masters = True, slaves = True):
+        if self.envIsUp:
+            return  # env is already up
+        for shard in self.shards:
+            shard.startEnv(masters, slaves)
+
+        ccs_bdb_config = {'shard_key_regex': '012.*\{(?<tag>.*)\}.*00a(?<tag>.*)',
+                          'sharding': 'enabled' if self.shardsCount > 0 else 'disabled'}
+
+        extra_keys = {}
+        if self.moduleConfig:
+            ccs_bdb_config['module_list'] = self.moduleConfig['module_name']
+            extra_keys['module:%s' % self.moduleConfig['module_name']] = {'module_name': self.moduleConfig['module_name'],
+                                                                          'commands': ','.join([c['command_name'] for c in self.moduleConfig['commands']])}
+            for c in self.moduleConfig['commands']:
+                key_name = 'module_command:%s' % c['command_name']
+                extra_keys[key_name] = {}
+                for key, val in c.items():
+                    extra_keys[key_name][key] = val
+
+        self.ccs.Start(self.shards, bdb_fields=ccs_bdb_config, legacy_hash_slots=False, extra_keys=extra_keys)
+        self.dmc.Start()
+        con = self.getConnection()
+        wait_for_conn(con, command='sping', shouldBe=['SPONG 0' for i in self.shards])
+        self.envIsUp = True
+        self.envIsHealthy = True
+
+    def stopEnv(self, masters = True, slaves = True):
+        for shard in self.shards:
+            shard.stopEnv(masters, slaves)
+            self.envIsUp = self.envIsUp or shard.envIsUp
+            self.envIsHealthy = self.envIsHealthy and shard.envIsUp
+        self.ccs.Stop()
+        self.dmc.Stop()
+
+    def getConnection(self, shardId=1):
+        return redis.Redis('localhost', self.DMC_PORT)
+
+    def getSlaveConnection(self):
+        raise Exception('unsupported')
+
+    def flush(self):
+        self.getConnection().flushall()
+
+    def dumpAndReload(self, restart=False, shardId=None, timeout_sec=40):
+        if shardId is None:
+            for shard in self.shards:
+                shard.dumpAndReload(restart=restart)
+            self.dmc.Stop()
+            self.dmc.Start()
+            con = self.getConnection()
+            wait_for_conn(con, command='sping', shouldBe=['SPONG 0' for i in self.shards])
+        else:
+            self.shards[shardId - 1].dumpAndReload(restart=restart, shardId=None)
+
+    def broadcast(self, *cmd):
+        for shard in self.shards:
+            shard.broadcast(*cmd)
+
+    def checkExitCode(self):
+        for shard in self.shards:
+            if not shard.checkExitCode():
+                return False
+        return True
+
+    def exists(self, val):
+        return self.getConnection().exists(val)
+
+    def hmset(self, *args):
+        return self.getConnection().hmset(*args)
+
+    def keys(self, reg):
+        return self.getConnection().keys(reg)
+
+    def isUp(self):
+        raise Exception('unsupported operation')

RLTest/Enterprise/__init__.py

@@ -0,0 +1,5 @@
+from .EnterpriseClusterEnv import EnterpriseClusterEnv
+
+__all__ = [
+    'EnterpriseClusterEnv'
+]

RLTest/Enterprise/binaryrepo.py

@@ -0,0 +1,71 @@
+from __future__ import print_function
+import os.path
+import shutil
+import distro
+import subprocess
+import sys
+
+from RLTest.utils import Colors
+
+
+OS_NAME = distro.linux_distribution()[2]
+REPO_ROOT = os.path.expanduser('~/.RLTest')
+ENTERPRISE_VERSION = '5.2.0'
+ENTERPRISE_SUB_VERSION = '14'
+ENTERPRISE_TAR_FILE_NAME = 'redislabs-%s-%s-%s-amd64.tar' % (
+    ENTERPRISE_VERSION, ENTERPRISE_SUB_VERSION, OS_NAME)
+ENTERPRISE_URL = 'https://s3.amazonaws.com/rlec-downloads/%s/%s' % (
+    ENTERPRISE_VERSION, ENTERPRISE_TAR_FILE_NAME)
+DEBIAN_PKG_NAME = 'redislabs_%s-%s~%s_amd64.deb' % (
+        ENTERPRISE_VERSION, ENTERPRISE_SUB_VERSION, OS_NAME)
+
+
+class BinaryRepository(object):
+    """
+    Installation manager for Redis Enterprise.
+    The long-term idea behind this is to allow for managing different versions, etc.
+    but for now it's just here to cut some code out of the main executable
+    """
+    def __init__(self, root=REPO_ROOT, url=ENTERPRISE_URL, debname=DEBIAN_PKG_NAME):
+        self.root = root
+        self.url = url
+        self.debname = debname
+
+    def download_binaries(self, binariesName='binaries.tar'):
+        print(Colors.Yellow('installing enterprise binaries'))
+        print(Colors.Yellow('creating RLTest working dir: %s' % self.root))
+        try:
+            shutil.rmtree(self.root)
+            os.makedirs(self.root)
+        except Exception:
+            pass
+
+        print(Colors.Yellow('download binaries'))
+        args = ['wget', self.url, '-O', os.path.join(self.root, binariesName)]
+        process = subprocess.Popen(args=args, stdout=sys.stdout,
+                                        stderr=sys.stdout)
+        process.wait()
+        if process.poll() != 0:
+            raise Exception('failed to download enterprise binaries from s3')
+
+        print(Colors.Yellow('extracting binaries'))
+
+        args = ['tar', '-xvf', os.path.join(self.root, binariesName),
+                '--directory', self.root, self.debname]
+        process = subprocess.Popen(args=args, stdout=sys.stdout, stderr=sys.stdout)
+        process.wait()
+        if process.poll() != 0:
+            raise Exception(
+                'failed to extract binaries to %s' % self.self.root)
+
+        # TODO: Support centos that does not have dpkg command
+        args = ['dpkg', '-x', os.path.join(self.root, self.debname),
+                self.root]
+        process = subprocess.Popen(args=args, stdout=sys.stdout,
+                                        stderr=sys.stdout)
+        process.wait()
+        if process.poll() != 0:
+            raise Exception(
+                'failed to extract binaries to %s' % self.self.root)
+
+        print(Colors.Yellow('finished installing enterprise binaries'))

RLTest/__init__.py

@@ -0,0 +1,10 @@
+from RLTest.env import Env, Defaults
+from RLTest.redis_std import StandardEnv
+from ._version import __version__
+
+__all__ = [
+    'Defaults',
+    'Env',
+    'StandardEnv'
+]
+