RLTest 0.7.14__py3-none-any.whl

RLTest/exists_redis.py

@@ -0,0 +1,122 @@
+
+from __future__ import print_function
+import redis
+import subprocess
+import sys
+import os
+import platform
+from .utils import Colors, wait_for_conn
+
+
+MASTER = 1
+SLAVE = 2
+
+
+class ExistsRedisEnv(object):
+    def __init__(self, addr='localhost:6379', password = None, **kwargs):
+        self.host, self.port = addr.split(':')
+        self.port = int(self.port)
+        self.password = password
+        self.useTLS = kwargs['useTLS']
+        self.decodeResponses = kwargs.get('decodeResponses', False)
+
+    @property
+    def has_interactive_debugger(self):
+        return False
+
+    def _printEnvData(self, prefix='', role=MASTER):
+        print(Colors.Yellow(prefix + 'addr: %s:%d' % (self.host, self.port)))
+
+    def printEnvData(self, prefix=''):
+        print(Colors.Yellow(prefix + 'master:'))
+        self._printEnvData(prefix + '\t', MASTER)
+
+    def startEnv(self, masters = True, slaves = True):
+        if not self.isUp():
+            raise Exception('env is not up')
+
+    def stopEnv(self, masters = True, slaves = True):
+        pass
+
+    def getConnection(self, shardId=1):
+        return redis.StrictRedis(self.host, self.port, password=self.password, decode_responses=self.decodeResponses)
+
+    def getSlaveConnection(self):
+        raise Exception('asked for slave connection but no slave exists')
+
+    # List of nodes that initial bootstrapping can be done from
+    def getMasterNodesList(self):
+        node_info = {"host": None, "port": None, "unix_socket_path": None, "password": None}
+        node_info["password"] = self.password
+        node_info["host"] = self.host
+        node_info["port"] = self.port
+        return [node_info]
+
+    # List containing a connection for each of the master nodes
+    def getOSSMasterNodesConnectionList(self):
+        return [self.getConnection()]
+
+    def getConnectionByKey(self, key, command):
+        return self.getConnection()
+
+    def _waitForBgsaveToFinish(self):
+        # on new Redis version (6.2 and above)
+        # flush trigger background rdb save
+        # waiting for rdbsave to finish
+        while True:
+            if not self.getConnection().execute_command('info', 'Persistence')['rdb_bgsave_in_progress']:
+                break
+
+    def flush(self):
+        self.getConnection().flushall()
+        self._waitForBgsaveToFinish()
+
+    def _waitForChild(self, conns):
+        import time
+        # Wait until file is available
+        for con in conns:
+            while True:
+                info = con.info('persistence')
+                if info['aof_rewrite_scheduled'] or info['aof_rewrite_in_progress']:
+                    time.sleep(0.1)
+                else:
+                    break
+
+    def dumpAndReload(self, restart=False, shardId=1, timeout_sec=0):
+        self._waitForBgsaveToFinish()
+        conn = self.getConnection()
+        conn.save()
+        try:
+            conn.execute_command('DEBUG', 'RELOAD')
+        except redis.RedisError as err:
+            raise err
+
+    def broadcast(self, *cmd):
+        try:
+            self.getConnection().execute_command(*cmd)
+        except Exception as e:
+            print(e)
+
+    def checkExitCode(self):
+        return True
+
+    def isUp(self):
+        return self.getConnection().ping()
+
+    def isTLS(self):
+        return self.useTLS
+
+    def exists(self, val):
+        return self.getConnection().exists(val)
+
+    def hmset(self, *args):
+        return self.getConnection().hmset(*args)
+
+    def keys(self, reg):
+        return self.getConnection().keys(reg)
+
+    def isUnixSocket(self):
+        return False
+
+    def isTcp(self):
+        return True

RLTest/loader.py

@@ -0,0 +1,174 @@
+from __future__ import print_function
+import os
+import sys
+import importlib.util
+import inspect
+from RLTest.utils import Colors
+
+
+class TestFunction(object):
+    is_class = False
+
+    def __init__(self, filename, symbol, modulename):
+        self.filename = filename
+        self.symbol = symbol
+        self.modulename = modulename
+        self.is_method = False
+        self.name = '{}:{}'.format(self.modulename, symbol)
+
+    def initialize(self):
+        module_spec = importlib.util.spec_from_file_location(self.modulename, self.filename)
+        module = importlib.util.module_from_spec(module_spec)
+        sys.modules[self.modulename] = module
+        module_spec.loader.exec_module(module)
+        obj = getattr(module, self.symbol)
+        self.target = obj
+
+    def shortname(self):
+        return self.target.__name__
+
+class TestMethod(object):
+    is_class = False
+
+    def __init__(self, obj, name):
+        self.target = obj
+        self.name = name
+        self.is_method = True
+
+    def initialize(self):
+        pass
+
+    def shortname(self):
+        return self.target.__name__
+
+class TestClass(object):
+    is_class = True
+
+    def __init__(self, filename, symbol, modulename, functions):
+        self.filename = filename
+        self.symbol = symbol
+        self.modulename = modulename
+        self.functions = functions
+        self.name = '{}:{}'.format(self.modulename, symbol)
+
+    def initialize(self):
+        module_spec = importlib.util.spec_from_file_location(self.modulename, self.filename)
+        module = importlib.util.module_from_spec(module_spec)
+        sys.modules[self.modulename] = module
+        module_spec.loader.exec_module(module)
+        obj = getattr(module, self.symbol)
+        self.clsname = obj.__name__
+        self.cls = obj
+
+    def create_instance(self, *args, **kwargs):
+        return self.cls(*args, **kwargs)
+
+    def get_functions(self, instance):
+        fns = []
+        for mname in self.functions:
+            bound = getattr(instance, mname)
+            if not callable(bound):
+                continue
+            fns.append(TestMethod(bound,
+                                  name='{}:{}.{}'.format(self.modulename, self.clsname, mname)))
+        return fns
+
+
+class TestLoader(object):
+    def __init__(self):
+        self.tests = []
+
+    def load_spec(self, arg):
+        # if arg is a list, load its elements
+        if isinstance(arg, list):
+            for spec in arg:
+                self.load_spec(spec)
+            return
+
+        # See what kind of spec this is!
+        """
+        Load tests from single argument form, e.g. foo.py:BarBaz
+        """
+        if ':' in arg:
+            filename, varname = arg.split(':')
+        else:
+            filename = arg
+            varname = None
+            if os.path.isdir(filename):
+                sys.path.append(filename)
+                self.scan_dir(filename)
+                return
+
+        # Ensure the path is in sys.path
+        dirname = os.path.abspath(os.path.dirname(filename))
+        if dirname not in sys.path:
+            sys.path.append(dirname)
+
+        module_name, _ = os.path.splitext(os.path.basename(filename))
+        toplevel_filter, subfilter = None, None
+        if varname:
+            if '.' in varname:
+                toplevel_filter, subfilter = varname.split('.')
+            else:
+                toplevel_filter = varname
+
+        self.load_files(dirname, module_name, toplevel_filter, subfilter)
+
+    def load_files(self, module_dir, module_name, toplevel_filter=None, subfilter=None):
+        filename = '%s/%s.py' % (module_dir, module_name)
+        try:
+            module_spec = importlib.util.spec_from_file_location(module_name, filename)
+            module = importlib.util.module_from_spec(module_spec)
+            sys.modules[module_name] = module
+            module_spec.loader.exec_module(module)
+            for symbol in dir(module):
+                if not self.filter_modulevar(symbol, toplevel_filter):
+                    continue
+
+                obj = getattr(module, symbol)
+                if inspect.isclass(obj):
+                    methnames = [mname for mname in dir(obj)
+                                    if self.filter_method(mname, subfilter)]
+                    self.tests.append(TestClass(filename, symbol, module_name, methnames))
+                elif inspect.isfunction(obj):
+                    self.tests.append(TestFunction(filename, symbol, module_name))
+        except OSError as e:
+            print(Colors.Red("Can't access file %s." % filename))
+            raise e
+        except Exception as e:
+            print(Colors.Red("Problems in file %s: %s" % (filename, e)))
+            raise e
+
+    def scan_dir(self, testdir):
+        for filename in os.listdir(testdir):
+            if filename.startswith('test') and filename.endswith('.py'):
+                module_name, ext = os.path.splitext(filename)
+                self.load_files(testdir, module_name)
+
+    def filter_modulevar(self, candidate, toplevel_filter):
+        if not candidate.lower().startswith('test'):
+            return False
+        if toplevel_filter and candidate != toplevel_filter:
+            return False
+        return True
+
+    def filter_method(self, candidate, subfilter):
+        if not candidate.lower().startswith('test'):
+            return False
+        if subfilter and candidate != subfilter:
+            return False
+
+        return True
+
+    def __iter__(self):
+        return iter(self.tests)
+
+    def print_tests(self):
+        tests = []
+        for t in self.tests:
+            if t.is_class:
+                for m in t.functions:
+                    tests.append(f"{t.name}.{m}")
+            else:
+                tests.append(t.name)
+        print(*sorted(tests), sep='\n')

RLTest/random_port.py

@@ -0,0 +1,63 @@
+import random
+import socket
+import struct
+import fcntl
+import os
+import errno
+import json
+
+def _check_alive(pid):
+    try:
+        os.kill(pid, 0)
+        return True
+    except OSError as e:
+        if e.errno == errno.EPERM:
+            return True
+        return False
+
+
+def register_port(port):
+    fp = open('/tmp/rltest_portfile.lock', 'a+')
+    fcntl.flock(fp.fileno(), fcntl.LOCK_EX)
+    fp.seek(0, 2)  # seek from end
+    if fp.tell() == 0:
+        entries = {}
+    else:
+        fp.seek(0, 0)
+        entries = json.load(fp)
+    # remove not responsive processes
+    entries = { p:pid for p, pid in entries.items() if _check_alive(pid) }
+
+    if str(port) in entries:
+        ret = False
+    else:
+        entries[str(port)] = os.getpid()
+        ret = True
+
+    fp.seek(0, 0)
+    fp.truncate()
+    json.dump(entries, fp)
+    fp.close()
+    return ret
+
+
+def get_random_port():
+    for _ in range(10000):
+        p = random.randint(10000, 20000)
+        # Try to open and bind the socket
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
+        try:
+            s.bind(('', p))
+            s.close()
+            if not register_port(p):
+                continue
+            return p
+        except Exception as e:
+            if hasattr(e, 'errno') and e.errno in (errno.EADDRINUSE, errno.EADDRNOTAVAIL):
+                pass
+            else:
+                raise e
+
+    raise Exception('Could not find open port to listen on!')

RLTest/redis_cluster.py

@@ -0,0 +1,241 @@
+from __future__ import print_function
+
+from .redis_std import StandardEnv
+from redis.cluster import ClusterNode
+import redis
+import time
+from RLTest.utils import Colors
+
+
+class ClusterEnv(object):
+    def __init__(self, **kwargs):
+        self.shards = []
+        self.envIsUp = False
+        self.envIsHealthy = False
+        self.modulePath = kwargs['modulePath']
+        self.moduleArgs = kwargs['moduleArgs']
+        self.password = kwargs['password']
+        self.shardsCount = kwargs.pop('shardsCount')
+        useSlaves = kwargs.get('useSlaves', False)
+        self.useTLS = kwargs['useTLS']
+        self.decodeResponses = kwargs.get('decodeResponses', False)
+        self.tlsPassphrase = kwargs.get('tlsPassphrase', None)
+        self.protocol = kwargs.get('protocol', 2)
+        self.terminateRetries = kwargs.get('terminateRetries', None)
+        self.terminateRetrySecs = kwargs.get('terminateRetrySecs', None)
+        startPort = kwargs.pop('port', 10000)
+        totalRedises = self.shardsCount * (2 if useSlaves else 1)
+        randomizePorts = kwargs.pop('randomizePorts', False)
+        for i in range(0, totalRedises, (2 if useSlaves else 1)):
+            port = 0 if randomizePorts else startPort
+            shard = StandardEnv(port=port, serverId=(i + 1),
+                                clusterEnabled=True, **kwargs)
+            self.shards.append(shard)
+            startPort += 2
+
+    def printEnvData(self, prefix=''):
+        print(Colors.Yellow(prefix + 'info:'))
+        print(Colors.Yellow(prefix + '\tshards count:%d' % len(self.shards)))
+        if self.modulePath:
+            print(Colors.Yellow(prefix + '\tzip module path:%s' % self.modulePath))
+        if self.moduleArgs:
+            print(Colors.Yellow(prefix + '\tmodule args:%s' % self.moduleArgs))
+        for i, shard in enumerate(self.shards):
+            print(Colors.Yellow(prefix + 'shard: %d' % (i + 1)))
+            shard.printEnvData(prefix + '\t')
+
+    def getInformationBeforeDispose(self):
+        return [shard.getInformationBeforeDispose() for shard in self.shards]
+
+    def getInformationAfterDispose(self):
+        return [shard.getInformationAfterDispose() for shard in self.shards]  
+
+    def waitCluster(self, timeout_sec=40):
+        st = time.time()
+        ok = 0
+
+        while st + timeout_sec > time.time():
+            ok = 0
+            for shard in self.shards:
+                con = shard.getConnection()
+                try:
+                    status = con.execute_command('CLUSTER', 'INFO')
+                except Exception as e:
+                    print('got error on cluster info, will try again, %s' % str(e))
+                    continue
+                if 'cluster_state:ok' in str(status):
+                    ok += 1
+            if ok == len(self.shards):
+                for shard in self.shards:
+                    try:
+                        shard.getConnection().execute_command('FT.CLUSTERREFRESH')
+                    except Exception:
+                        pass
+                    try:
+                        shard.getConnection().execute_command('SEARCH.CLUSTERREFRESH')
+                    except Exception:
+                        pass
+                return
+
+            time.sleep(0.1)
+        raise RuntimeError(
+            "Cluster OK wait loop timed out after %s seconds" % timeout_sec)
+
+    def startEnv(self, masters=True, slaves=True):
+        if self.envIsUp == True:
+            return  # env is already up
+        try:
+            for shard in self.shards:
+                shard.startEnv(masters, slaves)
+        except Exception:
+            for shard in self.shards:
+                shard.stopEnv()
+            raise
+
+        slots_per_node = int(16384 / len(self.shards)) + 1
+        for i, shard in enumerate(self.shards):
+            con = shard.getConnection()
+            for s in self.shards:
+                con.execute_command('CLUSTER', 'MEET',
+                                    '127.0.0.1', s.getMasterPort())
+
+            start_slot = i * slots_per_node
+            end_slot = start_slot + slots_per_node
+            if end_slot > 16384:
+                end_slot = 16384
+
+            try:
+                con.execute_command('CLUSTER', 'ADDSLOTS', *(str(x)
+                                    for x in range(start_slot, end_slot)))
+            except Exception:
+                pass
+
+        self.waitCluster()
+        self.envIsUp = True
+        self.envIsHealthy = True
+
+    def stopEnvWithSegFault(self, masters=True, slaves=True):
+        for shard in self.shards:
+            shard.stopEnvWithSegFault(masters, slaves)
+
+    def stopEnv(self, masters=True, slaves=True):
+        self.envIsUp = False
+        self.envIsHealthy = False
+        for shard in self.shards:
+            shard.stopEnv(masters, slaves)
+            self.envIsUp = self.envIsUp or shard.envIsUp
+            self.envIsHealthy = self.envIsHealthy and shard.envIsUp
+
+    def getConnection(self, shardId=1):
+        return self.shards[shardId - 1].getConnection()
+
+    def getClusterConnection(self):
+        statupNode = [ClusterNode(a['host'], a['port']) for a in self.getMasterNodesList()]
+        if self.useTLS:
+            return redis.RedisCluster(
+                ssl=True,
+                ssl_keyfile=self.shards[0].getTLSKeyFile(),
+                ssl_certfile=self.shards[0].getTLSCertFile(),
+                ssl_cert_reqs=None,
+                ssl_ca_certs=self.shards[0].getTLSCACertFile(),
+                ssl_password=self.tlsPassphrase,
+                password=self.password,
+                startup_nodes=statupNode,
+                decode_responses=self.decodeResponses,
+                protocol=self.protocol,
+                terminateRetries=self.terminateRetries, terminateRetrySecs=self.terminateRetrySecs
+            )
+        else:
+            return redis.RedisCluster(
+                startup_nodes=statupNode,
+                decode_responses=self.decodeResponses, password=self.password,
+                protocol=self.protocol,
+                terminateRetries=self.terminateRetries, terminateRetrySecs=self.terminateRetrySecs)
+
+    def getSlaveConnection(self):
+        raise Exception('unsupported')
+
+    # List of nodes that initial bootstrapping can be done from
+    def getMasterNodesList(self):
+        full_master_list = []
+        for shard in self.shards:
+            node_info_list = shard.getMasterNodesList()
+            full_master_list.append(node_info_list[0])
+        return full_master_list
+
+    # List containing a connection for each of the master nodes
+    def getOSSMasterNodesConnectionList(self):
+        full_master_connection_list = []
+        for shard in self.shards:
+            full_master_connection_list.append(shard.getConnection())
+        return full_master_connection_list
+
+    # Gets a cluster connection by key. On std redis the default connection is returned.
+    def getConnectionByKey(self, key, command):
+        clusterConn = self.getClusterConnection()
+        target_node = clusterConn._determine_nodes(command, key) # we will always which will give us the node responsible for the key
+        return clusterConn.get_redis_connection(target_node[0])
+
+    def addShardToCluster(self, redisBinaryPath, output_files_format, **kwargs):
+        kwargs.pop('port')
+        port = self.shards[-1].port + 2  # use a fresh port
+        self.shardsCount += 1
+        new_shard = StandardEnv(redisBinaryPath, port, outputFilesFormat=output_files_format,
+                                serverId=self.shardsCount, clusterEnabled=True, **kwargs)
+        try:
+            new_shard.startEnv()
+        except Exception:
+            new_shard.stopEnv()
+            raise
+        self.shards.append(new_shard)
+        # Notify other shards that the new shard is available and wait for the topology change to be acknowledged.
+        conn = new_shard.getConnection()
+        for s in self.shards:
+            conn.execute_command('CLUSTER', 'MEET', '127.0.0.1', s.getMasterPort())
+        self.waitCluster()
+
+    def flush(self):
+        self.getClusterConnection().flushall()
+
+    def dumpAndReload(self, restart=False, shardId=None, timeout_sec=40):
+        if shardId is None:
+            for shard in self.shards:
+                shard.dumpAndReload(restart=restart)
+            self.waitCluster(timeout_sec=timeout_sec)
+        else:
+            self.shards[shardId -
+                        1].dumpAndReload(restart=restart, shardId=None, timeout_sec=timeout_sec)
+
+    def broadcast(self, *cmd):
+        for shard in self.shards:
+            shard.broadcast(*cmd)
+
+    def checkExitCode(self):
+        for shard in self.shards:
+            if not shard.checkExitCode():
+                return False
+        return True
+
+    def isUp(self):
+        return self.envIsUp or self.envIsHealthy and self.waitCluster()
+
+    def isHealthy(self):
+        return self.envIsHealthy
+
+    def isUnixSocket(self):
+        return False
+
+    def isTcp(self):
+        return True
+
+    def isTLS(self):
+        return self.useTLS
+
+    def exists(self, val):
+        return self.getClusterConnection().exists(val)
+
+    def hmset(self, *args):
+        return self.getClusterConnection().hmset(*args)
+
+    def keys(self, reg):
+        return self.getClusterConnection().keys(reg)