synapse 2.176.0__py311-none-any.whl → 2.178.0__py311-none-any.whl

synapse/lib/config.py

@@ -392,10 +392,16 @@ class Config(c_abc.MutableMapping):
         else:
             return
 
-    def reqConfValu(self, key):
+    def reqConfValu(self, key):  # pragma: no cover
         '''
-        Get a configuration value.  If that value is not present in the schema
-        or is not set, then raise an exception.
+        Deprecated. Use ``req(key)`` API instead.
+        '''
+        s_common.deprecated('Config.reqConfValu(), use req() instead.')
+        return self.req(key)
+
+    def req(self, key):
+        '''
+        Get a configuration value. If that value is not present in the schema or is not set, then raise an exception.
 
         Args:
             key (str): The key to require.
@@ -403,17 +409,15 @@ class Config(c_abc.MutableMapping):
         Returns:
             The requested value.
         '''
-        # Ensure that the key is in self.json_schema
         if key not in self.json_schema.get('properties', {}):
-            raise s_exc.BadArg(mesg='Required key is not present in the configuration schema.',
-                               key=key)
+            raise s_exc.BadArg(mesg=f'The {key} configuration option is not present in the configuration schema.',
+                               name=key)
 
-        # Ensure that the key is present in self.conf
-        if key not in self.conf:
-            raise s_exc.NeedConfValu(mesg='Required key is not present in configuration data.',
-                                     key=key)
+        valu = self.conf.get(key, s_common.novalu)
+        if valu is not s_common.novalu:
+            return valu
 
-        return self.conf.get(key)
+        raise s_exc.NeedConfValu(mesg=f'The {key} configuration option is required.', name=key)
 
     def reqKeyValid(self, key, value):
         '''

synapse/lib/coro.py

@@ -148,6 +148,19 @@ async def ornot(func, *args, **kwargs):
         return await retn
     return retn
 
+bgtasks = set()
+def create_task(coro):
+
+    task = asyncio.get_running_loop().create_task(coro)
+    bgtasks.add(task)
+
+    def done(t):
+        bgtasks.remove(t)
+
+    task.add_done_callback(done)
+
+    return task
+
 class GenrHelp:
 
     def __init__(self, genr):

synapse/lib/grammar.py

@@ -15,6 +15,8 @@ formrestr = r'[a-z_][a-z0-9_]*(:[a-z0-9_]+)+'
 formre = regex.compile(formrestr)
 tagrestr = r'(\w+\.)*\w+'
 tagre = regex.compile(tagrestr)
+edgerestr = r'[\w\.:]{1,200}'
+edgere = regex.compile(edgerestr)
 basepropnopivpropstr = r'[a-z_][a-z0-9_]*(?:(\:|\.)[a-z_][a-z0-9_]*)*'
 basepropnopivpropre = regex.compile(basepropnopivpropstr)
 
@@ -38,6 +40,9 @@ def isFormName(name):
 def isBasePropNoPivprop(name):
     return basepropnopivpropre.fullmatch(name) is not None
 
+def isEdgeVerb(verb):
+    return edgere.fullmatch(verb) is not None
+
 floatre = regex.compile(r'\s*-?\d+(\.\d+)?([eE][-+]\d+)?')
 
 def parse_float(text, off):

synapse/lib/hive.py

@@ -91,7 +91,7 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
     An optionally persistent atomically accessed tree which implements
     primitives for use in making distributed/clustered services.
     '''
-    async def __anit__(self, conf=None, nexsroot=None):
+    async def __anit__(self, conf=None, nexsroot=None, cell=None):
 
         await s_nexus.Pusher.__anit__(self, 'hive', nexsroot=nexsroot)
 
@@ -100,6 +100,7 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
         if conf is None:
             conf = {}
 
+        self.cell = cell
         self.conf = conf
         self.nodes = {}  # full=Node()
 
@@ -340,6 +341,18 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
 
     @s_nexus.Pusher.onPush('hive:set')
     async def _set(self, full, valu):
+        if self.cell is not None:
+            if full[0] == 'auth':
+                if len(full) == 5:
+                    _, _, iden, dtyp, name = full
+                    if dtyp == 'vars':
+                        await self.cell.auth._hndlsetUserVarValu(iden, name, valu)
+                    elif dtyp == 'profile':
+                        await self.cell.auth._hndlsetUserProfileValu(iden, name, valu)
+
+            elif full[0] == 'cellvers':
+                self.cell.setCellVers(full[-1], valu, nexs=False)
+
         node = await self._getHiveNode(full)
 
         oldv = node.valu
@@ -377,6 +390,14 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
     @s_nexus.Pusher.onPush('hive:pop')
     async def _pop(self, full):
 
+        if self.cell is not None and full[0] == 'auth':
+            if len(full) == 5:
+                _, _, iden, dtyp, name = full
+                if dtyp == 'vars':
+                    await self.cell.auth._hndlpopUserVarValu(iden, name)
+                elif dtyp == 'profile':
+                    await self.cell.auth._hndlpopUserProfileValu(iden, name)
+
         node = self.nodes.get(full)
         if node is None:
             return
@@ -434,10 +455,10 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
 
 class SlabHive(Hive):
 
-    async def __anit__(self, slab, db=None, conf=None, nexsroot=None):
+    async def __anit__(self, slab, db=None, conf=None, nexsroot=None, cell=None):
         self.db = db
         self.slab = slab
-        await Hive.__anit__(self, conf=conf, nexsroot=nexsroot)
+        await Hive.__anit__(self, conf=conf, nexsroot=nexsroot, cell=cell)
         self.slab.onfini(self.fini)
 
     async def _storLoadHive(self):

synapse/lib/hiveauth.py

@@ -277,7 +277,6 @@ class Auth(s_nexus.Pusher):
 
         return user
 
-    @s_nexus.Pusher.onPushAuto('user:name')
     async def setUserName(self, iden, name):
         if not isinstance(name, str):
             raise s_exc.BadArg(mesg='setUserName() name must be a string')
@@ -302,7 +301,6 @@ class Auth(s_nexus.Pusher):
         }
         await self.feedBeholder('user:name', beheld)
 
-    @s_nexus.Pusher.onPushAuto('role:name')
     async def setRoleName(self, iden, name):
         if not isinstance(name, str):
             raise s_exc.BadArg(mesg='setRoleName() name must be a string')
@@ -346,7 +344,6 @@ class Auth(s_nexus.Pusher):
 
             await self.fire('cell:beholder', **behold)
 
-    @s_nexus.Pusher.onPushAuto('user:info')
     async def setUserInfo(self, iden, name, valu, gateiden=None, logged=True, mesg=None):
 
         user = await self.reqUser(iden)
@@ -373,7 +370,6 @@ class Auth(s_nexus.Pusher):
         # since any user info *may* effect auth
         user.clearAuthCache()
 
-    @s_nexus.Pusher.onPushAuto('role:info')
     async def setRoleInfo(self, iden, name, valu, gateiden=None, logged=True, mesg=None):
         role = await self.reqRole(iden)
 
@@ -528,7 +524,6 @@ class Auth(s_nexus.Pusher):
 
         return user
 
-    @s_nexus.Pusher.onPush('user:add')
     async def _addUser(self, iden, name):
 
         user = self.usersbyname.get(name)
@@ -554,7 +549,6 @@ class Auth(s_nexus.Pusher):
 
         return self.role(iden)
 
-    @s_nexus.Pusher.onPush('role:add')
     async def _addRole(self, iden, name):
 
         role = self.rolesbyname.get(name)
@@ -574,7 +568,6 @@ class Auth(s_nexus.Pusher):
         await self.reqUser(iden)
         return await self._push('user:del', iden)
 
-    @s_nexus.Pusher.onPush('user:del')
     async def _delUser(self, iden):
 
         if iden == self.rootuser.iden:
@@ -608,7 +601,6 @@ class Auth(s_nexus.Pusher):
         await self.reqRole(iden)
         return await self._push('role:del', iden)
 
-    @s_nexus.Pusher.onPush('role:del')
     async def _delRole(self, iden):
 
         if iden == self.allrole.iden:
@@ -838,10 +830,7 @@ class HiveRole(HiveRuler):
         }
 
     async def _setRulrInfo(self, name, valu, gateiden=None, nexs=True, mesg=None):
-        if nexs:
-            return await self.auth.setRoleInfo(self.iden, name, valu, gateiden=gateiden, mesg=mesg)
-        else:
-            return await self.auth._hndlsetRoleInfo(self.iden, name, valu, gateiden=gateiden, logged=nexs, mesg=mesg)
+        return await self.auth.setRoleInfo(self.iden, name, valu, gateiden=gateiden, logged=nexs, mesg=mesg)
 
     async def setName(self, name):
         return await self.auth.setRoleName(self.iden, name)
@@ -929,10 +918,7 @@ class HiveUser(HiveRuler):
         }
 
     async def _setRulrInfo(self, name, valu, gateiden=None, nexs=True, mesg=None):
-        if nexs:
-            return await self.auth.setUserInfo(self.iden, name, valu, gateiden=gateiden, mesg=mesg)
-        else:
-            return await self.auth._hndlsetUserInfo(self.iden, name, valu, gateiden=gateiden, logged=nexs, mesg=mesg)
+        return await self.auth.setUserInfo(self.iden, name, valu, gateiden=gateiden, logged=nexs, mesg=mesg)
 
     async def setName(self, name):
         return await self.auth.setUserName(self.iden, name)
@@ -1232,10 +1218,7 @@ class HiveUser(HiveRuler):
 
         roles.remove(role.iden)
         mesg = {'name': 'role:revoke', 'iden': self.iden, 'role': role.pack()}
-        if nexs:
-            await self.auth.setUserInfo(self.iden, 'roles', roles, mesg=mesg)
-        else:
-            await self.auth._hndlsetUserInfo(self.iden, 'roles', roles, logged=nexs, mesg=mesg)
+        await self.auth.setUserInfo(self.iden, 'roles', roles, logged=nexs, mesg=mesg)
 
     def isLocked(self):
         return self.info.get('locked')
@@ -1271,18 +1254,12 @@ class HiveUser(HiveRuler):
     async def setAdmin(self, admin, gateiden=None, logged=True):
         if not isinstance(admin, bool):
             raise s_exc.BadArg(mesg='setAdmin requires a boolean')
-        if logged:
-            await self.auth.setUserInfo(self.iden, 'admin', admin, gateiden=gateiden)
-        else:
-            await self.auth._hndlsetUserInfo(self.iden, 'admin', admin, gateiden=gateiden, logged=logged)
+        await self.auth.setUserInfo(self.iden, 'admin', admin, gateiden=gateiden, logged=logged)
 
     async def setLocked(self, locked, logged=True):
         if not isinstance(locked, bool):
             raise s_exc.BadArg(mesg='setLocked requires a boolean')
-        if logged:
-            await self.auth.setUserInfo(self.iden, 'locked', locked)
-        else:
-            await self.auth._hndlsetUserInfo(self.iden, 'locked', locked, logged=logged)
+        await self.auth.setUserInfo(self.iden, 'locked', locked, logged=logged)
 
     async def setArchived(self, archived):
         if not isinstance(archived, bool):
@@ -1347,7 +1324,4 @@ class HiveUser(HiveRuler):
             shadow = await s_passwd.getShadowV2(passwd=passwd)
         else:
             raise s_exc.BadArg(mesg='Password must be a string')
-        if nexs:
-            await self.auth.setUserInfo(self.iden, 'passwd', shadow)
-        else:
-            await self.auth._hndlsetUserInfo(self.iden, 'passwd', shadow, logged=nexs)
+        await self.auth.setUserInfo(self.iden, 'passwd', shadow, logged=nexs)

synapse/lib/layer.py

@@ -1431,9 +1431,6 @@ class Layer(s_nexus.Pusher):
         self.growsize = self.layrinfo.get('growsize')
         self.logedits = self.layrinfo.get('logedits')
 
-        self.mapasync = core.conf.get('layer:lmdb:map_async')
-        self.maxreplaylog = core.conf.get('layer:lmdb:max_replay_log')
-
         # slim hooks to avoid async/fire
         self.nodeAddHook = None
         self.nodeDelHook = None
@@ -2034,7 +2031,7 @@ class Layer(s_nexus.Pusher):
                                                  'stortype': stortype})
 
     async def pack(self):
-        ret = self.layrinfo.pack()
+        ret = deepcopy(self.layrinfo)
         if ret.get('mirror'):
             ret['mirror'] = s_urlhelp.sanitizeUrl(ret['mirror'])
         ret['offset'] = await self.getEditIndx()
@@ -2728,8 +2725,6 @@ class Layer(s_nexus.Pusher):
         slabopts = {
             'readahead': True,
             'lockmemory': self.lockmemory,
-            'map_async': self.mapasync,
-            'max_replay_log': self.maxreplaylog,
         }
 
         if self.growsize is not None:
@@ -2810,9 +2805,11 @@ class Layer(s_nexus.Pusher):
 
         # TODO when we can set more props, we may need to parse values.
         if valu is None:
-            await self.layrinfo.pop(name)
+            self.layrinfo.pop(name, None)
         else:
-            await self.layrinfo.set(name, valu)
+            self.layrinfo[name] = valu
+
+        self.core.layerdefs.set(self.iden, self.layrinfo)
 
         await self.core.feedBeholder('layer:set', {'iden': self.iden, 'name': name, 'valu': valu}, gates=[self.iden])
         return valu
@@ -4453,7 +4450,8 @@ class Layer(s_nexus.Pusher):
 
     @s_nexus.Pusher.onPush('layer:set:modelvers')
     async def _setModelVers(self, vers):
-        await self.layrinfo.set('model:version', vers)
+        self.layrinfo['model:version'] = vers
+        self.core.layerdefs.set(self.iden, self.layrinfo)
 
     async def getStorNodes(self):
         '''

synapse/lib/link.py

@@ -21,7 +21,7 @@ async def connect(host, port, ssl=None, hostname=None, linkinfo=None):
     '''
     Async connect and return a Link().
     '''
-    info = {'host': host, 'port': port, 'ssl': ssl, 'hostname': hostname}
+    info = {'host': host, 'port': port, 'ssl': ssl, 'hostname': hostname, 'tls': bool(ssl)}
     if linkinfo is not None:
         info.update(linkinfo)
 
@@ -137,6 +137,25 @@ class Link(s_base.Base):
 
         self.info = info
 
+        self._addrinfo = {}
+        # _addrinfo is populated in this order so that as first hit tls links (prod deployments)
+        # then unix links (unit tests with local sockets, container healthchecks, local tools )
+        # then tcp links ( unit tests and legacy deployments )
+        if self.info.get('tls'):
+            self._addrinfo['family'] = 'tls'
+            self._addrinfo['addr'] = self.sock.getpeername()
+        elif self.info.get('unix'):
+            self._addrinfo['family'] = 'unix'
+            # Unix sockets don't use getpeername
+            self._addrinfo['addr'] = self.sock.getsockname()
+        else:
+            self._addrinfo['family'] = 'tcp'
+            self._addrinfo['addr'] = self.sock.getpeername()
+        if self.sock.family == socket.AF_INET:
+            self._addrinfo['ipver'] = 'ipv4'
+        elif self.sock.family == socket.AF_INET6:
+            self._addrinfo['ipver'] = 'ipv6'
+
         self.unpk = s_msgpack.Unpk()
 
         async def fini():
@@ -145,7 +164,7 @@ class Link(s_base.Base):
                 self.reader._transport.abort()
             try:
                 await self.writer.wait_closed()
-            except (BrokenPipeError, ConnectionResetError) as e:
+            except Exception as e:
                 logger.debug('Link error waiting on close: %s', str(e))
 
         self.onfini(fini)
@@ -212,22 +231,7 @@ class Link(s_base.Base):
         '''
         Get a summary of address information related to the link.
         '''
-        ret = {'family': 'tcp',
-               'addr': self.sock.getpeername(),
-               }
-        # Set family information
-        if self.info.get('unix'):
-            ret['family'] = 'unix'
-            # Unix sockets don't use getpeername
-            ret['addr'] = self.sock.getsockname()
-        elif self.info.get('tls'):
-            ret['family'] = 'tls'
-        # Set ipver if needed
-        if self.sock.family == socket.AF_INET:
-            ret['ipver'] = 'ipv4'
-        if self.sock.family == socket.AF_INET6:
-            ret['ipver'] = 'ipv6'
-        return ret
+        return dict(self._addrinfo)
 
     async def send(self, byts):
         self.writer.write(byts)

synapse/lib/lmdbslab.py

@@ -166,6 +166,148 @@ class SlabDict:
         self.set(name, curv)
         return curv
 
+class SafeKeyVal:
+    '''
+    Key/value storage that does not store items in memory and ensures keys are < 512 characters in length.
+
+        Note:
+            The key size limitation includes the length of any prefixes added by
+            using getSubKeyVal().
+    '''
+    def __init__(self, slab, name, prefix=''):
+
+        self.prefix = prefix
+        self._prefix = prefix.encode('utf8')
+        self.preflen = len(self._prefix)
+
+        if self.preflen > 510:
+            mesg = 'SafeKeyVal prefix lengths must be < 511 bytes.'
+            raise s_exc.BadArg(mesg, prefix=self._prefix[:1024])
+
+        self.name = name
+        self.slab = slab
+        self.valudb = slab.initdb(name)
+
+    def getSubKeyVal(self, prefix):
+
+        if not prefix or not isinstance(prefix, str):
+            mesg = 'SafeKeyVal.getSubKeyVal() requires a string prefix of at least one character.'
+            raise s_exc.BadArg(mesg, prefix=prefix)
+
+        if self.prefix:
+            prefix = self.prefix + prefix
+
+        return SafeKeyVal(self.slab, self.name, prefix=prefix)
+
+    def reqValidName(self, name):
+
+        _name = name.encode('utf-8')
+
+        if self._prefix:
+            _name = self._prefix + _name
+
+        if len(_name) > 511:
+            maxlen = 511 - self.preflen
+            mesg = f'SafeKeyVal keys with prefix {self.prefix} must be less < {maxlen} bytes in length.'
+            raise s_exc.BadArg(mesg, prefix=self.prefix, name=name[:1024])
+        return _name
+
+    def get(self, name, defv=None):
+        '''
+        Get the value for a key.
+
+        Note:
+            This may only be used for keys < 512 characters in length.
+        '''
+        name = self.reqValidName(name)
+
+        if (byts := self.slab.get(name, db=self.valudb)) is None:
+            return defv
+        return s_msgpack.un(byts)
+
+    def set(self, name, valu):
+
+        name = self.reqValidName(name)
+
+        self.slab.put(name, s_msgpack.en(valu), db=self.valudb)
+        return valu
+
+    def pop(self, name, defv=None):
+
+        name = self.reqValidName(name)
+
+        if (byts := self.slab.pop(name, db=self.valudb)) is not None:
+            return s_msgpack.un(byts)
+        return defv
+
+    def delete(self, name):
+        '''
+        Delete a key.
+        '''
+        name = self.reqValidName(name)
+        return self.slab.delete(name, db=self.valudb)
+
+    async def truncate(self, pref=''):
+        '''
+        Delete all keys.
+        '''
+        pref = self.reqValidName(pref)
+
+        if not pref:
+            genr = self.slab.scanKeys(db=self.valudb)
+        else:
+            genr = self.slab.scanKeysByPref(pref, db=self.valudb)
+
+        for lkey in genr:
+            self.slab.delete(lkey, db=self.valudb)
+            await asyncio.sleep(0)
+
+    def items(self, pref=''):
+
+        pref = self.reqValidName(pref)
+
+        if not pref:
+            genr = self.slab.scanByFull(db=self.valudb)
+        else:
+            genr = self.slab.scanByPref(pref, db=self.valudb)
+
+        if self.prefix:
+            for lkey, byts in genr:
+                yield lkey[self.preflen:].decode('utf8'), s_msgpack.un(byts)
+            return
+
+        for lkey, byts in genr:
+            yield lkey.decode('utf8'), s_msgpack.un(byts)
+
+    def keys(self, pref=''):
+
+        pref = self.reqValidName(pref)
+
+        if not pref:
+            genr = self.slab.scanKeys(db=self.valudb)
+        else:
+            genr = self.slab.scanKeysByPref(pref, db=self.valudb)
+
+        if self.prefix:
+            for lkey in genr:
+                yield lkey[self.preflen:].decode('utf8')
+            return
+
+        for lkey in genr:
+            yield lkey.decode('utf8')
+
+    def values(self, pref=''):
+
+        pref = self.reqValidName(pref)
+
+        if not pref:
+            genr = self.slab.scanByFull(db=self.valudb)
+        else:
+            genr = self.slab.scanByPref(pref, db=self.valudb)
+
+        for lkey, byts in genr:
+            yield s_msgpack.un(byts)
+
 class SlabAbrv:
     '''
     A utility for translating arbitrary bytes into fixed with id bytes
@@ -696,7 +838,7 @@ class Slab(s_base.Base):
                 'recovering': slab.recovering,
                 'maxsize': slab.maxsize,
                 'growsize': slab.growsize,
-                'mapasync': slab.mapasync,
+                'mapasync': True,
 
             })
         return retn
@@ -709,6 +851,8 @@ class Slab(s_base.Base):
         kwargs.setdefault('lockmemory', False)
         kwargs.setdefault('map_async', True)
 
+        assert kwargs.get('map_async')
+
         opts = kwargs
 
         self.path = path
@@ -753,8 +897,6 @@ class Slab(s_base.Base):
                 logger.info(f'SYN_LOCKMEM_DISABLE envar set, skipping lockmem for {self.path}')
                 self.lockmemory = False
 
-        self.mapasync = opts.setdefault('map_async', True)
-
         self.mapsize = _mapsizeround(mapsize)
         if self.maxsize is not None:
             self.mapsize = min(self.mapsize, self.maxsize)
@@ -845,6 +987,13 @@ class Slab(s_base.Base):
         self.onfini(mq)
         return mq
 
+    def getSafeKeyVal(self, name, prefix='', create=True):
+        if not create and not self.dbexists(name):
+            mesg = f'Database {name=} does not exist.'
+            raise s_exc.BadArg(mesg=mesg)
+
+        return SafeKeyVal(self, name, prefix=prefix)
+
     def statinfo(self):
         return {
             'locking_memory': self.locking_memory,  # whether the memory lock loop was started and hasn't ended

synapse/lib/modelrev.py

@@ -821,7 +821,7 @@ class ModelRev:
         # that we are not able to rev ourselves and bail...
 
         layers = []
-        for layr in self.core.layers.values():
+        for layr in list(self.core.layers.values()):
 
             if layr.fresh:
                 await layr.setModelVers(version)