synapse 2.176.0__py311-none-any.whl → 2.177.0__py311-none-any.whl

synapse/lib/hiveauth.py

@@ -277,7 +277,6 @@ class Auth(s_nexus.Pusher):
 
         return user
 
-    @s_nexus.Pusher.onPushAuto('user:name')
     async def setUserName(self, iden, name):
         if not isinstance(name, str):
             raise s_exc.BadArg(mesg='setUserName() name must be a string')
@@ -302,7 +301,6 @@ class Auth(s_nexus.Pusher):
         }
         await self.feedBeholder('user:name', beheld)
 
-    @s_nexus.Pusher.onPushAuto('role:name')
     async def setRoleName(self, iden, name):
         if not isinstance(name, str):
             raise s_exc.BadArg(mesg='setRoleName() name must be a string')
@@ -346,7 +344,6 @@ class Auth(s_nexus.Pusher):
 
             await self.fire('cell:beholder', **behold)
 
-    @s_nexus.Pusher.onPushAuto('user:info')
     async def setUserInfo(self, iden, name, valu, gateiden=None, logged=True, mesg=None):
 
         user = await self.reqUser(iden)
@@ -373,7 +370,6 @@ class Auth(s_nexus.Pusher):
         # since any user info *may* effect auth
         user.clearAuthCache()
 
-    @s_nexus.Pusher.onPushAuto('role:info')
     async def setRoleInfo(self, iden, name, valu, gateiden=None, logged=True, mesg=None):
         role = await self.reqRole(iden)
 
@@ -528,7 +524,6 @@ class Auth(s_nexus.Pusher):
 
         return user
 
-    @s_nexus.Pusher.onPush('user:add')
     async def _addUser(self, iden, name):
 
         user = self.usersbyname.get(name)
@@ -554,7 +549,6 @@ class Auth(s_nexus.Pusher):
 
         return self.role(iden)
 
-    @s_nexus.Pusher.onPush('role:add')
     async def _addRole(self, iden, name):
 
         role = self.rolesbyname.get(name)
@@ -574,7 +568,6 @@ class Auth(s_nexus.Pusher):
         await self.reqUser(iden)
         return await self._push('user:del', iden)
 
-    @s_nexus.Pusher.onPush('user:del')
     async def _delUser(self, iden):
 
         if iden == self.rootuser.iden:
@@ -608,7 +601,6 @@ class Auth(s_nexus.Pusher):
         await self.reqRole(iden)
         return await self._push('role:del', iden)
 
-    @s_nexus.Pusher.onPush('role:del')
     async def _delRole(self, iden):
 
         if iden == self.allrole.iden:
@@ -838,10 +830,7 @@ class HiveRole(HiveRuler):
         }
 
     async def _setRulrInfo(self, name, valu, gateiden=None, nexs=True, mesg=None):
-        if nexs:
-            return await self.auth.setRoleInfo(self.iden, name, valu, gateiden=gateiden, mesg=mesg)
-        else:
-            return await self.auth._hndlsetRoleInfo(self.iden, name, valu, gateiden=gateiden, logged=nexs, mesg=mesg)
+        return await self.auth.setRoleInfo(self.iden, name, valu, gateiden=gateiden, logged=nexs, mesg=mesg)
 
     async def setName(self, name):
         return await self.auth.setRoleName(self.iden, name)
@@ -929,10 +918,7 @@ class HiveUser(HiveRuler):
         }
 
     async def _setRulrInfo(self, name, valu, gateiden=None, nexs=True, mesg=None):
-        if nexs:
-            return await self.auth.setUserInfo(self.iden, name, valu, gateiden=gateiden, mesg=mesg)
-        else:
-            return await self.auth._hndlsetUserInfo(self.iden, name, valu, gateiden=gateiden, logged=nexs, mesg=mesg)
+        return await self.auth.setUserInfo(self.iden, name, valu, gateiden=gateiden, logged=nexs, mesg=mesg)
 
     async def setName(self, name):
         return await self.auth.setUserName(self.iden, name)
@@ -1232,10 +1218,7 @@ class HiveUser(HiveRuler):
 
         roles.remove(role.iden)
         mesg = {'name': 'role:revoke', 'iden': self.iden, 'role': role.pack()}
-        if nexs:
-            await self.auth.setUserInfo(self.iden, 'roles', roles, mesg=mesg)
-        else:
-            await self.auth._hndlsetUserInfo(self.iden, 'roles', roles, logged=nexs, mesg=mesg)
+        await self.auth.setUserInfo(self.iden, 'roles', roles, logged=nexs, mesg=mesg)
 
     def isLocked(self):
         return self.info.get('locked')
@@ -1271,18 +1254,12 @@ class HiveUser(HiveRuler):
     async def setAdmin(self, admin, gateiden=None, logged=True):
         if not isinstance(admin, bool):
             raise s_exc.BadArg(mesg='setAdmin requires a boolean')
-        if logged:
-            await self.auth.setUserInfo(self.iden, 'admin', admin, gateiden=gateiden)
-        else:
-            await self.auth._hndlsetUserInfo(self.iden, 'admin', admin, gateiden=gateiden, logged=logged)
+        await self.auth.setUserInfo(self.iden, 'admin', admin, gateiden=gateiden, logged=logged)
 
     async def setLocked(self, locked, logged=True):
         if not isinstance(locked, bool):
             raise s_exc.BadArg(mesg='setLocked requires a boolean')
-        if logged:
-            await self.auth.setUserInfo(self.iden, 'locked', locked)
-        else:
-            await self.auth._hndlsetUserInfo(self.iden, 'locked', locked, logged=logged)
+        await self.auth.setUserInfo(self.iden, 'locked', locked, logged=logged)
 
     async def setArchived(self, archived):
         if not isinstance(archived, bool):
@@ -1347,7 +1324,4 @@ class HiveUser(HiveRuler):
             shadow = await s_passwd.getShadowV2(passwd=passwd)
         else:
             raise s_exc.BadArg(mesg='Password must be a string')
-        if nexs:
-            await self.auth.setUserInfo(self.iden, 'passwd', shadow)
-        else:
-            await self.auth._hndlsetUserInfo(self.iden, 'passwd', shadow, logged=nexs)
+        await self.auth.setUserInfo(self.iden, 'passwd', shadow, logged=nexs)

synapse/lib/layer.py

@@ -2034,7 +2034,7 @@ class Layer(s_nexus.Pusher):
                                                  'stortype': stortype})
 
     async def pack(self):
-        ret = self.layrinfo.pack()
+        ret = deepcopy(self.layrinfo)
         if ret.get('mirror'):
             ret['mirror'] = s_urlhelp.sanitizeUrl(ret['mirror'])
         ret['offset'] = await self.getEditIndx()
@@ -2810,9 +2810,11 @@ class Layer(s_nexus.Pusher):
 
         # TODO when we can set more props, we may need to parse values.
         if valu is None:
-            await self.layrinfo.pop(name)
+            self.layrinfo.pop(name, None)
         else:
-            await self.layrinfo.set(name, valu)
+            self.layrinfo[name] = valu
+
+        self.core.layerdefs.set(self.iden, self.layrinfo)
 
         await self.core.feedBeholder('layer:set', {'iden': self.iden, 'name': name, 'valu': valu}, gates=[self.iden])
         return valu
@@ -4453,7 +4455,8 @@ class Layer(s_nexus.Pusher):
 
     @s_nexus.Pusher.onPush('layer:set:modelvers')
     async def _setModelVers(self, vers):
-        await self.layrinfo.set('model:version', vers)
+        self.layrinfo['model:version'] = vers
+        self.core.layerdefs.set(self.iden, self.layrinfo)
 
     async def getStorNodes(self):
         '''

synapse/lib/link.py

@@ -21,7 +21,7 @@ async def connect(host, port, ssl=None, hostname=None, linkinfo=None):
     '''
     Async connect and return a Link().
     '''
-    info = {'host': host, 'port': port, 'ssl': ssl, 'hostname': hostname}
+    info = {'host': host, 'port': port, 'ssl': ssl, 'hostname': hostname, 'tls': bool(ssl)}
     if linkinfo is not None:
         info.update(linkinfo)
 
@@ -137,6 +137,25 @@ class Link(s_base.Base):
 
         self.info = info
 
+        self._addrinfo = {}
+        # _addrinfo is populated in this order so that as first hit tls links (prod deployments)
+        # then unix links (unit tests with local sockets, container healthchecks, local tools )
+        # then tcp links ( unit tests and legacy deployments )
+        if self.info.get('tls'):
+            self._addrinfo['family'] = 'tls'
+            self._addrinfo['addr'] = self.sock.getpeername()
+        elif self.info.get('unix'):
+            self._addrinfo['family'] = 'unix'
+            # Unix sockets don't use getpeername
+            self._addrinfo['addr'] = self.sock.getsockname()
+        else:
+            self._addrinfo['family'] = 'tcp'
+            self._addrinfo['addr'] = self.sock.getpeername()
+        if self.sock.family == socket.AF_INET:
+            self._addrinfo['ipver'] = 'ipv4'
+        elif self.sock.family == socket.AF_INET6:
+            self._addrinfo['ipver'] = 'ipv6'
+
         self.unpk = s_msgpack.Unpk()
 
         async def fini():
@@ -212,22 +231,7 @@ class Link(s_base.Base):
         '''
         Get a summary of address information related to the link.
         '''
-        ret = {'family': 'tcp',
-               'addr': self.sock.getpeername(),
-               }
-        # Set family information
-        if self.info.get('unix'):
-            ret['family'] = 'unix'
-            # Unix sockets don't use getpeername
-            ret['addr'] = self.sock.getsockname()
-        elif self.info.get('tls'):
-            ret['family'] = 'tls'
-        # Set ipver if needed
-        if self.sock.family == socket.AF_INET:
-            ret['ipver'] = 'ipv4'
-        if self.sock.family == socket.AF_INET6:
-            ret['ipver'] = 'ipv6'
-        return ret
+        return dict(self._addrinfo)
 
     async def send(self, byts):
         self.writer.write(byts)

synapse/lib/lmdbslab.py

@@ -166,6 +166,148 @@ class SlabDict:
         self.set(name, curv)
         return curv
 
+class SafeKeyVal:
+    '''
+    Key/value storage that does not store items in memory and ensures keys are < 512 characters in length.
+
+        Note:
+            The key size limitation includes the length of any prefixes added by
+            using getSubKeyVal().
+    '''
+    def __init__(self, slab, name, prefix=''):
+
+        self.prefix = prefix
+        self._prefix = prefix.encode('utf8')
+        self.preflen = len(self._prefix)
+
+        if self.preflen > 510:
+            mesg = 'SafeKeyVal prefix lengths must be < 511 bytes.'
+            raise s_exc.BadArg(mesg, prefix=self._prefix[:1024])
+
+        self.name = name
+        self.slab = slab
+        self.valudb = slab.initdb(name)
+
+    def getSubKeyVal(self, prefix):
+
+        if not prefix or not isinstance(prefix, str):
+            mesg = 'SafeKeyVal.getSubKeyVal() requires a string prefix of at least one character.'
+            raise s_exc.BadArg(mesg, prefix=prefix)
+
+        if self.prefix:
+            prefix = self.prefix + prefix
+
+        return SafeKeyVal(self.slab, self.name, prefix=prefix)
+
+    def reqValidName(self, name):
+
+        _name = name.encode('utf-8')
+
+        if self._prefix:
+            _name = self._prefix + _name
+
+        if len(_name) > 511:
+            maxlen = 511 - self.preflen
+            mesg = f'SafeKeyVal keys with prefix {self.prefix} must be less < {maxlen} bytes in length.'
+            raise s_exc.BadArg(mesg, prefix=self.prefix, name=name[:1024])
+        return _name
+
+    def get(self, name, defv=None):
+        '''
+        Get the value for a key.
+
+        Note:
+            This may only be used for keys < 512 characters in length.
+        '''
+        name = self.reqValidName(name)
+
+        if (byts := self.slab.get(name, db=self.valudb)) is None:
+            return defv
+        return s_msgpack.un(byts)
+
+    def set(self, name, valu):
+
+        name = self.reqValidName(name)
+
+        self.slab.put(name, s_msgpack.en(valu), db=self.valudb)
+        return valu
+
+    def pop(self, name, defv=None):
+
+        name = self.reqValidName(name)
+
+        if (byts := self.slab.pop(name, db=self.valudb)) is not None:
+            return s_msgpack.un(byts)
+        return defv
+
+    def delete(self, name):
+        '''
+        Delete a key.
+        '''
+        name = self.reqValidName(name)
+        return self.slab.delete(name, db=self.valudb)
+
+    async def truncate(self, pref=''):
+        '''
+        Delete all keys.
+        '''
+        pref = self.reqValidName(pref)
+
+        if not pref:
+            genr = self.slab.scanKeys(db=self.valudb)
+        else:
+            genr = self.slab.scanKeysByPref(pref, db=self.valudb)
+
+        for lkey in genr:
+            self.slab.delete(lkey, db=self.valudb)
+            await asyncio.sleep(0)
+
+    def items(self, pref=''):
+
+        pref = self.reqValidName(pref)
+
+        if not pref:
+            genr = self.slab.scanByFull(db=self.valudb)
+        else:
+            genr = self.slab.scanByPref(pref, db=self.valudb)
+
+        if self.prefix:
+            for lkey, byts in genr:
+                yield lkey[self.preflen:].decode('utf8'), s_msgpack.un(byts)
+            return
+
+        for lkey, byts in genr:
+            yield lkey.decode('utf8'), s_msgpack.un(byts)
+
+    def keys(self, pref=''):
+
+        pref = self.reqValidName(pref)
+
+        if not pref:
+            genr = self.slab.scanKeys(db=self.valudb)
+        else:
+            genr = self.slab.scanKeysByPref(pref, db=self.valudb)
+
+        if self.prefix:
+            for lkey in genr:
+                yield lkey[self.preflen:].decode('utf8')
+            return
+
+        for lkey in genr:
+            yield lkey.decode('utf8')
+
+    def values(self, pref=''):
+
+        pref = self.reqValidName(pref)
+
+        if not pref:
+            genr = self.slab.scanByFull(db=self.valudb)
+        else:
+            genr = self.slab.scanByPref(pref, db=self.valudb)
+
+        for lkey, byts in genr:
+            yield s_msgpack.un(byts)
+
 class SlabAbrv:
     '''
     A utility for translating arbitrary bytes into fixed with id bytes
@@ -845,6 +987,13 @@ class Slab(s_base.Base):
         self.onfini(mq)
         return mq
 
+    def getSafeKeyVal(self, name, prefix='', create=True):
+        if not create and not self.dbexists(name):
+            mesg = f'Database {name=} does not exist.'
+            raise s_exc.BadArg(mesg=mesg)
+
+        return SafeKeyVal(self, name, prefix=prefix)
+
     def statinfo(self):
         return {
             'locking_memory': self.locking_memory,  # whether the memory lock loop was started and hasn't ended

synapse/lib/modelrev.py

@@ -821,7 +821,7 @@ class ModelRev:
         # that we are not able to rev ourselves and bail...
 
         layers = []
-        for layr in self.core.layers.values():
+        for layr in list(self.core.layers.values()):
 
             if layr.fresh:
                 await layr.setModelVers(version)

synapse/lib/schemas.py

@@ -288,6 +288,93 @@ _stormPoolOptsSchema = {
 }
 reqValidStormPoolOpts = s_config.getJsValidator(_stormPoolOptsSchema)
 
+_authRulesSchema = {
+    'type': 'array',
+    'items': {
+        'type': 'array',
+        'items': [
+            {'type': 'boolean'},
+            {'type': 'array', 'items': {'type': 'string'}},
+        ],
+        'minItems': 2,
+        'maxItems': 2,
+    }
+}
+reqValidRules = s_config.getJsValidator(_authRulesSchema)
+
+_passwdPolicySchema = {
+    'type': 'object',
+    'properties': {
+        'complexity': {
+            'type': ['object', 'null'],
+            'properties': {
+                'length': {
+                    'type': ['number', 'null'],
+                    'minimum': 1,
+                    'description': 'Minimum password character length.',
+                },
+                'sequences': {
+                    'type': ['number', 'null'],
+                    'minimum': 2,
+                    'description': 'Maximum sequence length in a password. Sequences can be letters or number, forward or reverse.',
+                },
+                'upper:count': {
+                    'type': ['number', 'null'],
+                    'description': 'The minimum number of uppercase characters required in password.',
+                },
+                'upper:valid': {
+                    'type': ['string', 'null'],
+                    'minLength': 1,
+                    'description': 'All valid uppercase characters.',
+                },
+                'lower:count': {
+                    'type': ['number', 'null'],
+                    'minimum': 0,
+                    'description': 'The minimum number of lowercase characters required in password.',
+                },
+                'lower:valid': {
+                    'type': ['string', 'null'],
+                    'minLength': 1,
+                    'description': 'All valid lowercase characters.',
+                },
+                'special:count': {
+                    'type': ['number', 'null'],
+                    'minimum': 0,
+                    'description': 'The minimum number of special characters required in password.',
+                },
+                'special:valid': {
+                    'type': ['string', 'null'],
+                    'minLength': 1,
+                    'description': 'All valid special characters.',
+                },
+                'number:count': {
+                    'type': ['number', 'null'],
+                    'minimum': 0,
+                    'description': 'The minimum number of digit characters required in password.',
+                },
+                'number:valid': {
+                    'type': ['string', 'null'],
+                    'minLength': 1,
+                    'description': 'All valid digit characters.',
+                },
+            },
+            'additionalProperties': False,
+        },
+        'attempts': {
+            'type': ['number', 'null'],
+            'minimum': 1,
+            'description': 'Maximum number of incorrect attempts before locking user account.',
+        },
+        'previous': {
+            'type': ['number', 'null'],
+            'minimum': 1,
+            'description': 'Number of previous passwords to disallow.',
+        },
+    },
+    'additionalProperties': False,
+}
+reqValidPasswdPolicy = s_config.getJsValidator(_passwdPolicySchema)
+
 # These types are order sensitive
 _changelogTypes = {'migration': 'Automatic Migrations',
                    'model': 'Model Changes',
@@ -318,3 +405,52 @@ _changelogSchema = {
     'required': ['type', 'desc']
 }
 _reqChanglogSchema = s_config.getJsValidator(_changelogSchema)
+
+tabularConfSchema = {
+    'type': 'object',
+    'properties': {
+        'separators': {
+            'type': 'object',
+            'properties': {
+                'row:outline': {'type': 'boolean', 'default': False,
+                                'description': 'Add the row separator before the header data and after each row.'},
+                'column:outline': {'type': 'boolean', 'default': False,
+                                   'description': 'Add the column separator to the beginning and end of each row.'},
+                'header:row': {'type': 'string', 'default': '=',
+                               'description': 'The string to use to create a separator row when printing the header.'},
+                'data:row': {'type': 'string', 'default': '-',
+                             'description': 'The string to use to create a separator row when printing data rows.'},
+                'column': {'type': 'string', 'default': '|',
+                           'description': 'The string to use to separate columns.'},
+            },
+            'additionalProperties': False,
+        },
+        'columns': {
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                    'name': {'type': 'string',
+                             'description': 'The column name which will be used in the header row.'},
+                    'width': {'type': 'number', 'default': None, 'exclusiveMinimum': 0,
+                              'description': 'If not provided each cell will expand to fit the data.'},
+                    'justify': {'type': 'string', 'default': 'left', 'enum': ['left', 'center', 'right'],
+                                'description': 'Justification for the header titles and data rows.'},
+                    'overflow': {'type': 'string', 'default': 'trim', 'enum': ['wrap', 'trim'],
+                                 'description': 'For text exceeding the width, '
+                                                'either wrap text in multiple lines or trim and append "...".'},
+                    'newlines': {'type': 'string', 'default': 'replace', 'enum': ['replace', 'split'],
+                                 'description': 'Replace newlines with a space or split into multiple lines.'
+                                               'Split is only applied if width is undefined.'},
+                },
+                'required': ['name'],
+                'minItems': 1,
+                'additionalProperties': False,
+            },
+        },
+    },
+    'required': ['columns'],
+    'additionalProperties': False,
+}
+
+reqValidTabularConf = s_config.getJsValidator(tabularConfSchema)