synapse 2.180.1__py311-none-any.whl → 2.182.0__py311-none-any.whl

synapse/lib/ast.py

@@ -1369,14 +1369,14 @@ class SwitchCase(Oper):
         self.defcase = None
 
         for cent in self.kids[1:]:
+            *vals, subq = cent.kids
 
-            # if they only have one kid, it's a default case.
-            if len(cent.kids) == 1:
-                self.defcase = cent.kids[0]
+            if cent.defcase:
+                self.defcase = subq
                 continue
 
-            valu = cent.kids[0].value()
-            self.cases[valu] = cent.kids[1]
+            for valu in vals:
+                self.cases[valu.value()] = subq
 
     async def run(self, runt, genr):
         count = 0
@@ -1410,9 +1410,10 @@ class SwitchCase(Oper):
             async for item in subq.inline(runt, s_common.agen()):
                 yield item
 
-
 class CaseEntry(AstNode):
-    pass
+    def __init__(self, astinfo, kids=(), defcase=False):
+        AstNode.__init__(self, astinfo, kids=kids)
+        self.defcase = defcase
 
 class LiftOper(Oper):
 
@@ -4899,6 +4900,7 @@ class Function(AstNode):
                 subr.funcscope = True
 
                 try:
+                    await asyncio.sleep(0)
                     async for item in subr.execute():
                         await asyncio.sleep(0)
 
@@ -4913,10 +4915,13 @@ class Function(AstNode):
                 subr.funcscope = True
                 try:
                     if self.hasemit:
+                        await asyncio.sleep(0)
                         async with contextlib.aclosing(await subr.emitter()) as agen:
                             async for item in agen:
                                 yield item
+                                await asyncio.sleep(0)
                     else:
+                        await asyncio.sleep(0)
                         async with contextlib.aclosing(subr.execute()) as agen:
                             async for node, path in agen:
                                 yield node, path

synapse/lib/auth.py

@@ -579,11 +579,28 @@ class Auth(s_nexus.Pusher):
         await self.feedBeholder('user:add', user.pack())
 
     async def addRole(self, name, iden=None):
+        '''
+        Add a Role to the Auth system.
+
+        Args:
+            name (str): The name of the role.
+            iden (str): A optional iden to use as the role iden.
+
+        Returns:
+            Role: A Role.
+        '''
         if self.roleidenbynamecache.get(name) is not None:
             raise s_exc.DupRoleName(mesg=f'Duplicate role name, {name=} already exists.', name=name)
 
         if iden is None:
             iden = s_common.guid()
+        else:
+            if not s_common.isguid(iden):
+                raise s_exc.BadArg(name='iden', arg=iden, mesg=f'Argument {iden} it not a valid iden.')
+
+            if self.rolebyidencache.get(iden) is not None:
+                raise s_exc.DupIden(name=name, iden=iden,
+                                    mesg=f'Role already exists for {iden=}.')
 
         await self._push('role:add', iden, name)
 

synapse/lib/cell.py

@@ -443,8 +443,8 @@ class CellApi(s_base.Base):
         return await self.cell.delUser(iden)
 
     @adminapi(log=True)
-    async def addRole(self, name):
-        return await self.cell.addRole(name)
+    async def addRole(self, name, iden=None):
+        return await self.cell.addRole(name, iden=iden)
 
     @adminapi(log=True)
     async def delRole(self, iden):
@@ -1637,12 +1637,10 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                        f'{disk.free / disk.total * 100:.2f}%), setting Cell to read-only.'
                 logger.error(mesg)
 
-            elif nexsroot.readonly:
-
-                await nexsroot.delWriteHold(diskspace)
+            elif nexsroot.readonly and await nexsroot.delWriteHold(diskspace):
 
                 mesg = f'Free space on {self.dirn} above minimum threshold (currently ' \
-                       f'{disk.free / disk.total * 100:.2f}%), re-enabling writes.'
+                       f'{disk.free / disk.total * 100:.2f}%), removing free space write hold.'
                 logger.error(mesg)
 
             await self._checkspace.timewait(timeout=self.FREE_SPACE_CHECK_FREQ)
@@ -1851,7 +1849,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
     @s_nexus.Pusher.onPushAuto('drive:data:set')
     async def setDriveData(self, iden, versinfo, data):
-        return self.drive.setItemData(iden, versinfo, data)
+        return await self.drive.setItemData(iden, versinfo, data)
 
     async def delDriveData(self, iden, vers=None):
         if vers is None:
@@ -2883,8 +2881,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         logger.info(f'Deleted user={name}',
                    extra=await self.getLogExtra(target_user=iden, target_username=name, status='DELETE'))
 
-    async def addRole(self, name):
-        role = await self.auth.addRole(name)
+    async def addRole(self, name, iden=None):
+        role = await self.auth.addRole(name, iden=iden)
         logger.info(f'Added role={name}',
                     extra=await self.getLogExtra(target_role=role.iden, target_rolename=role.name, status='CREATE'))
         return role.pack()

synapse/lib/chop.py

@@ -1,5 +1,4 @@
 import binascii
-import ipaddress
 
 import regex
 

synapse/lib/drive.py

@@ -147,7 +147,7 @@ class Drive(s_base.Base):
             rows.append((LKEY_INFO + oldb, s_msgpack.en(oldpinfo)))
 
         self.slab.delete(LKEY_DIRN + oldb + oldname.encode(), db=self.dbname)
-        self.slab.putmulti(rows, db=self.dbname)
+        await self.slab.putmulti(rows, db=self.dbname)
 
         pathinfo.append(info)
         return pathinfo
@@ -167,7 +167,7 @@ class Drive(s_base.Base):
         if byts is not None:
             return s_msgpack.un(byts)
 
-    def _addStepInfo(self, parbidn, parinfo, info):
+    async def _addStepInfo(self, parbidn, parinfo, info):
 
         newbidn = s_common.uhex(info.get('iden'))
 
@@ -190,7 +190,7 @@ class Drive(s_base.Base):
             typekey = LKEY_INFO_BYTYPE + typename.encode() + b'\x00' + newbidn
             rows.append((typekey, b'\x01'))
 
-        self.slab.putmulti(rows, db=self.dbname)
+        await self.slab.putmulti(rows, db=self.dbname)
 
     def setItemPerm(self, iden, perm):
         return self._setItemPerm(s_common.uhex(iden), perm)
@@ -290,7 +290,7 @@ class Drive(s_base.Base):
             mesg = f'A drive entry with ID {iden} already exists.'
             raise s_exc.DupIden(mesg=mesg)
 
-        self._addStepInfo(parbidn, parinfo, info)
+        await self._addStepInfo(parbidn, parinfo, info)
 
         pathinfo.append(info)
         return pathinfo
@@ -384,10 +384,10 @@ class Drive(s_base.Base):
 
             yield info
 
-    def setItemData(self, iden, versinfo, data):
-        return self._setItemData(s_common.uhex(iden), versinfo, data)
+    async def setItemData(self, iden, versinfo, data):
+        return await self._setItemData(s_common.uhex(iden), versinfo, data)
 
-    def _setItemData(self, bidn, versinfo, data):
+    async def _setItemData(self, bidn, versinfo, data):
 
         info = self._reqItemInfo(bidn)
 
@@ -419,7 +419,7 @@ class Drive(s_base.Base):
             info.update(versinfo)
             rows.append((LKEY_INFO + bidn, s_msgpack.en(info)))
 
-        self.slab.putmulti(rows, db=self.dbname)
+        await self.slab.putmulti(rows, db=self.dbname)
 
         return info, versinfo
 

synapse/lib/layer.py

@@ -62,7 +62,6 @@ import shutil
 import struct
 import asyncio
 import logging
-import ipaddress
 import contextlib
 import collections
 
@@ -87,6 +86,8 @@ import synapse.lib.slabseqn as s_slabseqn
 
 from synapse.lib.msgpack import deepcopy
 
+ipaddress = s_common.ipaddress
+
 logger = logging.getLogger(__name__)
 
 import synapse.lib.msgpack as s_msgpack
@@ -2156,7 +2157,7 @@ class Layer(s_nexus.Pusher):
 
                     if len(tostor) >= 10000:
                         logger.warning(f'...syncing 10k nodes @{count}')
-                        self.layrslab.putmulti(tostor, db=self.bybuidv3)
+                        await self.layrslab.putmulti(tostor, db=self.bybuidv3)
                         tostor.clear()
 
                 lastbuid = buid
@@ -2197,7 +2198,7 @@ class Layer(s_nexus.Pusher):
             count += 1
             tostor.append((lastbuid, s_msgpack.en(sode)))
         if tostor:
-            self.layrslab.putmulti(tostor, db=self.bybuidv3)
+            await self.layrslab.putmulti(tostor, db=self.bybuidv3)
 
         logger.warning('...removing old bybuid index')
         self.layrslab.dropdb('bybuid')
@@ -2623,8 +2624,8 @@ class Layer(s_nexus.Pusher):
 
         logger.warning(f'Adding n1+n2 index to edges in layer {self.iden}')
 
-        def commit():
-            self.layrslab.putmulti(putkeys, db=self.edgesn1n2)
+        async def commit():
+            await self.layrslab.putmulti(putkeys, db=self.edgesn1n2)
             putkeys.clear()
 
         putkeys = []
@@ -2635,10 +2636,10 @@ class Layer(s_nexus.Pusher):
 
             putkeys.append((n1buid + n2buid, venc))
             if len(putkeys) > MIGR_COMMIT_SIZE:
-                commit()
+                await commit()
 
         if len(putkeys):
-            commit()
+            await commit()
 
         self.meta.set('version', 10)
         self.layrvers = 10
@@ -2649,8 +2650,8 @@ class Layer(s_nexus.Pusher):
 
         logger.warning(f'Adding byform index to layer {self.iden}')
 
-        def commit():
-            self.layrslab.putmulti(putkeys, db=self.byform)
+        async def commit():
+            await self.layrslab.putmulti(putkeys, db=self.byform)
             putkeys.clear()
 
         putkeys = []
@@ -2662,10 +2663,10 @@ class Layer(s_nexus.Pusher):
             putkeys.append((abrv, buid))
 
             if len(putkeys) > MIGR_COMMIT_SIZE:
-                commit()
+                await commit()
 
         if putkeys:
-            commit()
+            await commit()
 
         self.meta.set('version', 11)
         self.layrvers = 11
@@ -2899,7 +2900,7 @@ class Layer(s_nexus.Pusher):
             self.buidcache[buid] = sode
             kvlist.append((buid, s_msgpack.en(sode)))
 
-        self.layrslab.putmulti(kvlist, db=self.bybuidv3)
+        self.layrslab._putmulti(kvlist, db=self.bybuidv3)
         self.dirty.clear()
 
     def getStorNodeCount(self):
@@ -3050,6 +3051,47 @@ class Layer(s_nexus.Pusher):
 
         return count
 
+    async def iterPropValues(self, formname, propname, stortype):
+        try:
+            abrv = self.getPropAbrv(formname, propname)
+        except s_exc.NoSuchAbrv:
+            return
+
+        if stortype & 0x8000:
+            stortype = STOR_TYPE_MSGP
+
+        stor = self.stortypes[stortype]
+        abrvlen = len(abrv)
+
+        async for lkey in s_coro.pause(self.layrslab.scanKeysByPref(abrv, db=self.byprop, nodup=True)):
+
+            indx = lkey[abrvlen:]
+            valu = stor.decodeIndx(indx)
+            if valu is not s_common.novalu:
+                yield indx, valu
+                continue
+
+            buid = self.layrslab.get(lkey, db=self.byprop)
+            if buid is not None:
+                sode = self._getStorNode(buid)
+                if sode is not None:
+                    if propname is None:
+                        valt = sode.get('valu')
+                    else:
+                        valt = sode['props'].get(propname)
+
+                    if valt is not None:
+                        yield indx, valt[0]
+
+    async def iterPropIndxBuids(self, formname, propname, indx):
+        try:
+            abrv = self.getPropAbrv(formname, propname)
+        except s_exc.NoSuchAbrv:
+            return
+
+        async for _, buid in s_coro.pause(self.layrslab.scanByDups(abrv + indx, db=self.byprop)):
+            yield buid
+
     async def liftByTag(self, tag, form=None, reverse=False):
 
         try:
@@ -3769,7 +3811,7 @@ class Layer(s_nexus.Pusher):
             kvpairs.append((tp_abrv + indx, buid))
             kvpairs.append((ftp_abrv + indx, buid))
 
-        self.layrslab.putmulti(kvpairs, db=self.bytagprop)
+        await self.layrslab.putmulti(kvpairs, db=self.bytagprop)
 
         return (
             (EDIT_TAGPROP_SET, (tag, prop, valu, oldv, stortype), ()),

synapse/lib/lmdbslab.py

@@ -417,7 +417,7 @@ class HotKeyVal(s_base.Base):
         if not tups:
             return
 
-        self.slab.putmulti(tups, db=self.db)
+        self.slab._putmulti(tups, db=self.db)
         self.dirty.clear()
 
     def pack(self):
@@ -1651,7 +1651,28 @@ class Slab(s_base.Base):
         except lmdb.MapFullError:
             return self._handle_mapfull()
 
-    def putmulti(self, kvpairs, dupdata=False, append=False, db=None):
+    async def putmulti(self, kvpairs, dupdata=False, append=False, db=None):
+
+        # Use a fast path when we have a small amount of data to prevent creating new
+        # list objects when we don't have to.
+        if isinstance(kvpairs, (list, tuple)) and len(kvpairs) <= self.max_xactops_len:
+            ret = self._putmulti(kvpairs, dupdata=dupdata, append=append, db=db)
+            await asyncio.sleep(0)
+            return ret
+
+        # Otherwise, we chunk the large data or a generator into lists no greater than
+        # max_xactops_len and allow the slab the opportunity to commit between chunks.
+        # This helps to avoid a situation where a large generator or list of kvpairs
+        # could cause a greedy commit operation from happening.
+        consumed, added = 0, 0
+        for chunk in s_common.chunks(kvpairs, self.max_xactops_len):
+            rc, ra = self._putmulti(chunk, dupdata=dupdata, append=append, db=db)
+            consumed = consumed + rc
+            added = added + ra
+            await asyncio.sleep(0)
+        return (consumed, added)
+
+    def _putmulti(self, kvpairs: list, dupdata: bool =False, append: bool =False, db: str =None):
         '''
         Returns:
             Tuple of number of items consumed, number of items added
@@ -1669,7 +1690,7 @@ class Slab(s_base.Base):
             self.dirty = True
 
             if not self.recovering:
-                self._logXactOper(self.putmulti, kvpairs, dupdata=dupdata, append=append, db=db)
+                self._logXactOper(self._putmulti, kvpairs, dupdata=dupdata, append=append, db=db)
 
             with self.xact.cursor(db=realdb) as curs:
                 return curs.putmulti(kvpairs, dupdata=dupdata, append=append)
@@ -1677,7 +1698,7 @@ class Slab(s_base.Base):
         except lmdb.MapFullError:
             return self._handle_mapfull()
 
-    def copydb(self, sourcedbname, destslab, destdbname=None, progresscb=None):
+    async def copydb(self, sourcedbname, destslab, destdbname=None, progresscb=None):
         '''
         Copy an entire database in this slab to a new database in potentially another slab.
 
@@ -1707,7 +1728,7 @@ class Slab(s_base.Base):
         rowcount = 0
 
         for chunk in s_common.chunks(self.scanByFull(db=sourcedbname), COPY_CHUNKSIZE):
-            ccount, acount = destslab.putmulti(chunk, dupdata=True, append=True, db=destdbname)
+            ccount, acount = await destslab.putmulti(chunk, dupdata=True, append=True, db=destdbname)
             if ccount != len(chunk) or acount != len(chunk):
                 raise s_exc.BadCoreStore(mesg='Unexpected number of values written')  # pragma: no cover
 

synapse/lib/modelrev.py

@@ -2,13 +2,14 @@ import regex
 import logging
 
 import synapse.exc as s_exc
+import synapse.assets as s_assets
 import synapse.common as s_common
 
 import synapse.lib.layer as s_layer
 
 logger = logging.getLogger(__name__)
 
-maxvers = (0, 2, 27)
+maxvers = (0, 2, 30)
 
 class ModelRev:
 
@@ -41,6 +42,9 @@ class ModelRev:
             ((0, 2, 25), self.revModel_0_2_25),
             ((0, 2, 26), self.revModel_0_2_26),
             ((0, 2, 27), self.revModel_0_2_27),
+            # Model revision 0.2.28 skipped
+            ((0, 2, 29), self.revModel_0_2_29),
+            ((0, 2, 30), self.revModel_0_2_30),
         )
 
     async def _uniqSortArray(self, todoprops, layers):
@@ -787,6 +791,29 @@ class ModelRev:
     async def revModel_0_2_27(self, layers):
         await self._normPropValu(layers, 'it:dev:repo:commit:id')
 
+    async def revModel_0_2_28(self, layers):
+
+        opts = {'vars': {
+            'layridens': [layr.iden for layr in layers],
+        }}
+
+        text = s_assets.getStorm('migrations', 'model-0.2.28.storm')
+        await self.runStorm(text, opts=opts)
+
+    async def revModel_0_2_29(self, layers):
+        await self._propToForm(layers, 'ou:industry:type', 'ou:industry:type:taxonomy')
+
+    async def revModel_0_2_30(self, layers):
+        await self._normFormSubs(layers, 'inet:ipv4', cmprvalu='192.0.0.0/24')
+        await self._normFormSubs(layers, 'inet:ipv6', cmprvalu='64:ff9b:1::/48')
+        await self._normFormSubs(layers, 'inet:ipv6', cmprvalu='2002::/16')
+        await self._normFormSubs(layers, 'inet:ipv6', cmprvalu='2001:1::1/128')
+        await self._normFormSubs(layers, 'inet:ipv6', cmprvalu='2001:1::2/128')
+        await self._normFormSubs(layers, 'inet:ipv6', cmprvalu='2001:3::/32')
+        await self._normFormSubs(layers, 'inet:ipv6', cmprvalu='2001:4:112::/48')
+        await self._normFormSubs(layers, 'inet:ipv6', cmprvalu='2001:20::/28')
+        await self._normFormSubs(layers, 'inet:ipv6', cmprvalu='2001:30::/28')
+
     async def runStorm(self, text, opts=None):
         '''
         Run storm code in a schedcoro and log the output messages.

synapse/lib/modules.py

@@ -7,6 +7,7 @@ coremods = (
     'synapse.models.syn.SynModule',
     'synapse.models.auth.AuthModule',
     'synapse.models.base.BaseModule',
+    'synapse.models.math.MathModule',
     'synapse.models.risk.RiskModule',
     'synapse.models.person.PsModule',
     'synapse.models.files.FileModule',

synapse/lib/nexus.py

@@ -204,7 +204,7 @@ class NexsRoot(s_base.Base):
             olddb = self.nexsslab.initdb('nexs:indx')
             self.nexsslab.dropdb(olddb)
             db = newslab.initdb('nexs:indx')
-            newslab.copydb('nexs:indx', self.nexsslab, destdbname='nexs:indx')
+            await newslab.copydb('nexs:indx', self.nexsslab, destdbname='nexs:indx')
             newslab.dropdb(db)
 
         self.nexshot.set('version', 2)

synapse/lib/node.py

@@ -664,6 +664,11 @@ class Node:
         for name in self.props.keys():
             edits.extend(await self._getPropDelEdits(name, init=True))
 
+        # Only remove nodedata if we're in a layer that doesn't have the full node
+        if self.snap.wlyr.iden != self.bylayer['ndef']:
+            async for name in self.iterDataKeys():
+                edits.append((s_layer.EDIT_NODEDATA_DEL, (name, None), ()))
+
         edits.append(
             (s_layer.EDIT_NODE_DEL, (formvalu, self.form.type.stortype), ()),
         )

synapse/lib/parser.py

@@ -69,6 +69,7 @@ terminalEnglishMap = {
     'LISTTOKN': 'unquoted list value',
     'LPAR': '(',
     'LSQB': '[',
+    'MCASEBARE': 'case multi-value',
     'MODSET': '+= or -=',
     'NONQUOTEWORD': 'unquoted value',
     'NOT': 'not',
@@ -427,29 +428,35 @@ class AstConverter(lark.Transformer):
         return s_ast.VarDeref(astinfo, kids=(kids[0], newkid))
 
     @lark.v_args(meta=True)
-    def switchcase(self, meta, kids):
-
+    def caseentry(self, meta, kids):
+        assert kids and len(kids) >= 2
         astinfo = self.metaToAstInfo(meta)
+        newkids = self._convert_children(kids)
 
-        newkids = []
+        defcase = False
 
-        it = iter(kids)
+        if len(kids) == 2 and kids[0].type == 'DEFAULTCASE':
+            defcase = True
+            # Strip off the "Const: *" node
+            newkids = [newkids[1]]
 
-        varvalu = next(it)
-        newkids.append(varvalu)
+        return s_ast.CaseEntry(astinfo, kids=newkids, defcase=defcase)
 
-        for casekid, sqkid in zip(it, it):
-            subquery = self._convert_child(sqkid)
-            if casekid.type == 'DEFAULTCASE':
-                caseinfo = self.metaToAstInfo(casekid)
-                caseentry = s_ast.CaseEntry(caseinfo, kids=[subquery])
-            else:
-                casekid = self._convert_child(casekid)
-                caseentry = s_ast.CaseEntry(casekid.astinfo, kids=[casekid, subquery])
+    @lark.v_args(meta=True)
+    def switchcase(self, meta, kids):
+        kids = self._convert_children(kids)
+
+        astinfo = self.metaToAstInfo(meta)
+
+        # Check that we only have one default case
+        defcase = [k for k in kids[1:] if k.defcase]
 
-            newkids.append(caseentry)
+        deflen = len(defcase)
+        if deflen > 1:
+            mesg = f'Switch statements cannot have more than one default case. Found {deflen}.'
+            raise self.raiseBadSyntax(mesg, astinfo)
 
-        return s_ast.SwitchCase(astinfo, newkids)
+        return s_ast.SwitchCase(astinfo, kids)
 
     @lark.v_args(meta=True)
     def liftreverse(self, meta, kids):

synapse/lib/scrape.py

@@ -7,7 +7,6 @@ import collections
 
 import idna
 import regex
-import ipaddress
 import unicodedata
 
 import synapse.exc as s_exc
@@ -21,6 +20,7 @@ import synapse.lib.msgpack as s_msgpack
 
 import synapse.lib.crypto.coin as s_coin
 
+ipaddress = s_common.ipaddress
 
 logger = logging.getLogger(__name__)
 

synapse/lib/slabseqn.py

@@ -108,7 +108,7 @@ class SlabSeqn:
     def stat(self):
         return self.slab.stat(db=self.db)
 
-    def save(self, items):
+    async def save(self, items):
         '''
         Save a series of items to a sequence.
 
@@ -136,7 +136,7 @@ class SlabSeqn:
 
             rows.append((lkey, byts))
 
-        retn = self.slab.putmulti(rows, append=True, db=self.db)
+        retn = await self.slab.putmulti(rows, append=True, db=self.db)
         took = s_common.mononow() - abstick
 
         assert retn, "Not adding the largest indices"