synapse 2.197.0__py311-none-any.whl → 2.198.0__py311-none-any.whl

synapse/axon.py

@@ -1017,6 +1017,7 @@ class Axon(s_cell.Cell):
         '''
         for item in self.axonhist.carve(tick, tock=tock):
             yield item
+            await asyncio.sleep(0)
 
     async def hashes(self, offs, wait=False, timeout=None):
         '''
@@ -1086,10 +1087,12 @@ class Axon(s_cell.Cell):
 
             async for byts in self._getBytsOffsSize(sha256, offs, size):
                 yield byts
+                await asyncio.sleep(0)
 
         else:
             async for byts in self._get(sha256):
                 yield byts
+                await asyncio.sleep(0)
 
     async def _get(self, sha256):
 

synapse/common.py

@@ -1222,6 +1222,9 @@ def trimText(text: str, n: int = 256, placeholder: str = '...') -> str:
     assert n > plen
     return f'{text[:mlen]}{placeholder}'
 
+def queryhash(text):
+    return hashlib.md5(text.encode(errors='surrogatepass'), usedforsecurity=False).hexdigest()
+
 def _patch_http_cookies():
     '''
     Patch stdlib http.cookies._unquote from the 3.11.10 implementation if

synapse/cortex.py

@@ -4656,10 +4656,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         if not path:
             return await self.cellapi.anit(self, link, user)
 
-        # allow an admin to directly open the cortex hive
-        # (perhaps this should be a Cell() level pattern)
         if path[0] == 'hive' and user.isAdmin():
-            s_common.deprecated('Cortex /hive telepath path', curv='2.167.0')
+            s_common.deprecated('Cortex /hive telepath path', curv='2.198.0', eolv='2.199.0')
             return await self.hiveapi.anit(self.hive, user)
 
         if path[0] == 'layer':

synapse/lib/aha.py

@@ -1592,6 +1592,7 @@ class AhaCell(s_cell.Cell):
             self.slab.delete(iden, db='aha:provs')
             provinfo = s_msgpack.un(byts)
             logger.info(f'Deleted service provisioning service={provinfo.get("conf").get("aha:name")}, iden={iden.decode()}')
+            await asyncio.sleep(0)
 
     @s_nexus.Pusher.onPushAuto('aha:enroll:clear')
     async def clearAhaUserEnrolls(self):
@@ -1599,6 +1600,7 @@ class AhaCell(s_cell.Cell):
             self.slab.delete(iden, db='aha:enrolls')
             userinfo = s_msgpack.un(byts)
             logger.info(f'Deleted user enrollment username={userinfo.get("name")}, iden={iden.decode()}')
+            await asyncio.sleep(0)
 
     @s_nexus.Pusher.onPushAuto('aha:clone:clear')
     async def clearAhaClones(self):
@@ -1606,6 +1608,7 @@ class AhaCell(s_cell.Cell):
             self.slab.delete(lkey, db='aha:clones')
             cloninfo = s_msgpack.un(byts)
             logger.info(f'Deleted AHA clone enrollment username={cloninfo.get("host")}, iden={s_common.ehex(lkey)}')
+            await asyncio.sleep(0)
 
     @s_nexus.Pusher.onPushAuto('aha:svc:prov:del')
     async def delAhaSvcProv(self, iden):

synapse/lib/ast.py

@@ -62,7 +62,7 @@ class AstNode:
 
     def getPosInfo(self):
         return {
-            'hash': hashlib.md5(self.astinfo.text.encode(), usedforsecurity=False).hexdigest(),
+            'hash': s_common.queryhash(self.astinfo.text),
             'lines': (self.astinfo.sline, self.astinfo.eline),
             'columns': (self.astinfo.scol, self.astinfo.ecol),
             'offsets': (self.astinfo.soff, self.astinfo.eoff),
@@ -699,7 +699,92 @@ class SubGraph:
                 yield node, path
 
 class Oper(AstNode):
-    pass
+
+    async def yieldFromValu(self, runt, valu, vkid):
+
+        viewiden = runt.snap.view.iden
+
+        # there is nothing in None... ;)
+        if valu is None:
+            return
+
+        # a little DWIM on what we get back...
+        # ( most common case will be stormtypes libs agenr -> iden|buid )
+        # buid list -> nodes
+        if isinstance(valu, bytes):
+            node = await runt.snap.getNodeByBuid(valu)
+            if node is not None:
+                yield node
+
+            return
+
+        # iden list -> nodes
+        if isinstance(valu, str):
+            try:
+                buid = s_common.uhex(valu)
+            except binascii.Error:
+                mesg = 'Yield string must be iden in hexdecimal. Got: %r' % (valu,)
+                raise vkid.addExcInfo(s_exc.BadLiftValu(mesg=mesg))
+
+            node = await runt.snap.getNodeByBuid(buid)
+            if node is not None:
+                yield node
+
+            return
+
+        if isinstance(valu, types.AsyncGeneratorType):
+            try:
+                async for item in valu:
+                    async for node in self.yieldFromValu(runt, item, vkid):
+                        yield node
+            finally:
+                await valu.aclose()
+            return
+
+        if isinstance(valu, types.GeneratorType):
+            try:
+                for item in valu:
+                    async for node in self.yieldFromValu(runt, item, vkid):
+                        yield node
+            finally:
+                valu.close()
+            return
+
+        if isinstance(valu, (list, tuple, set)):
+            for item in valu:
+                async for node in self.yieldFromValu(runt, item, vkid):
+                    yield node
+            return
+
+        if isinstance(valu, s_stormtypes.Node):
+            valu = valu.valu
+            if valu.snap.view.iden != viewiden:
+                mesg = f'Node is not from the current view. Node {valu.iden()} is from {valu.snap.view.iden} expected {viewiden}'
+                raise vkid.addExcInfo(s_exc.BadLiftValu(mesg=mesg))
+            yield valu
+            return
+
+        if isinstance(valu, s_node.Node):
+            if valu.snap.view.iden != viewiden:
+                mesg = f'Node is not from the current view. Node {valu.iden()} is from {valu.snap.view.iden} expected {viewiden}'
+                raise vkid.addExcInfo(s_exc.BadLiftValu(mesg=mesg))
+            yield valu
+            return
+
+        if isinstance(valu, (s_stormtypes.List, s_stormtypes.Set)):
+            for item in valu.valu:
+                async for node in self.yieldFromValu(runt, item, vkid):
+                    yield node
+            return
+
+        if isinstance(valu, s_stormtypes.Prim):
+            async with contextlib.aclosing(valu.nodes()) as genr:
+                async for node in genr:
+                    if node.snap.view.iden != viewiden:
+                        mesg = f'Node is not from the current view. Node {node.iden()} is from {node.snap.view.iden} expected {viewiden}'
+                        raise vkid.addExcInfo(s_exc.BadLiftValu(mesg=mesg))
+                    yield node
+                return
 
 class SubQuery(Oper):
 
@@ -1534,106 +1619,21 @@ class YieldValu(Oper):
     async def run(self, runt, genr):
 
         node = None
+        vkid = self.kids[0]
 
         async for node, path in genr:
-            valu = await self.kids[0].compute(runt, path)
-            async with contextlib.aclosing(self.yieldFromValu(runt, valu)) as agen:
+            valu = await vkid.compute(runt, path)
+            async with contextlib.aclosing(self.yieldFromValu(runt, valu, vkid)) as agen:
                 async for subn in agen:
                     yield subn, runt.initPath(subn)
             yield node, path
 
         if node is None and self.kids[0].isRuntSafe(runt):
-            valu = await self.kids[0].compute(runt, None)
-            async with contextlib.aclosing(self.yieldFromValu(runt, valu)) as agen:
+            valu = await vkid.compute(runt, None)
+            async with contextlib.aclosing(self.yieldFromValu(runt, valu, vkid)) as agen:
                 async for subn in agen:
                     yield subn, runt.initPath(subn)
 
-    async def yieldFromValu(self, runt, valu):
-
-        viewiden = runt.snap.view.iden
-
-        # there is nothing in None... ;)
-        if valu is None:
-            return
-
-        # a little DWIM on what we get back...
-        # ( most common case will be stormtypes libs agenr -> iden|buid )
-        # buid list -> nodes
-        if isinstance(valu, bytes):
-            node = await runt.snap.getNodeByBuid(valu)
-            if node is not None:
-                yield node
-
-            return
-
-        # iden list -> nodes
-        if isinstance(valu, str):
-            try:
-                buid = s_common.uhex(valu)
-            except binascii.Error:
-                mesg = 'Yield string must be iden in hexdecimal. Got: %r' % (valu,)
-                raise self.kids[0].addExcInfo(s_exc.BadLiftValu(mesg=mesg))
-
-            node = await runt.snap.getNodeByBuid(buid)
-            if node is not None:
-                yield node
-
-            return
-
-        if isinstance(valu, types.AsyncGeneratorType):
-            try:
-                async for item in valu:
-                    async for node in self.yieldFromValu(runt, item):
-                        yield node
-            finally:
-                await valu.aclose()
-            return
-
-        if isinstance(valu, types.GeneratorType):
-            try:
-                for item in valu:
-                    async for node in self.yieldFromValu(runt, item):
-                        yield node
-            finally:
-                valu.close()
-            return
-
-        if isinstance(valu, (list, tuple, set)):
-            for item in valu:
-                async for node in self.yieldFromValu(runt, item):
-                    yield node
-            return
-
-        if isinstance(valu, s_stormtypes.Node):
-            valu = valu.valu
-            if valu.snap.view.iden != viewiden:
-                mesg = f'Node is not from the current view. Node {valu.iden()} is from {valu.snap.view.iden} expected {viewiden}'
-                raise s_exc.BadLiftValu(mesg=mesg)
-            yield valu
-            return
-
-        if isinstance(valu, s_node.Node):
-            if valu.snap.view.iden != viewiden:
-                mesg = f'Node is not from the current view. Node {valu.iden()} is from {valu.snap.view.iden} expected {viewiden}'
-                raise s_exc.BadLiftValu(mesg=mesg)
-            yield valu
-            return
-
-        if isinstance(valu, (s_stormtypes.List, s_stormtypes.Set)):
-            for item in valu.valu:
-                async for node in self.yieldFromValu(runt, item):
-                    yield node
-            return
-
-        if isinstance(valu, s_stormtypes.Prim):
-            async with contextlib.aclosing(valu.nodes()) as genr:
-                async for node in genr:
-                    if node.snap.view.iden != viewiden:
-                        mesg = f'Node is not from the current view. Node {node.iden()} is from {node.snap.view.iden} expected {viewiden}'
-                        raise s_exc.BadLiftValu(mesg=mesg)
-                    yield node
-                return
-
 class LiftTag(LiftOper):
 
     async def lift(self, runt, path):
@@ -4370,6 +4370,83 @@ class EditPropSet(Edit):
 
             await asyncio.sleep(0)
 
+class EditPropSetMulti(Edit):
+
+    async def run(self, runt, genr):
+
+        self.reqNotReadOnly(runt)
+
+        rval = self.kids[2]
+        oper = await self.kids[1].compute(runt, None)
+
+        isadd = '+' in oper
+        excignore = (s_exc.BadTypeValu,) if '?' in oper else ()
+
+        async for node, path in genr:
+
+            propname = await self.kids[0].compute(runt, path)
+            name = await tostr(propname)
+
+            prop = node.form.props.get(name)
+            if prop is None:
+                if (exc := await s_stormtypes.typeerr(propname, str)) is None:
+                    exc = s_exc.NoSuchProp.init(f'{node.form.name}:{name}')
+
+                raise self.kids[0].addExcInfo(exc)
+
+            runt.confirmPropSet(prop)
+
+            if not prop.type.isarray:
+                mesg = f'Property set using ({oper}) is only valid on arrays.'
+                exc = s_exc.StormRuntimeError(mesg=mesg)
+                raise self.kids[0].addExcInfo(exc)
+
+            if isinstance(rval, SubQuery):
+                valu = await rval.compute_array(runt, path)
+            else:
+                valu = await rval.compute(runt, path)
+
+            if valu is None:
+                yield node, path
+                await asyncio.sleep(0)
+                continue
+
+            atyp = prop.type.arraytype
+            isndef = isinstance(atyp, s_types.Ndef)
+            valu = await s_stormtypes.tostor(valu, isndef=isndef)
+
+            if (arry := node.get(name)) is None:
+                arry = ()
+
+            arry = list(arry)
+
+            try:
+                for item in valu:
+                    await asyncio.sleep(0)
+
+                    try:
+                        norm, info = atyp.norm(item)
+                    except excignore:
+                        continue
+
+                    if isadd:
+                        arry.append(norm)
+                    else:
+                        try:
+                            arry.remove(norm)
+                        except ValueError:
+                            pass
+
+            except TypeError:
+                styp = await s_stormtypes.totype(valu, basetypes=True)
+                mesg = f"'{styp}' object is not iterable: {s_common.trimText(repr(valu))}"
+                raise rval.addExcInfo(s_exc.StormRuntimeError(mesg=mesg, type=styp)) from None
+
+            await node.set(name, arry)
+
+            yield node, path
+            await asyncio.sleep(0)
+
 class EditPropDel(Edit):
 
     async def run(self, runt, genr):
@@ -4578,17 +4655,27 @@ class EditEdgeAdd(Edit):
 
         self.reqNotReadOnly(runt)
 
-        # SubQuery -> Query
-        query = self.kids[1].kids[0]
+        constverb = False
+        if self.kids[0].isconst:
+            constverb = True
+            verb = await tostr(await self.kids[0].compute(runt, None))
+            runt.layerConfirm(('node', 'edge', 'add', verb))
+        else:
+            hits = set()
+            def allowed(x):
+                if x in hits:
+                    return
 
-        hits = set()
+                runt.layerConfirm(('node', 'edge', 'add', x))
+                hits.add(x)
 
-        def allowed(x):
-            if x in hits:
-                return
+        isvar = False
+        vkid = self.kids[1]
 
-            runt.layerConfirm(('node', 'edge', 'add', x))
-            hits.add(x)
+        if not isinstance(vkid, SubQuery):
+            isvar = True
+        else:
+            query = vkid.kids[0]
 
         async for node, path in genr:
 
@@ -4596,38 +4683,44 @@ class EditEdgeAdd(Edit):
                 mesg = f'Edges cannot be used with runt nodes: {node.form.full}'
                 raise self.addExcInfo(s_exc.IsRuntForm(mesg=mesg, form=node.form.full))
 
-            iden = node.iden()
-            verb = await tostr(await self.kids[0].compute(runt, path))
-
-            allowed(verb)
-
-            async with runt.getSubRuntime(query) as subr:
-
-                if self.n2:
-                    async for subn, subp in subr.execute():
-                        if subn.form.isrunt:
-                            mesg = f'Edges cannot be used with runt nodes: {subn.form.full}'
-                            raise self.addExcInfo(s_exc.IsRuntForm(mesg=mesg, form=subn.form.full))
-
-                        await subn.addEdge(verb, iden)
-
-                else:
-                    async with node.snap.getEditor() as editor:
-                        proto = editor.loadNode(node)
+            if not constverb:
+                verb = await tostr(await self.kids[0].compute(runt, path))
+                allowed(verb)
+
+            if isvar:
+                valu = await vkid.compute(runt, path)
+                async with contextlib.aclosing(self.yieldFromValu(runt, valu, vkid)) as agen:
+                    if self.n2:
+                        iden = node.iden()
+                        async for subn in agen:
+                            await subn.addEdge(verb, iden, extra=self.addExcInfo)
+                    else:
+                        async with node.snap.getEditor() as editor:
+                            proto = editor.loadNode(node)
+                            async for subn in agen:
+                                if subn.form.isrunt:
+                                    mesg = f'Edges cannot be used with runt nodes: {subn.form.full}'
+                                    raise self.addExcInfo(s_exc.IsRuntForm(mesg=mesg, form=subn.form.full))
+
+                                await proto.addEdge(verb, subn.iden())
+                                await asyncio.sleep(0)
 
+            else:
+                async with runt.getSubRuntime(query) as subr:
+                    if self.n2:
+                        iden = node.iden()
                         async for subn, subp in subr.execute():
-                            if subn.form.isrunt:
-                                mesg = f'Edges cannot be used with runt nodes: {subn.form.full}'
-                                raise self.addExcInfo(s_exc.IsRuntForm(mesg=mesg, form=subn.form.full))
-
-                            await proto.addEdge(verb, subn.iden())
-                            await asyncio.sleep(0)
-
-                            if len(proto.edges) >= 1000:
-                                nodeedits = editor.getNodeEdits()
-                                if nodeedits:
-                                    await node.snap.applyNodeEdits(nodeedits)
-                                proto.edges.clear()
+                            await subn.addEdge(verb, iden, extra=self.addExcInfo)
+                    else:
+                        async with node.snap.getEditor() as editor:
+                            proto = editor.loadNode(node)
+                            async for subn, subp in subr.execute():
+                                if subn.form.isrunt:
+                                    mesg = f'Edges cannot be used with runt nodes: {subn.form.full}'
+                                    raise self.addExcInfo(s_exc.IsRuntForm(mesg=mesg, form=subn.form.full))
+
+                                await proto.addEdge(verb, subn.iden())
+                                await asyncio.sleep(0)
 
             yield node, path
 
@@ -4641,16 +4734,27 @@ class EditEdgeDel(Edit):
 
         self.reqNotReadOnly(runt)
 
-        query = self.kids[1].kids[0]
+        isvar = False
+        vkid = self.kids[1]
 
-        hits = set()
+        if not isinstance(vkid, SubQuery):
+            isvar = True
+        else:
+            query = vkid.kids[0]
 
-        def allowed(x):
-            if x in hits:
-                return
+        constverb = False
+        if self.kids[0].isconst:
+            constverb = True
+            verb = await tostr(await self.kids[0].compute(runt, None))
+            runt.layerConfirm(('node', 'edge', 'del', verb))
+        else:
+            hits = set()
+            def allowed(x):
+                if x in hits:
+                    return
 
-            runt.layerConfirm(('node', 'edge', 'del', x))
-            hits.add(x)
+                runt.layerConfirm(('node', 'edge', 'del', x))
+                hits.add(x)
 
         async for node, path in genr:
 
@@ -4658,36 +4762,44 @@ class EditEdgeDel(Edit):
                 mesg = f'Edges cannot be used with runt nodes: {node.form.full}'
                 raise self.addExcInfo(s_exc.IsRuntForm(mesg=mesg, form=node.form.full))
 
-            iden = node.iden()
-            verb = await tostr(await self.kids[0].compute(runt, path))
-
-            allowed(verb)
-
-            async with runt.getSubRuntime(query) as subr:
-                if self.n2:
-                    async for subn, subp in subr.execute():
-                        if subn.form.isrunt:
-                            mesg = f'Edges cannot be used with runt nodes: {subn.form.full}'
-                            raise self.addExcInfo(s_exc.IsRuntForm(mesg=mesg, form=subn.form.full))
-                        await subn.delEdge(verb, iden)
-
-                else:
-                    async with node.snap.getEditor() as editor:
-                        proto = editor.loadNode(node)
+            if not constverb:
+                verb = await tostr(await self.kids[0].compute(runt, path))
+                allowed(verb)
+
+            if isvar:
+                valu = await vkid.compute(runt, path)
+                async with contextlib.aclosing(self.yieldFromValu(runt, valu, vkid)) as agen:
+                    if self.n2:
+                        iden = node.iden()
+                        async for subn in agen:
+                            await subn.delEdge(verb, iden, extra=self.addExcInfo)
+                    else:
+                        async with node.snap.getEditor() as editor:
+                            proto = editor.loadNode(node)
+                            async for subn in agen:
+                                if subn.form.isrunt:
+                                    mesg = f'Edges cannot be used with runt nodes: {subn.form.full}'
+                                    raise self.addExcInfo(s_exc.IsRuntForm(mesg=mesg, form=subn.form.full))
+
+                                await proto.delEdge(verb, subn.iden())
+                                await asyncio.sleep(0)
 
+            else:
+                async with runt.getSubRuntime(query) as subr:
+                    if self.n2:
+                        iden = node.iden()
                         async for subn, subp in subr.execute():
-                            if subn.form.isrunt:
-                                mesg = f'Edges cannot be used with runt nodes: {subn.form.full}'
-                                raise self.addExcInfo(s_exc.IsRuntForm(mesg=mesg, form=subn.form.full))
-
-                            await proto.delEdge(verb, subn.iden())
-                            await asyncio.sleep(0)
-
-                            if len(proto.edgedels) >= 1000:
-                                nodeedits = editor.getNodeEdits()
-                                if nodeedits:
-                                    await node.snap.applyNodeEdits(nodeedits)
-                                proto.edgedels.clear()
+                            await subn.delEdge(verb, iden, extra=self.addExcInfo)
+                    else:
+                        async with node.snap.getEditor() as editor:
+                            proto = editor.loadNode(node)
+                            async for subn, subp in subr.execute():
+                                if subn.form.isrunt:
+                                    mesg = f'Edges cannot be used with runt nodes: {subn.form.full}'
+                                    raise self.addExcInfo(s_exc.IsRuntForm(mesg=mesg, form=subn.form.full))
+
+                                await proto.delEdge(verb, subn.iden())
+                                await asyncio.sleep(0)
 
             yield node, path