synapse 2.154.1__py311-none-any.whl → 2.156.0__py311-none-any.whl

synapse/lib/ast.py

@@ -330,6 +330,7 @@ class SubGraph:
                     'degrees': 1,
 
                     'edges': True,
+                    'edgelimit': 3000,
                     'filterinput': True,
                     'yieldfiltered': False,
 
@@ -377,6 +378,7 @@ class SubGraph:
         self.rules.setdefault('edges', True)
         self.rules.setdefault('degrees', 1)
         self.rules.setdefault('maxsize', 100000)
+        self.rules.setdefault('edgelimit', 3000)
 
         self.rules.setdefault('filterinput', True)
         self.rules.setdefault('yieldfiltered', False)
@@ -408,7 +410,7 @@ class SubGraph:
         self.omits[node.buid] = False
         return False
 
-    async def pivots(self, runt, node, path):
+    async def pivots(self, runt, node, path, existing):
 
         if self.rules.get('refs'):
 
@@ -420,6 +422,13 @@ class SubGraph:
 
                 yield (pivonode, path.fork(pivonode), {'type': 'prop', 'prop': propname})
 
+            for iden in existing:
+                buid = s_common.uhex(iden)
+                othr = await node.snap.getNodeByBuid(buid)
+                for propname, ndef in othr.getNodeRefs():
+                    if ndef == node.ndef:
+                        yield (othr, path, {'type': 'prop', 'prop': propname, 'reverse': True})
+
         for pivq in self.rules.get('pivots'):
             indx = 0
             async for node, path in node.storm(runt, pivq):
@@ -442,10 +451,25 @@ class SubGraph:
                 yield (node, path, {'type': 'rules', 'scope': scope, 'index': indx})
                 indx += 1
 
+    async def _edgefallback(self, runt, results, node):
+        async for buid01 in results:
+            await asyncio.sleep(0)
+
+            iden01 = s_common.ehex(buid01)
+            async for verb in node.iterEdgeVerbs(buid01):
+                await asyncio.sleep(0)
+                yield (iden01, {'type': 'edge', 'verb': verb})
+
+            # for existing nodes, we need to add n2 -> n1 edges in reverse
+            async for verb in runt.snap.iterEdgeVerbs(buid01, node.buid):
+                await asyncio.sleep(0)
+                yield (iden01, {'type': 'edge', 'verb': verb, 'reverse': True})
+
     async def run(self, runt, genr):
 
         # NOTE: this function must agressively yield the ioloop
 
+        edgelimit = self.rules.get('edgelimit')
         doedges = self.rules.get('edges')
         degrees = self.rules.get('degrees')
         maxsize = self.rules.get('maxsize')
@@ -465,9 +489,57 @@ class SubGraph:
             results = await stack.enter_async_context(await s_spooled.Set.anit(dirn=core.dirn, cell=core))
             revpivs = await stack.enter_async_context(await s_spooled.Dict.anit(dirn=core.dirn, cell=core))
 
+            revedge = await stack.enter_async_context(await s_spooled.Dict.anit(dirn=core.dirn, cell=core))
+            edgecounts = await stack.enter_async_context(await s_spooled.Dict.anit(dirn=core.dirn, cell=core))
+            n1delayed = await stack.enter_async_context(await s_spooled.Set.anit(dirn=core.dirn, cell=core))
+            n2delayed = await stack.enter_async_context(await s_spooled.Set.anit(dirn=core.dirn, cell=core))
+
             # load the existing graph as already done
             [await results.add(s_common.uhex(b)) for b in existing]
 
+            if doedges:
+                for b in existing:
+                    ecnt = 0
+                    cache = collections.defaultdict(list)
+                    async for verb, n2iden in runt.snap.iterNodeEdgesN1(s_common.uhex(b)):
+                        await asyncio.sleep(0)
+
+                        if s_common.uhex(n2iden) in results:
+                            continue
+
+                        ecnt += 1
+                        if ecnt > edgelimit:
+                            break
+
+                        cache[n2iden].append(verb)
+
+                    if ecnt > edgelimit:
+                        # don't let it into the cache.
+                        # We've hit a potential death star and need to deal with it specially
+                        await n1delayed.add(b)
+                        continue
+
+                    for n2iden, verbs in cache.items():
+                        await asyncio.sleep(0)
+                        if n2delayed.has(n2iden):
+                            continue
+
+                        if not revedge.has(n2iden):
+                            await revedge.set(n2iden, {})
+
+                        re = revedge.get(n2iden)
+                        if b not in re:
+                            re[b] = []
+
+                        count = edgecounts.get(n2iden, defv=0) + len(verbs)
+                        if count > edgelimit:
+                            await n2delayed.add(n2iden)
+                            revedge.pop(n2iden)
+                        else:
+                            await edgecounts.set(n2iden, count)
+                            re[b] += verbs
+                            await revedge.set(n2iden, re)
+
             async def todogenr():
 
                 async for node, path in genr:
@@ -482,7 +554,8 @@ class SubGraph:
 
                 await asyncio.sleep(0)
 
-                if node.buid in done:
+                buid = node.buid
+                if buid in done:
                     continue
 
                 count += 1
@@ -491,8 +564,8 @@ class SubGraph:
                     await runt.snap.warn(f'Graph projection hit max size {maxsize}. Truncating results.')
                     break
 
-                await done.add(node.buid)
-                intodo.discard(node.buid)
+                await done.add(buid)
+                intodo.discard(buid)
 
                 omitted = False
                 if dist > 0 or filterinput:
@@ -504,8 +577,9 @@ class SubGraph:
                 # we must traverse the pivots for the node *regardless* of degrees
                 # due to needing to tie any leaf nodes to nodes that were already yielded
 
-                edges = list(revpivs.get(node.buid, defv=()))
-                async for pivn, pivp, pinfo in self.pivots(runt, node, path):
+                nodeiden = node.iden()
+                edges = list(revpivs.get(buid, defv=()))
+                async for pivn, pivp, pinfo in self.pivots(runt, node, path, existing):
 
                     await asyncio.sleep(0)
 
@@ -514,7 +588,7 @@ class SubGraph:
                     else:
                         pinfo['reverse'] = True
                         pivedges = revpivs.get(pivn.buid, defv=())
-                        await revpivs.set(pivn.buid, pivedges + ((node.iden(), pinfo),))
+                        await revpivs.set(pivn.buid, pivedges + ((nodeiden, pinfo),))
 
                     # we dont pivot from omitted nodes
                     if omitted:
@@ -528,28 +602,82 @@ class SubGraph:
                     if pivn.buid in intodo:
                         continue
 
+                    # no need to pivot to existing nodes
+                    if pivn.iden() in existing:
+                        continue
+
                     # do we have room to go another degree out?
                     if degrees is None or dist < degrees:
                         todo.append((pivn, pivp, dist + 1))
                         await intodo.add(pivn.buid)
 
                 if doedges:
-
-                    async for buid01 in results:
-
+                    ecnt = 0
+                    cache = collections.defaultdict(list)
+                    await results.add(buid)
+                    # Try to lift and cache the potential edges for a node so that if we end up
+                    # seeing n2 later, we won't have to go back and check for it
+                    async for verb, n2iden in runt.snap.iterNodeEdgesN1(buid):
                         await asyncio.sleep(0)
-
-                        iden01 = s_common.ehex(buid01)
-                        async for verb in node.iterEdgeVerbs(buid01):
-                            await asyncio.sleep(0)
-                            edges.append((iden01, {'type': 'edge', 'verb': verb}))
-
-                        # for existing nodes, we need to add n2 -> n1 edges in reverse
-                        async for verb in runt.snap.iterEdgeVerbs(buid01, node.buid):
+                        if ecnt > edgelimit:
+                            break
+
+                        ecnt += 1
+                        cache[n2iden].append(verb)
+
+                    if ecnt > edgelimit:
+                        # The current node in the pipeline has too many edges from it, so it's
+                        # less prohibitive to just check against the graph
+                        await n1delayed.add(nodeiden)
+                        async for e in self._edgefallback(runt, results, node):
+                            edges.append(e)
+                    else:
+                        for n2iden, verbs in cache.items():
                             await asyncio.sleep(0)
-                            edges.append((iden01, {'type': 'edge', 'verb': verb, 'reverse': True}))
 
-                    await results.add(node.buid)
+                            if n2delayed.has(n2iden):
+                                continue
+
+                            if not revedge.has(n2iden):
+                                await revedge.set(n2iden, {})
+
+                            re = revedge.get(n2iden)
+                            if nodeiden not in re:
+                                re[nodeiden] = []
+
+                            count = edgecounts.get(n2iden, defv=0) + len(verbs)
+                            if count > edgelimit:
+                                await n2delayed.add(n2iden)
+                                revedge.pop(n2iden)
+                            else:
+                                await edgecounts.set(n2iden, count)
+                                re[nodeiden] += verbs
+                                await revedge.set(n2iden, re)
+
+                        if revedge.has(nodeiden):
+                            for n2iden, verbs in revedge.get(nodeiden).items():
+                                for verb in verbs:
+                                    await asyncio.sleep(0)
+                                    edges.append((n2iden, {'type': 'edge', 'verb': verb, 'reverse': True}))
+
+                        if n2delayed.has(nodeiden):
+                            async for buid01 in results:
+                                async for verb in runt.snap.iterEdgeVerbs(buid01, buid):
+                                    await asyncio.sleep(0)
+                                    edges.append((s_common.ehex(buid01), {'type': 'edge', 'verb': verb, 'reverse': True}))
+                        for n2iden, verbs in cache.items():
+                            if s_common.uhex(n2iden) not in results:
+                                continue
+
+                            for v in verbs:
+                                await asyncio.sleep(0)
+                                edges.append((n2iden, {'type': 'edge', 'verb': v}))
+
+                        async for n1iden in n1delayed:
+                            n1buid = s_common.uhex(n1iden)
+                            async for verb in runt.snap.iterEdgeVerbs(n1buid, buid):
+                                await asyncio.sleep(0)
+                                edges.append((n1iden, {'type': 'edge', 'verb': verb, 'reverse': True}))
 
                 path.metadata['edges'] = edges
                 yield node, path
@@ -568,6 +696,9 @@ class SubQuery(Oper):
         if len(kids):
             self.text = kids[0].getAstText()
 
+    def isRuntSafe(self, runt):
+        return True
+
     async def run(self, runt, genr):
 
         subq = self.kids[0]
@@ -676,6 +807,38 @@ class InitBlock(AstNode):
             async for innr in subq.run(runt, s_common.agen()):
                 yield innr
 
+class EmptyBlock(AstNode):
+    '''
+    An AST node that only runs if there are not inbound nodes in the pipeline. It is
+    capable of yielding nodes into the pipeline.
+
+    Example:
+
+        Using an empty block::
+
+            empty {
+                // the pipeline is empty so this block will execute
+            }
+
+            [foo:bar=*]
+            empty {
+                // there is a node in the pipeline so this block will not run
+            }
+    '''
+    async def run(self, runt, genr):
+
+        subq = self.kids[0]
+        self.reqRuntSafe(runt, 'Empty block query must be runtsafe')
+
+        empty = True
+        async for item in genr:
+            empty = False
+            yield item
+
+        if empty:
+            async for subn in subq.run(runt, s_common.agen()):
+                yield subn
+
 class FiniBlock(AstNode):
     '''
     An AST node that runs only once after all nodes have been consumed.
@@ -728,7 +891,7 @@ class TryCatch(AstNode):
                 async for subi in block.run(runt, agen):
                     yield subi
 
-        if count == 0 and self.isRuntSafe(runt):
+        if count == 0:
             try:
                 async for item in self.kids[0].run(runt, genr):
                     yield item
@@ -1738,6 +1901,9 @@ class N1WalkNPivo(PivotOut):
 
         async for node, path in genr:
 
+            if self.isjoin:
+                yield node, path
+
             async for item in self.getPivsOut(runt, node, path):
                 yield item
 
@@ -1857,6 +2023,9 @@ class N2WalkNPivo(PivotIn):
 
         async for node, path in genr:
 
+            if self.isjoin:
+                yield node, path
+
             async for item in self.getPivsIn(runt, node, path):
                 yield item
 
@@ -1900,7 +2069,8 @@ class PivotInFrom(PivotOper):
                 yield node, path
 
             if not isinstance(node.form.type, s_types.Edge):
-                continue
+                mesg = f'Pivot in from a specific form cannot be used with nodes of type {node.form.type.name}'
+                raise self.addExcInfo(s_exc.StormRuntimeError(mesg=mesg, name=node.form.type.name))
 
             # dont bother traversing edges to the wrong form
             if node.get('n1:form') != form.name:
@@ -2762,6 +2932,9 @@ class RelPropCond(Cond):
             if not isinstance(xval, s_node.Node):
                 xval = await s_stormtypes.tostor(xval)
 
+            if xval is None:
+                return False
+
             ctor = prop.type.getCmprCtor(cmpr)
             if ctor is None:
                 raise self.kids[1].addExcInfo(s_exc.NoSuchCmpr(cmpr=cmpr, name=prop.type.name))
@@ -2881,6 +3054,8 @@ class PropValue(Value):
         return False
 
     async def getPropAndValu(self, runt, path):
+        if not path:
+            return None, None
 
         name = await self.kids[0].compute(runt, path)
 
@@ -2894,6 +3069,10 @@ class PropValue(Value):
                                                     name=name, form=path.node.form.name))
 
             valu = path.node.get(name)
+            if isinstance(valu, (dict, list, tuple)):
+                # these get special cased because changing them affects the node
+                # while it's in the pipeline but the modification doesn't get stored
+                valu = s_msgpack.deepcopy(valu)
             return prop, valu
 
         # handle implicit pivot properties
@@ -2915,6 +3094,10 @@ class PropValue(Value):
                                                 name=name, form=node.form.name))
 
             if i >= imax:
+                if isinstance(valu, (dict, list, tuple)):
+                    # these get special cased because changing them affects the node
+                    # while it's in the pipeline but the modification doesn't get stored
+                    valu = s_msgpack.deepcopy(valu)
                 return prop, valu
 
             form = runt.model.forms.get(prop.type.name)
@@ -3766,6 +3949,13 @@ class EditUnivDel(Edit):
 
 class N1Walk(Oper):
 
+    def __init__(self, astinfo, kids=(), isjoin=False):
+        Oper.__init__(self, astinfo, kids=kids)
+        self.isjoin = isjoin
+
+    def repr(self):
+        return f'{self.__class__.__name__}: {self.kids}, isjoin={self.isjoin}'
+
     async def walkNodeEdges(self, runt, node, verb=None):
         async for _, iden in node.iterEdgesN1(verb=verb):
             buid = s_common.uhex(iden)
@@ -3824,6 +4014,9 @@ class N1Walk(Oper):
 
         async for node, path in genr:
 
+            if self.isjoin:
+                yield node, path
+
             verbs = await self.kids[0].compute(runt, path)
             verbs = await s_stormtypes.toprim(verbs)
 

synapse/lib/cell.py

@@ -2771,6 +2771,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         self.addHttpApi('/robots.txt', s_httpapi.RobotHandler, {'cell': self})
         self.addHttpApi('/api/v1/login', s_httpapi.LoginV1, {'cell': self})
+        self.addHttpApi('/api/v1/logout', s_httpapi.LogoutV1, {'cell': self})
+
         self.addHttpApi('/api/v1/active', s_httpapi.ActiveV1, {'cell': self})
         self.addHttpApi('/api/v1/healthcheck', s_httpapi.HealthCheckV1, {'cell': self})
 
@@ -3418,20 +3420,45 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             ahaname = provconf.get('aha:name')
             ahanetw = provconf.get('aha:network')
 
-            if not certdir.getCaCertPath(ahanetw):
-                certdir.saveCaCertByts(await prov.getCaCert())
+            _crt = certdir.getCaCertPath(ahanetw)
+            if _crt:
+                logger.debug(f'Removing existing CA crt: {_crt}')
+                os.unlink(_crt)
+            certdir.saveCaCertByts(await prov.getCaCert())
 
             await self._bootProvConf(provconf)
 
             hostname = f'{ahaname}.{ahanetw}'
-            if certdir.getHostCertPath(hostname) is None:
-                hostcsr = certdir.genHostCsr(hostname)
-                certdir.saveHostCertByts(await prov.signHostCsr(hostcsr))
+            _crt = certdir.getHostCertPath(hostname)
+            if _crt:
+                logger.debug(f'Removing existing host crt {_crt}')
+                os.unlink(_crt)
+            _kp = certdir.getHostKeyPath(hostname)
+            if _kp:
+                logger.debug(f'Removing existing host key {_kp}')
+                os.unlink(_kp)
+            _csr = certdir.getHostCsrPath(hostname)
+            if _csr:
+                logger.debug(f'Removing existing host csr {_csr}')
+                os.unlink(_csr)
+            hostcsr = certdir.genHostCsr(hostname)
+            certdir.saveHostCertByts(await prov.signHostCsr(hostcsr))
 
             userfull = f'{ahauser}@{ahanetw}'
-            if certdir.getUserCertPath(userfull) is None:
-                usercsr = certdir.genUserCsr(userfull)
-                certdir.saveUserCertByts(await prov.signUserCsr(usercsr))
+            _crt = certdir.getUserCertPath(userfull)
+            if _crt:
+                logger.debug(f'Removing existing user crt {_crt}')
+                os.unlink(_crt)
+            _kp = certdir.getUserKeyPath(userfull)
+            if _kp:
+                logger.debug(f'Removing existing user key {_kp}')
+                os.unlink(_kp)
+            _csr = certdir.getUserCsrPath(userfull)
+            if _csr:
+                logger.debug(f'Removing existing user csr {_csr}')
+                os.unlink(_csr)
+            usercsr = certdir.genUserCsr(userfull)
+            certdir.saveUserCertByts(await prov.signUserCsr(usercsr))
 
         with s_common.genfile(self.dirn, 'prov.done') as fd:
             fd.write(providen.encode())

synapse/lib/certdir.py

@@ -944,6 +944,17 @@ class CertDir:
             if os.path.isfile(path):
                 return path
 
+    def getUserCsrPath(self, name):
+        for cdir in self.certdirs:
+            path = s_common.genpath(cdir, 'users', '%s.csr' % name)
+            if os.path.isfile(path):
+                return path
+
+    def getHostCsrPath(self, name):
+        for cdir in self.certdirs:
+            path = s_common.genpath(cdir, 'hosts', '%s.csr' % name)
+            if os.path.isfile(path):
+                return path
     def importFile(self, path, mode, outp=None):
         '''
         Imports certs and keys into the Synapse cert directory

synapse/lib/cmdr.py

@@ -1,10 +1,8 @@
 import synapse.lib.cli as s_cli
 
 import synapse.cmds.boss as s_cmds_boss
-import synapse.cmds.cron as s_cmds_cron
 import synapse.cmds.hive as s_cmds_hive
 import synapse.cmds.cortex as s_cmds_cortex
-import synapse.cmds.trigger as s_cmds_trigger
 
 cmdsbycell = {
     'cell': (
@@ -14,14 +12,11 @@ cmdsbycell = {
     ),
 
     'cortex': (
-        s_cmds_cron.At,
-        s_cmds_cron.Cron,
         s_cmds_cortex.Log,
         s_cmds_boss.PsCmd,
         s_cmds_boss.KillCmd,
         s_cmds_hive.HiveCmd,
         s_cmds_cortex.StormCmd,
-        s_cmds_trigger.Trigger,
     ),
 }
 

synapse/lib/gis.py

@@ -1,7 +1,7 @@
 import math
 
 '''
-Synapse module with helpers for earth based geo-spacial calculations.
+Synapse module with helpers for earth based geospatial calculations.
 '''
 
 # base earth geo distances will be in mm
@@ -66,7 +66,7 @@ def haversine(px, py, r=r_mm):
 
 def bbox(lat, lon, dist):
     '''
-    Calculate a min/max bounding box for the circle defined by lalo/dist.
+    Calculate a min/max bounding box for the circle defined by lat/lon/dist.
 
     Args:
         lat (float): The latitude in degrees

synapse/lib/httpapi.py

@@ -518,7 +518,7 @@ class StormV1(StormHandler):
         if opts is None:
             return
 
-        opts.setdefault('editformat', 'splices')
+        opts.setdefault('editformat', 'nodeedits')
 
         async for mesg in self.getCore().storm(query, opts=opts):
             self.write(json.dumps(mesg))
@@ -612,48 +612,6 @@ class ReqValidStormV1(StormHandler):
         else:
             return self.sendRestRetn(valid)
 
-class WatchSockV1(WebSocket):
-    '''
-    A web-socket based API endpoint for distributing cortex tag events.
-
-    Deprecated.
-    '''
-    async def onWatchMesg(self, byts):
-        # Note: This API handler is intended to be used on a heavy Cortex object.
-        try:
-
-            wdef = json.loads(byts)
-            iden = wdef.get('view', self.cell.view.iden)
-
-            perm = ('watch', 'view', iden)
-            await self._reqUserAllow(perm)
-
-            async with self.cell.watcher(wdef) as watcher:
-
-                await self.xmit('init')
-
-                async for mesg in watcher:
-                    await self.xmit(mesg[0], **mesg[1])
-
-                # pragma: no cover
-                # (this would only happen on slow-consumer)
-                await self.xmit('fini')
-
-        except s_exc.SynErr as e:
-
-            text = e.get('mesg', str(e))
-            await self.xmit('errx', code=e.__class__.__name__, mesg=text)
-
-        except asyncio.CancelledError:  # pragma: no cover  TODO:  remove once >= py 3.8 only
-            raise
-
-        except Exception as e:
-            await self.xmit('errx', code=e.__class__.__name__, mesg=str(e))
-
-    async def on_message(self, byts):
-        s_common.deprdate('/api/v1/watch HTTP API', s_common._splicedepr)
-        self.cell.schedCoro(self.onWatchMesg(byts))
-
 class BeholdSockV1(WebSocket):
 
     async def onInitMessage(self, byts):
@@ -724,6 +682,19 @@ class LoginV1(Handler):
 
         return self.sendRestRetn(await authcell.getUserDef(iden))
 
+class LogoutV1(Handler):
+
+    async def get(self):
+        sess = await self.sess(gen=False)
+        if sess is not None:
+            self.web_useriden = sess.info.get('user')
+            self.web_username = sess.info.get('username', '<no username>')
+            await self.getAuthCell().delHttpSess(sess.iden)
+
+        self.clear_cookie('sess')
+
+        self.sendRestRetn(True)
+
 class AuthUsersV1(Handler):
 
     async def get(self):