synapse 2.155.0__py311-none-any.whl → 2.156.0__py311-none-any.whl

synapse/lib/ast.py

@@ -696,6 +696,9 @@ class SubQuery(Oper):
         if len(kids):
             self.text = kids[0].getAstText()
 
+    def isRuntSafe(self, runt):
+        return True
+
     async def run(self, runt, genr):
 
         subq = self.kids[0]
@@ -804,6 +807,38 @@ class InitBlock(AstNode):
             async for innr in subq.run(runt, s_common.agen()):
                 yield innr
 
+class EmptyBlock(AstNode):
+    '''
+    An AST node that only runs if there are not inbound nodes in the pipeline. It is
+    capable of yielding nodes into the pipeline.
+
+    Example:
+
+        Using an empty block::
+
+            empty {
+                // the pipeline is empty so this block will execute
+            }
+
+            [foo:bar=*]
+            empty {
+                // there is a node in the pipeline so this block will not run
+            }
+    '''
+    async def run(self, runt, genr):
+
+        subq = self.kids[0]
+        self.reqRuntSafe(runt, 'Empty block query must be runtsafe')
+
+        empty = True
+        async for item in genr:
+            empty = False
+            yield item
+
+        if empty:
+            async for subn in subq.run(runt, s_common.agen()):
+                yield subn
+
 class FiniBlock(AstNode):
     '''
     An AST node that runs only once after all nodes have been consumed.
@@ -856,7 +891,7 @@ class TryCatch(AstNode):
                 async for subi in block.run(runt, agen):
                     yield subi
 
-        if count == 0 and self.isRuntSafe(runt):
+        if count == 0:
             try:
                 async for item in self.kids[0].run(runt, genr):
                     yield item
@@ -1866,6 +1901,9 @@ class N1WalkNPivo(PivotOut):
 
         async for node, path in genr:
 
+            if self.isjoin:
+                yield node, path
+
             async for item in self.getPivsOut(runt, node, path):
                 yield item
 
@@ -1985,6 +2023,9 @@ class N2WalkNPivo(PivotIn):
 
         async for node, path in genr:
 
+            if self.isjoin:
+                yield node, path
+
             async for item in self.getPivsIn(runt, node, path):
                 yield item
 
@@ -3028,6 +3069,10 @@ class PropValue(Value):
                                                     name=name, form=path.node.form.name))
 
             valu = path.node.get(name)
+            if isinstance(valu, (dict, list, tuple)):
+                # these get special cased because changing them affects the node
+                # while it's in the pipeline but the modification doesn't get stored
+                valu = s_msgpack.deepcopy(valu)
             return prop, valu
 
         # handle implicit pivot properties
@@ -3049,6 +3094,10 @@ class PropValue(Value):
                                                 name=name, form=node.form.name))
 
             if i >= imax:
+                if isinstance(valu, (dict, list, tuple)):
+                    # these get special cased because changing them affects the node
+                    # while it's in the pipeline but the modification doesn't get stored
+                    valu = s_msgpack.deepcopy(valu)
                 return prop, valu
 
             form = runt.model.forms.get(prop.type.name)
@@ -3900,6 +3949,13 @@ class EditUnivDel(Edit):
 
 class N1Walk(Oper):
 
+    def __init__(self, astinfo, kids=(), isjoin=False):
+        Oper.__init__(self, astinfo, kids=kids)
+        self.isjoin = isjoin
+
+    def repr(self):
+        return f'{self.__class__.__name__}: {self.kids}, isjoin={self.isjoin}'
+
     async def walkNodeEdges(self, runt, node, verb=None):
         async for _, iden in node.iterEdgesN1(verb=verb):
             buid = s_common.uhex(iden)
@@ -3958,6 +4014,9 @@ class N1Walk(Oper):
 
         async for node, path in genr:
 
+            if self.isjoin:
+                yield node, path
+
             verbs = await self.kids[0].compute(runt, path)
             verbs = await s_stormtypes.toprim(verbs)
 

synapse/lib/cell.py

@@ -3420,20 +3420,45 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             ahaname = provconf.get('aha:name')
             ahanetw = provconf.get('aha:network')
 
-            if not certdir.getCaCertPath(ahanetw):
-                certdir.saveCaCertByts(await prov.getCaCert())
+            _crt = certdir.getCaCertPath(ahanetw)
+            if _crt:
+                logger.debug(f'Removing existing CA crt: {_crt}')
+                os.unlink(_crt)
+            certdir.saveCaCertByts(await prov.getCaCert())
 
             await self._bootProvConf(provconf)
 
             hostname = f'{ahaname}.{ahanetw}'
-            if certdir.getHostCertPath(hostname) is None:
-                hostcsr = certdir.genHostCsr(hostname)
-                certdir.saveHostCertByts(await prov.signHostCsr(hostcsr))
+            _crt = certdir.getHostCertPath(hostname)
+            if _crt:
+                logger.debug(f'Removing existing host crt {_crt}')
+                os.unlink(_crt)
+            _kp = certdir.getHostKeyPath(hostname)
+            if _kp:
+                logger.debug(f'Removing existing host key {_kp}')
+                os.unlink(_kp)
+            _csr = certdir.getHostCsrPath(hostname)
+            if _csr:
+                logger.debug(f'Removing existing host csr {_csr}')
+                os.unlink(_csr)
+            hostcsr = certdir.genHostCsr(hostname)
+            certdir.saveHostCertByts(await prov.signHostCsr(hostcsr))
 
             userfull = f'{ahauser}@{ahanetw}'
-            if certdir.getUserCertPath(userfull) is None:
-                usercsr = certdir.genUserCsr(userfull)
-                certdir.saveUserCertByts(await prov.signUserCsr(usercsr))
+            _crt = certdir.getUserCertPath(userfull)
+            if _crt:
+                logger.debug(f'Removing existing user crt {_crt}')
+                os.unlink(_crt)
+            _kp = certdir.getUserKeyPath(userfull)
+            if _kp:
+                logger.debug(f'Removing existing user key {_kp}')
+                os.unlink(_kp)
+            _csr = certdir.getUserCsrPath(userfull)
+            if _csr:
+                logger.debug(f'Removing existing user csr {_csr}')
+                os.unlink(_csr)
+            usercsr = certdir.genUserCsr(userfull)
+            certdir.saveUserCertByts(await prov.signUserCsr(usercsr))
 
         with s_common.genfile(self.dirn, 'prov.done') as fd:
             fd.write(providen.encode())

synapse/lib/certdir.py

@@ -944,6 +944,17 @@ class CertDir:
             if os.path.isfile(path):
                 return path
 
+    def getUserCsrPath(self, name):
+        for cdir in self.certdirs:
+            path = s_common.genpath(cdir, 'users', '%s.csr' % name)
+            if os.path.isfile(path):
+                return path
+
+    def getHostCsrPath(self, name):
+        for cdir in self.certdirs:
+            path = s_common.genpath(cdir, 'hosts', '%s.csr' % name)
+            if os.path.isfile(path):
+                return path
     def importFile(self, path, mode, outp=None):
         '''
         Imports certs and keys into the Synapse cert directory

synapse/lib/cmdr.py

@@ -1,10 +1,8 @@
 import synapse.lib.cli as s_cli
 
 import synapse.cmds.boss as s_cmds_boss
-import synapse.cmds.cron as s_cmds_cron
 import synapse.cmds.hive as s_cmds_hive
 import synapse.cmds.cortex as s_cmds_cortex
-import synapse.cmds.trigger as s_cmds_trigger
 
 cmdsbycell = {
     'cell': (
@@ -14,14 +12,11 @@ cmdsbycell = {
     ),
 
     'cortex': (
-        s_cmds_cron.At,
-        s_cmds_cron.Cron,
         s_cmds_cortex.Log,
         s_cmds_boss.PsCmd,
         s_cmds_boss.KillCmd,
         s_cmds_hive.HiveCmd,
         s_cmds_cortex.StormCmd,
-        s_cmds_trigger.Trigger,
     ),
 }
 

synapse/lib/gis.py

@@ -1,7 +1,7 @@
 import math
 
 '''
-Synapse module with helpers for earth based geo-spacial calculations.
+Synapse module with helpers for earth based geospatial calculations.
 '''
 
 # base earth geo distances will be in mm
@@ -66,7 +66,7 @@ def haversine(px, py, r=r_mm):
 
 def bbox(lat, lon, dist):
     '''
-    Calculate a min/max bounding box for the circle defined by lalo/dist.
+    Calculate a min/max bounding box for the circle defined by lat/lon/dist.
 
     Args:
         lat (float): The latitude in degrees

synapse/lib/httpapi.py

@@ -518,7 +518,7 @@ class StormV1(StormHandler):
         if opts is None:
             return
 
-        opts.setdefault('editformat', 'splices')
+        opts.setdefault('editformat', 'nodeedits')
 
         async for mesg in self.getCore().storm(query, opts=opts):
             self.write(json.dumps(mesg))
@@ -612,48 +612,6 @@ class ReqValidStormV1(StormHandler):
         else:
             return self.sendRestRetn(valid)
 
-class WatchSockV1(WebSocket):
-    '''
-    A web-socket based API endpoint for distributing cortex tag events.
-
-    Deprecated.
-    '''
-    async def onWatchMesg(self, byts):
-        # Note: This API handler is intended to be used on a heavy Cortex object.
-        try:
-
-            wdef = json.loads(byts)
-            iden = wdef.get('view', self.cell.view.iden)
-
-            perm = ('watch', 'view', iden)
-            await self._reqUserAllow(perm)
-
-            async with self.cell.watcher(wdef) as watcher:
-
-                await self.xmit('init')
-
-                async for mesg in watcher:
-                    await self.xmit(mesg[0], **mesg[1])
-
-                # pragma: no cover
-                # (this would only happen on slow-consumer)
-                await self.xmit('fini')
-
-        except s_exc.SynErr as e:
-
-            text = e.get('mesg', str(e))
-            await self.xmit('errx', code=e.__class__.__name__, mesg=text)
-
-        except asyncio.CancelledError:  # pragma: no cover  TODO:  remove once >= py 3.8 only
-            raise
-
-        except Exception as e:
-            await self.xmit('errx', code=e.__class__.__name__, mesg=str(e))
-
-    async def on_message(self, byts):
-        s_common.deprdate('/api/v1/watch HTTP API', s_common._splicedepr)
-        self.cell.schedCoro(self.onWatchMesg(byts))
-
 class BeholdSockV1(WebSocket):
 
     async def onInitMessage(self, byts):

synapse/lib/layer.py

@@ -170,19 +170,6 @@ class LayerApi(s_cell.CellApi):
         async for item in self.layr.syncNodeEdits2(offs, wait=wait):
             yield item
 
-    async def splices(self, offs=None, size=None):
-        '''
-        This API is deprecated.
-
-        Yield (offs, splice) tuples from the nodeedit log starting from the given offset.
-
-        Nodeedits will be flattened into splices before being yielded.
-        '''
-        s_common.deprdate('Layer.splices() telepath API', s_common._splicedepr)
-        await self._reqUserAllowed(self.liftperm)
-        async for item in self.layr.splices(offs=offs, size=size):
-            yield item
-
     async def getEditIndx(self):
         '''
         Returns what will be the *next* nodeedit log index.
@@ -2023,17 +2010,14 @@ class Layer(s_nexus.Pusher):
         ret['totalsize'] = await self.getLayerSize()
         return ret
 
-    async def truncate(self):
-        self._reqNotReadOnly()
-        return await self._push('layer:truncate')
-
     @s_nexus.Pusher.onPush('layer:truncate')
     async def _truncate(self):
         '''
         Nuke all the contents in the layer, leaving an empty layer
         NOTE: This internal API is deprecated but is kept for Nexus event backward compatibility
         '''
-        s_common.deprdate('Layer.truncate() API', s_common._splicedepr)
+        # TODO: Remove this in 3.0.0
+        s_common.deprecated('layer:truncate Nexus handler', curv='2.156.0')
 
         self.dirty.clear()
         self.buidcache.clear()
@@ -2044,8 +2028,6 @@ class Layer(s_nexus.Pusher):
 
         await self._initLayerStorage()
 
-    # async def wipe(self, meta): ...
-
     async def iterWipeNodeEdits(self):
 
         await self._saveDirtySodes()
@@ -2914,6 +2896,7 @@ class Layer(s_nexus.Pusher):
             abrv = self.tagabrv.bytsToAbrv(tagname.encode())
             if formname is not None:
                 abrv += self.getPropAbrv(formname, None)
+                return self.layrslab.count(abrv, db=self.bytag)
 
         except s_exc.NoSuchAbrv:
             return 0
@@ -2931,6 +2914,44 @@ class Layer(s_nexus.Pusher):
 
         return await self.layrslab.countByPref(abrv, db=self.byprop, maxsize=maxsize)
 
+    def getPropValuCount(self, formname, propname, stortype, valu):
+        try:
+            abrv = self.getPropAbrv(formname, propname)
+        except s_exc.NoSuchAbrv:
+            return 0
+
+        if stortype & 0x8000:
+            stortype = STOR_TYPE_MSGP
+
+        count = 0
+        for indx in self.getStorIndx(stortype, valu):
+            count += self.layrslab.count(abrv + indx, db=self.byprop)
+
+        return count
+
+    async def getPropArrayCount(self, formname, propname=None):
+        '''
+        Return the number of invidiual value rows in the layer for the given array form/prop.
+        '''
+        try:
+            abrv = self.getPropAbrv(formname, propname)
+        except s_exc.NoSuchAbrv:
+            return 0
+
+        return await self.layrslab.countByPref(abrv, db=self.byarray)
+
+    def getPropArrayValuCount(self, formname, propname, stortype, valu):
+        try:
+            abrv = self.getPropAbrv(formname, propname)
+        except s_exc.NoSuchAbrv:
+            return 0
+
+        count = 0
+        for indx in self.getStorIndx(stortype, valu):
+            count += self.layrslab.count(abrv + indx, db=self.byarray)
+
+        return count
+
     async def getUnivPropCount(self, propname, maxsize=None):
         '''
         Return the number of universal property rows in the layer for the given prop.
@@ -2942,6 +2963,29 @@ class Layer(s_nexus.Pusher):
 
         return await self.layrslab.countByPref(abrv, db=self.byprop, maxsize=maxsize)
 
+    async def getTagPropCount(self, form, tag, prop):
+        '''
+        Return the number of property rows in the layer for the given form/tag/prop.
+        '''
+        try:
+            abrv = self.getTagPropAbrv(form, tag, prop)
+        except s_exc.NoSuchAbrv:
+            return 0
+
+        return await self.layrslab.countByPref(abrv, db=self.bytagprop)
+
+    def getTagPropValuCount(self, form, tag, prop, stortype, valu):
+        try:
+            abrv = self.getTagPropAbrv(form, tag, prop)
+        except s_exc.NoSuchAbrv:
+            return 0
+
+        count = 0
+        for indx in self.getStorIndx(stortype, valu):
+            count += self.layrslab.count(abrv + indx, db=self.bytagprop)
+
+        return count
+
     async def liftByTag(self, tag, form=None, reverse=False):
 
         try:
@@ -4151,76 +4195,6 @@ class Layer(s_nexus.Pusher):
             yield buid, s_msgpack.un(byts)
             await asyncio.sleep(0)
 
-    async def splices(self, offs=None, size=None):
-        '''
-        This API is deprecated.
-
-        Yield (offs, splice) tuples from the nodeedit log starting from the given offset.
-
-        Nodeedits will be flattened into splices before being yielded.
-        '''
-        s_common.deprdate('Layer.splices() API', s_common._splicedepr)
-        if not self.logedits:
-            return
-
-        if offs is None:
-            offs = (0, 0, 0)
-
-        if size is not None:
-
-            count = 0
-            async for offset, nodeedits, meta in self.iterNodeEditLog(offs[0]):
-                async for splice in self.makeSplices(offset, nodeedits, meta):
-
-                    if splice[0] < offs:
-                        continue
-
-                    if count >= size:
-                        return
-
-                    yield splice
-                    count = count + 1
-        else:
-            async for offset, nodeedits, meta in self.iterNodeEditLog(offs[0]):
-                async for splice in self.makeSplices(offset, nodeedits, meta):
-
-                    if splice[0] < offs:
-                        continue
-
-                    yield splice
-
-    async def splicesBack(self, offs=None, size=None):
-
-        s_common.deprdate('Layer.splicesBack() API', s_common._splicedepr)
-        if not self.logedits:
-            return
-
-        if offs is None:
-            offs = (await self.getEditIndx(), 0, 0)
-
-        if size is not None:
-
-            count = 0
-            async for offset, nodeedits, meta in self.iterNodeEditLogBack(offs[0]):
-                async for splice in self.makeSplices(offset, nodeedits, meta, reverse=True):
-
-                    if splice[0] > offs:
-                        continue
-
-                    if count >= size:
-                        return
-
-                    yield splice
-                    count += 1
-        else:
-            async for offset, nodeedits, meta in self.iterNodeEditLogBack(offs[0]):
-                async for splice in self.makeSplices(offset, nodeedits, meta, reverse=True):
-
-                    if splice[0] > offs:
-                        continue
-
-                    yield splice
-
     async def iterNodeEditLog(self, offs=0):
         '''
         Iterate the node edit log and yield (offs, edits, meta) tuples.
@@ -4312,117 +4286,6 @@ class Layer(s_nexus.Pusher):
             if count % 1000 == 0:
                 yield (curoff, (None, None, EDIT_PROGRESS, (), ()))
 
-    async def makeSplices(self, offs, nodeedits, meta, reverse=False):
-        '''
-        Flatten a set of nodeedits into splices.
-        '''
-        if meta is None:
-            meta = {}
-
-        user = meta.get('user')
-        time = meta.get('time')
-        prov = meta.get('prov')
-
-        if reverse:
-            nodegenr = reversed(list(enumerate(nodeedits)))
-        else:
-            nodegenr = enumerate(nodeedits)
-
-        for nodeoffs, (buid, form, edits) in nodegenr:
-
-            formvalu = None
-
-            if reverse:
-                editgenr = reversed(list(enumerate(edits)))
-            else:
-                editgenr = enumerate(edits)
-
-            for editoffs, (edit, info, _) in editgenr:
-
-                if edit in (EDIT_NODEDATA_SET, EDIT_NODEDATA_DEL, EDIT_EDGE_ADD, EDIT_EDGE_DEL):
-                    continue
-
-                spliceoffs = (offs, nodeoffs, editoffs)
-
-                props = {
-                    'time': time,
-                    'user': user,
-                }
-
-                if prov is not None:
-                    props['prov'] = prov
-
-                if edit == EDIT_NODE_ADD:
-                    formvalu, stortype = info
-                    props['ndef'] = (form, formvalu)
-
-                    yield (spliceoffs, ('node:add', props))
-                    continue
-
-                if edit == EDIT_NODE_DEL:
-                    formvalu, stortype = info
-                    props['ndef'] = (form, formvalu)
-
-                    yield (spliceoffs, ('node:del', props))
-                    continue
-
-                if formvalu is None:
-                    formvalu = await self.getNodeValu(buid)
-
-                props['ndef'] = (form, formvalu)
-
-                if edit == EDIT_PROP_SET:
-                    prop, valu, oldv, stortype = info
-                    props['prop'] = prop
-                    props['valu'] = valu
-                    props['oldv'] = oldv
-
-                    yield (spliceoffs, ('prop:set', props))
-                    continue
-
-                if edit == EDIT_PROP_DEL:
-                    prop, valu, stortype = info
-                    props['prop'] = prop
-                    props['valu'] = valu
-
-                    yield (spliceoffs, ('prop:del', props))
-                    continue
-
-                if edit == EDIT_TAG_SET:
-                    tag, valu, oldv = info
-                    props['tag'] = tag
-                    props['valu'] = valu
-                    props['oldv'] = oldv
-
-                    yield (spliceoffs, ('tag:add', props))
-                    continue
-
-                if edit == EDIT_TAG_DEL:
-                    tag, valu = info
-                    props['tag'] = tag
-                    props['valu'] = valu
-
-                    yield (spliceoffs, ('tag:del', props))
-                    continue
-
-                if edit == EDIT_TAGPROP_SET:
-                    tag, prop, valu, oldv, stortype = info
-                    props['tag'] = tag
-                    props['prop'] = prop
-                    props['valu'] = valu
-                    props['oldv'] = oldv
-
-                    yield (spliceoffs, ('tag:prop:set', props))
-                    continue
-
-                if edit == EDIT_TAGPROP_DEL:
-                    tag, prop, valu, stortype = info
-                    props['tag'] = tag
-                    props['prop'] = prop
-                    props['valu'] = valu
-
-                    yield (spliceoffs, ('tag:prop:del', props))
-
     @contextlib.asynccontextmanager
     async def getNodeEditWindow(self):
         if not self.logedits:

synapse/lib/lmdbslab.py

@@ -1196,6 +1196,17 @@ class Slab(s_base.Base):
         with self.xact.cursor(db=realdb) as curs:
             return curs.set_key_dup(lkey, lval)
 
+    def count(self, lkey, db=None):
+        realdb, dupsort = self.dbnames[db]
+        with self.xact.cursor(db=realdb) as curs:
+            if not curs.set_key(lkey):
+                return 0
+
+            if not dupsort:
+                return 1
+
+            return curs.count()
+
     def prefexists(self, byts, db=None):
         '''
         Returns True if a prefix exists in the db.

synapse/lib/node.py

@@ -613,17 +613,15 @@ class Node:
             * delete all the tags (bottom up)
                 * fire onDelTag() handlers
                 * delete tag properties from storage
-                * log tag:del splices
 
             * delete all secondary properties
                 * fire onDelProp handler
                 * delete secondary property from storage
-                * log prop:del splices
 
             * delete the primary property
                 * fire onDel handlers for the node
                 * delete primary property from storage
-                * log node:del splices
+
         '''
 
         formname, formvalu = self.ndef