synapse 2.154.1__py311-none-any.whl → 2.156.0__py311-none-any.whl

synapse/lib/layer.py

@@ -170,19 +170,6 @@ class LayerApi(s_cell.CellApi):
         async for item in self.layr.syncNodeEdits2(offs, wait=wait):
             yield item
 
-    async def splices(self, offs=None, size=None):
-        '''
-        This API is deprecated.
-
-        Yield (offs, splice) tuples from the nodeedit log starting from the given offset.
-
-        Nodeedits will be flattened into splices before being yielded.
-        '''
-        s_common.deprdate('Layer.splices() telepath API', s_common._splicedepr)
-        await self._reqUserAllowed(self.liftperm)
-        async for item in self.layr.splices(offs=offs, size=size):
-            yield item
-
     async def getEditIndx(self):
         '''
         Returns what will be the *next* nodeedit log index.
@@ -2023,17 +2010,14 @@ class Layer(s_nexus.Pusher):
         ret['totalsize'] = await self.getLayerSize()
         return ret
 
-    async def truncate(self):
-        self._reqNotReadOnly()
-        return await self._push('layer:truncate')
-
     @s_nexus.Pusher.onPush('layer:truncate')
     async def _truncate(self):
         '''
         Nuke all the contents in the layer, leaving an empty layer
         NOTE: This internal API is deprecated but is kept for Nexus event backward compatibility
         '''
-        s_common.deprdate('Layer.truncate() API', s_common._splicedepr)
+        # TODO: Remove this in 3.0.0
+        s_common.deprecated('layer:truncate Nexus handler', curv='2.156.0')
 
         self.dirty.clear()
         self.buidcache.clear()
@@ -2044,8 +2028,6 @@ class Layer(s_nexus.Pusher):
 
         await self._initLayerStorage()
 
-    # async def wipe(self, meta): ...
-
     async def iterWipeNodeEdits(self):
 
         await self._saveDirtySodes()
@@ -2914,6 +2896,7 @@ class Layer(s_nexus.Pusher):
             abrv = self.tagabrv.bytsToAbrv(tagname.encode())
             if formname is not None:
                 abrv += self.getPropAbrv(formname, None)
+                return self.layrslab.count(abrv, db=self.bytag)
 
         except s_exc.NoSuchAbrv:
             return 0
@@ -2931,6 +2914,44 @@ class Layer(s_nexus.Pusher):
 
         return await self.layrslab.countByPref(abrv, db=self.byprop, maxsize=maxsize)
 
+    def getPropValuCount(self, formname, propname, stortype, valu):
+        try:
+            abrv = self.getPropAbrv(formname, propname)
+        except s_exc.NoSuchAbrv:
+            return 0
+
+        if stortype & 0x8000:
+            stortype = STOR_TYPE_MSGP
+
+        count = 0
+        for indx in self.getStorIndx(stortype, valu):
+            count += self.layrslab.count(abrv + indx, db=self.byprop)
+
+        return count
+
+    async def getPropArrayCount(self, formname, propname=None):
+        '''
+        Return the number of invidiual value rows in the layer for the given array form/prop.
+        '''
+        try:
+            abrv = self.getPropAbrv(formname, propname)
+        except s_exc.NoSuchAbrv:
+            return 0
+
+        return await self.layrslab.countByPref(abrv, db=self.byarray)
+
+    def getPropArrayValuCount(self, formname, propname, stortype, valu):
+        try:
+            abrv = self.getPropAbrv(formname, propname)
+        except s_exc.NoSuchAbrv:
+            return 0
+
+        count = 0
+        for indx in self.getStorIndx(stortype, valu):
+            count += self.layrslab.count(abrv + indx, db=self.byarray)
+
+        return count
+
     async def getUnivPropCount(self, propname, maxsize=None):
         '''
         Return the number of universal property rows in the layer for the given prop.
@@ -2942,6 +2963,29 @@ class Layer(s_nexus.Pusher):
 
         return await self.layrslab.countByPref(abrv, db=self.byprop, maxsize=maxsize)
 
+    async def getTagPropCount(self, form, tag, prop):
+        '''
+        Return the number of property rows in the layer for the given form/tag/prop.
+        '''
+        try:
+            abrv = self.getTagPropAbrv(form, tag, prop)
+        except s_exc.NoSuchAbrv:
+            return 0
+
+        return await self.layrslab.countByPref(abrv, db=self.bytagprop)
+
+    def getTagPropValuCount(self, form, tag, prop, stortype, valu):
+        try:
+            abrv = self.getTagPropAbrv(form, tag, prop)
+        except s_exc.NoSuchAbrv:
+            return 0
+
+        count = 0
+        for indx in self.getStorIndx(stortype, valu):
+            count += self.layrslab.count(abrv + indx, db=self.bytagprop)
+
+        return count
+
     async def liftByTag(self, tag, form=None, reverse=False):
 
         try:
@@ -4151,76 +4195,6 @@ class Layer(s_nexus.Pusher):
             yield buid, s_msgpack.un(byts)
             await asyncio.sleep(0)
 
-    async def splices(self, offs=None, size=None):
-        '''
-        This API is deprecated.
-
-        Yield (offs, splice) tuples from the nodeedit log starting from the given offset.
-
-        Nodeedits will be flattened into splices before being yielded.
-        '''
-        s_common.deprdate('Layer.splices() API', s_common._splicedepr)
-        if not self.logedits:
-            return
-
-        if offs is None:
-            offs = (0, 0, 0)
-
-        if size is not None:
-
-            count = 0
-            async for offset, nodeedits, meta in self.iterNodeEditLog(offs[0]):
-                async for splice in self.makeSplices(offset, nodeedits, meta):
-
-                    if splice[0] < offs:
-                        continue
-
-                    if count >= size:
-                        return
-
-                    yield splice
-                    count = count + 1
-        else:
-            async for offset, nodeedits, meta in self.iterNodeEditLog(offs[0]):
-                async for splice in self.makeSplices(offset, nodeedits, meta):
-
-                    if splice[0] < offs:
-                        continue
-
-                    yield splice
-
-    async def splicesBack(self, offs=None, size=None):
-
-        s_common.deprdate('Layer.splicesBack() API', s_common._splicedepr)
-        if not self.logedits:
-            return
-
-        if offs is None:
-            offs = (await self.getEditIndx(), 0, 0)
-
-        if size is not None:
-
-            count = 0
-            async for offset, nodeedits, meta in self.iterNodeEditLogBack(offs[0]):
-                async for splice in self.makeSplices(offset, nodeedits, meta, reverse=True):
-
-                    if splice[0] > offs:
-                        continue
-
-                    if count >= size:
-                        return
-
-                    yield splice
-                    count += 1
-        else:
-            async for offset, nodeedits, meta in self.iterNodeEditLogBack(offs[0]):
-                async for splice in self.makeSplices(offset, nodeedits, meta, reverse=True):
-
-                    if splice[0] > offs:
-                        continue
-
-                    yield splice
-
     async def iterNodeEditLog(self, offs=0):
         '''
         Iterate the node edit log and yield (offs, edits, meta) tuples.
@@ -4312,117 +4286,6 @@ class Layer(s_nexus.Pusher):
             if count % 1000 == 0:
                 yield (curoff, (None, None, EDIT_PROGRESS, (), ()))
 
-    async def makeSplices(self, offs, nodeedits, meta, reverse=False):
-        '''
-        Flatten a set of nodeedits into splices.
-        '''
-        if meta is None:
-            meta = {}
-
-        user = meta.get('user')
-        time = meta.get('time')
-        prov = meta.get('prov')
-
-        if reverse:
-            nodegenr = reversed(list(enumerate(nodeedits)))
-        else:
-            nodegenr = enumerate(nodeedits)
-
-        for nodeoffs, (buid, form, edits) in nodegenr:
-
-            formvalu = None
-
-            if reverse:
-                editgenr = reversed(list(enumerate(edits)))
-            else:
-                editgenr = enumerate(edits)
-
-            for editoffs, (edit, info, _) in editgenr:
-
-                if edit in (EDIT_NODEDATA_SET, EDIT_NODEDATA_DEL, EDIT_EDGE_ADD, EDIT_EDGE_DEL):
-                    continue
-
-                spliceoffs = (offs, nodeoffs, editoffs)
-
-                props = {
-                    'time': time,
-                    'user': user,
-                }
-
-                if prov is not None:
-                    props['prov'] = prov
-
-                if edit == EDIT_NODE_ADD:
-                    formvalu, stortype = info
-                    props['ndef'] = (form, formvalu)
-
-                    yield (spliceoffs, ('node:add', props))
-                    continue
-
-                if edit == EDIT_NODE_DEL:
-                    formvalu, stortype = info
-                    props['ndef'] = (form, formvalu)
-
-                    yield (spliceoffs, ('node:del', props))
-                    continue
-
-                if formvalu is None:
-                    formvalu = await self.getNodeValu(buid)
-
-                props['ndef'] = (form, formvalu)
-
-                if edit == EDIT_PROP_SET:
-                    prop, valu, oldv, stortype = info
-                    props['prop'] = prop
-                    props['valu'] = valu
-                    props['oldv'] = oldv
-
-                    yield (spliceoffs, ('prop:set', props))
-                    continue
-
-                if edit == EDIT_PROP_DEL:
-                    prop, valu, stortype = info
-                    props['prop'] = prop
-                    props['valu'] = valu
-
-                    yield (spliceoffs, ('prop:del', props))
-                    continue
-
-                if edit == EDIT_TAG_SET:
-                    tag, valu, oldv = info
-                    props['tag'] = tag
-                    props['valu'] = valu
-                    props['oldv'] = oldv
-
-                    yield (spliceoffs, ('tag:add', props))
-                    continue
-
-                if edit == EDIT_TAG_DEL:
-                    tag, valu = info
-                    props['tag'] = tag
-                    props['valu'] = valu
-
-                    yield (spliceoffs, ('tag:del', props))
-                    continue
-
-                if edit == EDIT_TAGPROP_SET:
-                    tag, prop, valu, oldv, stortype = info
-                    props['tag'] = tag
-                    props['prop'] = prop
-                    props['valu'] = valu
-                    props['oldv'] = oldv
-
-                    yield (spliceoffs, ('tag:prop:set', props))
-                    continue
-
-                if edit == EDIT_TAGPROP_DEL:
-                    tag, prop, valu, stortype = info
-                    props['tag'] = tag
-                    props['prop'] = prop
-                    props['valu'] = valu
-
-                    yield (spliceoffs, ('tag:prop:del', props))
-
     @contextlib.asynccontextmanager
     async def getNodeEditWindow(self):
         if not self.logedits:

synapse/lib/lmdbslab.py

@@ -1196,6 +1196,17 @@ class Slab(s_base.Base):
         with self.xact.cursor(db=realdb) as curs:
             return curs.set_key_dup(lkey, lval)
 
+    def count(self, lkey, db=None):
+        realdb, dupsort = self.dbnames[db]
+        with self.xact.cursor(db=realdb) as curs:
+            if not curs.set_key(lkey):
+                return 0
+
+            if not dupsort:
+                return 1
+
+            return curs.count()
+
     def prefexists(self, byts, db=None):
         '''
         Returns True if a prefix exists in the db.

synapse/lib/node.py

@@ -613,17 +613,15 @@ class Node:
             * delete all the tags (bottom up)
                 * fire onDelTag() handlers
                 * delete tag properties from storage
-                * log tag:del splices
 
             * delete all secondary properties
                 * fire onDelProp handler
                 * delete secondary property from storage
-                * log prop:del splices
 
             * delete the primary property
                 * fire onDel handlers for the node
                 * delete primary property from storage
-                * log node:del splices
+
         '''
 
         formname, formvalu = self.ndef

synapse/lib/parser.py

@@ -113,10 +113,13 @@ terminalEnglishMap = {
     '_EDGEN1INIT': '-(',
     '_EDGEN2INIT': '<(',
     '_EDGEN2FINI': ')-',
+    '_EDGEN1JOINFINI': ')+>',
+    '_EDGEN2JOININIT': '<+(',
     '_ELSE': 'else',
     '_EMBEDQUERYSTART': '${',
     '_EXPRCOLONNOSPACE': ':',
     '_EMIT': 'emit',
+    '_EMPTY': 'empty',
     '_FINI': 'fini',
     '_HASH': '#',
     '_HASHSPACE': '#',
@@ -131,6 +134,8 @@ terminalEnglishMap = {
     '_RIGHTJOIN': '-+>',
     '_RIGHTPIVOT': '->',
     '_STOP': 'stop',
+    '_WALKNJOINN1': '--+>',
+    '_WALKNJOINN2': '<+--',
     '_WALKNPIVON1': '-->',
     '_WALKNPIVON2': '<--',
     '$END': 'end of input',
@@ -640,6 +645,7 @@ ruleClassMap = {
     'editparens': s_ast.EditParens,
     'emit': s_ast.Emit,
     'initblock': s_ast.InitBlock,
+    'emptyblock': s_ast.EmptyBlock,
     'finiblock': s_ast.FiniBlock,
     'formname': s_ast.FormName,
     'editpropdel': lambda astinfo, kids: s_ast.EditPropDel(astinfo, kids[1:]),
@@ -690,8 +696,12 @@ ruleClassMap = {
     'liftbyformtagprop': s_ast.LiftFormTagProp,
     'looklist': s_ast.LookList,
     'lookup': s_ast.Lookup,
+    'n1join': lambda astinfo, kids: s_ast.N1Walk(astinfo, kids, isjoin=True),
+    'n2join': lambda astinfo, kids: s_ast.N2Walk(astinfo, kids, isjoin=True),
     'n1walk': s_ast.N1Walk,
     'n2walk': s_ast.N2Walk,
+    'n1walknjoin': lambda astinfo, kids: s_ast.N1WalkNPivo(astinfo, kids, isjoin=True),
+    'n2walknjoin': lambda astinfo, kids: s_ast.N2WalkNPivo(astinfo, kids, isjoin=True),
     'n1walknpivo': s_ast.N1WalkNPivo,
     'n2walknpivo': s_ast.N2WalkNPivo,
     'notcond': s_ast.NotCond,

synapse/lib/slabseqn.py

@@ -123,6 +123,7 @@ class SlabSeqn:
 
         size = 0
         tick = s_common.now()
+        abstick = s_common.mononow()
 
         for item in items:
 
@@ -136,7 +137,7 @@ class SlabSeqn:
             rows.append((lkey, byts))
 
         retn = self.slab.putmulti(rows, append=True, db=self.db)
-        took = s_common.now() - tick
+        took = s_common.mononow() - abstick
 
         assert retn, "Not adding the largest indices"
 

synapse/lib/snap.py

@@ -301,11 +301,7 @@ class ProtoNode:
         if self.node is not None:
             return self.node.get(name)
 
-    async def set(self, name, valu, norminfo=None):
-
-        prop = self.form.props.get(name)
-        if prop is None:
-            return False
+    async def _set(self, prop, valu, norminfo=None):
 
         if prop.locked:
             mesg = f'Prop {prop.full} is locked due to deprecation.'
@@ -339,7 +335,7 @@ class ProtoNode:
                 await self.ctx.snap._raiseOnStrict(s_exc.IsDeprLocked, mesg)
                 return False
 
-        curv = self.get(name)
+        curv = self.get(prop.name)
         if curv == valu:
             return False
 
@@ -351,7 +347,20 @@ class ProtoNode:
         if self.node is not None:
             await self.ctx.snap.core._callPropSetHook(self.node, prop, valu)
 
-        self.props[name] = valu
+        self.props[prop.name] = valu
+
+        return valu, norminfo
+
+    async def set(self, name, valu, norminfo=None):
+        prop = self.form.props.get(name)
+        if prop is None:
+            return False
+
+        retn = await self._set(prop, valu, norminfo=norminfo)
+        if retn is False:
+            return False
+
+        (valu, norminfo) = retn
 
         propform = self.ctx.snap.core.model.form(prop.type.name)
         if propform is not None:
@@ -372,6 +381,37 @@ class ProtoNode:
 
         return True
 
+    async def getSetOps(self, name, valu, norminfo=None):
+        prop = self.form.props.get(name)
+        if prop is None:
+            return ()
+
+        retn = await self._set(prop, valu, norminfo=norminfo)
+        if retn is False:
+            return ()
+
+        (valu, norminfo) = retn
+        ops = []
+
+        propform = self.ctx.snap.core.model.form(prop.type.name)
+        if propform is not None:
+            ops.append(self.ctx.getAddNodeOps(propform.name, valu, norminfo=norminfo))
+
+        # TODO can we mandate any subs are returned pre-normalized?
+        propsubs = norminfo.get('subs')
+        if propsubs is not None:
+            for subname, subvalu in propsubs.items():
+                full = f'{prop.name}:{subname}'
+                if self.form.props.get(full) is not None:
+                    ops.append(self.getSetOps(full, subvalu))
+
+        propadds = norminfo.get('adds')
+        if propadds is not None:
+            for addname, addvalu, addinfo in propadds:
+                ops.append(self.ctx.getAddNodeOps(addname, addvalu, norminfo=addinfo))
+
+        return ops
+
 class SnapEditor:
     '''
     A SnapEditor allows tracking node edits with subs/deps as a transaction.
@@ -394,17 +434,12 @@ class SnapEditor:
                 nodeedits.append(nodeedit)
         return nodeedits
 
-    async def addNode(self, formname, valu, props=None, norminfo=None):
+    async def _addNode(self, form, valu, props=None, norminfo=None):
 
         self.snap.core._checkMaxNodes()
 
-        form = self.snap.core.model.form(formname)
-        if form is None:
-            mesg = f'No form named {formname} for valu={valu}.'
-            return await self.snap._raiseOnStrict(s_exc.NoSuchForm, mesg)
-
         if form.isrunt:
-            mesg = f'Cannot make runt nodes: {formname}.'
+            mesg = f'Cannot make runt nodes: {form.name}.'
             return await self.snap._raiseOnStrict(s_exc.IsRuntForm, mesg)
 
         if form.locked:
@@ -417,15 +452,73 @@ class SnapEditor:
             except s_exc.BadTypeValu as e:
                 e.errinfo['form'] = form.name
                 if self.snap.strict: raise e
-                await self.snap.warn(f'addNode() BadTypeValu {formname}={valu} {e}')
+                await self.snap.warn(f'addNode() BadTypeValu {form.name}={valu} {e}')
                 return None
 
+        return valu, norminfo
+
+    async def addNode(self, formname, valu, props=None, norminfo=None):
+
+        form = self.snap.core.model.form(formname)
+        if form is None:
+            mesg = f'No form named {formname} for valu={valu}.'
+            return await self.snap._raiseOnStrict(s_exc.NoSuchForm, mesg)
+
+        retn = await self._addNode(form, valu, props=props, norminfo=norminfo)
+        if retn is None:
+            return None
+
+        valu, norminfo = retn
+
         protonode = await self._initProtoNode(form, valu, norminfo)
         if props is not None:
             [await protonode.set(p, v) for (p, v) in props.items()]
 
         return protonode
 
+    async def getAddNodeOps(self, formname, valu, props=None, norminfo=None):
+
+        form = self.snap.core.model.form(formname)
+        if form is None:
+            mesg = f'No form named {formname} for valu={valu}.'
+            await self.snap._raiseOnStrict(s_exc.NoSuchForm, mesg)
+            return()
+
+        retn = await self._addNode(form, valu, props=props, norminfo=norminfo)
+        if retn is None:
+            return ()
+
+        norm, norminfo = retn
+
+        ndef = (form.name, norm)
+
+        protonode = self.protonodes.get(ndef)
+        if protonode is not None:
+            return ()
+
+        buid = s_common.buid(ndef)
+        node = await self.snap.getNodeByBuid(buid)
+        if node is not None:
+            return ()
+
+        protonode = ProtoNode(self, buid, form, norm, node)
+
+        self.protonodes[ndef] = protonode
+
+        ops = []
+
+        subs = norminfo.get('subs')
+        if subs is not None:
+            for prop, valu in subs.items():
+                ops.append(protonode.getSetOps(prop, valu))
+
+        adds = norminfo.get('adds')
+        if adds is not None:
+            for addname, addvalu, addinfo in adds:
+                ops.append(self.getAddNodeOps(addname, addvalu, norminfo=addinfo))
+
+        return ops
+
     def loadNode(self, node):
         protonode = self.protonodes.get(node.ndef)
         if protonode is None:
@@ -448,14 +541,25 @@ class SnapEditor:
 
         self.protonodes[ndef] = protonode
 
+        ops = collections.deque()
+
         subs = norminfo.get('subs')
         if subs is not None:
-            [await protonode.set(p, v) for (p, v) in subs.items()]
+            for prop, valu in subs.items():
+                ops.append(protonode.getSetOps(prop, valu))
+
+            while ops:
+                oset = ops.popleft()
+                ops.extend(await oset)
 
         adds = norminfo.get('adds')
         if adds is not None:
             for addname, addvalu, addinfo in adds:
-                await self.addNode(addname, addvalu, norminfo=addinfo)
+                ops.append(self.getAddNodeOps(addname, addvalu, norminfo=addinfo))
+
+            while ops:
+                oset = ops.popleft()
+                ops.extend(await oset)
 
         return protonode
 
@@ -469,8 +573,6 @@ class Snap(s_base.Base):
 
     Transactions produce the following EventBus events:
 
-    (...any splice...)
-    ('log', {'level': 'mesg': })
     ('print', {}),
     '''
     tagcachesize = 1000
@@ -598,7 +700,6 @@ class Snap(s_base.Base):
 
             yield pode
 
-    @s_coro.genrhelp
     async def storm(self, text, opts=None, user=None):
         '''
         Execute a storm query and yield (Node(), Path()) tuples.
@@ -616,7 +717,6 @@ class Snap(s_base.Base):
             async for x in runt.execute():
                 yield x
 
-    @s_coro.genrhelp
     async def eval(self, text, opts=None, user=None):
         '''
         Run a storm query and yield Node() objects.

synapse/lib/spooled.py

@@ -143,6 +143,15 @@ class Dict(Spooled):
             self.len = len(self.realdict)
             self.realdict.clear()
 
+    def pop(self, key, defv=None):
+        if self.fallback:
+            ret = self.slab.pop(s_msgpack.en(key))
+            if ret is None:
+                return defv
+            self.len -= 1
+            return s_msgpack.un(ret)
+        return self.realdict.pop(key, defv)
+
     def has(self, key):
         if self.fallback:
             return self.slab.has(s_msgpack.en(key))