synapse 2.198.0__py311-none-any.whl → 2.200.0__py311-none-any.whl

synapse/cortex.py

@@ -20,7 +20,6 @@ import synapse.lib.base as s_base
 import synapse.lib.cell as s_cell
 import synapse.lib.chop as s_chop
 import synapse.lib.coro as s_coro
-import synapse.lib.hive as s_hive
 import synapse.lib.view as s_view
 import synapse.lib.cache as s_cache
 import synapse.lib.const as s_const
@@ -855,7 +854,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     cellapi = CoreApi
     viewapi = s_view.ViewApi
     layerapi = s_layer.LayerApi
-    hiveapi = s_hive.HiveApi
 
     viewctor = s_view.View.anit
     layrctor = s_layer.Layer.anit
@@ -4656,10 +4654,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         if not path:
             return await self.cellapi.anit(self, link, user)
 
-        if path[0] == 'hive' and user.isAdmin():
-            s_common.deprecated('Cortex /hive telepath path', curv='2.198.0', eolv='2.199.0')
-            return await self.hiveapi.anit(self.hive, user)
-
         if path[0] == 'layer':
 
             if len(path) == 1:

synapse/datamodel.py

@@ -1,6 +1,7 @@
 '''
 An API to assist with the creation and enforcement of cortex data models.
 '''
+import sys
 import asyncio
 import logging
 import collections
@@ -15,6 +16,7 @@ import synapse.lib.cache as s_cache
 import synapse.lib.types as s_types
 import synapse.lib.dyndeps as s_dyndeps
 import synapse.lib.grammar as s_grammar
+import synapse.lib.msgpack as s_msgpack
 
 logger = logging.getLogger(__name__)
 
@@ -142,6 +144,9 @@ class Prop:
             async def depfunc(node, oldv):
                 mesg = f'The property {self.full} is deprecated or using a deprecated type and will be removed in 3.0.0'
                 await node.snap.warnonce(mesg)
+                if __debug__:
+                    sys.audit('synapse.datamodel.Prop.deprecated', mesg, self.full)
+
             self.onSet(depfunc)
 
     def __repr__(self):
@@ -306,6 +311,8 @@ class Form:
             async def depfunc(node):
                 mesg = f'The form {self.full} is deprecated or using a deprecated type and will be removed in 3.0.0'
                 await node.snap.warnonce(mesg)
+                if __debug__:
+                    sys.audit('synapse.datamodel.Form.deprecated', mesg, self.full)
             self.onAdd(depfunc)
 
     def getStorNode(self, form):
@@ -1107,7 +1114,7 @@ class Model:
 
     def _prepFormIface(self, form, iface):
 
-        template = iface.get('template', {})
+        template = s_msgpack.deepcopy(iface.get('template', {}))
         template.update(form.type.info.get('template', {}))
 
         def convert(item):

synapse/lib/hive.py

@@ -86,7 +86,7 @@ class Node(s_base.Base):
         for name, node in self.kids.items():
             yield name, node
 
-class Hive(s_nexus.Pusher, s_telepath.Aware):
+class Hive(s_nexus.Pusher):
     '''
     An optionally persistent atomically accessed tree which implements
     primitives for use in making distributed/clustered services.
@@ -95,8 +95,6 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
 
         await s_nexus.Pusher.__anit__(self, 'hive', nexsroot=nexsroot)
 
-        s_telepath.Aware.__init__(self)
-
         if conf is None:
             conf = {}
 
@@ -161,24 +159,6 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
             for cullname in culls:
                 await node.pop((cullname,))
 
-    async def getHiveAuth(self):
-        '''
-        Retrieve a HiveAuth for hive standalone or non-cell uses.
-
-        Note:
-            This is for the hive's own auth, or for non-cell auth.  It isn't the same auth as for a cell
-        '''
-        import synapse.lib.hiveauth as s_hiveauth
-        if self.auth is None:
-
-            path = tuple(self.conf.get('auth:path').split('/'))
-
-            node = await self.open(path)
-            self.auth = await s_hiveauth.Auth.anit(node, nexsroot=self.nexsroot)
-            self.onfini(self.auth.fini)
-
-        return self.auth
-
     async def _onHiveFini(self):
         await self.root.fini()
 
@@ -423,27 +403,6 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
 
         return node.valu
 
-    async def getTeleApi(self, link, mesg, path):
-        s_common.deprecated('Hive.getTeleApi', curv='2.198.0', eolv='2.199.0')
-        auth = await self.getHiveAuth()
-
-        if not self.conf.get('auth:en'):
-            user = await auth.getUserByName('root')
-            return await HiveApi.anit(self, user)
-
-        name, info = mesg[1].get('auth')
-
-        user = await auth.getUserByName(name)
-        if user is None:
-            raise s_exc.NoSuchUser(name=name)
-
-        # passwd None always fails...
-        passwd = info.get('passwd')
-        if not await user.tryPasswd(passwd):
-            raise s_exc.AuthDeny(mesg='Invalid password', user=user.iden, username=user.name)
-
-        return await HiveApi.anit(self, user)
-
     async def _storLoadHive(self):
         pass
 
@@ -480,224 +439,6 @@ class SlabHive(Hive):
         lkey = '\x00'.join(full).encode('utf8')
         self.slab.pop(lkey, db=self.db)
 
-class HiveApi(s_base.Base):
-
-    async def __anit__(self, hive, user):
-
-        await s_base.Base.__anit__(self)
-
-        self.hive = hive
-        self.user = user
-
-        self.msgq = asyncio.Queue(maxsize=10000)
-
-        self.onfini(self._onHapiFini)
-
-    async def loadHiveTree(self, tree, path=(), trim=False):
-        s_common.deprecated('HiveApi.loadHiveTree', curv='2.167.0')
-        return await self.hive.loadHiveTree(tree, path=path, trim=trim)
-
-    async def saveHiveTree(self, path=()):
-        s_common.deprecated('HiveApi.saveHiveTree', curv='2.167.0')
-        return await self.hive.saveHiveTree(path=path)
-
-    async def treeAndSync(self, path, iden):
-        s_common.deprecated('HiveApi.treeAndSync', curv='2.167.0')
-
-        node = await self.hive.open(path)
-
-        # register handlers...
-        node.on('hive:add', self._onHiveEdit, base=self)
-        node.on('hive:set', self._onHiveEdit, base=self)
-        node.on('hive:pop', self._onHiveEdit, base=self)
-
-        # serialize the subtree into a message and return
-        # via the mesg queue so there is no get/update race
-        root = (node.valu, {})
-
-        todo = collections.deque([(node, root)])
-
-        # breadth first generator
-        while todo:
-
-            node, pode = todo.popleft()
-
-            for name, kidn in node.kids.items():
-
-                kidp = (kidn.valu, {})
-                pode[1][name] = kidp
-
-                todo.append((kidn, kidp))
-
-        await self.msgq.put(('hive:tree', {'path': path, 'tree': root}))
-        await self.msgq.put(('hive:sync', {'iden': iden}))
-        return
-
-    async def setAndSync(self, path, valu, iden, nexs=False):
-        s_common.deprecated('HiveApi.setAndSync', curv='2.167.0')
-
-        valu = await self.hive.set(path, valu, nexs=nexs)
-        await self.msgq.put(('hive:sync', {'iden': iden}))
-        return valu
-
-    async def addAndSync(self, path, valu, iden):
-        s_common.deprecated('HiveApi.addAndSync', curv='2.167.0')
-
-        valu = await self.hive.add(path, valu)
-        await self.msgq.put(('hive:sync', {'iden': iden}))
-        return valu
-
-    async def popAndSync(self, path, iden, nexs=False):
-        s_common.deprecated('HiveApi.popAndSync', curv='2.167.0')
-
-        valu = await self.hive.pop(path, nexs=nexs)
-        await self.msgq.put(('hive:sync', {'iden': iden}))
-        return valu
-
-    async def _onHapiFini(self):
-        await self.msgq.put(None)
-
-    async def _onHiveEdit(self, mesg):
-        self.msgq.put_nowait(mesg)
-
-    async def get(self, full):
-        s_common.deprecated('HiveApi.get', curv='2.167.0')
-        return await self.hive.get(full)
-
-    async def edits(self):
-        s_common.deprecated('HiveApi.edits', curv='2.167.0')
-
-        while not self.isfini:
-
-            item = await self.msgq.get()
-            if item is None:
-                return
-
-            yield item
-
-class TeleHive(Hive):
-    '''
-    A Hive that acts as a consistent read cache for a telepath proxy Hive
-    '''
-
-    async def __anit__(self, proxy):
-
-        self.proxy = proxy
-
-        await Hive.__anit__(self)
-
-        self.lock = asyncio.Lock()
-
-        self.syncevents = {}  # iden: asyncio.Event()
-
-        # fire a task to sync the sections of the tree we open
-        self.schedCoro(self._runHiveLoop())
-
-        self.mesgbus = await s_base.Base.anit()
-        self.mesgbus.on('hive:set', self._onHiveSet)
-        self.mesgbus.on('hive:pop', self._onHivePop)
-        self.mesgbus.on('hive:tree', self._onHiveTree)
-        self.mesgbus.on('hive:sync', self._onHiveSync)
-
-        self.onfini(self.mesgbus.fini)
-
-        self.onfini(proxy.fini)
-
-    async def _onHiveSync(self, mesg):
-
-        iden = mesg[1].get('iden')
-        evnt = self.syncevents.pop(iden, None)
-        if evnt is None:
-            return
-
-        evnt.set()
-
-    def _getSyncIden(self):
-        iden = s_common.guid()
-        evnt = asyncio.Event()
-        self.syncevents[iden] = evnt
-        return iden, evnt
-
-    async def _runHiveLoop(self):
-        while not self.isfini:
-            async for mesg in self.proxy.edits():
-                await self.mesgbus.dist(mesg)
-
-    async def _onHiveSet(self, mesg):
-        path = mesg[1].get('path')
-        valu = mesg[1].get('valu')
-        await Hive.set(self, path, valu)
-
-    async def _onHivePop(self, mesg):
-        path = mesg[1].get('path')
-        await Hive.pop(self, path)
-
-    async def _onHiveTree(self, mesg):
-
-        # get an entire tree update at once
-        path = mesg[1].get('path')
-        tree = mesg[1].get('tree')
-
-        node = await Hive.open(self, path)
-
-        todo = collections.deque([(node, path, tree)])
-
-        while todo:
-
-            node, path, (valu, kids) = todo.popleft()
-
-            # do *not* go through the set() API
-            node.valu = valu
-            for name, kidt in kids.items():
-
-                kidp = path + (name,)
-                kidn = await Hive.open(self, kidp)
-
-                todo.append((kidn, kidp, kidt))
-
-    async def set(self, path, valu, nexs=False):
-        iden, evnt = self._getSyncIden()
-        valu = await self.proxy.setAndSync(path, valu, iden, nexs=nexs)
-        await evnt.wait()
-        return valu
-
-    async def add(self, path, valu):
-        iden, evnt = self._getSyncIden()
-        valu = await self.proxy.addAndSync(path, valu, iden)
-        await evnt.wait()
-        return valu
-
-    async def pop(self, path, nexs=False):
-        iden, evnt = self._getSyncIden()
-        valu = await self.proxy.popAndSync(path, iden, nexs=nexs)
-        await evnt.wait()
-        return valu
-
-    async def get(self, path):
-        return await self.proxy.get(path)
-
-    async def open(self, path):
-
-        # try once pre-lock for speed
-        node = self.nodes.get(path)
-        if node is not None:
-            return node
-
-        async with self.lock:
-
-            # try again with lock to avoid race
-            node = self.nodes.get(path)
-            if node is not None:
-                return node
-
-            iden, evnt = self._getSyncIden()
-
-            await self.proxy.treeAndSync(path, iden)
-
-            await evnt.wait()
-
-            return self.nodes.get(path)
-
 class HiveDict(s_base.Base):
     '''
     '''
@@ -758,11 +499,6 @@ def iterpath(path):
     for i in range(len(path)):
         yield path[:i + 1]
 
-async def openurl(url, **opts):
-    s_common.deprecated('synapse.lib.hive.openurl()', curv='2.198.0', eolv='2.199.0')
-    prox = await s_telepath.openurl(url, **opts)
-    return await TeleHive.anit(prox)
-
 async def opendir(dirn, conf=None):
     slab = await s_slab.Slab.anit(dirn, map_size=s_const.gibibyte)
     db = slab.initdb('hive')

synapse/lib/snap.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import sys
 import types
 import asyncio
 import logging
@@ -12,14 +11,12 @@ import synapse.exc as s_exc
 import synapse.common as s_common
 
 import synapse.lib.base as s_base
-import synapse.lib.coro as s_coro
 import synapse.lib.node as s_node
 import synapse.lib.time as s_time
 import synapse.lib.cache as s_cache
 import synapse.lib.layer as s_layer
 import synapse.lib.storm as s_storm
 import synapse.lib.types as s_types
-import synapse.lib.spooled as s_spooled
 
 logger = logging.getLogger(__name__)
 
@@ -138,16 +135,30 @@ class ProtoNode:
 
         if not await self.ctx.snap.hasNodeEdge(self.buid, verb, s_common.uhex(n2iden)):
             self.edges.add(tupl)
-
             if len(self.edges) >= 1000:
-                nodeedits = self.ctx.getNodeEdits()
-                await self.ctx.snap.applyNodeEdits(nodeedits)
-                self.edges.clear()
-
+                await self.flushEdits()
             return True
 
         return False
 
+    async def flushEdits(self):
+        if (nodeedit := self.getNodeEdit()) is not None:
+            nodecache = {self.buid: self.node}
+            nodes = await self.ctx.snap.applyNodeEdits((nodeedit,), nodecache=nodecache, meta=self.ctx.meta)
+
+            if self.node is None:
+                if nodes and nodes[0].buid == self.buid:
+                    self.node = nodes[0]
+                else:  # pragma: no cover
+                    self.node = await self.ctx.snap.getNodeByBuid(self.buid)
+
+            self.tags.clear()
+            self.props.clear()
+            self.tagprops.clear()
+            self.edges.clear()
+            self.edgedels.clear()
+            self.nodedata.clear()
+
     async def delEdge(self, verb, n2iden):
 
         if not isinstance(verb, str):
@@ -175,12 +186,8 @@ class ProtoNode:
 
         if await self.ctx.snap.layers[-1].hasNodeEdge(self.buid, verb, s_common.uhex(n2iden)):
             self.edgedels.add(tupl)
-
             if len(self.edgedels) >= 1000:
-                nodeedits = self.ctx.getNodeEdits()
-                await self.ctx.snap.applyNodeEdits(nodeedits)
-                self.edgedels.clear()
-
+                await self.flushEdits()
             return True
 
         return False
@@ -429,6 +436,9 @@ class ProtoNode:
                 full = f'{prop.name}:{subname}'
                 subprop = self.form.props.get(full)
                 if subprop is not None and not subprop.locked:
+                    if subprop.deprecated:
+                        self.ctx.snap._skipPropDeprWarn(subprop.full)
+
                     await self.set(full, subvalu)
 
         propadds = norminfo.get('adds')
@@ -438,11 +448,14 @@ class ProtoNode:
 
         return True
 
-    async def getSetOps(self, name, valu, norminfo=None):
+    async def getSetSubOps(self, name, valu, norminfo=None):
         prop = self.form.props.get(name)
-        if prop is None:
+        if prop is None or prop.locked:
             return ()
 
+        if prop.deprecated:
+            self.ctx.snap._skipPropDeprWarn(prop.full)
+
         retn = await self._set(prop, valu, norminfo=norminfo)
         if retn is False:
             return ()
@@ -459,9 +472,7 @@ class ProtoNode:
         if propsubs is not None:
             for subname, subvalu in propsubs.items():
                 full = f'{prop.name}:{subname}'
-                subprop = self.form.props.get(full)
-                if subprop is not None and not subprop.locked:
-                    ops.append(self.getSetOps(full, subvalu))
+                ops.append(self.getSetSubOps(full, subvalu))
 
         propadds = norminfo.get('adds')
         if propadds is not None:
@@ -474,7 +485,8 @@ class SnapEditor:
     '''
     A SnapEditor allows tracking node edits with subs/deps as a transaction.
     '''
-    def __init__(self, snap):
+    def __init__(self, snap, meta=None):
+        self.meta = meta
         self.snap = snap
         self.protonodes = {}
         self.maxnodes = snap.core.maxnodes
@@ -492,6 +504,19 @@ class SnapEditor:
                 nodeedits.append(nodeedit)
         return nodeedits
 
+    async def flushEdits(self):
+        nodecache = {}
+        nodeedits = []
+        for protonode in self.protonodes.values():
+            if (nodeedit := protonode.getNodeEdit()) is not None:
+                nodeedits.append(nodeedit)
+                nodecache[protonode.buid] = protonode.node
+
+        if nodeedits:
+            await self.snap.applyNodeEdits(nodeedits, nodecache=nodecache, meta=self.meta)
+
+        self.protonodes.clear()
+
     async def _addNode(self, form, valu, props=None, norminfo=None):
 
         self.snap.core._checkMaxNodes()
@@ -568,7 +593,7 @@ class SnapEditor:
         subs = norminfo.get('subs')
         if subs is not None:
             for prop, valu in subs.items():
-                ops.append(protonode.getSetOps(prop, valu))
+                ops.append(protonode.getSetSubOps(prop, valu))
 
         adds = norminfo.get('adds')
         if adds is not None:
@@ -604,7 +629,7 @@ class SnapEditor:
         subs = norminfo.get('subs')
         if subs is not None:
             for prop, valu in subs.items():
-                ops.append(protonode.getSetOps(prop, valu))
+                ops.append(protonode.getSetSubOps(prop, valu))
 
             while ops:
                 oset = ops.popleft()
@@ -861,6 +886,10 @@ class Snap(s_base.Base):
         self._warnonce_keys.add(mesg)
         await self.warn(mesg, log, **info)
 
+    def _skipPropDeprWarn(self, name):
+        mesg = f'The property {name} is deprecated or using a deprecated type and will be removed in 3.0.0'
+        self._warnonce_keys.add(mesg)
+
     async def getNodeByBuid(self, buid):
         '''
         Retrieve a node tuple by binary id.
@@ -1218,11 +1247,13 @@ class Snap(s_base.Base):
         if nodes:
             return nodes[0]
 
-    async def applyNodeEdits(self, edits, nodecache=None):
+    async def applyNodeEdits(self, edits, nodecache=None, meta=None):
         '''
         Sends edits to the write layer and evaluates the consequences (triggers, node object updates)
         '''
-        meta = await self.getSnapMeta()
+        if meta is None:
+            meta = await self.getSnapMeta()
+
         saveoff, changes, nodes = await self._applyNodeEdits(edits, meta, nodecache=nodecache)
         return nodes
 
@@ -1572,7 +1603,8 @@ class Snap(s_base.Base):
         proptype = prop.type
         for prop in prop.getAlts():
             if prop.type.isarray and prop.type.arraytype == proptype:
-                if valu in node.get(prop.name):
+                arryvalu = node.get(prop.name)
+                if arryvalu is not None and valu in arryvalu:
                     return True
             else:
                 if node.get(prop.name) == valu:
@@ -1660,10 +1692,6 @@ class Snap(s_base.Base):
         return tagnode
 
     async def _raiseOnStrict(self, ctor, mesg, **info):
-        if __debug__:
-            if issubclass(ctor, s_exc.IsDeprLocked):
-                sys.audit('synapse.exc.IsDeprLocked', (mesg, info))
-
         if self.strict:
             raise ctor(mesg=mesg, **info)
         await self.warn(mesg)

synapse/lib/storm.py

@@ -2077,7 +2077,7 @@ class Runtime(s_base.Base):
 
 class Parser:
 
-    def __init__(self, prog=None, descr=None, root=None, model=None):
+    def __init__(self, prog=None, descr=None, root=None, model=None, cdef=None):
 
         if root is None:
             root = self
@@ -2088,6 +2088,7 @@ class Parser:
 
         self.prog = prog
         self.descr = descr
+        self.cdef = cdef
 
         self.exc = None
 
@@ -2365,6 +2366,21 @@ class Parser:
 
         self._printf(f'Usage: {self.prog} [options] {posargs}')
 
+        if self.cdef is not None and (endpoints := self.cdef.get('endpoints')):
+            self._printf('')
+            self._printf('Endpoints:')
+            self._printf('')
+            base_w = 32
+            wrap_w = 120 - base_w
+            for endpoint in endpoints:
+                path = endpoint['path']
+                desc = endpoint.get('desc', '')
+                base = f'    {path}'
+                wrap_desc = self._wrap_text(desc, wrap_w) if desc else ['']
+                self._printf(f'{base:<{base_w-2}}: {wrap_desc[0]}')
+                for ln in wrap_desc[1:]:
+                    self._printf(f'{"":<{base_w}}{ln}')
+
         options = [x for x in self.allargs if x[0][0].startswith('-')]
 
         self._printf('')
@@ -2655,6 +2671,7 @@ class PureCmd(Cmd):
         if inputs:
             pars.set_inputs(inputs)
 
+        pars.cdef = self.cdef
         return pars
 
     async def execStormCmd(self, runt, genr):
@@ -3653,23 +3670,26 @@ class MergeCmd(Cmd):
 
             genr = diffgenr()
 
-        async with await runt.snap.view.parent.snap(user=runt.user.iden) as snap:
+        async with await runt.snap.view.parent.snap(user=runt.user) as snap:
             snap.strict = False
 
             snap.on('warn', runt.snap.dist)
 
+            meta = {'user': runt.user.iden}
+
+            if doapply:
+                editor = s_snap.SnapEditor(snap, meta=meta)
+
             async for node, path in genr:
 
                 # the timestamp for the adds/subs of each node merge will match
                 nodeiden = node.iden()
-                meta = {'user': runt.user.iden, 'time': s_common.now()}
+
+                meta['time'] = s_common.now()
 
                 sodes = await node.getStorNodes()
                 sode = sodes[0]
 
-                if doapply:
-                    editor = s_snap.SnapEditor(snap)
-
                 subs = []
 
                 # check all node perms first
@@ -3818,13 +3838,11 @@ class MergeCmd(Cmd):
                     subs.append((s_layer.EDIT_NODE_DEL, valu, ()))
 
                 if doapply:
-                    addedits = editor.getNodeEdits()
-                    if addedits:
-                        await runt.snap.view.parent.storNodeEdits(addedits, meta=meta)
+                    await editor.flushEdits()
 
                     if subs:
                         subedits = [(node.buid, node.form.name, subs)]
-                        await runt.snap.view.storNodeEdits(subedits, meta=meta)
+                        await runt.snap.applyNodeEdits(subedits, nodecache={node.buid: node}, meta=meta)
 
                 runt.snap.clearCachedNode(node.buid)
                 yield await runt.snap.getNodeByBuid(node.buid), path

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 198, 0)
+version = (2, 200, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = '047e5e996d0bcaad4ada4ea390a726aca1abba9b'
+commit = 'ab9ca513fe33d52df7a3122c0bf4eab6951b53d3'

synapse/lib/view.py

@@ -1474,7 +1474,8 @@ class View(s_nexus.Pusher):  # type: ignore
 
             meta = await snap.getSnapMeta()
             async for nodeedits in fromlayr.iterLayerNodeEdits():
-                await self.parent.storNodeEdits([nodeedits], meta)
+                meta['time'] = s_common.now()
+                await snap.saveNodeEdits([nodeedits], meta)
 
     async def swapLayer(self):
         oldlayr = self.layers[0]