synapse 2.199.0__tar.gz → 2.200.0__tar.gz

{synapse-2.199.0/synapse.egg-info → synapse-2.200.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: synapse
-Version: 2.199.0
+Version: 2.200.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License: Apache License 2.0
@@ -23,7 +23,7 @@ Requires-Dist: pyOpenSSL<24.3.0,>=24.0.0
 Requires-Dist: cryptography<44.0.0,>=43.0.1
 Requires-Dist: msgpack<1.2.0,>=1.0.5
 Requires-Dist: xxhash<3.6.0,>=1.4.4
-Requires-Dist: lmdb<1.5.0,>=1.2.1
+Requires-Dist: lmdb<1.7.0,>=1.2.1
 Requires-Dist: tornado<7.0.0,>=6.2.0
 Requires-Dist: regex>=2022.9.11
 Requires-Dist: PyYAML<6.1.0,>=5.4

{synapse-2.199.0 → synapse-2.200.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.199.0'
+version = '2.200.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]
@@ -27,7 +27,7 @@ dependencies = [
     'cryptography>=43.0.1,<44.0.0',
     'msgpack>=1.0.5,<1.2.0',
     'xxhash>=1.4.4,<3.6.0',
-    'lmdb>=1.2.1,<1.5.0',
+    'lmdb>=1.2.1,<1.7.0',
     'tornado>=6.2.0,<7.0.0',
     'regex>=2022.9.11',
     'PyYAML>=5.4,<6.1.0',

{synapse-2.199.0 → synapse-2.200.0}/synapse/cortex.py

@@ -20,7 +20,6 @@ import synapse.lib.base as s_base
 import synapse.lib.cell as s_cell
 import synapse.lib.chop as s_chop
 import synapse.lib.coro as s_coro
-import synapse.lib.hive as s_hive
 import synapse.lib.view as s_view
 import synapse.lib.cache as s_cache
 import synapse.lib.const as s_const
@@ -855,7 +854,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     cellapi = CoreApi
     viewapi = s_view.ViewApi
     layerapi = s_layer.LayerApi
-    hiveapi = s_hive.HiveApi
 
     viewctor = s_view.View.anit
     layrctor = s_layer.Layer.anit
@@ -4656,10 +4654,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         if not path:
             return await self.cellapi.anit(self, link, user)
 
-        if path[0] == 'hive' and user.isAdmin():
-            s_common.deprecated('Cortex /hive telepath path', curv='2.198.0', eolv='2.199.0')
-            return await self.hiveapi.anit(self.hive, user)
-
         if path[0] == 'layer':
 
             if len(path) == 1:

{synapse-2.199.0 → synapse-2.200.0}/synapse/datamodel.py

@@ -1,6 +1,7 @@
 '''
 An API to assist with the creation and enforcement of cortex data models.
 '''
+import sys
 import asyncio
 import logging
 import collections
@@ -15,6 +16,7 @@ import synapse.lib.cache as s_cache
 import synapse.lib.types as s_types
 import synapse.lib.dyndeps as s_dyndeps
 import synapse.lib.grammar as s_grammar
+import synapse.lib.msgpack as s_msgpack
 
 logger = logging.getLogger(__name__)
 
@@ -142,6 +144,9 @@ class Prop:
             async def depfunc(node, oldv):
                 mesg = f'The property {self.full} is deprecated or using a deprecated type and will be removed in 3.0.0'
                 await node.snap.warnonce(mesg)
+                if __debug__:
+                    sys.audit('synapse.datamodel.Prop.deprecated', mesg, self.full)
+
             self.onSet(depfunc)
 
     def __repr__(self):
@@ -306,6 +311,8 @@ class Form:
             async def depfunc(node):
                 mesg = f'The form {self.full} is deprecated or using a deprecated type and will be removed in 3.0.0'
                 await node.snap.warnonce(mesg)
+                if __debug__:
+                    sys.audit('synapse.datamodel.Form.deprecated', mesg, self.full)
             self.onAdd(depfunc)
 
     def getStorNode(self, form):
@@ -1107,7 +1114,7 @@ class Model:
 
     def _prepFormIface(self, form, iface):
 
-        template = iface.get('template', {})
+        template = s_msgpack.deepcopy(iface.get('template', {}))
         template.update(form.type.info.get('template', {}))
 
         def convert(item):

{synapse-2.199.0 → synapse-2.200.0}/synapse/lib/hive.py

@@ -86,7 +86,7 @@ class Node(s_base.Base):
         for name, node in self.kids.items():
             yield name, node
 
-class Hive(s_nexus.Pusher, s_telepath.Aware):
+class Hive(s_nexus.Pusher):
     '''
     An optionally persistent atomically accessed tree which implements
     primitives for use in making distributed/clustered services.
@@ -95,8 +95,6 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
 
         await s_nexus.Pusher.__anit__(self, 'hive', nexsroot=nexsroot)
 
-        s_telepath.Aware.__init__(self)
-
         if conf is None:
             conf = {}
 
@@ -161,24 +159,6 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
             for cullname in culls:
                 await node.pop((cullname,))
 
-    async def getHiveAuth(self):
-        '''
-        Retrieve a HiveAuth for hive standalone or non-cell uses.
-
-        Note:
-            This is for the hive's own auth, or for non-cell auth.  It isn't the same auth as for a cell
-        '''
-        import synapse.lib.hiveauth as s_hiveauth
-        if self.auth is None:
-
-            path = tuple(self.conf.get('auth:path').split('/'))
-
-            node = await self.open(path)
-            self.auth = await s_hiveauth.Auth.anit(node, nexsroot=self.nexsroot)
-            self.onfini(self.auth.fini)
-
-        return self.auth
-
     async def _onHiveFini(self):
         await self.root.fini()
 
@@ -423,27 +403,6 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
 
         return node.valu
 
-    async def getTeleApi(self, link, mesg, path):
-        s_common.deprecated('Hive.getTeleApi', curv='2.198.0', eolv='2.199.0')
-        auth = await self.getHiveAuth()
-
-        if not self.conf.get('auth:en'):
-            user = await auth.getUserByName('root')
-            return await HiveApi.anit(self, user)
-
-        name, info = mesg[1].get('auth')
-
-        user = await auth.getUserByName(name)
-        if user is None:
-            raise s_exc.NoSuchUser(name=name)
-
-        # passwd None always fails...
-        passwd = info.get('passwd')
-        if not await user.tryPasswd(passwd):
-            raise s_exc.AuthDeny(mesg='Invalid password', user=user.iden, username=user.name)
-
-        return await HiveApi.anit(self, user)
-
     async def _storLoadHive(self):
         pass
 
@@ -480,224 +439,6 @@ class SlabHive(Hive):
         lkey = '\x00'.join(full).encode('utf8')
         self.slab.pop(lkey, db=self.db)
 
-class HiveApi(s_base.Base):
-
-    async def __anit__(self, hive, user):
-
-        await s_base.Base.__anit__(self)
-
-        self.hive = hive
-        self.user = user
-
-        self.msgq = asyncio.Queue(maxsize=10000)
-
-        self.onfini(self._onHapiFini)
-
-    async def loadHiveTree(self, tree, path=(), trim=False):
-        s_common.deprecated('HiveApi.loadHiveTree', curv='2.167.0')
-        return await self.hive.loadHiveTree(tree, path=path, trim=trim)
-
-    async def saveHiveTree(self, path=()):
-        s_common.deprecated('HiveApi.saveHiveTree', curv='2.167.0')
-        return await self.hive.saveHiveTree(path=path)
-
-    async def treeAndSync(self, path, iden):
-        s_common.deprecated('HiveApi.treeAndSync', curv='2.167.0')
-
-        node = await self.hive.open(path)
-
-        # register handlers...
-        node.on('hive:add', self._onHiveEdit, base=self)
-        node.on('hive:set', self._onHiveEdit, base=self)
-        node.on('hive:pop', self._onHiveEdit, base=self)
-
-        # serialize the subtree into a message and return
-        # via the mesg queue so there is no get/update race
-        root = (node.valu, {})
-
-        todo = collections.deque([(node, root)])
-
-        # breadth first generator
-        while todo:
-
-            node, pode = todo.popleft()
-
-            for name, kidn in node.kids.items():
-
-                kidp = (kidn.valu, {})
-                pode[1][name] = kidp
-
-                todo.append((kidn, kidp))
-
-        await self.msgq.put(('hive:tree', {'path': path, 'tree': root}))
-        await self.msgq.put(('hive:sync', {'iden': iden}))
-        return
-
-    async def setAndSync(self, path, valu, iden, nexs=False):
-        s_common.deprecated('HiveApi.setAndSync', curv='2.167.0')
-
-        valu = await self.hive.set(path, valu, nexs=nexs)
-        await self.msgq.put(('hive:sync', {'iden': iden}))
-        return valu
-
-    async def addAndSync(self, path, valu, iden):
-        s_common.deprecated('HiveApi.addAndSync', curv='2.167.0')
-
-        valu = await self.hive.add(path, valu)
-        await self.msgq.put(('hive:sync', {'iden': iden}))
-        return valu
-
-    async def popAndSync(self, path, iden, nexs=False):
-        s_common.deprecated('HiveApi.popAndSync', curv='2.167.0')
-
-        valu = await self.hive.pop(path, nexs=nexs)
-        await self.msgq.put(('hive:sync', {'iden': iden}))
-        return valu
-
-    async def _onHapiFini(self):
-        await self.msgq.put(None)
-
-    async def _onHiveEdit(self, mesg):
-        self.msgq.put_nowait(mesg)
-
-    async def get(self, full):
-        s_common.deprecated('HiveApi.get', curv='2.167.0')
-        return await self.hive.get(full)
-
-    async def edits(self):
-        s_common.deprecated('HiveApi.edits', curv='2.167.0')
-
-        while not self.isfini:
-
-            item = await self.msgq.get()
-            if item is None:
-                return
-
-            yield item
-
-class TeleHive(Hive):
-    '''
-    A Hive that acts as a consistent read cache for a telepath proxy Hive
-    '''
-
-    async def __anit__(self, proxy):
-
-        self.proxy = proxy
-
-        await Hive.__anit__(self)
-
-        self.lock = asyncio.Lock()
-
-        self.syncevents = {}  # iden: asyncio.Event()
-
-        # fire a task to sync the sections of the tree we open
-        self.schedCoro(self._runHiveLoop())
-
-        self.mesgbus = await s_base.Base.anit()
-        self.mesgbus.on('hive:set', self._onHiveSet)
-        self.mesgbus.on('hive:pop', self._onHivePop)
-        self.mesgbus.on('hive:tree', self._onHiveTree)
-        self.mesgbus.on('hive:sync', self._onHiveSync)
-
-        self.onfini(self.mesgbus.fini)
-
-        self.onfini(proxy.fini)
-
-    async def _onHiveSync(self, mesg):
-
-        iden = mesg[1].get('iden')
-        evnt = self.syncevents.pop(iden, None)
-        if evnt is None:
-            return
-
-        evnt.set()
-
-    def _getSyncIden(self):
-        iden = s_common.guid()
-        evnt = asyncio.Event()
-        self.syncevents[iden] = evnt
-        return iden, evnt
-
-    async def _runHiveLoop(self):
-        while not self.isfini:
-            async for mesg in self.proxy.edits():
-                await self.mesgbus.dist(mesg)
-
-    async def _onHiveSet(self, mesg):
-        path = mesg[1].get('path')
-        valu = mesg[1].get('valu')
-        await Hive.set(self, path, valu)
-
-    async def _onHivePop(self, mesg):
-        path = mesg[1].get('path')
-        await Hive.pop(self, path)
-
-    async def _onHiveTree(self, mesg):
-
-        # get an entire tree update at once
-        path = mesg[1].get('path')
-        tree = mesg[1].get('tree')
-
-        node = await Hive.open(self, path)
-
-        todo = collections.deque([(node, path, tree)])
-
-        while todo:
-
-            node, path, (valu, kids) = todo.popleft()
-
-            # do *not* go through the set() API
-            node.valu = valu
-            for name, kidt in kids.items():
-
-                kidp = path + (name,)
-                kidn = await Hive.open(self, kidp)
-
-                todo.append((kidn, kidp, kidt))
-
-    async def set(self, path, valu, nexs=False):
-        iden, evnt = self._getSyncIden()
-        valu = await self.proxy.setAndSync(path, valu, iden, nexs=nexs)
-        await evnt.wait()
-        return valu
-
-    async def add(self, path, valu):
-        iden, evnt = self._getSyncIden()
-        valu = await self.proxy.addAndSync(path, valu, iden)
-        await evnt.wait()
-        return valu
-
-    async def pop(self, path, nexs=False):
-        iden, evnt = self._getSyncIden()
-        valu = await self.proxy.popAndSync(path, iden, nexs=nexs)
-        await evnt.wait()
-        return valu
-
-    async def get(self, path):
-        return await self.proxy.get(path)
-
-    async def open(self, path):
-
-        # try once pre-lock for speed
-        node = self.nodes.get(path)
-        if node is not None:
-            return node
-
-        async with self.lock:
-
-            # try again with lock to avoid race
-            node = self.nodes.get(path)
-            if node is not None:
-                return node
-
-            iden, evnt = self._getSyncIden()
-
-            await self.proxy.treeAndSync(path, iden)
-
-            await evnt.wait()
-
-            return self.nodes.get(path)
-
 class HiveDict(s_base.Base):
     '''
     '''
@@ -758,11 +499,6 @@ def iterpath(path):
     for i in range(len(path)):
         yield path[:i + 1]
 
-async def openurl(url, **opts):
-    s_common.deprecated('synapse.lib.hive.openurl()', curv='2.198.0', eolv='2.199.0')
-    prox = await s_telepath.openurl(url, **opts)
-    return await TeleHive.anit(prox)
-
 async def opendir(dirn, conf=None):
     slab = await s_slab.Slab.anit(dirn, map_size=s_const.gibibyte)
     db = slab.initdb('hive')

{synapse-2.199.0 → synapse-2.200.0}/synapse/lib/snap.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import sys
 import types
 import asyncio
 import logging
@@ -12,14 +11,12 @@ import synapse.exc as s_exc
 import synapse.common as s_common
 
 import synapse.lib.base as s_base
-import synapse.lib.coro as s_coro
 import synapse.lib.node as s_node
 import synapse.lib.time as s_time
 import synapse.lib.cache as s_cache
 import synapse.lib.layer as s_layer
 import synapse.lib.storm as s_storm
 import synapse.lib.types as s_types
-import synapse.lib.spooled as s_spooled
 
 logger = logging.getLogger(__name__)
 
@@ -1606,7 +1603,8 @@ class Snap(s_base.Base):
         proptype = prop.type
         for prop in prop.getAlts():
             if prop.type.isarray and prop.type.arraytype == proptype:
-                if valu in node.get(prop.name):
+                arryvalu = node.get(prop.name)
+                if arryvalu is not None and valu in arryvalu:
                     return True
             else:
                 if node.get(prop.name) == valu:
@@ -1694,10 +1692,6 @@ class Snap(s_base.Base):
         return tagnode
 
     async def _raiseOnStrict(self, ctor, mesg, **info):
-        if __debug__:
-            if issubclass(ctor, s_exc.IsDeprLocked):
-                sys.audit('synapse.exc.IsDeprLocked', (mesg, info))
-
         if self.strict:
             raise ctor(mesg=mesg, **info)
         await self.warn(mesg)

{synapse-2.199.0 → synapse-2.200.0}/synapse/lib/storm.py

@@ -2077,7 +2077,7 @@ class Runtime(s_base.Base):
 
 class Parser:
 
-    def __init__(self, prog=None, descr=None, root=None, model=None):
+    def __init__(self, prog=None, descr=None, root=None, model=None, cdef=None):
 
         if root is None:
             root = self
@@ -2088,6 +2088,7 @@ class Parser:
 
         self.prog = prog
         self.descr = descr
+        self.cdef = cdef
 
         self.exc = None
 
@@ -2365,6 +2366,21 @@ class Parser:
 
         self._printf(f'Usage: {self.prog} [options] {posargs}')
 
+        if self.cdef is not None and (endpoints := self.cdef.get('endpoints')):
+            self._printf('')
+            self._printf('Endpoints:')
+            self._printf('')
+            base_w = 32
+            wrap_w = 120 - base_w
+            for endpoint in endpoints:
+                path = endpoint['path']
+                desc = endpoint.get('desc', '')
+                base = f'    {path}'
+                wrap_desc = self._wrap_text(desc, wrap_w) if desc else ['']
+                self._printf(f'{base:<{base_w-2}}: {wrap_desc[0]}')
+                for ln in wrap_desc[1:]:
+                    self._printf(f'{"":<{base_w}}{ln}')
+
         options = [x for x in self.allargs if x[0][0].startswith('-')]
 
         self._printf('')
@@ -2655,6 +2671,7 @@ class PureCmd(Cmd):
         if inputs:
             pars.set_inputs(inputs)
 
+        pars.cdef = self.cdef
         return pars
 
     async def execStormCmd(self, runt, genr):

{synapse-2.199.0 → synapse-2.200.0}/synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 199, 0)
+version = (2, 200, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'c256bd46a9df8621fd05fd6a0f36ca476ad29942'
+commit = 'ab9ca513fe33d52df7a3122c0bf4eab6951b53d3'

{synapse-2.199.0 → synapse-2.200.0}/synapse/tests/test_cortex.py

@@ -4629,12 +4629,6 @@ class CortexBasicTest(s_t_utils.SynTest):
             self.len(2, await core.nodes('[ inet:dns:a=(vertex.link,1.2.3.4) inet:dns:a=(woot.com,5.6.7.8)]'))
             self.len(4, await core.nodes('inet:dns:a inet:fqdn=:fqdn'))
 
-    async def test_cortex_hive(self):
-        async with self.getTestCore() as core:
-            await core.hive.set(('visi',), 200)
-            async with core.getLocalProxy(share='cortex/hive') as hive:
-                self.eq(200, await hive.get(('visi',)))
-
     async def test_cortex_delnode_perms(self):
 
         async with self.getTestCore() as core:

{synapse-2.199.0 → synapse-2.200.0}/synapse/tests/test_datamodel.py

@@ -44,6 +44,8 @@ class DataModelTest(s_t_utils.SynTest):
 
     async def test_datamodel_basics(self):
         async with self.getTestCore() as core:
+            iface = core.model.ifaces.get('phys:object')
+            self.eq('object', iface['template']['phys:object'])
             core.model.addType('woot:one', 'guid', {}, {
                 'display': {
                     'columns': (

{synapse-2.199.0 → synapse-2.200.0}/synapse/tests/test_lib_cell.py

@@ -1203,12 +1203,12 @@ class CellTest(s_t_utils.SynTest):
 
     async def test_cell_hiveapi(self):
 
-        async with self.getTestCore() as core:
+        async with self.getTestCell() as cell:
 
-            await core.setHiveKey(('foo', 'bar'), 10)
-            await core.setHiveKey(('foo', 'baz'), 30)
+            await cell.setHiveKey(('foo', 'bar'), 10)
+            await cell.setHiveKey(('foo', 'baz'), 30)
 
-            async with core.getLocalProxy() as proxy:
+            async with cell.getLocalProxy() as proxy:
                 self.eq((), await proxy.getHiveKeys(('lulz',)))
                 self.eq((('bar', 10), ('baz', 30)), await proxy.getHiveKeys(('foo',)))
 

{synapse-2.199.0 → synapse-2.200.0}/synapse/tests/test_lib_hive.py

@@ -1,12 +1,8 @@
-import asyncio
-
 import synapse.exc as s_exc
 import synapse.common as s_common
 
 import synapse.tests.utils as s_test
 
-import synapse.lib.hive as s_hive
-
 tree0 = {
     'kids': {
         'hehe': {'value': 'haha'},
@@ -73,40 +69,6 @@ class HiveTest(s_test.SynTest):
 
                 self.none(await hive.get(('foo', 'bar', 'lulz')))
 
-    async def test_hive_telepath(self):
-
-        # confirm that the primitives used by higher level APIs
-        # work using telepath remotes and property synchronize.
-
-        async with self.getTestHiveDmon() as dmon:
-
-            turl = self.getTestUrl(dmon, 'hive')
-
-            async with await s_hive.openurl(turl) as hive0:
-
-                path = ('foo', 'bar')
-
-                evnt = asyncio.Event()
-
-                def onedit(mesg):
-                    evnt.set()
-
-                node0 = await hive0.open(path)
-                node0.on('hive:set', onedit)
-
-                async with await s_hive.openurl(turl) as hive1:
-
-                    node1 = await hive1.open(path)
-                    await node1.set(200)
-
-                    await evnt.wait()
-
-                    self.eq(200, node0.valu)
-
-                    self.eq(201, await node0.add(1))
-                    self.eq(202, await node1.add(1))
-                    self.eq(203, await node0.add(1))
-
     async def test_hive_dir(self):
 
         async with self.getTestHive() as hive:

{synapse-2.199.0 → synapse-2.200.0}/synapse/tests/test_lib_storm.py

@@ -163,6 +163,10 @@ class StormTest(s_t_utils.SynTest):
             with self.raises(s_exc.BadTypeValu):
                 await core.nodes('inet:flow:from=({"name": "vertex", "type": "newp"})')
 
+            await core.nodes('[ ou:org=({"name": "origname"}) ]')
+            self.len(1, await core.nodes('ou:org=({"name": "origname"}) [ :name=newname ]'))
+            self.len(0, await core.nodes('ou:org=({"name": "origname"})'))
+
     async def test_lib_storm_jsonexpr(self):
         async with self.getTestCore() as core:
 
@@ -4121,6 +4125,9 @@ class StormTest(s_t_utils.SynTest):
                     {'name': 'woot', 'cmdinputs': (
                         {'form': 'hehe:haha'},
                         {'form': 'hoho:lol', 'help': 'We know whats up'}
+                    ), 'endpoints': (
+                        {'path': '/v1/test/one', 'desc': 'My multi-line endpoint description which spans multiple lines and has a second line. This is the second line of the description.'},
+                        {'path': '/v1/test/two', 'host': 'vertex.link', 'desc': 'Single line endpoint description.'},
                     )},
                 ),
             }
@@ -4128,6 +4135,9 @@ class StormTest(s_t_utils.SynTest):
             msgs = await core.stormlist('woot --help')
             helptext = '\n'.join([m[1].get('mesg') for m in msgs if m[0] == 'print'])
             self.isin('Inputs:\n\n    hehe:haha\n    hoho:lol  - We know whats up', helptext)
+            self.isin('Endpoints:\n\n    /v1/test/one              : My multi-line endpoint description which spans multiple lines and has a second line.', helptext)
+            self.isin('This is the second line of the description.', helptext)
+            self.isin('/v1/test/two              : Single line endpoint description.', helptext)
 
     async def test_storm_help_cmd(self):