synapse 2.208.0__tar.gz → 2.209.0__tar.gz

{synapse-2.208.0/synapse.egg-info → synapse-2.209.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 2.208.0
+Version: 2.209.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0

{synapse-2.208.0 → synapse-2.209.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.208.0'
+version = '2.209.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]

{synapse-2.208.0 → synapse-2.209.0}/synapse/lib/layer.py

@@ -1569,7 +1569,10 @@ class Layer(s_nexus.Pusher):
                 await self.initUpstreamSync(uplayr)
 
     async def initLayerPassive(self):
+        await self._stopMirror()
+        self._stopUpstream()
 
+    async def _stopMirror(self):
         if self.leadtask is not None:
             self.leadtask.cancel()
             self.leadtask = None
@@ -1578,6 +1581,7 @@ class Layer(s_nexus.Pusher):
             await self.leader.fini()
             self.leader = None
 
+    def _stopUpstream(self):
         [t.cancel() for t in self.activetasks]
         self.activetasks.clear()
 
@@ -2786,6 +2790,11 @@ class Layer(s_nexus.Pusher):
     async def setLayerInfo(self, name, valu):
         if name != 'readonly':
             self._reqNotReadOnly()
+
+        if name in ('mirror', 'upstream') and valu is not None:
+            mesg = 'Layer only supports setting "mirror" and "upstream" to None.'
+            raise s_exc.BadOptValu(mesg=mesg)
+
         return await self._push('layer:set', name, valu)
 
     @s_nexus.Pusher.onPush('layer:set')
@@ -2793,17 +2802,25 @@ class Layer(s_nexus.Pusher):
         '''
         Set a mutable layer property.
         '''
-        if name not in ('name', 'desc', 'logedits', 'readonly'):
+        if name not in ('name', 'desc', 'logedits', 'readonly', 'mirror', 'upstream'):
             mesg = f'{name} is not a valid layer info key'
             raise s_exc.BadOptValu(mesg=mesg)
 
         if name == 'logedits':
             valu = bool(valu)
             self.logedits = valu
+
         elif name == 'readonly':
             valu = bool(valu)
             self.readonly = valu
 
+        elif name == 'mirror' and valu is None:
+            await self._stopMirror()
+            self.ismirror = False
+
+        elif name == 'upstream' and valu is None:
+            self._stopUpstream()
+
         # TODO when we can set more props, we may need to parse values.
         if valu is None:
             self.layrinfo.pop(name, None)
@@ -4382,6 +4399,11 @@ class Layer(s_nexus.Pusher):
             async for verb, n2iden in self.iterNodeEdgesN1(buid):
                 edits.append((EDIT_EDGE_ADD, (verb, n2iden), ()))
 
+                if len(edits) >= 100:
+                    yield nodeedit
+                    edits = []
+                    nodeedit = (buid, form, edits)
+
             yield nodeedit
 
     async def initUpstreamSync(self, url):

{synapse-2.208.0 → synapse-2.209.0}/synapse/lib/rstorm.py

@@ -168,7 +168,9 @@ class StormOutput(s_cmds_cortex.StormCmd):
         if self.ctx.pop('storm-fail', None):
             s_cmds_cortex.StormCmd._onErr(self, mesg, opts)
             return
-        raise s_exc.StormRuntimeError(mesg=mesg)
+        (errname, errinfo) = mesg[1]
+        errinfo.setdefault('_errname', errname)
+        raise s_exc.StormRuntimeError(**errinfo)
 
     async def runCmdOpts(self, opts):
 

{synapse-2.208.0 → synapse-2.209.0}/synapse/lib/stormtypes.py

@@ -7669,10 +7669,18 @@ class Layer(Prim):
                 valu = None
             else:
                 valu = await tostr(await toprim(valu), noneok=True)
+
         elif name == 'logedits':
             valu = await tobool(valu)
+
         elif name == 'readonly':
             valu = await tobool(valu)
+
+        elif name in ('mirror', 'upstream'):
+            if (valu := await toprim(valu)) is not None:
+                mesg = 'Layer only supports setting "mirror" and "upstream" to null.'
+                raise s_exc.BadOptValu(mesg=mesg)
+
         else:
             mesg = f'Layer does not support setting: {name}'
             raise s_exc.BadOptValu(mesg=mesg)

{synapse-2.208.0 → synapse-2.209.0}/synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 208, 0)
+version = (2, 209, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = '2cc5a903edc504ca95e4a6b905fdb468b7e3b361'
+commit = '0e5089f5b2e5da0d93c93a99943ebbd5fdb6cb4c'

{synapse-2.208.0 → synapse-2.209.0}/synapse/lib/view.py

@@ -412,7 +412,7 @@ class View(s_nexus.Pusher):  # type: ignore
 
                     nodeedits.append(nodeedit)
 
-                    if len(nodeedits) == 10:
+                    if len(nodeedits) == 5:
                         yield nodeedits
                         nodeedits.clear()
 
@@ -432,7 +432,7 @@ class View(s_nexus.Pusher):  # type: ignore
 
                     meta['time'] = s_common.now()
 
-                    await snap.saveNodeEdits(edits, meta)
+                    await snap._applyNodeEdits(edits, meta)
                     await asyncio.sleep(0)
 
                     count += len(edits)
@@ -1467,8 +1467,6 @@ class View(s_nexus.Pusher):  # type: ignore
         Merge this view into its parent.  All changes made to this view will be applied to the parent.  Parent's
         triggers will be run.
         '''
-        fromlayr = self.layers[0]
-
         if useriden is None:
             user = await self.core.auth.getUserByName('root')
         else:
@@ -1481,10 +1479,25 @@ class View(s_nexus.Pusher):  # type: ignore
 
         async with await self.parent.snap(user=user) as snap:
 
+            async def chunked():
+                nodeedits = []
+
+                async for nodeedit in self.layers[0].iterLayerNodeEdits():
+
+                    nodeedits.append(nodeedit)
+
+                    if len(nodeedits) == 5:
+                        yield nodeedits
+                        nodeedits.clear()
+
+                if nodeedits:
+                    yield nodeedits
+
             meta = await snap.getSnapMeta()
-            async for nodeedits in fromlayr.iterLayerNodeEdits():
+            async for edits in chunked():
                 meta['time'] = s_common.now()
-                await snap.saveNodeEdits([nodeedits], meta)
+                await snap._applyNodeEdits(edits, meta)
+                await asyncio.sleep(0)
 
     async def swapLayer(self):
         oldlayr = self.layers[0]

{synapse-2.208.0 → synapse-2.209.0}/synapse/tests/test_lib_rstorm.py

@@ -119,7 +119,7 @@ clear_storm_opts = '''
 .. storm-cortex:: default
 .. storm-opts:: {"vars": {"foobar": "bar"}}
 .. storm-clear-http:: true
-.. storm:: $lib.print($lib.str.concat($foobar, "bizboz"))
+.. storm:: $lib.print(`{$foobar}bizboz`)
 '''
 
 stormenv = '''
@@ -381,8 +381,9 @@ class RStormLibTest(s_test.SynTest):
             path = s_common.genpath(dirn, 'clear_storm_opts.rst')
             with s_common.genfile(path) as fd:
                 fd.write(clear_storm_opts.encode())
-            with self.raises(s_exc.StormRuntimeError):
-                text = await get_rst_text(path)
+            with self.raises(s_exc.StormRuntimeError) as cm:
+                await get_rst_text(path)
+            self.eq('Missing variable: foobar', cm.exception.get('mesg'))
 
             # boom1 test
             path = s_common.genpath(dirn, 'boom1.rst')

{synapse-2.208.0 → synapse-2.209.0}/synapse/tests/test_lib_stormtypes.py

@@ -3802,7 +3802,6 @@ class StormTypesTest(s_test.SynTest):
         async with self.getTestCore() as core:
             async with self.getTestCore() as core2:
 
-                await core2.nodes('[ inet:ipv4=1.2.3.4 ]')
                 url = core2.getLocalUrl('*/layer')
 
                 layriden = core2.view.layers[0].iden
@@ -3819,8 +3818,141 @@ class StormTypesTest(s_test.SynTest):
 
                 layr = core.getLayer(uplayr)
 
-                evnt = await layr.waitUpstreamOffs(layriden, offs)
-                self.true(await asyncio.wait_for(evnt.wait(), timeout=6))
+                async def query(q):
+                    '''
+                    Run a query on core2 and wait for it to sync to layr from core
+                    '''
+                    nodes = await core2.nodes(q)
+                    offs = await core2.view.layers[0].getEditIndx()
+                    evnt = await layr.waitUpstreamOffs(layriden, offs)
+                    self.true(await asyncio.wait_for(evnt.wait(), timeout=6))
+                    return nodes
+
+                vdef = {
+                    'layers': [layr.iden]
+                }
+
+                view00 = await core.addView(vdef)
+                self.nn(view00)
+
+                # No foobar in core
+                opts = {'view': view00.get('iden')}
+                nodes = await core.nodes('it:dev:str=foobar', opts=opts)
+                self.len(0, nodes)
+
+                # Add foobar in core2
+                nodes = await query('[ it:dev:str=foobar ]')
+                self.len(1, nodes)
+
+                # foobar shows up in core
+                nodes = await core.nodes('it:dev:str=foobar', opts=opts)
+                self.len(1, nodes)
+
+                self.len(1, layr.activetasks)
+
+                # The upstream key only accepts null
+                q = f'layer.set {uplayr} upstream (true)'
+                msgs = await core.stormlist(q)
+                self.stormIsInErr('Layer only supports setting "mirror" and "upstream" to null.', msgs)
+
+                with self.raises(s_exc.BadOptValu) as exc:
+                    await layr.setLayerInfo('upstream', False)
+                self.eq(exc.exception.get('mesg'), 'Layer only supports setting "mirror" and "upstream" to None.', msgs)
+
+                # Now remove the upstream configuration
+                q = f'layer.set {uplayr} upstream (null)'
+                msgs = await core.stormlist(q)
+                self.stormHasNoWarnErr(msgs)
+
+                layr = core.getLayer(uplayr)
+                self.len(0, layr.activetasks)
+                self.none(layr.layrinfo.get('upstream'))
+
+                with self.raises(TimeoutError):
+                    await query('[ it:dev:str=newp ]')
+
+                # No newp in core because layer upstream is disabled
+                nodes = await core.nodes('it:dev:str=newp', opts=opts)
+                self.len(0, nodes)
+
+    async def test_storm_lib_layer_mirror(self):
+        async with self.getTestCore() as core:
+            async with self.getTestCore() as core2:
+
+                url = core2.getLocalUrl('*/layer')
+
+                layers = set(core.layers.keys())
+                q = f'layer.add --mirror {url}'
+                mesgs = await core.stormlist(q)
+                uplayr = list(set(core.layers.keys()) - layers)[0]
+
+                q = f'layer.set {uplayr} name "woot woot"'
+                mesgs = await core.stormlist(q)
+                self.stormIsInPrint('(name: woot woot)', mesgs)
+
+                layr = core.getLayer(uplayr)
+
+                async def query(q):
+                    '''
+                    Run a query on core2 and wait for it to sync to layr from core
+                    '''
+                    nodes = await core2.nodes(q)
+                    offs = await core2.view.layers[0].getEditOffs()
+                    self.true(await layr.waitEditOffs(offs, timeout=10))
+                    return nodes
+
+                vdef = {
+                    'layers': [layr.iden]
+                }
+
+                view00 = await core.addView(vdef)
+                self.nn(view00)
+
+                # No foobar in core
+                opts = {'view': view00.get('iden')}
+                nodes = await core.nodes('it:dev:str=foobar', opts=opts)
+                self.len(0, nodes)
+
+                # Add foobar in core2
+                nodes = await query('[ it:dev:str=foobar ]')
+                self.len(1, nodes)
+
+                # foobar shows up in core
+                nodes = await core.nodes('it:dev:str=foobar', opts=opts)
+                self.len(1, nodes)
+
+                self.true(layr.ismirror)
+                self.nn(layr.leadtask)
+                self.nn(layr.leader)
+                self.len(0, layr.activetasks)
+
+                # The mirror key only accepts null
+                q = f'layer.set {uplayr} mirror (true)'
+                msgs = await core.stormlist(q)
+                self.stormIsInErr('Layer only supports setting "mirror" and "upstream" to null.', msgs)
+
+                with self.raises(s_exc.BadOptValu) as exc:
+                    await layr.setLayerInfo('mirror', False)
+                self.eq(exc.exception.get('mesg'), 'Layer only supports setting "mirror" and "upstream" to None.', msgs)
+
+                # Now remove the mirror configuration
+                q = f'layer.set {uplayr} mirror (null)'
+                msgs = await core.stormlist(q)
+                self.stormHasNoWarnErr(msgs)
+
+                layr = core.getLayer(uplayr)
+                self.none(layr.layrinfo.get('mirror'))
+                self.none(layr.leadtask)
+                self.none(layr.leader)
+                self.false(layr.ismirror)
+
+                # Add newp in core2
+                nodes = await query('[ it:dev:str=newp ]')
+                self.len(1, nodes)
+
+                # No newp in core because layer mirroring is disabled
+                nodes = await core.nodes('it:dev:str=newp', opts=opts)
+                self.len(0, nodes)
 
     async def test_storm_lib_view(self):
 

{synapse-2.208.0 → synapse-2.209.0}/synapse/tests/test_lib_view.py

@@ -310,6 +310,28 @@ class ViewTest(s_t_utils.SynTest):
             # But not the same layer twice
             await self.asyncraises(s_exc.DupIden, core.view.addLayer(layriden))
 
+            # Nodes with a large number of edge edits may be chunked by iterLayerNodeEdits
+            await core.nodes('[ test:str=foo ] for $i in $lib.range(102) {[ test:int=$i ]}')
+
+            vdef2 = await core.view.fork()
+            opts = {'view': vdef2['iden']}
+            await core.nodes('[ test:str=foo +(refs)> { for $i in $lib.range(102) { test:int=$i } } ]', opts=opts)
+
+            strt = core.nexsroot.nexslog.index()
+            await core.nodes('$lib.view.get().merge()', opts=opts)
+
+            self.len(102, await core.nodes('test:str=foo -(refs)> test:int'))
+
+            edits = [edit async for edit in core.nexsroot.nexslog.iter(strt)]
+            self.len(1, edits)
+
+            nodeedit = edits[0][1][2][0]
+
+            # We should have two chunks of edits for the same buid due to the number of edges
+            self.eq(nodeedit[0][0], nodeedit[1][0])
+            self.len(100, nodeedit[0][2])
+            self.len(2, nodeedit[1][2])
+
     async def test_view_merge_ival(self):
 
         async with self.getTestCore() as core:

{synapse-2.208.0 → synapse-2.209.0/synapse.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 2.208.0
+Version: 2.209.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0