synapse 2.168.0__py311-none-any.whl → 2.170.0__py311-none-any.whl

synapse/tests/test_cortex.py

@@ -2430,6 +2430,41 @@ class CortexTest(s_t_utils.SynTest):
             msgs = await core.stormlist('test:int :loc -> test:newp')
             self.stormIsInErr('No property named test:newp', msgs)
 
+            # ndef pivots
+            await core.nodes('''
+                [
+                    ( test:str=ndefpivdst )
+                    ( test:str=ndefpivsrc :bar=(test:str, ndefpivdst) )
+                    ( test:str=ndefpivprp :bar=(test:str, ndefpivdst) )
+                ]
+            ''')
+
+            nodes = await core.nodes('test:str=ndefpivsrc -> test:str')
+            self.eq(['ndefpivdst'], [n.ndef[1] for n in nodes])
+
+            nodes = await core.nodes('test:str=ndefpivsrc -> test:str:bar')
+            self.len(0, nodes)
+
+            nodes = await core.nodes('test:str=ndefpivdst -> test:str:bar')
+            self.sorteq(['ndefpivprp', 'ndefpivsrc'], [n.ndef[1] for n in nodes])
+
+            nodes = await core.nodes('test:str=ndefpivsrc :bar -> * +test:str')
+            self.eq(['ndefpivdst'], [n.ndef[1] for n in nodes])
+
+            nodes = await core.nodes('test:str=ndefpivsrc :bar -> test:str:bar')
+            self.sorteq(['ndefpivprp', 'ndefpivsrc'], [n.ndef[1] for n in nodes])
+
+            nodes = await core.nodes('test:str=ndefpivsrc :bar -> test:str')
+            self.eq(['ndefpivdst'], [n.ndef[1] for n in nodes])
+
+            nodes = await core.nodes('test:str=ndefpivsrc :bar -> test:int')
+            self.len(0, nodes)
+
+            await core.nodes('test:str=ndefpivdst delnode')
+            msgs = await core.stormlist('test:str=ndefpivsrc :bar -> test:str')
+            self.len(0, [m for m in msgs if m[0] == 'node'])
+            self.stormIsInWarn("Missing node corresponding to ndef ('test:str', 'ndefpivdst')", msgs)
+
             # Bad pivot syntax go here
             for q in ['test:pivcomp :lulz <- *',
                       'test:pivcomp :lulz <+- *',
@@ -6257,12 +6292,14 @@ class CortexBasicTest(s_t_utils.SynTest):
 
             # Can't delete the default view
             await self.asyncraises(s_exc.SynErr, core.delView(core.view.iden))
+            await self.asyncraises(s_exc.SynErr, core._delViewWithLayer(core.view.iden, None, None))
 
             # Can't delete a layer in a view
             await self.asyncraises(s_exc.SynErr, core.delLayer(core.view.layers[0].iden))
 
             # Can't delete a nonexistent view
             await self.asyncraises(s_exc.NoSuchView, core.delView('XXX'))
+            await self.asyncraises(s_exc.NoSuchView, core.delViewWithLayer('XXX'))
 
             # Can't delete a nonexistent layer
             await self.asyncraises(s_exc.NoSuchLayer, core.delLayer('XXX'))
@@ -6275,6 +6312,68 @@ class CortexBasicTest(s_t_utils.SynTest):
             await core.delView(view2_iden)
             await self.asyncraises(s_exc.NoSuchView, core.delView(view2_iden))
 
+            layr = await core.addLayer()
+            layriden = layr['iden']
+            vdef3 = {'layers': (layriden,)}
+            view3_iden = (await core.addView(vdef3)).get('iden')
+
+            opts = {'view': view3_iden}
+            await core.callStorm('$lib.view.get().set(protected, $lib.true)', opts=opts)
+
+            await self.asyncraises(s_exc.CantDelView, core.delViewWithLayer(view3_iden))
+
+            await core.callStorm('$lib.view.get().set(protected, $lib.false)', opts=opts)
+
+            view3 = core.getView(view3_iden)
+            vdef4 = await view3.fork()
+
+            deadlayr = view3.layers[0].iden
+            view4_iden = vdef4.get('iden')
+            view4 = core.getView(view4_iden)
+
+            self.eq(view4.parent, view3)
+            self.len(2, view4.layers)
+
+            await core.auth.rootuser.setPasswd('secret')
+            host, port = await core.dmon.listen('tcp://127.0.0.1:0/')
+            layr2 = await core.callStorm('$layer=$lib.layer.add() return($layer)')
+            varz = {'iden': layriden, 'tgt': layr2.get('iden'), 'port': port}
+            opts = {'vars': varz, 'view': view3_iden}
+
+            pullq = '$layer=$lib.layer.get($iden).addPull(`tcp://root:secret@127.0.0.1:{$port}/*/layer/{$tgt}`)'
+            pushq = '$layer=$lib.layer.get($iden).addPush(`tcp://root:secret@127.0.0.1:{$port}/*/layer/{$tgt}`)'
+            msgs = await core.stormlist(pullq, opts=opts)
+            self.stormHasNoWarnErr(msgs)
+
+            msgs = await core.stormlist(pushq, opts=opts)
+            self.stormHasNoWarnErr(msgs)
+
+            coros = len(core.activecoros)
+
+            await core.delViewWithLayer(view3_iden)
+
+            # push/pull activecoros have been deleted
+            self.len(coros - 2, core.activecoros)
+
+            self.none(view4.parent)
+            self.len(1, view4.layers)
+            self.none(core.getLayer(deadlayr))
+
+            vdef5 = await view4.fork()
+            view5 = core.getView(vdef5.get('iden'))
+
+            usedlayr = view4.layers[0].iden
+            vdef6 = {'layers': (usedlayr,)}
+            view6 = core.getView((await core.addView(vdef6)).get('iden'))
+
+            await core.delViewWithLayer(view4_iden)
+
+            self.none(view5.parent)
+            self.len(1, view5.layers)
+
+            self.nn(core.getLayer(usedlayr))
+            self.eq([usedlayr], [lyr.iden for lyr in view6.layers])
+
     async def test_cortex_view_opts(self):
         '''
         Test that the view opts work

synapse/tests/test_lib_agenda.py

@@ -1,3 +1,4 @@
+import json
 import asyncio
 import hashlib
 import datetime
@@ -156,7 +157,7 @@ class AgendaTest(s_t_utils.SynTest):
         newts = ar.nexttime(now)
         self.eq(newts, datetime.datetime(year=2018, month=12, day=5, hour=7, minute=2, tzinfo=tz.utc).timestamp())
 
-    async def test_agenda(self):
+    async def test_agenda_base(self):
         MONO_DELT = 1543827303.0
         unixtime = datetime.datetime(year=2018, month=12, day=5, hour=7, minute=0, tzinfo=tz.utc).timestamp()
 
@@ -382,8 +383,21 @@ class AgendaTest(s_t_utils.SynTest):
                 self.true(appt.enabled)
                 self.eq(0, appt.startcount)
 
-                unixtime = datetime.datetime(year=2019, month=2, day=13, hour=10, minute=16, tzinfo=tz.utc).timestamp()
-                self.eq((12, 'bar'), await asyncio.wait_for(core.callStorm('return($lib.queue.gen(visi).pop(wait=$lib.true))'), timeout=5))
+                # Ensure structured logging captures the cron iden value
+                core.stormlog = True
+                with self.getStructuredAsyncLoggerStream('synapse.storm') as stream:
+                    unixtime = datetime.datetime(year=2019, month=2, day=13, hour=10, minute=16,
+                                                 tzinfo=tz.utc).timestamp()
+                    self.eq((12, 'bar'), await asyncio.wait_for(core.callStorm('return($lib.queue.gen(visi).pop(wait=$lib.true))'), timeout=5))
+                core.stormlog = False
+
+                data = stream.getvalue()
+                raw_mesgs = [m for m in data.split('\n') if m]
+                msgs = [json.loads(m) for m in raw_mesgs]
+                msgs = [m for m in msgs if m['text'] == '$lib.queue.gen(visi).put(bar)']
+                self.gt(len(msgs), 0)
+                for m in msgs:
+                    self.eq(m.get('cron'), appt.iden)
 
                 self.eq(1, appt.startcount)
 

synapse/tests/test_lib_ast.py

@@ -670,6 +670,28 @@ class AstTest(s_test.SynTest):
             self.len(1, nodes)
             self.eq('geo:nloc', nodes[0].ndef[0])
 
+            await core.nodes('[ test:str=ndefs :ndefs=((it:dev:int, 1), (it:dev:int, 2)) ]')
+            await core.nodes('test:str=ndefs [ :ndefs += (inet:fqdn, woot.com) ]')
+            self.len(1, nodes)
+
+            nodes = await core.nodes('it:dev:int=1 -> test:str:ndefs')
+            self.len(1, nodes)
+            self.eq('ndefs', nodes[0].ndef[1])
+            self.eq(nodes[0].getNodeRefs(), [
+                ('ndefs', ('it:dev:int', 1)),
+                ('ndefs', ('it:dev:int', 2)),
+                ('ndefs', ('inet:fqdn', 'woot.com'))
+            ])
+
+            nodes = await core.nodes('[ test:str = norefs ]')
+            self.eq(nodes[0].getNodeRefs(), [])
+
+            self.len(1, await core.nodes('it:dev:int=1 -> test:str'))
+            self.len(3, await core.nodes('test:str=ndefs -> *'))
+            self.len(2, await core.nodes('test:str=ndefs -> it:dev:int'))
+            self.len(3, await core.nodes('test:str=ndefs :ndefs -> *'))
+            self.len(2, await core.nodes('test:str=ndefs :ndefs -> it:dev:int'))
+
     async def test_ast_pivot(self):
         # a general purpose pivot test. come on in!
         async with self.getTestCore() as core:
@@ -2761,6 +2783,50 @@ class AstTest(s_test.SynTest):
                     self.stormHasNoWarnErr(msgs)
                     self.len(0, calls)
 
+    async def test_ast_tag_optimization(self):
+        calls = []
+        origtag = s_snap.Snap.nodesByTag
+
+        async def checkTag(self, tag, form=None, reverse=False):
+            calls.append(('tag', tag, form))
+            async for node in origtag(self, tag, form=form, reverse=reverse):
+                yield node
+
+        with mock.patch('synapse.lib.snap.Snap.nodesByTag', checkTag):
+            async with self.getTestCore() as core:
+                self.len(1, await core.nodes('[inet:asn=200 :name=visi]'))
+                self.len(1, await core.nodes('[test:int=12 +#visi]'))
+                self.len(1, await core.nodes('[test:int=99 +#visi]'))
+
+                nodes = await core.nodes('test:int +#$x', opts={'vars': {'x': 'visi'}})
+                self.len(2, nodes)
+                self.len(1, calls)
+                self.eq(('tag', 'visi', 'test:int'), calls[0])
+
+                calls = []
+                # not for non-runtsafe
+                nodes = await core.nodes('inet:asn:name $valu=:name test:int +#$valu')
+                self.len(2, nodes)
+                self.len(0, calls)
+
+                nodes = await core.nodes('''
+                    $tag = 'yeyeyeyeyeyeyeyeye'
+                    if $x {
+                        $tag = 'visi'
+                    } else {
+                        $tag = 'lolnope'
+                    }
+                    test:int +#$tag
+                ''', opts={'vars': {'x': True}})
+                self.len(2, nodes)
+                self.len(1, calls)
+                self.eq(('tag', 'visi', 'test:int'), calls[0])
+
+                calls = []
+                nodes = await core.nodes('test:int +#$x', opts={'vars': {'x': 'v*'}})
+                self.len(2, nodes)
+                self.len(0, calls)
+
     async def test_ast_cmdoper(self):
 
         async with self.getTestCore() as core:

synapse/tests/test_lib_cell.py

@@ -1,6 +1,7 @@
 import os
 import ssl
 import sys
+import json
 import time
 import base64
 import signal
@@ -33,6 +34,7 @@ import synapse.lib.version as s_version
 import synapse.lib.hiveauth as s_hiveauth
 import synapse.lib.lmdbslab as s_lmdbslab
 import synapse.lib.crypto.passwd as s_passwd
+import synapse.lib.platforms.linux as s_linux
 
 import synapse.tools.backup as s_tools_backup
 
@@ -496,52 +498,6 @@ class CellTest(s_t_utils.SynTest):
                 with self.raises(s_exc.AuthDeny):
                     await prox.setCellUser(visiiden)
 
-    async def test_cell_hiveboot(self):
-
-        with self.getTestDir() as dirn:
-
-            tree = {
-                'kids': {
-                    'hehe': {'value': 'haha'},
-                }
-            }
-
-            bootpath = os.path.join(dirn, 'hiveboot.yaml')
-
-            s_common.yamlsave(tree, bootpath)
-
-            with warnings.catch_warnings(record=True) as warns:
-                async with self.getTestCell(s_cell.Cell, dirn=dirn) as cell:
-                    self.eq('haha', await cell.hive.get(('hehe',)))
-
-                self.isin('Initial hive config from hiveboot.yaml', str(warns[0].message))
-
-            # test that the file does not load again
-            tree['kids']['redballoons'] = {'value': 99}
-            s_common.yamlsave(tree, bootpath)
-
-            async with self.getTestCell(s_cell.Cell, dirn=dirn) as cell:
-                self.none(await cell.hive.get(('redballoons',)))
-
-        # Do a full hive dump/load
-        with self.getTestDir() as dirn:
-            dir0 = s_common.genpath(dirn, 'cell00')
-            dir1 = s_common.genpath(dirn, 'cell01')
-            async with self.getTestCell(s_cell.Cell, dirn=dir0, conf={'auth:passwd': 'root'}) as cell00:
-                await cell00.hive.set(('beeps',), [1, 2, 'three'])
-
-                tree = await cell00.saveHiveTree()
-                s_common.yamlsave(tree, dir1, 'hiveboot.yaml')
-                with s_common.genfile(dir1, 'cell.guid') as fd:
-                    _ = fd.write(cell00.iden.encode())
-
-            async with self.getTestCell(s_cell.Cell, dirn=dir1) as cell01:
-                resp = await cell01.hive.get(('beeps',))
-                self.isinstance(resp, tuple)
-                self.eq(resp, (1, 2, 'three'))
-
-            self.eq(cell00.iden, cell01.iden)
-
     async def test_cell_getinfo(self):
         async with self.getTestCore() as cell:
             cell.COMMIT = 'mycommit'
@@ -2243,8 +2199,8 @@ class CellTest(s_t_utils.SynTest):
                 async with self.getTestCore(dirn=dirn, conf=conf) as core:
                     pass
 
-                stream.seek(0)
-                self.isin('onboot optimization complete!', stream.read())
+            stream.seek(0)
+            self.isin('onboot optimization complete!', stream.read())
 
             stat01 = os.stat(lmdbfile)
             self.ne(stat00.st_ino, stat01.st_ino)
@@ -2266,10 +2222,25 @@ class CellTest(s_t_utils.SynTest):
                     async with self.getTestCore(dirn=dirn, conf=conf) as core:
                         pass
 
-                    stream.seek(0)
-                    buf = stream.read()
-                    self.notin('onboot optimization complete!', buf)
-                    self.isin('not on the same volume', buf)
+                stream.seek(0)
+                buf = stream.read()
+                self.notin('onboot optimization complete!', buf)
+                self.isin('not on the same volume', buf)
+
+            # Local backup files are skipped
+            async with self.getTestCore(dirn=dirn) as core:
+                await core.runBackup()
+
+            with self.getAsyncLoggerStream('synapse.lib.cell') as stream:
+
+                conf = {'onboot:optimize': True}
+                async with self.getTestCore(dirn=dirn, conf=conf) as core:
+                    pass
+
+            stream.seek(0)
+            buf = stream.read()
+            self.isin('Skipping backup file', buf)
+            self.isin('onboot optimization complete!', buf)
 
     async def test_cell_gc(self):
         async with self.getTestCore() as core:
@@ -2642,3 +2613,113 @@ class CellTest(s_t_utils.SynTest):
 
             with self.raises(s_exc.NoSuchIden):
                 await cell.delUserApiKey(newp)
+
+    async def test_cell_check_sysctl(self):
+        sysctls = s_linux.getSysctls()
+
+        sysvals = s_cell.Cell.SYSCTL_VALS.copy()
+        sysvals['vm.dirty_expire_centisecs'] += 1
+        sysvals['vm.dirty_writeback_centisecs'] += 1
+
+        # Detect and report incorrect values
+        with self.getStructuredAsyncLoggerStream('synapse.lib.cell') as stream:
+            with mock.patch.object(s_cell.Cell, 'SYSCTL_VALS', sysvals):
+                async with self.getTestCore():
+                    pass
+
+        stream.seek(0)
+        data = stream.getvalue()
+        raw_mesgs = [m for m in data.split('\\n') if m]
+        msgs = [json.loads(m) for m in raw_mesgs]
+
+        self.len(1, msgs)
+
+        mesg = f'Sysctl values different than expected: {", ".join(sysvals)}. '
+        mesg += 'See https://synapse.docs.vertex.link/en/latest/synapse/devopsguide.html#performance-tuning '
+        mesg += 'for information about these sysctl parameters.'
+        self.eq(msgs[0]['message'], mesg)
+        self.eq(msgs[0]['sysctls'], [
+            {'name': 'vm.dirty_expire_centisecs', 'expected': 21, 'actual': sysctls['vm.dirty_expire_centisecs']},
+            {'name': 'vm.dirty_writeback_centisecs', 'expected': 21, 'actual': sysctls['vm.dirty_writeback_centisecs']},
+        ])
+
+        # Copy the current sysctl valus to the cell so the check passes
+        sysvals = {
+            'vm.dirty_expire_centisecs': sysctls['vm.dirty_expire_centisecs'],
+            'vm.dirty_writeback_centisecs': sysctls['vm.dirty_writeback_centisecs'],
+        }
+
+        # Detect correct values and stop the task
+        with self.getLoggerStream('synapse.lib.cell') as stream:
+            with mock.patch.object(s_cell.Cell, 'SYSCTL_VALS', sysvals):
+                async with self.getTestCore():
+                    pass
+
+        stream.seek(0)
+        data = stream.read()
+        self.len(0, data)
+
+        # Disable the sysctl check and don't check at all
+        with self.getLoggerStream('synapse.lib.cell') as stream:
+            conf = {'health:sysctl:checks': False}
+            async with self.getTestCore(conf=conf):
+                pass
+
+        stream.seek(0)
+        data = stream.read()
+        self.len(0, data, msg=data)
+
+    async def test_cell_version_regression(self):
+        oldver = (0, 1, 0)
+        newver = (0, 2, 0)
+
+        class TestCell(s_cell.Cell):
+            VERSION = newver
+
+        with self.getTestDir() as dirn:
+            async with self.getTestCell(TestCell, dirn=dirn):
+                pass
+
+            with self.raises(s_exc.BadVersion) as exc:
+                with mock.patch.object(TestCell, 'VERSION', oldver):
+                    with self.getLoggerStream('synapse.lib.cell') as stream:
+                        async with self.getTestCell(TestCell, dirn=dirn):
+                            pass
+
+            mesg = f'Cell version regression (testcell) is not allowed! Stored version: {newver}, current version: {oldver}.'
+            self.eq(exc.exception.get('mesg'), mesg)
+            self.eq(exc.exception.get('currver'), oldver)
+            self.eq(exc.exception.get('lastver'), newver)
+
+            stream.seek(0)
+            data = stream.read()
+            self.isin(mesg, data)
+
+            async with self.getTestCell(TestCell, dirn=dirn):
+                pass
+
+        with self.getTestDir() as dirn:
+            async with self.getTestCell(s_cell.Cell, dirn=dirn):
+                pass
+
+            synver = list(s_version.version)
+            synver[1] -= 1
+            synver = tuple(synver)
+
+            with self.raises(s_exc.BadVersion) as exc:
+                with mock.patch.object(s_version, 'version', synver):
+                    with self.getLoggerStream('synapse.lib.cell') as stream:
+                        async with self.getTestCell(s_cell.Cell, dirn=dirn):
+                            pass
+
+            mesg = f'Synapse version regression (cell) is not allowed! Stored version: {s_version.version}, current version: {synver}.'
+            self.eq(exc.exception.get('mesg'), mesg)
+            self.eq(exc.exception.get('currver'), synver)
+            self.eq(exc.exception.get('lastver'), s_version.version)
+
+            stream.seek(0)
+            data = stream.read()
+            self.isin(mesg, data)
+
+            async with self.getTestCell(s_cell.Cell, dirn=dirn):
+                pass

synapse/tests/test_lib_layer.py

@@ -26,7 +26,7 @@ class LayerTest(s_t_utils.SynTest):
 
     def checkLayrvers(self, core):
         for layr in core.layers.values():
-            self.eq(layr.layrvers, 10)
+            self.eq(layr.layrvers, 11)
 
     async def test_layer_verify(self):
 
@@ -1530,6 +1530,57 @@ class LayerTest(s_t_utils.SynTest):
             verbs = [verb async for verb in nodes0[0].iterEdgeVerbs(buid2)]
             self.len(0, verbs)
 
+    async def test_layer_v11(self):
+
+        try:
+
+            oldv = s_layer.MIGR_COMMIT_SIZE
+            s_layer.MIGR_COMMIT_SIZE = 1
+
+            async with self.getRegrCore('layer-v11') as core:
+
+                wlyrs_byview = await core.callStorm('''
+                    $wlyrs = ({})
+                    for $view in $lib.view.list() {
+                        $wlyrs.($view.get(name)) = $view.layers.0.iden
+                    }
+                    return($wlyrs)
+                ''')
+                self.len(8, wlyrs_byview)
+
+                layr = core.getLayer(iden=wlyrs_byview['default'])
+                await self.agenlen(2, layr.getStorNodesByForm('test:str'))
+                await self.agenlen(1, layr.getStorNodesByForm('syn:tag'))
+
+                layr = core.getLayer(iden=wlyrs_byview['prop'])
+                await self.agenlen(1, layr.getStorNodesByForm('test:str'))
+
+                layr = core.getLayer(iden=wlyrs_byview['tags'])
+                await self.agenlen(1, layr.getStorNodesByForm('test:str'))
+                await self.agenlen(1, layr.getStorNodesByForm('syn:tag'))
+
+                layr = core.getLayer(iden=wlyrs_byview['tagp'])
+                await self.agenlen(1, layr.getStorNodesByForm('test:str'))
+                await self.agenlen(0, layr.getStorNodesByForm('syn:tag'))
+
+                layr = core.getLayer(iden=wlyrs_byview['n1eg'])
+                await self.agenlen(1, layr.getStorNodesByForm('test:str'))
+                await self.agenlen(1, layr.getStorNodesByForm('test:int'))
+
+                layr = core.getLayer(iden=wlyrs_byview['n2eg'])
+                await self.agenlen(0, layr.getStorNodesByForm('test:str'))
+                await self.agenlen(1, layr.getStorNodesByForm('test:int'))
+
+                layr = core.getLayer(iden=wlyrs_byview['data'])
+                await self.agenlen(1, layr.getStorNodesByForm('test:str'))
+
+                layr = core.getLayer(iden=wlyrs_byview['noop'])
+                await self.agenlen(0, layr.getStorNodes())
+                await self.agenlen(0, layr.getStorNodesByForm('test:str'))
+
+        finally:
+            s_layer.MIGR_COMMIT_SIZE = oldv
+
     async def test_layer_logedits_default(self):
         async with self.getTestCore() as core:
             self.true(core.getLayer().logedits)