synapse 2.180.1__py311-none-any.whl → 2.182.0__py311-none-any.whl

synapse/tests/test_cortex.py

@@ -4827,6 +4827,45 @@ class CortexBasicTest(s_t_utils.SynTest):
             q = '$valu={[test:str=foo]} switch $valu { foo: {test:str=foo return($node.value()) } }'
             self.eq('foo', await core.callStorm(q))
 
+            # multi-value switch cases
+            q = '''
+            [test:str=$inval]
+            switch $node.value() {
+                "foo": { return($node.value()) }
+                ("boo", "bar"): { return($node.value()) }
+                (coo, car): { return($node.value()) }
+                ('doo', 'dar'): { return($node.value()) }
+                ("goo", 'gar', gaz): { return($node.value()) }
+            }
+            $lib.raise(BadArg, `Failed match on {$inval}`)
+            '''
+            for inval in ('foo', 'boo', 'bar', 'coo', 'car', 'doo', 'dar', 'goo', 'gar', 'gaz'):
+                valu = await core.callStorm(q, opts={'vars': {'inval': inval}})
+                self.eq(valu, inval)
+
+            # bare asterisk is allowed as a multi-value
+            valu = await core.callStorm('$foo="*" switch $foo { *:{ return(default) } (someval, *): { return(multi) } }')
+            self.eq(valu, 'multi')
+
+            # multiple default cases is invalid
+            msgs = await core.stormlist('$foo=foo switch $foo { *:{} *:{} }')
+            self.stormIsInErr('Switch statements cannot have more than one default case. Found 2.', msgs)
+
+            # multi-value case without a comma
+            msgs = await core.stormlist('$foo=foo switch $foo { (foo bar): { $lib.print(woot) } }')
+            self.stormIsInErr('Unexpected token', msgs)
+            self.stormIsInErr('expecting one of: case multi-value, double-quoted string, single-quoted string', msgs)
+
+            # multi-value case without a second value
+            msgs = await core.stormlist('$foo=foo switch $foo { (foo, ): { $lib.print(woot) } }')
+            self.stormIsInErr('Unexpected token', msgs)
+            self.stormIsInErr('expecting one of: case multi-value, double-quoted string, single-quoted string', msgs)
+
+            # multi-value case without a comma or second value
+            msgs = await core.stormlist('$foo=foo switch $foo { (foo): { $lib.print(woot) } }')
+            self.stormIsInErr('Unexpected token', msgs)
+            self.stormIsInErr('expecting one of: ,', msgs)
+
     async def test_storm_tagvar(self):
 
         async with self.getTestCore() as core:
@@ -7861,17 +7900,23 @@ class CortexBasicTest(s_t_utils.SynTest):
                 'view': view.iden,
             })
 
+            othr = s_common.guid()
             info4 = await core.addHttpExtApi({
+                'iden': othr,
                 'path': 'another/item',
                 'owner': unfo.get('iden'),
                 'view': view.iden,
             })
+            self.eq(info4.get('iden'), othr)
 
             iden = info.get('iden')
 
             adef = await core.getHttpExtApi(iden)
             self.eq(adef, info)
 
+            adef = await core.getHttpExtApi(othr)
+            self.eq(adef, info4)
+
             adef, args = await core.getHttpExtApiByPath('test/path/hehe/wow')
             self.eq(adef, info)
             self.eq(args, ('hehe', 'wow'))
@@ -7916,6 +7961,23 @@ class CortexBasicTest(s_t_utils.SynTest):
 
             # Sad path
 
+            with self.raises(s_exc.SchemaViolation):
+                await core.addHttpExtApi({
+                    'iden': 'lolnope',
+                    'path': 'not/gonna/happen',
+                    'owner': unfo.get('iden'),
+                    'view': view.iden
+                })
+
+            with self.raises(s_exc.DupIden) as ectx:
+                await core.addHttpExtApi({
+                    'iden': othr,
+                    'path': 'bad/dup',
+                    'owner': unfo.get('iden'),
+                    'view': view.iden
+                })
+            self.eq(ectx.exception.get('iden'), othr)
+
             with self.raises(s_exc.SynErr):
                 await core.setHttpApiIndx(newp, 0)
 
@@ -7953,6 +8015,10 @@ class CortexBasicTest(s_t_utils.SynTest):
 
     async def test_cortex_query_offload(self):
 
+        async def _hang(*args, **kwargs):
+            await asyncio.sleep(6)
+            return
+
         async with self.getTestAha() as aha:
 
             async with await s_base.Base.anit() as base:
@@ -8028,8 +8094,61 @@ class CortexBasicTest(s_t_utils.SynTest):
                     self.isin('Offloading Storm query', data)
                     self.notin('Timeout', data)
 
+                    with patch('synapse.cortex.CoreApi.getNexsIndx', _hang):
+
+                        with self.getLoggerStream('synapse') as stream:
+                            msgs = await alist(core00.storm('inet:asn=0'))
+                            self.len(1, [m for m in msgs if m[0] == 'node'])
+
+                        stream.seek(0)
+                        data = stream.read()
+                        self.notin('Offloading Storm query', data)
+                        self.isin('Timeout waiting for pool mirror [01.core.synapse] Nexus offset', data)
+                        self.notin('Timeout waiting for query mirror', data)
+
+                    await core00.stormpool.waitready(timeout=12)
+
+                    with patch('synapse.telepath.Proxy.getPoolLink', _hang):
+
+                        with self.getLoggerStream('synapse') as stream:
+                            msgs = await alist(core00.storm('inet:asn=0'))
+                            self.len(1, [m for m in msgs if m[0] == 'node'])
+
+                        stream.seek(0)
+                        data = stream.read()
+                        self.notin('Offloading Storm query', data)
+                        self.isin('Timeout waiting for pool mirror [01.core.synapse] Nexus offset', data)
+                        self.notin('Timeout waiting for query mirror', data)
+
+                    await core00.stormpool.waitready(timeout=12)
+
+                    with self.getLoggerStream('synapse') as stream:
+                        msgs = await alist(core00.storm('inet:asn=0'))
+                        self.len(1, [m for m in msgs if m[0] == 'node'])
+
+                    stream.seek(0)
+                    data = stream.read()
+                    self.isin('Offloading Storm query', data)
+                    self.notin('Timeout waiting for pool mirror', data)
+                    self.notin('Timeout waiting for query mirror', data)
+
                     core01.nexsroot.nexslog.indx = 0
 
+                    with patch('synapse.cortex.MAX_NEXUS_DELTA', 1):
+
+                        nexsoffs = await core00.getNexsIndx()
+
+                        with self.getLoggerStream('synapse') as stream:
+                            msgs = await alist(core00.storm('inet:asn=0'))
+                            self.len(1, [m for m in msgs if m[0] == 'node'])
+
+                        stream.seek(0)
+                        data = stream.read()
+                        explog = (f'Pool mirror [01.core.synapse] Nexus offset delta too large '
+                                  f'({nexsoffs} > 1), running query locally')
+                        self.isin(explog, data)
+                        self.notin('Offloading Storm query', data)
+
                     with self.getLoggerStream('synapse') as stream:
                         msgs = await alist(core00.storm('inet:asn=0'))
                         self.len(1, [m for m in msgs if m[0] == 'node'])
@@ -8076,6 +8195,16 @@ class CortexBasicTest(s_t_utils.SynTest):
                     with self.raises(s_exc.TimeOut):
                         await core00.count('inet:asn=0', opts=opts)
 
+                    core00.stormpool.ready.clear()
+
+                    with self.getLoggerStream('synapse') as stream:
+                        msgs = await alist(core00.storm('inet:asn=0'))
+                        self.len(1, [m for m in msgs if m[0] == 'node'])
+
+                    stream.seek(0)
+                    data = stream.read()
+                    self.isin('Timeout waiting for pool mirror, running query locally', data)
+
                     await core01.fini()
 
                     with self.getLoggerStream('synapse') as stream:

synapse/tests/test_datamodel.py

@@ -63,6 +63,12 @@ class DataModelTest(s_t_utils.SynTest):
             with self.raises(s_exc.BadFormDef):
                 core.model.addForm('woot:two', {}, ())
 
+            with self.raises(s_exc.NoSuchForm):
+                core.model.reqForm('newp:newp')
+
+            with self.raises(s_exc.NoSuchProp):
+                core.model.reqForm('inet:asn').reqProp('newp')
+
     async def test_datamodel_formname(self):
         modl = s_datamodel.Model()
         mods = (

synapse/tests/test_lib_cell.py

@@ -2355,6 +2355,18 @@ class CellTest(s_t_utils.SynTest):
 
             async with self.getTestCore() as core:
 
+                # This tmp_reason assertion seems counter-intuitive at first; but it's really
+                # asserting that the message which was incorrectly being logged is no longer logged.
+                log_enable_writes = f'Free space on {core.dirn} above minimum threshold'
+                with self.getAsyncLoggerStream('synapse.lib.cell', log_enable_writes) as stream:
+                    await core.nexsroot.addWriteHold(tmp_reason := 'something else')
+                    self.false(await stream.wait(1))
+                stream.seek(0)
+                self.eq(stream.read(), '')
+
+                await core.nexsroot.delWriteHold(tmp_reason)
+                revt.clear()
+
                 self.len(1, await core.nodes('[inet:fqdn=vertex.link]'))
 
                 with mock.patch('shutil.disk_usage', full_disk):

synapse/tests/test_lib_grammar.py

@@ -619,6 +619,10 @@ Queries = [
             switch $foo {
                 bar: { [ +#ohai ] break }
                 baz: { [ +#visi ] continue }
+                (far, faz): { [ +#multi.far ] continue }
+                ("gar", "gaz"): { [ +#multi.gar ] continue }
+                ('har', 'haz'): { [ +#multi.har ] continue }
+                ("kar", 'kaz', koo): { [ +#multi.kar ] continue }
             }
 
             [ inet:ipv4=5.6.7.8 ]
@@ -1251,7 +1255,9 @@ _ParseResults = [
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: c], SwitchCase: [VarValue: [Const: woot], CaseEntry: [Const: hehe, SubQuery: [Query: [EditTagAdd: [TagName: [Const: baz]]]]], CaseEntry: [SubQuery: [Query: [EditTagAdd: [TagName: [Const: jaz]]]]]]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: c], SwitchCase: [VarValue: [Const: woot], CaseEntry: [Const: hehe, SubQuery: [Query: [EditTagAdd: [TagName: [Const: baz]]]]], CaseEntry: [Const: haha hoho, SubQuery: [Query: [EditTagAdd: [TagName: [Const: faz]]]]], CaseEntry: [Const: lolz:lulz, SubQuery: [Query: [EditTagAdd: [TagName: [Const: jaz]]]]]]]',
     'Query: [EditNodeAdd: [FormName: [Const: inet:ipv4], Const: =, Const: 1.2.3.4], SwitchCase: [VarValue: [Const: foo], CaseEntry: [Const: bar, SubQuery: [Query: [EditTagAdd: [TagName: [Const: hehe, Const: haha]]]]], CaseEntry: [Const: baz faz, SubQuery: [Query: []]]]]',
-    'Query: [ForLoop: [Const: foo, VarValue: [Const: foos], SubQuery: [Query: [EditNodeAdd: [FormName: [Const: inet:ipv4], Const: =, Const: 1.2.3.4], SwitchCase: [VarValue: [Const: foo], CaseEntry: [Const: bar, SubQuery: [Query: [EditTagAdd: [TagName: [Const: ohai]], BreakOper: []]]], CaseEntry: [Const: baz, SubQuery: [Query: [EditTagAdd: [TagName: [Const: visi]], ContinueOper: []]]]], EditNodeAdd: [FormName: [Const: inet:ipv4], Const: =, Const: 5.6.7.8], EditTagAdd: [TagName: [Const: hehe]]]]]]',
+
+    'Query: [ForLoop: [Const: foo, VarValue: [Const: foos], SubQuery: [Query: [EditNodeAdd: [FormName: [Const: inet:ipv4], Const: =, Const: 1.2.3.4], SwitchCase: [VarValue: [Const: foo], CaseEntry: [Const: bar, SubQuery: [Query: [EditTagAdd: [TagName: [Const: ohai]], BreakOper: []]]], CaseEntry: [Const: baz, SubQuery: [Query: [EditTagAdd: [TagName: [Const: visi]], ContinueOper: []]]], CaseEntry: [Const: far, Const: faz, SubQuery: [Query: [EditTagAdd: [TagName: [Const: multi, Const: far]], ContinueOper: []]]], CaseEntry: [Const: gar, Const: gaz, SubQuery: [Query: [EditTagAdd: [TagName: [Const: multi, Const: gar]], ContinueOper: []]]], CaseEntry: [Const: har, Const: haz, SubQuery: [Query: [EditTagAdd: [TagName: [Const: multi, Const: har]], ContinueOper: []]]], CaseEntry: [Const: kar, Const: kaz, Const: koo, SubQuery: [Query: [EditTagAdd: [TagName: [Const: multi, Const: kar]], ContinueOper: []]]]], EditNodeAdd: [FormName: [Const: inet:ipv4], Const: =, Const: 5.6.7.8], EditTagAdd: [TagName: [Const: hehe]]]]]]',
+
     'Query: [SwitchCase: [VarValue: [Const: a], CaseEntry: [Const: a, SubQuery: [Query: []]]]]',
     'Query: [SwitchCase: [VarValue: [Const: a], CaseEntry: [Const: test:str, SubQuery: [Query: []]], CaseEntry: [SubQuery: [Query: []]]]]',
     'Query: [SwitchCase: [VarValue: [Const: a], CaseEntry: [Const: test:this:works:, SubQuery: [Query: []]], CaseEntry: [SubQuery: [Query: []]]]]',

synapse/tests/test_lib_layer.py

@@ -2238,3 +2238,38 @@ class LayerTest(s_t_utils.SynTest):
                 self.false(layr.nodeeditslab.lenv.flags()['readahead'])
                 self.false(layr.dataslab.readahead)
                 self.false(layr.dataslab.lenv.flags()['readahead'])
+
+    async def test_layer_delete_with_nodedata(self):
+
+        async with self.getTestCore() as core:
+
+            fork00 = await core.view.fork()
+            infork00 = {'view': fork00['iden']}
+            layr00 = core.getLayer(fork00['layers'][0]['iden'])
+
+            iden = await core.callStorm('[ inet:ipv4=1.2.3.4 ] return($node.iden())')
+
+            sodes = await s_t_utils.alist(layr00.getStorNodesByForm('inet:ipv4'))
+            self.len(0, sodes)
+
+            q = '''
+                inet:ipv4=1.2.3.4
+                $node.data.set("key", "valu")
+            '''
+            await core.callStorm(q, opts=infork00)
+
+            sodes = await s_t_utils.alist(layr00.getStorNodesByForm('inet:ipv4'))
+            self.len(1, sodes)
+
+            q = '''
+                view.exec $fork00 {
+                    yield $iden
+                    $lib.print($node)
+                    delnode --deledges --force
+                }
+            '''
+            opts = {'vars': {'iden': iden, 'fork00': fork00['iden']}}
+            await core.callStorm(q, opts=opts)
+
+            sodes = await s_t_utils.alist(layr00.getStorNodesByForm('inet:ipv4'))
+            self.len(0, sodes)

synapse/tests/test_lib_lmdbslab.py

@@ -295,8 +295,9 @@ class LmdbSlabTest(s_t_utils.SynTest):
             self.eq(items, ((b'\x00\x02', b'haha'), (b'\x00\x01', b'hehe')))
 
             # Copy a database inside the same slab
-            self.raises(s_exc.DataAlreadyExists, slab.copydb, foo, slab, 'bar')
-            self.eq(3, slab.copydb(foo, slab, 'foo2'))
+            with self.raises(s_exc.DataAlreadyExists):
+                await slab.copydb(foo, slab, 'bar')
+            self.eq(3, await slab.copydb(foo, slab, 'foo2'))
 
             # Increase the size of the new source DB to trigger a resize on the next copydb
             foo2 = slab.initdb('foo2')
@@ -312,7 +313,7 @@ class LmdbSlabTest(s_t_utils.SynTest):
             async with await s_lmdbslab.Slab.anit(path2, map_size=512 * 1024) as slab2:
                 with patch('synapse.lib.lmdbslab.PROGRESS_PERIOD', 2):
 
-                    self.eq(4, slab.copydb(foo2, slab2, destdbname='foo2', progresscb=progfunc))
+                    self.eq(4, await slab.copydb(foo2, slab2, destdbname='foo2', progresscb=progfunc))
                     self.gt(vardict.get('prog', 0), 0)
 
             # Test slab.drop and slab.dbexists
@@ -808,7 +809,8 @@ class LmdbSlabTest(s_t_utils.SynTest):
                 self.raises(s_exc.IsReadOnly, newdb.replace, b'1234', b'3456')
                 self.raises(s_exc.IsReadOnly, newdb.pop, b'1234')
                 self.raises(s_exc.IsReadOnly, newdb.delete, b'1234')
-                self.raises(s_exc.IsReadOnly, newdb.putmulti, ((b'1234', b'3456'),))
+                with self.raises(s_exc.IsReadOnly):
+                    await newdb.putmulti((b'1234', b'3456'))
 
                 # While we have the DB open in readonly, have another process write a bunch of data to cause the
                 # map size to be increased
@@ -836,7 +838,7 @@ class LmdbSlabTest(s_t_utils.SynTest):
                 # A putmulti across a grow
                 before_mapsize = slab.mapsize
                 kvpairs = [(x, x) for x in data]
-                retn = slab.putmulti(kvpairs)
+                retn = await slab.putmulti(kvpairs)
                 self.eq(retn, (1000, 1000))
 
                 after_mapsize1 = slab.mapsize
@@ -844,7 +846,7 @@ class LmdbSlabTest(s_t_utils.SynTest):
 
                 # A putmulti across a grow with a generator passed in
                 kvpairs = ((b' ' + x, x) for x in data)
-                retn = slab.putmulti(kvpairs)
+                retn = await slab.putmulti(kvpairs)
                 self.eq(retn, (1000, 1000))
                 after_mapsize2 = slab.mapsize
                 self.gt(after_mapsize2, after_mapsize1)
@@ -1011,7 +1013,7 @@ class LmdbSlabTest(s_t_utils.SynTest):
             async with await s_lmdbslab.Slab.anit(path, map_size=32000, growsize=5000) as slab:
                 slab.initdb('foo')
                 kvpairs = [(x, x) for x in data]
-                slab.putmulti(kvpairs)
+                await slab.putmulti(kvpairs)
                 slab.forcecommit()
                 before_mapsize = slab.mapsize
                 slab.dropdb('foo')
@@ -1029,7 +1031,7 @@ class LmdbSlabTest(s_t_utils.SynTest):
             async with await s_lmdbslab.Slab.anit(path, map_size=32000, growsize=5000) as slab:
                 slab.initdb('foo')
                 kvpairs = [(x, x) for x in data]
-                slab.putmulti(kvpairs)
+                await slab.putmulti(kvpairs)
                 slab.forcecommit()
 
         asyncio.run(workloop())
@@ -1047,7 +1049,7 @@ class LmdbSlabTest(s_t_utils.SynTest):
 
                 proc = mpctx.Process(target=self.make_slab, args=(path,))
                 proc.start()
-                proc.join(10)
+                proc.join(20)
                 self.nn(proc.exitcode)
                 slab.initdb('foo')
                 self.true(True)