synapse 2.192.0__py311-none-any.whl → 2.194.0__py311-none-any.whl

synapse/tests/test_lib_ast.py

@@ -131,7 +131,7 @@ class AstTest(s_test.SynTest):
             self.stormIsInWarn('Storm search interface is not enabled!', msgs)
 
         async with self.getTestCore() as core:
-            await core.loadStormPkg({
+            core.loadStormPkg({
                 'name': 'testsearch',
                 'modules': [
                     {'name': 'testsearch', 'interfaces': ['search'], 'storm': '''
@@ -292,11 +292,11 @@ class AstTest(s_test.SynTest):
         async with self.getTestCore() as core:
             q = '''
                 [test:str=another :hehe=asdf]
-                $s = $lib.text("Foo")
+                $s = ("Foo",)
                 $newvar=:hehe
                 -.created
-                $s.add("yar {x}", x=$newvar)
-                $lib.print($s.str())
+                $s.append("yar {x}", x=$newvar)
+                $lib.print($lib.str.join('', $s))
             '''
             mesgs = await core.stormlist(q)
             prints = [m[1]['mesg'] for m in mesgs if m[0] == 'print']
@@ -405,6 +405,93 @@ class AstTest(s_test.SynTest):
             q = 'test:str=foo $newp=($node.repr(), bar) [*$newp=foo]'
             await self.asyncraises(s_exc.StormRuntimeError, core.nodes(q))
 
+    async def test_ast_condsetoper(self):
+        async with self.getTestCore() as core:
+
+            q = '$var=hehe $foo=unset [test:str=foo :$var*unset=heval]'
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq('heval', nodes[0].get('hehe'))
+
+            q = '$var=hehe $foo=unset [test:str=foo :$var*unset=newp]'
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq('heval', nodes[0].get('hehe'))
+
+            q = '$var=hehe $foo=unset [test:str=foo :$var*$foo=newp]'
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq('heval', nodes[0].get('hehe'))
+
+            q = '$var=hehe $foo=always [test:str=foo :$var*$foo=yep]'
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq('yep', nodes[0].get('hehe'))
+
+            q = '[test:str=foo -:hehe]'
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.none(nodes[0].get('hehe'))
+
+            q = '$var=hehe $foo=never [test:str=foo :$var*$foo=yep]'
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.none(nodes[0].get('hehe'))
+
+            q = '$var=hehe $foo=unset [test:str=foo :$var*$foo=heval]'
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq('heval', nodes[0].get('hehe'))
+
+            with self.raises(s_exc.BadTypeValu):
+                q = '$var=tick $foo=always [test:str=foo :$var*$foo=heval]'
+                nodes = await core.nodes(q)
+
+            q = '$var=tick $foo=always [test:str=foo :$var*$foo?=heval]'
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.none(nodes[0].get('tick'))
+
+            q = '''
+            $opts=({"tick": "unset", "hehe": "always"})
+            [ test:str=foo
+                :hehe*$opts.hehe=newv
+                :tick*$opts.tick?=2020]
+            '''
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq('newv', nodes[0].get('hehe'))
+            tick = nodes[0].get('tick')
+            self.nn(tick)
+
+            q = '''
+            $opts=({"tick": "never", "hehe": "unset"})
+            [ test:str=foo
+                :hehe*$opts.hehe=newp
+                :tick*$opts.tick?=2020]
+            '''
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq('newv', nodes[0].get('hehe'))
+            self.eq(tick, nodes[0].get('tick'))
+
+            q = '$foo=always [test:str=foo :tick*$foo?=2021]'
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.ne(tick, nodes[0].get('tick'))
+
+            with self.raises(s_exc.IsReadOnly):
+                q = '[test:str=foo :hehe*unset=heval]'
+                nodes = await core.nodes(q, opts={'readonly': True})
+
+            with self.raises(s_exc.NoSuchProp):
+                q = '[test:str=foo :newp*unset=heval]'
+                nodes = await core.nodes(q)
+
+            with self.raises(s_exc.StormRuntimeError):
+                q = '$foo=newp [test:str=foo :hehe*$foo=heval]'
+                nodes = await core.nodes(q)
+
     async def test_ast_editparens(self):
 
         async with self.getTestCore() as core:
@@ -1070,10 +1157,10 @@ class AstTest(s_test.SynTest):
 
             nodes = await core.nodes('[ test:arrayprop="*" :ints=(1, 2, 3) ]')
             nodes = await core.nodes('[ test:arrayprop="*" :ints=(100, 101, 102) ]')
-            nodes = await core.nodes('test:arrayprop +:ints=$lib.list(1,2,3)')
+            nodes = await core.nodes('test:arrayprop +:ints=([1,2,3])')
             self.len(1, nodes)
 
-            nodes = await core.nodes('test:arrayprop:ints=$lib.list(1,2,3)')
+            nodes = await core.nodes('test:arrayprop:ints=([1,2,3])')
             self.len(1, nodes)
 
             with self.raises(s_exc.NoSuchProp):
@@ -1232,7 +1319,7 @@ class AstTest(s_test.SynTest):
             nodes = await core.nodes(q)
             self.len(1, nodes)
 
-            nodes = await core.nodes('[ test:arrayprop=* :strs={return ($lib.list(a,b,c,d))} ]')
+            nodes = await core.nodes('[ test:arrayprop=* :strs={return ((a,b,c,d))} ]')
             self.len(1, nodes)
             self.len(4, nodes[0].get('strs'))
 
@@ -1870,7 +1957,7 @@ class AstTest(s_test.SynTest):
             self.len(0, await core.nodes('init { function x() { return((0)) } }'))
 
             # Can't use a mutable variable as a default
-            q = '$var=$lib.list(1,2,3) function badargs(x=foo, y=$var) {} $badargs()'
+            q = '$var=([1,2,3]) function badargs(x=foo, y=$var) {} $badargs()'
             msgs = await core.stormlist(q)
             erfo = [m for m in msgs if m[0] == 'err'][0]
             self.eq(erfo[1][0], 'StormRuntimeError')
@@ -2648,12 +2735,12 @@ class AstTest(s_test.SynTest):
 
             q = 'function func(arg) { auth.user.addrule root $arg | return () } $func(hehe.haha)'
             msgs = await core.stormlist(q, opts={'readonly': True})
-            self.stormIsInErr('Function (_methUserAddRule) is not marked readonly safe.', msgs)
+            self.stormIsInErr('auth:user.addRule() is not marked readonly safe.', msgs)
 
     async def test_ast_yield(self):
 
         async with self.getTestCore() as core:
-            q = '$nodes = $lib.list() [ inet:asn=10 inet:asn=20 ] $nodes.append($node) | spin | yield $nodes'
+            q = '$nodes = () [ inet:asn=10 inet:asn=20 ] $nodes.append($node) | spin | yield $nodes'
             nodes = await core.nodes(q)
             self.len(2, nodes)
 
@@ -3119,6 +3206,48 @@ class AstTest(s_test.SynTest):
             off, end = errm[1][1]['highlight']['offsets']
             self.eq('newp', text[off:end])
 
+            visi = (await core.addUser('visi'))['iden']
+            text = '$users=$lib.auth.users.list() $lib.print($users.0.profile)'
+            msgs = await core.stormlist(text, opts={'user': visi})
+            errm = [m for m in msgs if m[0] == 'err'][0]
+            off, end = errm[1][1]['highlight']['offsets']
+            self.eq('lib.print($users.0.profile)', text[off:end])
+
+            text = '$lib.len(foo, bar)'
+            msgs = await core.stormlist(text)
+            errm = [m for m in msgs if m[0] == 'err'][0]
+            off, end = errm[1][1]['highlight']['offsets']
+            self.eq('lib.len(foo, bar)', text[off:end])
+            self.stormIsInErr('$lib.len()', msgs)
+
+            text = '$foo=$lib.pkg.get $foo()'
+            msgs = await core.stormlist(text)
+            errm = [m for m in msgs if m[0] == 'err'][0]
+            off, end = errm[1][1]['highlight']['offsets']
+            self.eq('foo()', text[off:end])
+            self.stormIsInErr('$lib.pkg.get()', msgs)
+
+            text = '$obj = $lib.pipe.gen(${ $obj.put() }) $obj.put(foo, bar, baz)'
+            msgs = await core.stormlist(text)
+            errm = [m for m in msgs if m[0] == 'err'][0]
+            off, end = errm[1][1]['highlight']['offsets']
+            self.eq('obj.put(foo, bar, baz)', text[off:end])
+            self.stormIsInErr('pipe.put()', msgs)
+
+            text = '$lib.gen.campaign(foo, bar, baz)'
+            msgs = await core.stormlist(text)
+            errm = [m for m in msgs if m[0] == 'err'][0]
+            off, end = errm[1][1]['highlight']['offsets']
+            self.eq('lib.gen.campaign(foo, bar, baz)', text[off:end])
+            self.stormIsInErr('$lib.gen.campaign()', msgs)
+
+            text = '$gen = $lib.gen.campaign $gen(foo, bar, baz)'
+            msgs = await core.stormlist(text)
+            errm = [m for m in msgs if m[0] == 'err'][0]
+            off, end = errm[1][1]['highlight']['offsets']
+            self.eq('gen(foo, bar, baz)', text[off:end])
+            self.stormIsInErr('$lib.gen.campaign()', msgs)
+
     async def test_ast_bulkedges(self):
 
         async with self.getTestCore() as core:
@@ -4395,3 +4524,165 @@ class AstTest(s_test.SynTest):
             text = '($x, $y) = (1)'
             with self.raises(s_exc.StormRuntimeError):
                 await core.nodes(text)
+
+    async def test_ast_functypes(self):
+
+        async with self.getTestCore() as core:
+
+            async def verify(q, isin=False):
+                msgs = await core.stormlist(q)
+                if isin:
+                    self.stormIsInPrint('yep', msgs)
+                else:
+                    self.stormNotInPrint('newp', msgs)
+                self.len(1, [m for m in msgs if m[0] == 'node'])
+                self.stormHasNoErr(msgs)
+
+            q = '''
+            function foo() {
+                for $n in { return((newp,)) } { $lib.print($n) }
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q)
+
+            q = '''
+            function foo() {
+                while { return((newp,)) } { $lib.print(newp) break }
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q)
+
+            q = '''
+            function foo() {
+                switch $lib.print({ return(newp) }) { *: { $lib.print(newp) } }
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q)
+
+            q = '''
+            function foo() {
+                switch $foo { *: { $lib.print(yep) return() } }
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q, isin=True)
+
+            q = '''
+            function foo() {
+                if { return(newp) } { $lib.print(newp) }
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q)
+
+            q = '''
+            function foo() {
+                if (false) { $lib.print(newp) }
+                elif { return(newp) } { $lib.print(newp) }
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q)
+
+            q = '''
+            function foo() {
+                if (false) { $lib.print(newp) }
+                elif (true) { $lib.print(yep) return() }
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q)
+
+            q = '''
+            function foo() {
+                if (false) { $lib.print(newp) }
+                elif (false) { $lib.print(newp) }
+                else { $lib.print(yep) return() }
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q, isin=True)
+
+            q = '''
+            function foo() {
+                [ it:dev:str=foo +(refs)> { $lib.print(newp) return() } ]
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q)
+
+            q = '''
+            function foo() {
+                $lib.print({ return(newp) })
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q)
+
+            q = '''
+            function foo() {
+                $x = { $lib.print(newp) return() }
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q)
+
+            q = '''
+            function foo() {
+                ($x, $y) = { $lib.print(newp) return((foo, bar)) }
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q)
+
+            q = '''
+            function foo() {
+                $x = ({})
+                $x.y = { $lib.print(newp) return((foo, bar)) }
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q)
+
+            q = '''
+            function foo() {
+                .created -({$lib.print(newp) return(refs)})> *
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q)
+
+            q = '''
+            function foo() {
+                try { $lib.raise(boom) } catch { $lib.print(newp) return(newp) } as e {}
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q)
+
+            q = '''
+            function foo() {
+                it:dev:str={ $lib.print(newp) return(test) }
+            }
+            [ it:dev:str=test ]
+            $foo()
+            '''
+            await verify(q)

synapse/tests/test_lib_auth.py

@@ -495,7 +495,7 @@ class AuthTest(s_test.SynTest):
             ])
 
             # Check sequences
-            seqmsg = f'Password must not contain forward/reverse sequences longer than 3 characters.'
+            seqmsg = 'Password must not contain forward/reverse sequences longer than 3 characters.'
             passwords = [
                 # letters
                 'abcA', 'dcbA', 'Abcd', 'Acba',
@@ -531,12 +531,8 @@ class AuthTest(s_test.SynTest):
                 'Password must contain at least 2 digit characters, 0 found.'
             ])
 
-            with self.raises(s_exc.BadArg) as exc:
-                await core.setUserPasswd(user.iden, None)
-            self.isin(
-                'Password must be at least 12 characters.',
-                exc.exception.get('failures')
-            )
+            # Setting password to None should work also
+            await core.setUserPasswd(user.iden, None)
 
             # Attempting to add a user with a bad passwd will add the user and fail to set the password
             with self.raises(s_exc.BadArg):
@@ -582,6 +578,9 @@ class AuthTest(s_test.SynTest):
             await core.setUserPasswd(user.iden, pass2)
             await core.setUserPasswd(user.iden, pass3)
 
+            # Setting password to None should work also
+            await core.setUserPasswd(user.iden, None)
+
             with self.raises(s_exc.BadArg) as exc:
                 await core.setUserPasswd(user.iden, pass1)
             self.eq(exc.exception.get('failures'), [

synapse/tests/test_lib_cell.py

@@ -1433,6 +1433,11 @@ class CellTest(s_t_utils.SynTest):
                         with mock.patch('os.stat', diffdev):
                             await self.asyncraises(s_exc.LowSpace, proxy.runBackup())
 
+                user = await core.auth.getUserByName('root')
+                with self.raises(s_exc.SynErr) as cm:
+                    await core.iterNewBackupArchive(user)
+                self.isin('This API must be called via a CellApi', cm.exception.get('mesg'))
+
             async def err(*args, **kwargs):
                 raise RuntimeError('boom')
 
@@ -2298,11 +2303,13 @@ class CellTest(s_t_utils.SynTest):
                             # Backup the mirror (core01) which points to the core00
                             async with await axon00.upload() as upfd:
                                 async with core01.getLocalProxy() as prox:
+                                    tot_chunks = 0
                                     async for chunk in prox.iterNewBackupArchive():
                                         await upfd.write(chunk)
+                                        tot_chunks += len(chunk)
 
                                     size, sha256 = await upfd.save()
-                                    await asyncio.sleep(0)
+                                    self.eq(size, tot_chunks)
 
                     furl = f'{url}{s_common.ehex(sha256)}'
                     purl = await aha.addAhaSvcProv('00.mynewcortex')
@@ -3276,3 +3283,66 @@ class CellTest(s_t_utils.SynTest):
                     with self.raises(s_exc.BadState) as cm:
                         await cell00.promote(graceful=True)
                     self.isin('02.cell is not the current leader', cm.exception.get('mesg'))
+
+    async def test_stream_backup_exception(self):
+
+        with self.getTestDir() as dirn:
+            backdirn = os.path.join(dirn, 'backups')
+            coredirn = os.path.join(dirn, 'cortex')
+
+            conf = {'backup:dir': backdirn}
+            s_common.yamlsave(conf, coredirn, 'cell.yaml')
+
+            async with self.getTestCore(dirn=coredirn) as core:
+                async with core.getLocalProxy() as proxy:
+
+                    await proxy.runBackup(name='bkup')
+
+                    mock_proc = mock.Mock()
+                    mock_proc.join = mock.Mock()
+
+                    async def mock_executor(func, *args, **kwargs):
+                        if isinstance(func, mock.Mock) and func is mock_proc.join:
+                            raise Exception('boom')
+                        return mock_proc
+
+                    with mock.patch('synapse.lib.cell.s_coro.executor', mock_executor):
+                        with self.getAsyncLoggerStream('synapse.lib.cell', 'Error during backup streaming') as stream:
+                            with self.raises(Exception) as cm:
+                                async for _ in proxy.iterBackupArchive('bkup'):
+                                    pass
+                            self.true(await stream.wait(timeout=6))
+
+    async def test_iter_new_backup_archive(self):
+
+        with self.getTestDir() as dirn:
+            backdirn = os.path.join(dirn, 'backups')
+            coredirn = os.path.join(dirn, 'cortex')
+
+            conf = {'backup:dir': backdirn}
+            s_common.yamlsave(conf, coredirn, 'cell.yaml')
+
+            async with self.getTestCore(dirn=coredirn) as core:
+                async with core.getLocalProxy() as proxy:
+
+                    async def mock_runBackup(*args, **kwargs):
+                        raise Exception('backup failed')
+
+                    with mock.patch.object(s_cell.Cell, 'runBackup', mock_runBackup):
+                        with self.getAsyncLoggerStream('synapse.lib.cell', 'Removing') as stream:
+                            with self.raises(s_exc.SynErr) as cm:
+                                async for _ in proxy.iterNewBackupArchive('failedbackup', remove=True):
+                                    pass
+
+                            self.isin('backup failed', str(cm.exception))
+                            self.true(await stream.wait(timeout=6))
+
+                            path = os.path.join(backdirn, 'failedbackup')
+                            self.false(os.path.exists(path))
+
+                    self.false(core.backupstreaming)
+
+                    core.backupstreaming = True
+                    with self.raises(s_exc.BackupAlreadyRunning):
+                        async for _ in proxy.iterNewBackupArchive('newbackup', remove=True):
+                            pass

synapse/tests/test_lib_grammar.py

@@ -728,6 +728,13 @@ Queries = [
     '$pvar=stuff test:arrayprop +:$pvar*[=neato]',
     '$pvar=ints test:arrayprop +:$pvar*[=$othervar]',
     '$foo = ({"foo": ${ inet:fqdn }})',
+    '[test:str=foo :hehe*unset=heval]',
+    '[test:str=foo :hehe*$foo=heval]',
+    '[test:str=foo :$foo*unset=heval]',
+    '[test:str=foo :$foo*$bar=heval]',
+    '[test:str=foo :$foo*$bar.baz=heval]',
+    '[test:str=foo :$foo*$bar.("baz")=heval]',
+    '[test:str=foo :$foo*$bar.baz()=heval]',
 ]
 
 # Generated with print_parse_list below
@@ -1358,6 +1365,13 @@ _ParseResults = [
     'Query: [SetVarOper: [Const: pvar, Const: stuff], LiftProp: [Const: test:arrayprop], FiltOper: [Const: +, ArrayCond: [RelProp: [VarValue: [Const: pvar]], Const: =, Const: neato]]]',
     'Query: [SetVarOper: [Const: pvar, Const: ints], LiftProp: [Const: test:arrayprop], FiltOper: [Const: +, ArrayCond: [RelProp: [VarValue: [Const: pvar]], Const: =, VarValue: [Const: othervar]]]]',
     'Query: [SetVarOper: [Const: foo, DollarExpr: [ExprDict: [Const: foo, EmbedQuery: inet:fqdn]]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [Const: hehe], CondSetOper: [Const: unset], Const: heval]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [Const: hehe], CondSetOper: [VarValue: [Const: foo]], Const: heval]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [Const: unset], Const: heval]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [VarValue: [Const: bar]], Const: heval]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [VarDeref: [VarValue: [Const: bar], Const: baz]], Const: heval]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [VarDeref: [VarValue: [Const: bar], DollarExpr: [Const: baz]]], Const: heval]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [FuncCall: [VarDeref: [VarValue: [Const: bar], Const: baz], CallArgs: [], CallKwargs: []]], Const: heval]]',
 ]
 
 class GrammarTest(s_t_utils.SynTest):

synapse/tests/test_lib_layer.py

@@ -54,7 +54,7 @@ class LayerTest(s_t_utils.SynTest):
             self.eq(errors[1][0], 'NoPropIndex')
 
             errors = await core.callStorm('''
-                $retn = $lib.list()
+                $retn = ()
                 for $mesg in $lib.layer.get().verify() {
                     $retn.append($mesg)
                 }

synapse/tests/test_lib_lmdbslab.py

@@ -338,9 +338,9 @@ class LmdbSlabTest(s_t_utils.SynTest):
 
             # Ensure that our envar override for memory locking is acknowledged
             with self.setTstEnvars(SYN_LOCKMEM_DISABLE='1'):
-                slab = await s_lmdbslab.Slab.anit(path, map_size=1000000, lockmemory=True)
-                self.false(slab.lockmemory)
-                self.none(slab.memlocktask)
+                async with await s_lmdbslab.Slab.anit(path, map_size=1000000, lockmemory=True) as slab:
+                    self.false(slab.lockmemory)
+                    self.none(slab.memlocktask)
 
     def simplenow(self):
         self._nowtime += 1000