synapse 2.152.0__py311-none-any.whl → 2.154.0__py311-none-any.whl

synapse/tests/test_lib_ast.py

@@ -1,17 +1,13 @@
 import json
 import math
 import asyncio
-import contextlib
-import collections
 
 from unittest import mock
 
 import synapse.exc as s_exc
 import synapse.common as s_common
-import synapse.cortex as s_cortex
 
 import synapse.lib.ast as s_ast
-import synapse.lib.base as s_base
 import synapse.lib.snap as s_snap
 
 import synapse.tests.utils as s_test
@@ -125,44 +121,6 @@ foo_stormpkg = {
     ],
 }
 
-@contextlib.asynccontextmanager
-async def matchContexts(testself):
-    origenter = s_base.Base.__aenter__
-    origexit = s_base.Base.__aexit__
-    origstorm = s_cortex.Cortex.storm
-    orignodes = s_cortex.Cortex.nodes
-
-    contexts = collections.defaultdict(int)
-
-    async def enter(self):
-        contexts[type(self)] += 1
-        return await origenter(self)
-
-    async def exit(self, exc, cls, tb):
-        contexts[type(self)] -= 1
-        await origexit(self, exc, cls, tb)
-
-    async def storm(self, text, opts=None):
-        async for mesg in origstorm(self, text, opts=opts):
-            yield mesg
-
-        for cont, refs in contexts.items():
-            testself.eq(0, refs)
-
-    async def nodes(self, text, opts=None):
-        nodes = await orignodes(self, text, opts=opts)
-
-        for cont, refs in contexts.items():
-            testself.eq(0, refs)
-
-        return nodes
-
-    with mock.patch('synapse.lib.base.Base.__aenter__', enter):
-        with mock.patch('synapse.lib.base.Base.__aexit__', exit):
-            with mock.patch('synapse.cortex.Cortex.nodes', nodes):
-                with mock.patch('synapse.cortex.Cortex.storm', storm):
-                    yield
-
 class AstTest(s_test.SynTest):
 
     async def test_mode_search(self):
@@ -2099,6 +2057,13 @@ class AstTest(s_test.SynTest):
             else { $lib.print(no) }
             '''
             msgs = await core.stormlist(q)
+            self.stormIsInPrint('yes', msgs)
+
+            q = '''test:str $data=$node.value()
+            if ($data ~= "(?-i:brown)") { $lib.print(yes) }
+            else { $lib.print(no) }
+            '''
+            msgs = await core.stormlist(q)
             self.stormIsInPrint('no', msgs)
 
             q = '''test:str $data=$node.value()
@@ -2320,7 +2285,7 @@ class AstTest(s_test.SynTest):
 
             await core.nodes('[ inet:fqdn=vertex.link ]')
 
-            async with matchContexts(self):
+            async with s_test.matchContexts(self):
 
                 await core.nodes("inet:fqdn -> { inet:fqdn=vertex.link } | limit 1")
                 await core.nodes("function x() { inet:fqdn=vertex.link } yield $x() | limit 1")
@@ -2561,3 +2526,133 @@ class AstTest(s_test.SynTest):
             self.len(0, nodes[0][1]['path']['edges'])
             # one for the refs edge (via doedges) and one for the rule..
             self.len(2, nodes[1][1]['path']['edges'])
+
+    async def test_ast_double_init_fini(self):
+        async with self.getTestCore() as core:
+            q = '''
+            init {$foo = bar $lib.print(`{$foo} {$wow}`) }
+            init {$baz = hehe $lib.print('second init!') }
+            $lib.print($baz)
+            '''
+            msgs = await core.stormlist(q, opts={'vars': {'wow': 'hehe'}})
+            pmesgs = [m[1].get('mesg') for m in msgs if m[0] == 'print']
+            self.eq(pmesgs, ['bar hehe', 'second init!', 'hehe'])
+
+            q = '''
+            init {$foo = bar $lib.print(`{$foo} {$wow}`) }
+            init {$baz = hehe $lib.print('second init!') }
+            $lib.print($baz)
+            [test:str=stuff]
+            $stuff = $node.value()
+            fini { $lib.print(fini1) }
+            fini { $lib.print(`fini {$stuff}`) }
+            '''
+            msgs = await core.stormlist(q, opts={'vars': {'wow': 'hehe', 'stuff': None}})
+            pmesgs = [m[1].get('mesg') for m in msgs if m[0] == 'print']
+            self.eq(pmesgs, ['bar hehe', 'second init!', 'hehe', 'fini1', 'fini stuff'])
+
+            q = '''
+            init { $foo = bar }
+            init { $baz = $lib.str.format('foo={foo}', foo=$foo) }
+            $lib.print($baz)
+            '''
+            msgs = await core.stormlist(q)
+            self.stormIsInPrint('foo=bar', msgs)
+
+    async def test_ast_tagfilters(self):
+
+        async with self.getTestCore() as core:
+
+            await core.addTagProp('score', ('int', {}), {})
+
+            await core.nodes('[ test:str=foo +#tagaa=2023 +#tagaa:score=5 <(foo)+ { test:str=foo } ]')
+            await core.nodes('[ test:str=bar +#tagab=2024 +#tagab:score=6 ]')
+            await core.nodes('[ test:str=baz +#tagba=2023 +#tagba:score=7 ]')
+            await core.nodes('[ test:str=faz +#tagbb=2024 +#tagbb:score=8 ]')
+
+            self.len(2, await core.nodes('test:str +#taga*'))
+            self.len(1, await core.nodes('test:str +#tagaa=2023'))
+            self.len(1, await core.nodes('test:str +#taga* <(*)- *'))
+            self.len(2, await core.nodes('$tag=taga* test:str +#$tag'))
+            self.len(1, await core.nodes('$tag=tagaa test:str +#$tag=2023'))
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('test:str +#taga*=2023')
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('test:str +#taga*@=2023')
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('test:str +#taga*>2023')
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('test:str +#taga*<(3+5)')
+
+            with self.raises(s_exc.NoSuchCmpr):
+                await core.nodes('test:str +#taga<(3+5)')
+
+            with self.raises(s_exc.NoSuchCmpr):
+                await core.nodes('test:str +#taga*min>=2023')
+
+            with self.raises(s_exc.StormRuntimeError):
+                await core.nodes('$tag=taga* test:str +#$tag=2023')
+
+            with self.raises(s_exc.StormRuntimeError):
+                await core.nodes('$tag=taga* test:str +#$"tag"=2023')
+
+            with self.raises(s_exc.StormRuntimeError):
+                await core.nodes('$tag=taga* test:str +#foo.$"tag"=2023')
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('$tag=taga* test:str +#foo*.$"tag"=2023')
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('$tag=taga test:str +#foo.$"tag".*=2023')
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('$tag=taga test:str +#foo.$"tag".*=2023')
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('$tag=taga test:str +#foo.$"tag".$"tag".*=2023')
+
+            self.len(2, await core.nodes('test:str +#taga*:score'))
+            self.len(1, await core.nodes('test:str +#tagaa:score=5'))
+            self.len(1, await core.nodes('test:str +#tagaa:score<(2+4)'))
+            self.len(1, await core.nodes('test:str +#tagaa:score*range=(4,6)'))
+            self.len(1, await core.nodes('test:str +#taga*:score <(*)- *'))
+            self.len(2, await core.nodes('$tag=taga* test:str +#$tag:score'))
+            self.len(1, await core.nodes('$tag=tagaa test:str +#$tag:score=5'))
+            self.len(1, await core.nodes('$tag=tagaa test:str +#$tag:score*range=(4,6)'))
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('test:str +#taga*:score=2023')
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('test:str +#taga*:score@=2023')
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('test:str +#taga*:score>2023')
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('test:str +#taga*:score<(3+5)')
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('test:str +#taga*:score*min>=2023')
+
+            with self.raises(s_exc.NoSuchCmpr):
+                await core.nodes('test:str +#taga:score*min>=2023')
+
+            with self.raises(s_exc.StormRuntimeError):
+                await core.nodes('$tag=taga* test:str +#$tag:score=2023')
+
+            with self.raises(s_exc.StormRuntimeError):
+                await core.nodes('$tag=taga* test:str +#foo.$"tag":score=2023')
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('$tag=taga* test:str +#foo*.$"tag":score=2023')
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('$tag=taga test:str +#foo.$"tag".*:score=2023')
+
+            with self.raises(s_exc.BadSyntax):
+                await core.nodes('$tag=taga test:str +#foo.$"tag".$"tag".*:score=2023')

synapse/tests/test_lib_autodoc.py

@@ -62,7 +62,7 @@ shave'''
         self.eq(s_autodoc.getArgLines({}), [])
         lines = s_autodoc.getArgLines(rtype)
         self.eq(lines, ['\n', 'Args:', '    foo (str): The foos!', '\n',
-                        '    bar: The bar. The input type may one one of the following: ``str``, ``int``.',
+                        '    bar: The bar. The input type may be one of the following: ``str``, ``int``.',
                         '\n', '    \\*\\*kwargs (any): Extra foobars.', '\n'])
 
         lines = s_autodoc.getArgLines({'args': [{'name': 'cmplx', 'type': {}, 'desc': 'unsupported'}]})

synapse/tests/test_lib_modelrev.py

@@ -419,3 +419,12 @@ class ModelRevTest(s_tests.SynTest):
 
             self.len(1, await core.nodes('risk:vulnname="woot woot"'))
             self.len(1, await core.nodes('risk:vuln:name="woot woot"'))
+
+    async def test_modelrev_0_2_22(self):
+
+        async with self.getRegrCore('model-0.2.22') as core:
+            nodes = await core.nodes('inet:ipv4=100.64.0.0/10')
+            self.len(257, nodes)
+
+            for node in nodes:
+                self.eq(node.props.get('type'), 'shared')

synapse/tests/test_lib_node.py

@@ -517,3 +517,58 @@ class NodeTest(s_t_utils.SynTest):
 
             msgs = await core.stormlist(edgeq)
             self.len(1, [m for m in msgs if m[0] == 'print'])
+
+    async def test_node_remove_missing_basetag(self):
+
+        async with self.getTestCore() as core:
+
+            base = await core.callStorm('return($lib.view.get().iden)')
+            fork = await core.callStorm('return($lib.view.get().fork().iden)')
+
+            await core.nodes('[test:str=neato +#foo.one]', opts={'view': base})
+            await core.nodes('test:str=neato | [ +#foo.two ]', opts={'view': fork})
+
+            await core.nodes('test:str=neato | [ -#foo ]', opts={'view': base})
+
+            othr = await core.nodes('test:str=neato', opts={'view': fork})
+            self.len(1, othr)
+            self.isin('foo.two', othr[0].tags)
+            self.notin('foo', othr[0].tags)
+
+            msgs = await core.stormlist('test:str=neato | [ -#foo ]', opts={'view': fork})
+            edits = [m[1] for m in msgs if m[0] == 'node:edits']
+            nodes = [m[1] for m in msgs if m[0] == 'node']
+            self.len(1, edits)
+            self.len(1, edits[0]['edits'][0][2])
+
+            self.len(1, nodes)
+            self.len(0, nodes[0][1]['tags'])
+
+            await core.nodes('[test:int=12 +#ping.pong.neato.burrito]', opts={'view': base})
+            await core.nodes('test:int=12 | [ +#ping.pong.awesome.possum ]', opts={'view': fork})
+
+            await core.nodes('test:int=12 | [ -#ping.pong]', opts={'view': base})
+
+            othr = await core.nodes('test:int=12', opts={'view': fork})
+            self.len(1, othr)
+            self.isin('ping', othr[0].tags)
+            self.isin('ping.pong.awesome', othr[0].tags)
+            self.isin('ping.pong.awesome.possum', othr[0].tags)
+
+            self.notin('ping.pong', othr[0].tags)
+
+            msgs = await core.stormlist('test:int=12 | [ -#ping.pong ]', opts={'view': fork})
+            edits = [m[1] for m in msgs if m[0] == 'node:edits']
+            nodes = [m[1] for m in msgs if m[0] == 'node']
+
+            self.len(1, edits)
+            self.len(2, edits[0]['edits'][0][2])
+
+            self.len(1, nodes)
+            self.len(1, nodes[0][1]['tags'])
+            self.isin('ping', nodes[0][1]['tags'])
+
+            nodes = await core.nodes('test:int=12 | [ -#p ]')
+            self.len(1, nodes)
+            self.len(1, nodes[0].tags)
+            self.isin('ping', nodes[0].tags)

synapse/tests/test_lib_storm.py

@@ -1213,6 +1213,19 @@ class StormTest(s_t_utils.SynTest):
             self.len(1, nodes)
             self.eq(nodes[0].ndef, ('test:str', 'pluto\udcbaneptune'))
 
+            nodes = await core.nodes('[ media:news=* :publisher:name=woot ] $name=:publisher:name [ :publisher={ gen.ou.org $name } ]')
+            self.len(1, nodes)
+            self.nn(nodes[0].get('publisher'))
+
+            # test regular expressions are case insensitive by default
+            self.len(1, await core.nodes('test:str~=Pluto'))
+            self.len(1, await core.nodes('test:str +test:str~=Pluto'))
+            self.true(await core.callStorm('return(("Foo" ~= "foo"))'))
+            self.len(0, await core.nodes('test:str~="(?-i:Pluto)"'))
+            self.len(0, await core.nodes('test:str +test:str~="(?-i:Pluto)"'))
+            self.false(await core.callStorm('return(("Foo" ~= "(?-i:foo)"))'))
+            self.true(await core.callStorm('return(("Foo" ~= "(?-i:Foo)"))'))
+
     async def test_storm_diff_merge(self):
 
         async with self.getTestCore() as core:
@@ -3641,7 +3654,7 @@ class StormTest(s_t_utils.SynTest):
             msgs = await core.stormlist('help list')
             self.stormIsInPrint('***\nlist\n****\nImplements the Storm API for a List instance.', msgs)
             self.stormIsInPrint('append(valu)\nAppend a value to the list.', msgs)
-            self.stormIsInPrint('auth.user.list : List all users.', msgs)
+            self.stormIsInPrint('auth.user.list     : List all users.', msgs)
 
             # email stor / gettr has a multi value return type
             msgs = await core.stormlist('help -v auth:user')

synapse/tests/test_lib_stormhttp.py

@@ -509,13 +509,15 @@ class StormHttpTest(s_test.SynTest):
             await visi.addRule((True, ('storm', 'lib', 'axon', 'wget')))
             await visi.addRule((True, ('storm', 'lib', 'axon', 'wput')))
 
+            errmsg = f'User {visi.name!r} ({visi.iden}) must have permission {{perm}}'
+
             asvisi = {'user': visi.iden}
             msgs = await core.stormlist('$lib.inet.http.get(http://vertex.link, proxy=$lib.false)', opts=asvisi)
-            self.stormIsInErr(s_exc.proxy_admin_mesg, msgs)
+            self.stormIsInErr(errmsg.format(perm='storm.lib.inet.http.proxy'), msgs)
 
             asvisi = {'user': visi.iden}
             msgs = await core.stormlist('$lib.inet.http.get(http://vertex.link, proxy=socks5://user:pass@127.0.0.1:1)', opts=asvisi)
-            self.stormIsInErr(s_exc.proxy_admin_mesg, msgs)
+            self.stormIsInErr(errmsg.format(perm='storm.lib.inet.http.proxy'), msgs)
 
             resp = await core.callStorm('return($lib.inet.http.get(http://vertex.link, proxy=socks5://user:pass@127.0.0.1:1))')
             self.eq('ProxyConnectionError', resp['err'][0])
@@ -523,15 +525,15 @@ class StormHttpTest(s_test.SynTest):
             # test $lib.axon proxy API
             asvisi = {'user': visi.iden}
             msgs = await core.stormlist('$lib.axon.wget(http://vertex.link, proxy=$lib.false)', opts=asvisi)
-            self.stormIsInErr(s_exc.proxy_admin_mesg, msgs)
+            self.stormIsInErr(errmsg.format(perm='storm.lib.inet.http.proxy'), msgs)
 
             asvisi = {'user': visi.iden}
             msgs = await core.stormlist('$lib.axon.wget(http://vertex.link, proxy=socks5://user:pass@127.0.0.1:1)', opts=asvisi)
-            self.stormIsInErr(s_exc.proxy_admin_mesg, msgs)
+            self.stormIsInErr(errmsg.format(perm='storm.lib.inet.http.proxy'), msgs)
 
             asvisi = {'user': visi.iden}
             msgs = await core.stormlist('$lib.axon.wput(asdf, http://vertex.link, proxy=socks5://user:pass@127.0.0.1:1)', opts=asvisi)
-            self.stormIsInErr(s_exc.proxy_admin_mesg, msgs)
+            self.stormIsInErr(errmsg.format(perm='storm.lib.inet.http.proxy'), msgs)
 
             resp = await core.callStorm('return($lib.axon.wget(http://vertex.link, proxy=socks5://user:pass@127.0.0.1:1))')
             self.false(resp.get('ok'))
@@ -544,6 +546,63 @@ class StormHttpTest(s_test.SynTest):
             self.false(resp.get('ok'))
             self.isin('connect to proxy 127.0.0.1:1', resp['mesg'])
 
+        async with self.getTestCore(conf=conf) as core:
+            # Proxy permission tests in this section
+
+            visi = await core.auth.addUser('visi')
+
+            await visi.addRule((True, ('storm', 'lib', 'axon', 'wget')))
+            await visi.addRule((True, ('storm', 'lib', 'axon', 'wput')))
+
+            _, sha256 = await core.axon.put(b'asdf')
+            sha256 = s_common.ehex(sha256)
+
+            host, port = await core.addHttpsPort(0)
+
+            q1 = f'return($lib.inet.http.get(https://loop.vertex.link:{port}, ssl_verify=$lib.false, proxy=$proxy))'
+            q2 = f'return($lib.axon.wget(https://loop.vertex.link:{port}, ssl=$lib.false, proxy=$proxy))'
+            q3 = f'return($lib.axon.wput({sha256}, https://loop.vertex.link:{port}, ssl=$lib.false, proxy=$proxy))'
+
+            for proxy in ('socks5://user:pass@127.0.0.1:1', False):
+                opts = {'vars': {'proxy': proxy}, 'user': visi.iden}
+
+                with self.raises(s_exc.AuthDeny):
+                    await core.callStorm(q1, opts=opts)
+
+                with self.raises(s_exc.AuthDeny):
+                    await core.callStorm(q2, opts=opts)
+
+                with self.raises(s_exc.AuthDeny):
+                    await core.callStorm(q3, opts=opts)
+
+            # Add permissions to use a proxy
+            await visi.addRule((True, ('storm', 'lib', 'inet', 'http', 'proxy')))
+
+            opts = {'vars': {'proxy': 'socks5://user:pass@127.0.0.1:1'}, 'user': visi.iden}
+
+            resp = await core.callStorm(q1, opts=opts)
+            self.eq('ProxyConnectionError', resp['err'][0])
+
+            resp = await core.callStorm(q2, opts=opts)
+            self.eq('ProxyConnectionError', resp['err'][0])
+
+            resp = await core.callStorm(q3, opts=opts)
+            self.eq('ProxyConnectionError', resp['err'][0])
+
+            opts = {'vars': {'proxy': False}, 'user': visi.iden}
+
+            resp = await core.callStorm(q1, opts=opts)
+            self.eq(resp['code'], 404)
+            self.eq(resp['reason'], 'Not Found')
+
+            resp = await core.callStorm(q2, opts=opts)
+            self.eq(resp['code'], 404)
+            self.eq(resp['reason'], 'Not Found')
+
+            resp = await core.callStorm(q3, opts=opts)
+            self.eq(resp['code'], 404)
+            self.eq(resp['reason'], 'Not Found')
+
     async def test_storm_http_connect(self):
 
         async with self.getTestCore() as core:
@@ -604,7 +663,7 @@ class StormHttpTest(s_test.SynTest):
             opts = {'user': visi.iden, 'vars': {'port': port, 'proxy': False}}
             with self.raises(s_exc.AuthDeny) as cm:
                 await core.callStorm(query, opts=opts)
-            self.eq(cm.exception.get('mesg'), s_exc.proxy_admin_mesg)
+            self.eq(cm.exception.get('mesg'), f'User {visi.name!r} ({visi.iden}) must have permission storm.lib.inet.http.proxy')
 
             await visi.setAdmin(True)
 

synapse/tests/test_lib_stormlib_auth.py

@@ -178,12 +178,21 @@ class StormLibAuthTest(s_test.SynTest):
             self.stormIsInPrint('allowed: true - Matched user rule (node) on gate 741529fa80e3fb42f63c5320e4bf348f.',
                 msgs, deguid=True)
 
-            q = '$lib.auth.users.byname(visi).setAdmin($admin, gateiden=$lib.view.get().layers.0.iden)'
-            await core.nodes(q, opts={'vars': {'admin': True}})
+            msgs = await core.stormlist('auth.user.mod visi --gate $lib.view.get().layers.0.iden')
+            self.stormIsInWarn('Granting/revoking admin status on an auth gate, requires the use of `--admin <true|false>` also.', msgs)
+
+            msgs = await core.stormlist('auth.user.mod visi --admin $lib.true --gate $lib.view.get().layers.0.iden')
+            self.stormIsInPrint('User (visi) admin status set to true for auth gate 741529fa80e3fb42f63c5320e4bf348f.',
+                                msgs, deguid=True)
             msgs = await core.stormlist('auth.user.allowed visi node.tag.del --gate $lib.view.get().layers.0.iden')
             self.stormIsInPrint('allowed: true - The user is an admin of auth gate 741529fa80e3fb42f63c5320e4bf348f',
                                 msgs, deguid=True)
-            await core.nodes(q, opts={'vars': {'admin': False}})
+            msgs = await core.stormlist('auth.user.mod visi --admin $lib.false --gate $lib.view.get().layers.0.iden')
+            self.stormIsInPrint('User (visi) admin status set to false for auth gate 741529fa80e3fb42f63c5320e4bf348f.',
+                                msgs, deguid=True)
+            msgs = await core.stormlist('auth.user.allowed visi node.tag.del --gate $lib.view.get().layers.0.iden')
+            self.stormIsInPrint('allowed: true - Matched user rule (node) on gate 741529fa80e3fb42f63c5320e4bf348f',
+                                msgs, deguid=True)
 
             await core.nodes('auth.role.addrule ninjas beep.sys --gate $lib.view.get().layers.0.iden')
             msgs = await core.stormlist('auth.user.allowed visi beep.sys --gate $lib.view.get().layers.0.iden')