synapse 2.221.0__py311-none-any.whl → 2.223.0__py311-none-any.whl

synapse/tests/test_lib_storm.py

@@ -1,6 +1,5 @@
 import copy
 import asyncio
-import datetime
 import itertools
 import urllib.parse as u_parse
 import unittest.mock as mock
@@ -17,6 +16,7 @@ import synapse.lib.storm as s_storm
 import synapse.lib.httpapi as s_httpapi
 import synapse.lib.msgpack as s_msgpack
 import synapse.lib.version as s_version
+import synapse.lib.stormtypes as s_stormtypes
 
 import synapse.tests.utils as s_t_utils
 from synapse.tests.utils import alist
@@ -1220,33 +1220,6 @@ class StormTest(s_t_utils.SynTest):
             ''')
             self.eq(email, 'visi@vertex.link')
 
-            pkg0 = {'name': 'hehe', 'version': '1.2.3'}
-            await core.addStormPkg(pkg0)
-            self.eq('1.2.3', await core.callStorm('return($lib.pkg.get(hehe).version)'))
-
-            self.eq(None, await core.callStorm('return($lib.pkg.get(nopkg))'))
-
-            pkg1 = {'name': 'haha', 'version': '1.2.3'}
-            await core.addStormPkg(pkg1)
-            msgs = await core.stormlist('pkg.list')
-            self.stormIsInPrint('haha', msgs)
-            self.stormIsInPrint('hehe', msgs)
-
-            self.true(await core.callStorm('return($lib.pkg.has(haha))'))
-
-            await core.delStormPkg('haha')
-            self.none(await core.callStorm('return($lib.pkg.get(haha))'))
-            self.false(await core.callStorm('return($lib.pkg.has(haha))'))
-
-            msgs = await core.stormlist('pkg.list --verbose')
-            self.stormIsInPrint('not available', msgs)
-
-            pkg2 = {'name': 'hoho', 'version': '4.5.6', 'build': {'time': 1732017600000}}
-            await core.addStormPkg(pkg2)
-            self.eq('4.5.6', await core.callStorm('return($lib.pkg.get(hoho).version)'))
-            msgs = await core.stormlist('pkg.list --verbose')
-            self.stormIsInPrint('2024-11-19 12:00:00', msgs)
-
             # test for $lib.queue.gen()
             self.eq(0, await core.callStorm('return($lib.queue.gen(woot).size())'))
             # and again to test *not* creating it...
@@ -1575,161 +1548,6 @@ class StormTest(s_t_utils.SynTest):
                 nodes = [x for x in msgs if x[0] == 'node']
                 self.len(0, nodes)
 
-            pkgdef = {
-                'name': 'foobar',
-                'version': '1.2.3',
-            }
-
-            await core.addStormPkg(pkgdef)
-
-            deps = await core.callStorm('return($lib.pkg.deps($pkgdef))', opts={'vars': {'pkgdef': pkgdef}})
-            self.eq({
-                'requires': (),
-                'conflicts': (),
-            }, deps)
-
-            pkgdef = {
-                'name': 'bazfaz',
-                'version': '2.2.2',
-                'depends': {
-                    'conflicts': (
-                        {'name': 'foobar'},
-                    ),
-                }
-            }
-
-            with self.raises(s_exc.StormPkgConflicts):
-                await core.addStormPkg(pkgdef)
-
-            deps = await core.callStorm('return($lib.pkg.deps($pkgdef))', opts={'vars': {'pkgdef': pkgdef}})
-            self.eq({
-                'requires': (),
-                'conflicts': (
-                    {'name': 'foobar', 'version': None, 'desc': None, 'ok': False, 'actual': '1.2.3'},
-                )
-            }, deps)
-
-            pkgdef = {
-                'name': 'bazfaz',
-                'version': '2.2.2',
-                'depends': {
-                    'conflicts': (
-                        {'name': 'foobar', 'version': '>=1.0.0', 'desc': 'foo'},
-                    ),
-                }
-            }
-
-            with self.raises(s_exc.StormPkgConflicts):
-                await core.addStormPkg(pkgdef)
-
-            deps = await core.callStorm('return($lib.pkg.deps($pkgdef))', opts={'vars': {'pkgdef': pkgdef}})
-            self.eq({
-                'requires': (),
-                'conflicts': (
-                    {'name': 'foobar', 'version': '>=1.0.0', 'desc': 'foo', 'ok': False, 'actual': '1.2.3'},
-                )
-            }, deps)
-
-            pkgdef = {
-                'name': 'bazfaz',
-                'version': '2.2.2',
-                'depends': {
-                    'requires': (
-                        {'name': 'foobar', 'version': '>=2.0.0,<3.0.0'},
-                    ),
-                }
-            }
-
-            with self.getAsyncLoggerStream('synapse.cortex', 'bazfaz requirement') as stream:
-                await core.addStormPkg(pkgdef)
-                self.true(await stream.wait(timeout=1))
-
-            pkgdef = {
-                'name': 'bazfaz',
-                'version': '2.2.2',
-                'depends': {
-                    'requires': (
-                        {'name': 'foobar', 'version': '>=2.0.0,<3.0.0', 'optional': True},
-                    ),
-                }
-            }
-
-            with self.getAsyncLoggerStream('synapse.cortex', 'bazfaz optional requirement') as stream:
-                await core.addStormPkg(pkgdef)
-                self.true(await stream.wait(timeout=1))
-
-            deps = await core.callStorm('return($lib.pkg.deps($pkgdef))', opts={'vars': {'pkgdef': pkgdef}})
-            self.eq({
-                'requires': (
-                    {'name': 'foobar', 'version': '>=2.0.0,<3.0.0', 'desc': None,
-                     'ok': False, 'actual': '1.2.3', 'optional': True},
-                ),
-                'conflicts': ()
-            }, deps)
-
-            pkgdef = {
-                'name': 'lolzlolz',
-                'version': '1.2.3',
-            }
-
-            await core.addStormPkg(pkgdef)
-
-            deps = await core.callStorm('return($lib.pkg.deps($pkgdef))', opts={'vars': {'pkgdef': pkgdef}})
-            self.eq({
-                'requires': (),
-                'conflicts': (),
-            }, deps)
-
-            pkgdef = {
-                'name': 'bazfaz',
-                'version': '2.2.2',
-                'depends': {
-                    'requires': (
-                        {'name': 'lolzlolz', 'version': '>=1.0.0,<2.0.0', 'desc': 'lol'},
-                    ),
-                    'conflicts': (
-                        {'name': 'foobar', 'version': '>=3.0.0'},
-                    ),
-                }
-            }
-
-            await core.addStormPkg(pkgdef)
-
-            deps = await core.callStorm('return($lib.pkg.deps($pkgdef))', opts={'vars': {'pkgdef': pkgdef}})
-            self.eq({
-                'requires': (
-                    {'name': 'lolzlolz', 'version': '>=1.0.0,<2.0.0', 'desc': 'lol', 'ok': True, 'actual': '1.2.3'},
-                ),
-                'conflicts': (
-                    {'name': 'foobar', 'version': '>=3.0.0', 'desc': None, 'ok': True, 'actual': '1.2.3'},
-                )
-            }, deps)
-
-            pkgdef = {
-                'name': 'zoinkszoinks',
-                'version': '2.2.2',
-                'depends': {
-                    'requires': (
-                        {'name': 'newpnewp', 'version': '1.2.3'},
-                    ),
-                    'conflicts': (
-                        {'name': 'newpnewp'},
-                    ),
-                }
-            }
-
-            await core.addStormPkg(pkgdef)
-
-            deps = await core.callStorm('return($lib.pkg.deps($pkgdef))', opts={'vars': {'pkgdef': pkgdef}})
-            self.eq({
-                'requires': (
-                    {'name': 'newpnewp', 'version': '1.2.3', 'desc': None, 'ok': False, 'actual': None},
-                ),
-                'conflicts': (
-                    {'name': 'newpnewp', 'version': None, 'desc': None, 'ok': True, 'actual': None},
-                )
-            }, deps)
-
             # force old-cron behavior which lacks a view
             await core.nodes('cron.add --hourly 03 { inet:ipv4 }')
             for (iden, cron) in core.agenda.list():
@@ -2980,72 +2798,6 @@ class StormTest(s_t_utils.SynTest):
             with self.raises(s_exc.NoSuchVar):
                 await core.callStorm('$foo = 10 $foo = $lib.undef return($foo)')
 
-    async def test_storm_pkg_load(self):
-        cont = s_common.guid()
-        pkg = {
-            'name': 'testload',
-            'version': '0.3.0',
-            'modules': (
-                {
-                    'name': 'testload',
-                    'storm': 'function x() { return((0)) }',
-                },
-            ),
-            'onload': f'[ ps:contact={cont} ] $lib.print(teststring) $lib.warn(testwarn, key=valu) return($path.vars.newp)'
-        }
-        class PkgHandler(s_httpapi.Handler):
-
-            async def get(self, name):
-                assert self.request.headers.get('X-Synapse-Version') == s_version.verstring
-
-                if name == 'notok':
-                    self.sendRestErr('FooBar', 'baz faz')
-                    return
-
-                self.sendRestRetn(pkg)
-
-        class PkgHandlerRaw(s_httpapi.Handler):
-            async def get(self, name):
-                assert self.request.headers.get('X-Synapse-Version') == s_version.verstring
-
-                self.set_header('Content-Type', 'application/json')
-                return self.write(pkg)
-
-        async with self.getTestCore() as core:
-            core.addHttpApi('/api/v1/pkgtest/(.*)', PkgHandler, {'cell': core})
-            core.addHttpApi('/api/v1/pkgtestraw/(.*)', PkgHandlerRaw, {'cell': core})
-            port = (await core.addHttpsPort(0, host='127.0.0.1'))[1]
-
-            msgs = await core.stormlist(f'pkg.load --ssl-noverify https://127.0.0.1:{port}/api/v1/newp/newp')
-            self.stormIsInWarn('pkg.load got HTTP code: 404', msgs)
-
-            msgs = await core.stormlist(f'pkg.load --ssl-noverify https://127.0.0.1:{port}/api/v1/pkgtest/notok')
-            self.stormIsInWarn('pkg.load got JSON error: FooBar', msgs)
-
-            # onload will on fire once. all other pkg.load events will effectively bounce
-            # because the pkg hasn't changed so no loading occurs
-            waiter = core.waiter(1, 'core:pkg:onload:complete')
-
-            with self.getAsyncLoggerStream('synapse.cortex') as stream:
-                msgs = await core.stormlist(f'pkg.load --ssl-noverify https://127.0.0.1:{port}/api/v1/pkgtest/yep')
-                self.stormIsInPrint('testload @0.3.0', msgs)
-
-                msgs = await core.stormlist(f'pkg.load --ssl-noverify --raw https://127.0.0.1:{port}/api/v1/pkgtestraw/yep')
-                self.stormIsInPrint('testload @0.3.0', msgs)
-
-            stream.seek(0)
-            buf = stream.read()
-            self.isin("testload onload output: teststring", buf)
-            self.isin("testload onload output: testwarn", buf)
-            self.isin("No var with name: newp", buf)
-            self.len(1, await core.nodes(f'ps:contact={cont}'))
-
-            evnts = await waiter.wait(timeout=4)
-            exp = [
-                ('core:pkg:onload:complete', {'pkg': 'testload'})
-            ]
-            self.eq(exp, evnts)
-
     async def test_storm_pkg_onload_active(self):
         pkg = {
             'name': 'testload',
@@ -3294,6 +3046,37 @@ class StormTest(s_t_utils.SynTest):
                         await stream.wait(timeout=10)
                         self.eq(8, await core.getStormPkgVar('testload', 'testload:version'))
 
+                    # init that advances the version
+
+                    pkg['version'] = '0.7.0'
+                    pkg['inits']['versions'].extend([
+                        {
+                            'version': 9,
+                            'name': 'init09',
+                            'query': '''
+                                $lib.globals.set(init09, $lib.time.now())
+                                $lib.pkg.vars(testload)."testload:version" = (10)
+                            ''',
+                        },
+                        {
+                            'version': 10,
+                            'name': 'init10',
+                            'query': '$lib.globals.set(init10, $lib.time.now())',
+                        },
+                        {
+                            'version': 11,
+                            'name': 'init11',
+                            'query': '$lib.globals.set(init11, $lib.time.now())',
+                        },
+                    ])
+
+                    await loadPkg(core, pkg)
+
+                    self.eq(11, await core.getStormPkgVar('testload', 'testload:version'))
+                    self.nn(await core.getStormVar('init09'))
+                    self.none(await core.getStormVar('init10'))
+                    self.nn(await core.getStormVar('init11'))
+
     async def test_storm_tree(self):
 
         async with self.getTestCore() as core:
@@ -4874,6 +4657,79 @@ class StormTest(s_t_utils.SynTest):
             self.len(1, links)
             self.eq({'type': 'runtime'}, links[0][1])
 
+    async def test_storm_derefprops(self):
+        async with self.getTestCore() as core:
+            await core.addTagProp('score', ('int', {}), {})
+
+            mesg = "Expected value of type 'str', got '"
+
+            # editnodeadd
+            msgs = await core.stormlist('$form = inet:fqdn [ *$form=foobar.com ]')
+            self.stormHasNoWarnErr(msgs)
+
+            invals = [10, None, False, [], {}]
+
+            for inval in invals:
+                opts = {'vars': {'form': inval}}
+                with self.raises(s_exc.StormRuntimeError) as exc:
+                    await core.nodes('[ *$form=valu ]', opts=opts)
+                self.true(exc.exception.get('mesg').startswith(mesg))
+
+            # liftprop
+            msgs = await core.stormlist('$form = inet:fqdn *$form')
+            self.stormHasNoWarnErr(msgs)
+
+            for inval in invals:
+                opts = {'vars': {'form': inval}}
+                with self.raises(s_exc.StormRuntimeError) as exc:
+                    await core.nodes('*$form', opts=opts)
+                self.true(exc.exception.get('mesg').startswith(mesg))
+
+            # liftpropby
+            msgs = await core.stormlist('$form = inet:fqdn *$form=foobar.com')
+            self.stormHasNoWarnErr(msgs)
+
+            for inval in invals:
+                opts = {'vars': {'form': inval}}
+                with self.raises(s_exc.StormRuntimeError) as exc:
+                    await core.nodes('*$form=newp', opts=opts)
+                self.true(exc.exception.get('mesg').startswith(mesg))
+
+            # liftformtag
+            msgs = await core.stormlist('$form = inet:fqdn *$form#foo')
+            self.stormHasNoWarnErr(msgs)
+
+            for inval in invals:
+                opts = {'vars': {'form': inval}}
+                with self.raises(s_exc.StormRuntimeError) as exc:
+                    await core.nodes('*$form#newp', opts=opts)
+                self.true(exc.exception.get('mesg').startswith(mesg))
+
+            # liftbyarray
+            msgs = await core.stormlist('$form = test:arrayform *$form*[=(10)]')
+            self.stormHasNoWarnErr(msgs)
+
+            for inval in invals:
+                opts = {'vars': {'form': inval}}
+                with self.raises(s_exc.StormRuntimeError) as exc:
+                    await core.nodes('*$form*[="newp"]', opts=opts)
+                self.true(exc.exception.get('mesg').startswith(mesg))
+
+            # formtagprop
+            msgs = await core.stormlist('$form = inet:fqdn *$form#foo:score')
+            self.stormHasNoWarnErr(msgs)
+
+            for inval in invals:
+                opts = {'vars': {'form': inval}}
+                with self.raises(s_exc.StormRuntimeError) as exc:
+                    await core.nodes('*$form#newp:score', opts=opts)
+                self.true(exc.exception.get('mesg').startswith(mesg))
+
+            # Check Storm Str types
+            name = s_stormtypes.Str('inet:fqdn')
+            msgs = await core.stormlist('*$form', opts={'vars': {'form': name}})
+            self.stormHasNoWarnErr(msgs)
+
     async def test_storm_nested_root(self):
         async with self.getTestCore() as core:
             self.eq(20, await core.callStorm('''

synapse/tests/test_lib_stormlib_auth.py

@@ -427,6 +427,90 @@ class StormLibAuthTest(s_test.SynTest):
                     opts=asuser
                 )
 
+    async def test_stormlib_auth_asroot_ondeny(self):
+        async with self.getTestCore() as core:
+
+            stormpkg = {
+                'name': 'authtest',
+                'version': '0.0.1',
+                'perms': (
+                    {'perm': ('authtest', 'perm'), 'desc': 'Elevate mod', 'gate': 'cortex'},
+                ),
+                'modules': (
+                    {
+                        'name': 'authtest.mod.default',
+                        'asroot:perms': (
+                            ('authtest', 'perm'),
+                        ),
+                        'storm': 'function func() { [ test:guid=* ] return($node) }',
+                    },
+                    {
+                        'name': 'authtest.mod.allow',
+                        'asroot:perms': (
+                            ('authtest', 'perm'),
+                        ),
+                        'asroot:ondeny:import': 'allow',
+                        'storm': '''
+                            function func() { [ test:guid=* ] return($node) }
+                            function func2() { [ test:str=newp ] return($node) }
+                        ''',
+                    },
+                    {
+                        'name': 'authtest.mod.warn',
+                        'asroot:perms': (
+                            ('authtest', 'perm'),
+                        ),
+                        'asroot:ondeny:import': 'warn',
+                        'storm': '''
+                            function func() { [ test:guid=* ] return($node) }
+                            function func2() { [ test:str=newp ] return($node) }
+                        ''',
+                    },
+                    {
+                        'name': 'authtest.mod.deny',
+                        'asroot:perms': (
+                            ('authtest', 'perm'),
+                        ),
+                        'asroot:ondeny:import': 'deny',
+                        'storm': 'function func() { [ test:guid=* ] return($node) }',
+                    },
+                ),
+            }
+
+            await core.stormlist('auth.user.add user')
+            await core.stormlist('auth.user.add lowuser')
+
+            await core.addStormPkg(stormpkg)
+
+            user = await core.auth.getUserByName('user')
+            asuser = {'user': user.iden}
+
+            lowuser = await core.auth.getUserByName('lowuser')
+            aslowuser = {'user': lowuser.iden}
+
+            await core.stormlist('auth.user.addrule user authtest.perm')
+            await core.stormlist('auth.user.addrule lowuser node.add.test:guid')
+
+            # user has no node perms, but has asroot so perms are elevated
+            self.len(1, await core.nodes('yield $lib.import(authtest.mod.default).func()', opts=asuser))
+            self.len(1, await core.nodes('yield $lib.import(authtest.mod.allow).func()', opts=asuser))
+            self.len(1, await core.nodes('yield $lib.import(authtest.mod.deny).func()', opts=asuser))
+            msgs = await core.stormlist('yield $lib.import(authtest.mod.warn).func()', opts=asuser)
+            self.stormHasNoWarnErr(msgs)
+            self.len(1, [m for m in msgs if m[0] == 'node'])
+
+            # lowuser does not have asroot perms and gets denied importing certain modules
+            await self.asyncraises(s_exc.AuthDeny, core.nodes('yield $lib.import(authtest.mod.deny).func()', opts=aslowuser))
+            await self.asyncraises(s_exc.AuthDeny, core.nodes('yield $lib.import(authtest.mod.default).func()', opts=aslowuser))
+            # ...and for modules they can import perms are not elevated so they can only add test:guid nodes
+            self.len(1, await core.nodes('yield $lib.import(authtest.mod.allow).func()', opts=aslowuser))
+            msgs = await core.stormlist('yield $lib.import(authtest.mod.warn).func()', opts=aslowuser)
+            self.stormIsInWarn('permissions will not be elevated.', msgs)
+            self.len(1, [m for m in msgs if m[0] == 'node'])
+            # ...and gets denied on adding test:str nodes
+            await self.asyncraises(s_exc.AuthDeny, core.nodes('yield $lib.import(authtest.mod.allow).func2()', opts=aslowuser))
+            await self.asyncraises(s_exc.AuthDeny, core.nodes('yield $lib.import(authtest.mod.warn).func2()', opts=aslowuser))
+
     async def test_stormlib_auth_userjson(self):
 
         async with self.getTestCore() as core:

synapse/tests/test_lib_stormlib_cortex.py

@@ -110,6 +110,7 @@ $request.reply(206, headers=$headers, body=({"no":"body"}))
                 self.eq(resp.status, http.HTTPStatus.OK)
                 data = await resp.json()
                 self.eq(data.get('method'), 'get')
+                self.eq(resp.headers.get('Content-Type'), 'application/json; charset=utf8')
 
                 resp = await sess.post(f'https://localhost:{hport}/api/ext/testpath00')
                 self.eq(resp.status, http.HTTPStatus.CREATED)

synapse/tests/test_lib_stormlib_mime.py

@@ -1,3 +1,5 @@
+import synapse.lib.stormlib.mime as s_mime
+
 import synapse.tests.utils as s_test
 
 html00 = '''
@@ -35,20 +37,42 @@ class StormlibMimeTest(s_test.SynTest):
 
     async def test_stormlib_mime_html(self):
 
+        # this is mostly for coverage because it looks like code coverage doesn't work for code executed under semafork()
+        self.eq('html5lib includes this\na title\nhello there\nother text', s_mime.htmlToText(html00))
+        self.eq('html5lib includes this|a title|hello there|other text', s_mime.htmlToText(html00, separator='|'))
+        self.eq('  foo  ', s_mime.htmlToText('<div> <p> foo </p> </div>', separator='', strip=False))
+
         async with self.getTestCore() as core:
 
             opts = {'vars': {'html': html00}}
             ret = await core.callStorm('return($lib.mime.html.totext($html))', opts=opts)
             self.eq('html5lib includes this\na title\nhello there\nother text', ret)
 
+            ret = await core.callStorm('return($lib.mime.html.totext($html, separator="|"))', opts=opts)
+            self.eq('html5lib includes this|a title|hello there|other text', ret)
+
             opts = {'vars': {'html': html01}}
             ret = await core.callStorm('return($lib.mime.html.totext($html))', opts=opts)
             self.eq('a bad tag\nanother bad tag\nmore text\nfor fun', ret)
 
+            ret = await core.callStorm('return($lib.mime.html.totext($html, separator=", "))', opts=opts)
+            self.eq('a bad tag, another bad tag, more text, for fun', ret)
+
+            ret = await core.callStorm('return($lib.mime.html.totext($html, separator=(null), strip=(false)))', opts=opts)
+            self.eq('\n  \n  \n  \n    a bad tag\n  \n  \n    \n        another bad tag\n    \n        more text\n    \n  \n\nfor fun\n', ret)
+
             opts = {'vars': {'html': '<div></div>'}}
             ret = await core.callStorm('return($lib.mime.html.totext($html))', opts=opts)
             self.eq('', ret)
 
+            opts = {'vars': {'html': '<div>    </div>'}}
+            ret = await core.callStorm('return($lib.mime.html.totext($html, strip=(false)))', opts=opts)
+            self.eq('    ', ret)
+
+            opts = {'vars': {'html': '<div> <p> foo </p> </div>'}}
+            ret = await core.callStorm('return($lib.mime.html.totext($html, separator=(null), strip=(false)))', opts=opts)
+            self.eq('  foo  ', ret)
+
             opts = {'vars': {'html': '...'}}
             ret = await core.callStorm('return($lib.mime.html.totext($html))', opts=opts)
             self.eq('...', ret)