synapse 2.190.0__py311-none-any.whl → 2.192.0__py311-none-any.whl

synapse/tests/test_model_syn.py

@@ -362,6 +362,93 @@ class SynModelTest(s_t_utils.SynTest):
                 nodes = await core.nodes('syn:tagprop')
                 self.len(0, nodes)
 
+        async with self.getTestCore() as core:
+                # Check we can iterate runt nodes while changing the underlying dictionary
+
+                numforms = len(core.model.forms)
+
+                q = '''
+                init {
+                    $forms = ()
+                    $count = (0)
+                }
+
+                syn:form
+
+                $forms.append(({'name': $node.repr(), 'doc': :doc }))
+
+                $count = ($count + 1)
+
+                if ($count = (2)) {
+                    $info = ({"doc": "test taxonomy", "interfaces": ["meta:taxonomy"]})
+                    $lib.model.ext.addForm(_test:taxonomy, taxonomy, ({}), $info)
+                }
+
+                spin |
+
+                fini { return($forms) }
+                '''
+
+                forms = await core.callStorm(q)
+                self.len(numforms, forms)
+                self.len(numforms + 1, core.model.forms)
+
+                numtypes = len(core.model.types)
+                q = '''
+                init {
+                    $types = ()
+                    $count = (0)
+                }
+
+                syn:type
+
+                $types.append(({'name': $node.repr(), 'doc': :doc }))
+
+                $count = ($count + 1)
+
+                if ($count = (2)) {
+                    $typeopts = ({"lower": true, "onespace": true})
+                    $typeinfo = ({"doc": "A test type doc."})
+                    $lib.model.ext.addType(_test:type, str, $typeopts, $typeinfo)
+                }
+
+                spin |
+
+                fini { return($types) }
+                '''
+
+                types = await core.callStorm(q)
+                self.len(numtypes, types)
+                self.len(numtypes + 1, core.model.types)
+
+                q = '''
+                init {
+                    $tagprops = ()
+                    $count = (0)
+                    $lib.model.ext.addTagProp(cypher, (str, ({})), ({}))
+                    $lib.model.ext.addTagProp(trinity, (str, ({})), ({}))
+                    $lib.model.ext.addTagProp(morpheus, (str, ({})), ({}))
+                }
+
+                syn:tagprop
+
+                $tagprops.append(({'name': $node.repr(), 'doc': :doc }))
+
+                $count = ($count + 1)
+
+                if ($count = (2)) {
+                    $lib.model.ext.addTagProp(neo, (str, ({})), ({}))
+                }
+
+                spin |
+
+                fini { return($tagprops) }
+                '''
+
+                tagprops = await core.callStorm(q)
+                self.len(3, tagprops)
+                self.len(4, core.model.tagprops)
+
     async def test_syn_trigger_runts(self):
         async with self.getTestCore() as core:
             nodes = await core.nodes('syn:trigger')
@@ -465,6 +552,42 @@ class SynModelTest(s_t_utils.SynTest):
             pode = nodes[0].pack()
             self.eq(pode[0][1], iden)
 
+        async with self.getTestCore() as core:
+                # Check we can iterate runt nodes while changing the underlying dictionary
+
+                tdef = {'cond': 'node:add', 'form': 'it:dev:str', 'storm': '[inet:user=1] | testcmd'}
+                await core.view.addTrigger(tdef)
+
+                tdef = {'cond': 'node:add', 'form': 'it:dev:str', 'storm': '[inet:user=2] | testcmd'}
+                await core.view.addTrigger(tdef)
+
+                q = '''
+                init {
+                    $trigs = ()
+                    $count = (0)
+                }
+
+                syn:trigger
+
+                $trigs.append(({'name': $node.repr(), 'doc': :doc }))
+
+                $count = ($count + 1)
+
+                if ($count = (2)) {
+                    $lib.trigger.add($tdef)
+                }
+
+                spin |
+
+                fini { return($trigs) }
+                '''
+
+                tdef = {'cond': 'node:add', 'form': 'it:dev:str', 'storm': '[inet:user=3] | testcmd'}
+                opts = {'vars': {'tdef': tdef}}
+                triggers = await core.callStorm(q, opts=opts)
+                self.len(2, triggers)
+                self.len(3, core.view.triggers.triggers)
+
     async def test_syn_cmd_runts(self):
 
         async with self.getTestDmon() as dmon:
@@ -554,6 +677,49 @@ class SynModelTest(s_t_utils.SynTest):
                 self.eq(nodes[0].get('output'), ('inet:fqdn',))
                 self.eq(nodes[0].get('nodedata'), (('foo', 'inet:ipv4'), ('bar', 'inet:fqdn')))
 
+        async with self.getTestCore() as core:
+                # Check we can iterate runt nodes while changing the underlying dictionary
+
+                numcmds = len(core.stormcmds)
+
+                stormpkg = {
+                    'name': 'stormpkg',
+                    'version': '1.2.3',
+                    'synapse_version': '>=2.8.0,<3.0.0',
+                    'commands': (
+                        {
+                         'name': 'pkgcmd.old',
+                         'storm': '$lib.print(hi)',
+                        },
+                    ),
+                }
+
+                q = '''
+                init {
+                    $cmds = ()
+                    $count = (0)
+                }
+
+                syn:cmd
+
+                $cmds.append(({'name': $node.repr(), 'doc': :doc }))
+
+                $count = ($count + 1)
+
+                if ($count = (2)) {
+                    $lib.pkg.add($pkgdef)
+                }
+
+                spin |
+
+                fini { return($cmds) }
+                '''
+
+                opts = {'vars': {'pkgdef': stormpkg}}
+                cmds = await core.callStorm(q, opts=opts)
+                self.len(numcmds, cmds)
+                self.len(numcmds + 1, core.stormcmds)
+
     async def test_syn_cron_runts(self):
 
         async with self.getTestCore() as core:
@@ -582,3 +748,35 @@ class SynModelTest(s_t_utils.SynTest):
             self.eq(nodes[0].ndef, ('syn:cron', iden))
             self.eq(nodes[0].get('doc'), 'hehe')
             self.eq(nodes[0].get('name'), 'haha')
+
+        async with self.getTestCore() as core:
+                # Check we can iterate runt nodes while changing the underlying dictionary
+
+                q = '''
+                init {
+                    $appts = ()
+                    $count = (0)
+
+                    cron.add --hour 1 --day 1 {#foo} |
+                    cron.add --hour 2 --day 1 {#foo} |
+                    cron.add --hour 3 --day 1 {#foo}
+                }
+
+                syn:cron
+
+                $appts.append(({'name': $node.repr(), 'doc': :doc }))
+
+                $count = ($count + 1)
+
+                if ($count = (2)) {
+                    cron.add  --hour 4 --day 1 {#foo}
+                }
+
+                spin |
+
+                fini { return($appts) }
+                '''
+
+                appts = await core.callStorm(q)
+                self.len(3, appts)
+                self.len(4, core.agenda.appts)

synapse/tests/test_servers_univ.py

@@ -56,18 +56,6 @@ class UnivServerTest(s_t_utils.SynTest):
                     async with await s_telepath.openurl(f'cell://{dirn}') as proxy:
                         self.eq('cell', await proxy.getCellType())
 
-                argv = [
-                    'synapse.tests.test_lib_cell.EchoAuth',
-                    '--telepath', 'tcp://127.0.0.1:0/',
-                    '--https', '0',
-                    '--name', 'univtest',
-                    dirn,
-                ]
-                # Or start the Cortex off a a EchoAuth (don't do this in practice...)
-                async with await s_s_univ.main(argv) as cell:
-                    async with await s_telepath.openurl(f'cell://{dirn}') as proxy:
-                        self.eq('echoauth', await proxy.getCellType())
-
                 argv = ['synapse.lib.newp.Newp']
                 with self.raises(s_exc.NoSuchCtor):
                     async with await s_s_univ.main(argv) as core:

synapse/tests/test_tools_apikey.py

@@ -0,0 +1,227 @@
+import datetime
+
+import synapse.exc as s_exc
+import synapse.common as s_common
+
+import synapse.lib.time as s_time
+import synapse.lib.output as s_output
+
+import synapse.tests.utils as s_test
+import synapse.tools.apikey as s_t_apikey
+
+async def getApiKeyByName(core, name):
+    keys = {k.get('name'): k async for k in core.getApiKeys()}
+    return keys.get(name)
+
+class ApiKeyTest(s_test.SynTest):
+
+    async def test_tools_apikey(self):
+        async with self.getTestCore() as core:
+
+            await core.auth.addUser('blackout')
+
+            rooturl = core.getLocalUrl()
+            blckurl = core.getLocalUrl(user='blackout')
+
+            # Add API keys
+            argv = (
+                '--svcurl', rooturl,
+                'add',
+                'rootkey00',
+                '-d', '120',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_apikey.main(argv, outp=outp))
+
+            self.isin('Successfully added API key with name=rootkey00.', str(outp))
+            rootkey00 = await getApiKeyByName(core, 'rootkey00')
+
+            self.isin(f'Iden: {rootkey00.get("iden")}', str(outp))
+            self.isin('  API Key: ', str(outp))
+            self.isin('  Name: rootkey00', str(outp))
+            self.isin(f'  Created: {s_time.repr(rootkey00.get("created"))}', str(outp))
+            self.isin(f'  Updated: {s_time.repr(rootkey00.get("updated"))}', str(outp))
+            self.isin(f'  Expires: {s_time.repr(rootkey00.get("expires"))}', str(outp))
+            self.eq(rootkey00.get('expires'), rootkey00.get('created') + 120000)
+
+            argv = (
+                '--svcurl', rooturl,
+                'add',
+                '-u', 'blackout',
+                'blckkey00',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_apikey.main(argv, outp=outp))
+
+            self.isin('Successfully added API key with name=blckkey00.', str(outp))
+            blckkey00 = await getApiKeyByName(core, 'blckkey00')
+
+            self.isin(f'Iden: {blckkey00.get("iden")}', str(outp))
+            self.isin('  API Key: ', str(outp))
+            self.isin('  Name: blckkey00', str(outp))
+            self.isin(f'  Created: {s_time.repr(blckkey00.get("created"))}', str(outp))
+            self.isin(f'  Updated: {s_time.repr(blckkey00.get("updated"))}', str(outp))
+            self.notin('  Expires: ', str(outp))
+
+            argv = (
+                '--svcurl', blckurl,
+                'add',
+                'blckkey01',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_apikey.main(argv, outp=outp))
+
+            self.isin('Successfully added API key with name=blckkey01.', str(outp))
+            blckkey01 = await getApiKeyByName(core, 'blckkey01')
+
+            self.isin(f'Iden: {blckkey01.get("iden")}', str(outp))
+            self.isin('  API Key: ', str(outp))
+            self.isin('  Name: blckkey01', str(outp))
+            self.isin(f'  Created: {s_time.repr(blckkey01.get("created"))}', str(outp))
+            self.isin(f'  Updated: {s_time.repr(blckkey01.get("updated"))}', str(outp))
+            self.notin('  Expires: ', str(outp))
+
+            # List API keys
+            argv = (
+                '--svcurl', rooturl,
+                'list',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_apikey.main(argv, outp=outp))
+
+            self.isin(f'Iden: {rootkey00.get("iden")}', str(outp))
+            self.notin('  API Key: ', str(outp))
+            self.isin('  Name: rootkey00', str(outp))
+            self.isin(f'  Created: {s_time.repr(rootkey00.get("created"))}', str(outp))
+            self.isin(f'  Updated: {s_time.repr(rootkey00.get("updated"))}', str(outp))
+            self.isin(f'  Expires: {s_time.repr(rootkey00.get("expires"))}', str(outp))
+            self.eq(rootkey00.get('expires'), rootkey00.get('created') + 120000)
+
+            argv = (
+                '--svcurl', rooturl,
+                'list',
+                '-u', 'blackout',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_apikey.main(argv, outp=outp))
+
+            self.isin(f'Iden: {blckkey00.get("iden")}', str(outp))
+            self.notin('  API Key: ', str(outp))
+            self.isin('  Name: blckkey00', str(outp))
+            self.isin(f'  Created: {s_time.repr(blckkey00.get("created"))}', str(outp))
+            self.isin(f'  Updated: {s_time.repr(blckkey00.get("updated"))}', str(outp))
+            self.notin('  Expires: ', str(outp))
+
+            self.isin(f'Iden: {blckkey01.get("iden")}', str(outp))
+            self.notin('  API Key: ', str(outp))
+            self.isin('  Name: blckkey01', str(outp))
+            self.isin(f'  Created: {s_time.repr(blckkey01.get("created"))}', str(outp))
+            self.isin(f'  Updated: {s_time.repr(blckkey01.get("updated"))}', str(outp))
+            self.notin('  Expires: ', str(outp))
+
+            argv = (
+                '--svcurl', blckurl,
+                'list',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_apikey.main(argv, outp=outp))
+
+            self.isin(f'Iden: {blckkey00.get("iden")}', str(outp))
+            self.notin('  API Key: ', str(outp))
+            self.isin('  Name: blckkey00', str(outp))
+            self.isin(f'  Created: {s_time.repr(blckkey00.get("created"))}', str(outp))
+            self.isin(f'  Updated: {s_time.repr(blckkey00.get("updated"))}', str(outp))
+            self.notin('  Expires: ', str(outp))
+
+            self.isin(f'Iden: {blckkey01.get("iden")}', str(outp))
+            self.notin('  API Key: ', str(outp))
+            self.isin('  Name: blckkey01', str(outp))
+            self.isin(f'  Created: {s_time.repr(blckkey01.get("created"))}', str(outp))
+            self.isin(f'  Updated: {s_time.repr(blckkey01.get("updated"))}', str(outp))
+            self.notin('  Expires: ', str(outp))
+
+            # Delete API keys
+            rootiden00 = rootkey00.get('iden')
+            argv = (
+                '--svcurl', rooturl,
+                'del',
+                rootiden00,
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_apikey.main(argv, outp=outp))
+            self.isin(f'Successfully deleted API key with iden={rootiden00}.', str(outp))
+
+            blckiden00 = blckkey00.get('iden')
+            argv = (
+                '--svcurl', rooturl,
+                'del',
+                blckiden00,
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_apikey.main(argv, outp=outp))
+            self.isin(f'Successfully deleted API key with iden={blckiden00}.', str(outp))
+
+            blckiden01 = blckkey01.get('iden')
+            argv = (
+                '--svcurl', blckurl,
+                'del',
+                blckiden01,
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_apikey.main(argv, outp=outp))
+            self.isin(f'Successfully deleted API key with iden={blckiden01}.', str(outp))
+
+            # List API keys again
+            argv = (
+                '--svcurl', rooturl,
+                'list',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_apikey.main(argv, outp=outp))
+            self.isin('No API keys found.', str(outp))
+
+            argv = (
+                '--svcurl', rooturl,
+                'list',
+                '-u', 'blackout',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_apikey.main(argv, outp=outp))
+            self.isin('No API keys found.', str(outp))
+
+            argv = (
+                '--svcurl', blckurl,
+                'list',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_apikey.main(argv, outp=outp))
+            self.isin('No API keys found.', str(outp))
+
+            # Check errors
+            argv = (
+                '--svcurl', rooturl,
+                'list',
+                '-u', 'newp',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(1, await s_t_apikey.main(argv, outp=outp))
+            self.isin('ERROR: NoSuchUser: No user named newp.', str(outp))
+
+            argv = (
+                '--svcurl', blckurl,
+                'list',
+                '-u', 'root',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(1, await s_t_apikey.main(argv, outp=outp))
+            self.isin('ERROR: AuthDeny: getUserInfo denied for non-admin and non-self', str(outp))
+
+            newpiden = s_common.guid()
+            argv = (
+                '--svcurl', rooturl,
+                'del',
+                newpiden,
+            )
+            outp = s_output.OutPutStr()
+            self.eq(1, await s_t_apikey.main(argv, outp=outp))
+            self.isin(f"ERROR: NoSuchIden: User API key with iden='{newpiden}' does not exist.", str(outp))

synapse/tests/test_utils.py

@@ -1,5 +1,6 @@
 import os
 import time
+import json
 import logging
 import unittest
 
@@ -109,13 +110,13 @@ class TestUtils(s_t_utils.SynTest):
     def test_syntest_logstream_event(self):
 
         @s_common.firethread
-        def logathing():
+        def logathing(mesg):
             time.sleep(0.01)
-            logger.error('StreamEvent Test Message')
+            logger.error(mesg)
 
         logger.error('notthere')
         with self.getLoggerStream('synapse.tests.test_utils', 'Test Message') as stream:
-            thr = logathing()
+            thr = logathing('StreamEvent Test Message')
             self.true(stream.wait(10))
             thr.join()
 
@@ -124,16 +125,34 @@ class TestUtils(s_t_utils.SynTest):
         self.isin('StreamEvent Test Message', mesgs)
         self.notin('notthere', mesgs)
 
+        with self.getLoggerStream('synapse.tests.test_utils', 'Test Message') as stream:
+            thr = logathing(json.dumps({'mesg': 'Test Message'}))
+            self.true(stream.wait(10))
+            thr.join()
+
+        msgs = stream.jsonlines()
+        self.len(1, msgs)
+        self.eq(msgs[0], {'mesg': 'Test Message'})
+
     def test_syntest_envars(self):
         os.environ['foo'] = '1'
         os.environ['bar'] = '2'
 
-        with self.setTstEnvars(foo=1, bar='joke', baz=1234) as cm:
+        with self.setTstEnvars(foo=1, bar='joke', baz=1234, FOO_THING=1, BAR_THING=0) as cm:
             self.none(cm)
             self.eq(os.environ.get('foo'), '1')
             self.eq(os.environ.get('bar'), 'joke')
             self.eq(os.environ.get('baz'), '1234')
 
+            self.thisEnvMust('FOO_THING', 'baz')
+            self.thisEnvMustNot('BAR_THING', 'NEWP_THING')
+            with self.raises(unittest.SkipTest):
+                self.thisEnvMust('MEWP_THING')
+            with self.raises(unittest.SkipTest):
+                self.thisEnvMust('BAR_THING')
+            with self.raises(unittest.SkipTest):
+                self.thisEnvMustNot('FOO_THING')
+
         self.eq(os.environ.get('foo'), '1')
         self.eq(os.environ.get('bar'), '2')
         self.none(os.environ.get('baz'))

synapse/tests/utils.py

@@ -21,9 +21,11 @@ import io
 import os
 import sys
 import copy
+import json
 import math
 import types
 import shutil
+import typing
 import asyncio
 import hashlib
 import inspect
@@ -97,6 +99,10 @@ def norm(z):
 def deguidify(x):
     return regex.sub('[0-9a-f]{32}', '*' * 32, x)
 
+def jsonlines(text: str):
+    lines = [k for k in text.split('\n') if k]
+    return [json.loads(line) for line in lines]
+
 async def waitForBehold(core, events):
     async for mesg in core.behold():
         for event in list(events):
@@ -789,6 +795,10 @@ class StreamEvent(io.StringIO, threading.Event):
         if self.mesg and self.mesg in s:
             self.set()
 
+    def jsonlines(self) -> typing.List[dict]:
+        '''Get the messages as jsonlines. May throw Json errors if the captured stream is not jsonlines.'''
+        return jsonlines(self.getvalue())
+
 class AsyncStreamEvent(io.StringIO, asyncio.Event):
     '''
     A combination of a io.StringIO object and an asyncio.Event object.
@@ -821,6 +831,10 @@ class AsyncStreamEvent(io.StringIO, asyncio.Event):
             return await asyncio.Event.wait(self)
         return await s_coro.event_wait(self, timeout=timeout)
 
+    def jsonlines(self) -> typing.List[dict]:
+        '''Get the messages as jsonlines. May throw Json errors if the captured stream is not jsonlines.'''
+        return jsonlines(self.getvalue())
+
 class HttpReflector(s_httpapi.Handler):
     '''Test handler which reflects get/post data back to the caller'''
 
@@ -1126,6 +1140,28 @@ class SynTest(unittest.TestCase):
         '''
         return TstOutPut()
 
+    def thisEnvMust(self, *envvars):
+        '''
+        Requires a host must have environment variables set to truthy values.
+
+        Args:
+            *envars: Environment variables to require being present.
+        '''
+        for envar in envvars:
+            if not s_common.envbool(envar):
+                self.skip(f'Envar {envar} is not set to a truthy value.')
+
+    def thisEnvMustNot(self, *envvars):
+        '''
+        Requires a host must not have environment variables set to truthy values.
+
+        Args:
+            *envars: Environment variables to require being absent or set to falsey values.
+        '''
+        for envar in envvars:
+            if s_common.envbool(envar):
+                self.skip(f'Envar {envar} is set to a truthy value.')
+
     def thisHostMust(self, **props):  # pragma: no cover
         '''
         Requires a host having a specific property.
@@ -1719,7 +1755,7 @@ class SynTest(unittest.TestCase):
             slogger.setLevel(level)
 
     @contextlib.contextmanager
-    def getAsyncLoggerStream(self, logname, mesg=''):
+    def getAsyncLoggerStream(self, logname, mesg='') -> contextlib.AbstractContextManager[StreamEvent, None, None]:
         '''
         Async version of getLoggerStream.
 
@@ -1764,7 +1800,7 @@ class SynTest(unittest.TestCase):
             slogger.setLevel(level)
 
     @contextlib.contextmanager
-    def getStructuredAsyncLoggerStream(self, logname, mesg=''):
+    def getStructuredAsyncLoggerStream(self, logname, mesg='') -> contextlib.AbstractContextManager[AsyncStreamEvent, None, None]:
         '''
         Async version of getLoggerStream which uses structured logging.
 
@@ -1787,9 +1823,7 @@ class SynTest(unittest.TestCase):
                     # Wait for the mesg to be written to the stream
                     await stream.wait(timeout=10)
 
-                data = stream.getvalue()
-                raw_mesgs = [m for m in data.split('\n') if m]
-                msgs = [json.loads(m) for m in raw_mesgs]
+                msgs = stream.jsonlines()
                 # Do something with messages
 
         Returns: