synapse 2.195.0__py311-none-any.whl → 2.196.0__py311-none-any.whl

synapse/tests/test_lib_aha.py

@@ -828,6 +828,33 @@ class AhaTest(s_test.SynTest):
                         async with await s_telepath.openurl(url) as prox:
                             self.fail(f'Connected to an expired clone URL {url}')  # pragma: no cover
 
+    async def test_lib_aha_mirrors(self):
+
+        async with self.getTestAha() as aha:
+            async with await s_base.Base.anit() as base:
+                with self.getTestDir() as dirn:
+                    user = 'synuser'
+                    dirn00 = s_common.genpath(dirn, 'cell00')
+                    dirn01 = s_common.genpath(dirn, 'cell01')
+
+                    core00 = await base.enter_context(self.addSvcToAha(aha, '00.cortex', s_cortex.Cortex, dirn=dirn00,
+                                                                       provinfo={'conf': {'aha:user': user}}))
+                    self.eq(core00.conf.get('aha:user'), user)
+
+                    core01 = await base.enter_context(self.addSvcToAha(aha, '01.cortex', s_cortex.Cortex, dirn=dirn01,
+                                                                       conf={'axon': 'aha://cortex...'},
+                                                                       provinfo={'conf': {'aha:user': user}}))
+                    self.eq(core01.conf.get('aha:user'), user)
+
+                    async with aha.getLocalProxy() as ahaproxy:
+                        self.eq(None, await ahaproxy.getAhaSvcMirrors('99.bogus'))
+                        self.len(1, await ahaproxy.getAhaSvcMirrors('00.cortex.synapse'))
+                        self.nn(await ahaproxy.getAhaServer(host=aha._getDnsName(), port=aha.sockaddr[1]))
+
+                        todo = s_common.todo('getCellInfo')
+                        res = await asyncio.wait_for(await aha.callAhaSvcApi('00.cortex.synapse', todo, timeout=3), 3)
+                        self.nn(res)
+
     async def test_aha_httpapi(self):
 
         async with self.getTestAha() as aha:
@@ -905,6 +932,10 @@ class AhaTest(s_test.SynTest):
                     info = await resp.json()
                     self.eq(info.get('status'), 'err')
                     self.eq(info.get('code'), 'SchemaViolation')
+                async with sess.post(url, json={'name': 'doom' * 16}) as resp:
+                    info = await resp.json()
+                    self.eq(info.get('status'), 'err')
+                    self.eq(info.get('code'), 'BadArg')
 
             # Not an admin
             await aha.addUser('lowuser', passwd='lowuser')
@@ -1093,6 +1124,35 @@ class AhaTest(s_test.SynTest):
                     self.stormHasNoWarnErr(msgs)
                     self.stormIsInPrint('Removed AHA service pool: pool00.synapse', msgs)
 
+    async def test_aha_svc_api_exception(self):
+
+        async with self.getTestAha() as aha:
+
+            async def mockGetAhaSvcProxy(*args, **kwargs):
+                raise s_exc.SynErr(mesg='proxy error')
+
+            aha.getAhaSvcProxy = mockGetAhaSvcProxy
+            name = '00.cortex.synapse'
+            todo = ('bogus', (), {})
+            retn = await asyncio.wait_for(await aha.callAhaSvcApi(name, todo), 3)
+
+            self.false(retn[0])
+            self.eq('SynErr', retn[1].get('err'))
+            self.eq('proxy error', retn[1].get('errinfo').get('mesg'))
+
+            bad_info = {
+                'urlinfo': {
+                    'host': 'nonexistent.host',
+                    'port': 12345,
+                    'scheme': 'ssl'
+                }
+            }
+
+            await aha.addAhaSvc(name, bad_info)
+            async for ok, info in aha.callAhaPeerGenr(name, ('nonexistent.method', (), {})):
+                self.false(ok)
+                self.isin('err', info)
+
     async def test_aha_reprovision(self):
 
         with self.withNexusReplay() as stack:
@@ -1317,3 +1377,184 @@ class AhaTest(s_test.SynTest):
                 async with await s_telepath.openurl(provurl) as prox:
                     info = await prox.getUserInfo()
                     self.eq(info.get('aha:user'), 'bob.grey')
+
+    async def test_aha_gather(self):
+
+        async with self.getTestAha() as aha:
+
+            async with aha.waiter(3, 'aha:svcadd', timeout=10):
+
+                conf = {'aha:provision': await aha.addAhaSvcProv('00.cell')}
+                cell00 = await aha.enter_context(self.getTestCell(conf=conf))
+
+                conf = {'aha:provision': await aha.addAhaSvcProv('01.cell', {'mirror': 'cell'})}
+                cell01 = await aha.enter_context(self.getTestCell(conf=conf))
+
+            await cell01.sync()
+
+            async with cell01.getLocalProxy() as proxy:
+                self.true(proxy._hasTeleFeat('dynmirror'))
+                self.true(proxy._hasTeleMeth('getNexsIndx'))
+
+            nexsindx = await cell00.getNexsIndx()
+
+            # test the call endpoint
+            todo = s_common.todo('getCellInfo')
+            items = dict([item async for item in aha.callAhaPeerApi(cell00.iden, todo, timeout=3)])
+            self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse'))
+            self.true(all(item[0] for item in items.values()))
+            self.eq(cell00.runid, items['00.cell.synapse'][1]['cell']['run'])
+            self.eq(cell01.runid, items['01.cell.synapse'][1]['cell']['run'])
+
+            todo = s_common.todo('newp')
+            items = dict([item async for item in aha.callAhaPeerApi(cell00.iden, todo, timeout=3)])
+            self.false(any(item[0] for item in items.values()))
+            self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse'))
+
+            # test the genr endpoint
+            reals = [item async for item in cell00.getNexusChanges(0, wait=False)]
+            todo = s_common.todo('getNexusChanges', 0, wait=False)
+            items = [item async for item in aha.callAhaPeerGenr(cell00.iden, todo, timeout=3) if item[1]]
+            self.len(nexsindx * 2, items)
+
+            # ensure we handle down services correctly
+            async with aha.waiter(1, 'aha:svcdown', timeout=10):
+                await cell01.fini()
+
+            # test the call endpoint
+            todo = s_common.todo('getCellInfo')
+            items = dict([item async for item in aha.callAhaPeerApi(cell00.iden, todo, timeout=3)])
+            self.sorteq(items.keys(), ('00.cell.synapse',))
+            self.true(all(item[0] for item in items.values()))
+            self.eq(cell00.runid, items['00.cell.synapse'][1]['cell']['run'])
+
+            # test the genr endpoint
+            todo = s_common.todo('getNexusChanges', 0, wait=False)
+            items = [item async for item in aha.callAhaPeerGenr(cell00.iden, todo, timeout=3) if item[1]]
+            self.len(nexsindx, items)
+            self.true(all(item[1][0] for item in items))
+
+            # test some of the gather API implementations...
+            purl00 = await aha.addAhaSvcProv('0.cell')
+            purl01 = await aha.addAhaSvcProv('1.cell', provinfo={'mirror': '0.cell'})
+
+            cell00 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl00}))
+            cell01 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl01}))
+
+            await cell01.sync()
+
+            async def sleep99(cell):
+                await cell.boss.promote('sleep99', cell.auth.rootuser)
+                await cell00.fire('sleep99')
+                await asyncio.sleep(99)
+
+            async with cell00.waiter(2, 'sleep99', timeout=2):
+                task00 = cell00.schedCoro(sleep99(cell00))
+                task01 = cell01.schedCoro(sleep99(cell01))
+
+            proxy = await aha.enter_context(aha.getLocalProxy())
+            tasks = [task async for task in cell00.getTasks(timeout=3)]
+            self.len(2, tasks)
+            self.eq(tasks[0]['service'], '0.cell.synapse')
+            self.eq(tasks[1]['service'], '1.cell.synapse')
+
+            self.eq(tasks[0], await cell00.getTask(tasks[0].get('iden')))
+            self.eq(tasks[1], await cell00.getTask(tasks[1].get('iden')))
+            self.none(await cell00.getTask(tasks[1].get('iden'), peers=False))
+
+            self.true(await cell00.killTask(tasks[0].get('iden')))
+
+            task01 = tasks[1].get('iden')
+            self.false(await cell00.killTask(task01, peers=False))
+
+            self.true(await cell00.killTask(task01))
+
+            self.none(await cell00.getTask(task01))
+            self.false(await cell00.killTask(task01))
+
+            self.none(await cell00.getAhaProxy(feats=(('newp', 9),)))
+
+    async def test_lib_aha_peer_api(self):
+
+        async with self.getTestAha() as aha:
+
+            purl00 = await aha.addAhaSvcProv('0.cell')
+            purl01 = await aha.addAhaSvcProv('1.cell', provinfo={'mirror': '0.cell'})
+            purl02 = await aha.addAhaSvcProv('2.cell', provinfo={'mirror': '0.cell'})
+
+            cell00 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl00}))
+            cell01 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl01}))
+            cell02 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl02}))
+
+            await cell01.sync()
+            await cell02.sync()
+
+            todo = s_common.todo('getCellInfo')
+            items = [item async for item in cell00.callPeerApi(todo)]
+            self.len(2, items)
+
+    async def test_lib_aha_peer_genr(self):
+
+        async with self.getTestAha() as aha:
+
+            purl00 = await aha.addAhaSvcProv('0.cell')
+            purl01 = await aha.addAhaSvcProv('1.cell', provinfo={'mirror': '0.cell'})
+
+            cell00 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl00}))
+            cell01 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl01}))
+
+            await cell01.sync()
+
+            todo = s_common.todo('getNexusChanges', 0, wait=False)
+            items = dict([item async for item in cell00.callPeerGenr(todo)])
+            self.len(1, items)
+
+            todo = s_common.todo('getNexusChanges', 0, wait=False)
+            items = dict([item async for item in cell00.callPeerGenr(todo, timeout=2)])
+            self.len(1, items)
+
+    async def test_lib_aha_call_aha_peer_api_isactive(self):
+
+        async with self.getTestAha() as aha0:
+
+            async with aha0.waiter(3, 'aha:svcadd', timeout=10):
+
+                conf = {'aha:provision': await aha0.addAhaSvcProv('00.cell')}
+                cell00 = await aha0.enter_context(self.getTestCell(conf=conf))
+
+                conf = {'aha:provision': await aha0.addAhaSvcProv('01.cell', {'mirror': 'cell'})}
+                cell01 = await aha0.enter_context(self.getTestCell(conf=conf))
+
+            await cell01.sync()
+
+            # test active AHA peer
+            todo = s_common.todo('getCellInfo')
+            items = dict([item async for item in aha0.callAhaPeerApi(cell00.iden, todo, timeout=3)])
+            self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse'))
+
+            todo = s_common.todo('getNexusChanges', 0, wait=False)
+            items = dict([item async for item in aha0.callAhaPeerGenr(cell00.iden, todo, timeout=3)])
+            self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse'))
+
+            async with aha0.getLocalProxy() as proxy0:
+                purl = await proxy0.addAhaClone('01.aha.loop.vertex.link')
+
+            conf1 = {'clone': purl}
+            async with self.getTestAha(conf=conf1) as aha1:
+
+                await aha1.sync()
+
+                self.eq(aha0.iden, aha1.iden)
+                self.nn(aha1.conf.get('mirror'))
+
+                self.true(aha0.isactive)
+                self.false(aha1.isactive)
+
+                # test non-active AHA peer
+                todo = s_common.todo('getCellInfo')
+                items = dict([item async for item in aha1.callAhaPeerApi(cell00.iden, todo, timeout=3)])
+                self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse'))
+
+                todo = s_common.todo('getNexusChanges', 0, wait=False)
+                items = dict([item async for item in aha1.callAhaPeerGenr(cell00.iden, todo, timeout=3)])
+                self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse'))

synapse/tests/test_lib_cell.py

@@ -784,6 +784,10 @@ class CellTest(s_t_utils.SynTest):
             cell.VERSION = (1, 2, 3)
             cell.VERSTRING = '1.2.3'
 
+            cell.features.update({
+                'testvalu': 2
+            })
+
             http_info = []
             host, port = await cell.addHttpsPort(0)
             http_info.append({'host': host, 'port': port})
@@ -807,6 +811,9 @@ class CellTest(s_t_utils.SynTest):
                 # A Cortex populated cellvers
                 self.isin('cortex:defaults', cnfo.get('cellvers', {}))
 
+                self.eq(info.get('features'), cell.features)
+                self.eq(info.get('features', {}).get('testvalu'), 2)
+
                 # Defaults aha data is
                 self.eq(cnfo.get('aha'), {'name': None, 'leader': None, 'network': None})
 
@@ -3294,6 +3301,51 @@ class CellTest(s_t_utils.SynTest):
                         await cell00.promote(graceful=True)
                     self.isin('02.cell is not the current leader', cm.exception.get('mesg'))
 
+    async def test_cell_get_aha_proxy(self):
+
+        async with self.getTestCell() as cell:
+
+            self.none(await cell.getAhaProxy())
+
+            class MockAhaClient:
+                def __init__(self, proxy=None):
+                    self._proxy = proxy
+
+                async def proxy(self, timeout=None):
+                    return self._proxy
+
+            with self.getAsyncLoggerStream('synapse.lib.cell', 'AHA client connection failed.') as stream:
+                cell.ahaclient = MockAhaClient()
+                self.none(await cell.getAhaProxy())
+                self.true(await stream.wait(timeout=1))
+
+            class MockProxyHasNot:
+                def _hasTeleFeat(self, name, vers):
+                    return False
+
+            cell.ahaclient = MockAhaClient(proxy=MockProxyHasNot())
+            self.none(await cell.getAhaProxy(feats=(('test', 1),)))
+
+            class MockProxyHas:
+                def _hasTeleFeat(self, name, vers):
+                    return True
+
+            mock_proxy = MockProxyHas()
+            cell.ahaclient = MockAhaClient(proxy=mock_proxy)
+            self.eq(await cell.getAhaProxy(), mock_proxy)
+            self.eq(await cell.getAhaProxy(feats=(('test', 1),)), mock_proxy)
+
+    async def test_lib_cell_sadaha(self):
+
+        async with self.getTestCell() as cell:
+
+            self.none(await cell.getAhaProxy())
+            cell.ahaclient = await s_telepath.Client.anit('cell:///tmp/newp')
+
+            # coverage for failure of aha client to connect
+            with self.raises(TimeoutError):
+                self.none(await cell.getAhaProxy(timeout=0.1))
+
     async def test_stream_backup_exception(self):
 
         with self.getTestDir() as dirn:
@@ -3356,3 +3408,12 @@ class CellTest(s_t_utils.SynTest):
                     with self.raises(s_exc.BackupAlreadyRunning):
                         async for _ in proxy.iterNewBackupArchive('newbackup', remove=True):
                             pass
+
+    async def test_cell_peer_noaha(self):
+
+        todo = s_common.todo('newp')
+        async with self.getTestCell() as cell:
+            async for item in cell.callPeerApi(todo):
+                pass
+            async for item in cell.callPeerGenr(todo):
+                pass

synapse/tests/test_lib_jsonstor.py

@@ -116,6 +116,7 @@ class JsonStorTest(s_test.SynTest):
                     self.eq({'hehe': 'haha', 'zip': {}}, await prox.getPathObj('foo/bar'))
 
                     self.false(await prox.delPathObjProp('newp/newp', 'newp'))
+                    self.false(await prox.delPathObjProp('foo/bar', 'nested/newp'))
 
                     await prox.delPathObj('foo/bar')
                     self.none(await prox.getPathObj('foo/bar'))

synapse/tests/test_lib_modelrev.py

@@ -1766,3 +1766,9 @@ class ModelRevTest(s_tests.SynTest):
             self.eq('foo bar', nodes[0].get('model'))
             nodes = await core.nodes('transport:sea:vessel')
             self.eq('foo bar', nodes[0].get('model'))
+
+    async def test_modelrev_0_2_33(self):
+        async with self.getRegrCore('model-0.2.33') as core:
+            nodes = await core.nodes('entity:name')
+            self.len(1, nodes)
+            self.eq('foo bar', nodes[0].repr())

synapse/tests/test_lib_nexus.py

@@ -318,7 +318,7 @@ class NexusTest(s_t_utils.SynTest):
                     cell01.nexsiden = 'newp'
                     with self.raises(s_exc.NoSuchIden) as cm:
                         await cell01.sync()
-                    self.eq(cm.exception.get('mesg'), 'No Nexus Pusher with iden newp.')
+                    self.eq(cm.exception.get('mesg'), "No Nexus Pusher with iden newp event='sync' args=() kwargs={}")
 
                     self.none(await cell00.nexsroot.nexslog.last())
                     self.none(await cell01.nexsroot.nexslog.last())
@@ -331,7 +331,7 @@ class NexusTest(s_t_utils.SynTest):
 
                     with self.raises(s_exc.NoSuchName) as cm:
                         await cell01._push('newp')
-                    self.eq(cm.exception.get('mesg'), 'No Nexus handler for event newp.')
+                    self.eq(cm.exception.get('mesg'), 'No event handler for event newp args=() kwargs={}')
 
                     self.eq(0, (await cell00.nexsroot.nexslog.last())[0])
                     self.eq(0, (await cell01.nexsroot.nexslog.last())[0])
@@ -379,3 +379,25 @@ class NexusTest(s_t_utils.SynTest):
                     self.isin(mesg, data)
 
                 self.true(restarted)
+
+    async def test_pusher_race(self):
+
+        evnt = asyncio.Event()
+
+        async with self.getTestCore() as core:
+            orig = core._nexshands['view:delwithlayer'][0]
+
+            async def holdlock(self, viewiden, layriden, nexsitem, newparent=None):
+                evnt.set()
+                await asyncio.sleep(1)
+                await orig(self, viewiden, layriden, nexsitem, newparent=newparent)
+
+            core._nexshands['view:delwithlayer'] = (holdlock, True)
+
+            forkiden = await core.callStorm('return($lib.view.get().fork().iden)')
+
+            core.schedCoro(core.delViewWithLayer(forkiden))
+            await asyncio.wait_for(evnt.wait(), timeout=10)
+
+            with self.raises(s_exc.NoSuchIden):
+                await core.nodes('[ it:dev:str=foo ]', opts={'view': forkiden})

synapse/tests/test_lib_stormlib_aha.py

@@ -6,6 +6,8 @@ import synapse.lib.cell as s_cell
 
 import synapse.tests.utils as s_test
 
+import unittest.mock as mock
+
 class AhaLibTest(s_test.SynTest):
 
     async def test_stormlib_aha_basics(self):
@@ -193,3 +195,189 @@ Connection information:
 
                 with self.raises(s_exc.NoSuchName):
                     await core00.callStorm('$lib.aha.del(axon...)')
+
+    async def test_stormlib_aha_mirror(self):
+
+        async with self.getTestAha() as aha:
+
+            with self.getTestDir() as dirn:
+
+                dirn00 = s_common.genpath(dirn, 'cell00')
+                dirn01 = s_common.genpath(dirn, 'cell01')
+                dirn02 = s_common.genpath(dirn, 'cell02')
+
+                async with aha.waiter(3, 'aha:svcadd', timeout=10):
+
+                    cell00 = await aha.enter_context(self.addSvcToAha(aha, '00.cell', s_cell.Cell, dirn=dirn00))
+                    cell01 = await aha.enter_context(self.addSvcToAha(aha, '01.cell', s_cell.Cell, dirn=dirn01,
+                                                                      provinfo={'mirror': 'cell'}))
+                    core00 = await aha.enter_context(self.addSvcToAha(aha, 'core', s_cortex.Cortex, dirn=dirn02))
+                    await cell01.sync()
+
+                # PeerGenr
+                resp = await core00.callStorm('''
+                    $resps = $lib.list()
+                    $todo = $lib.utils.todo('getTasks')
+                    for ($name, $info) in $lib.aha.callPeerGenr(cell..., $todo) {
+                        $resps.append(($name, $info))
+                    }
+                    return($resps)
+                ''')
+                self.len(0, resp)
+
+                resp = await core00.callStorm('''
+                    $resps = $lib.list()
+                    $todo = $lib.utils.todo('getNexusChanges', (0), wait=(false))
+                    for ($name, $info) in $lib.aha.callPeerGenr(cell..., $todo) {
+                        $resps.append(($name, $info))
+                    }
+                    return($resps)
+                ''')
+                self.len(4, resp)
+
+                cell00_rid = (await cell00.getCellInfo())['cell']['run']
+                resp = await core00.callStorm('''
+                    $resps = $lib.list()
+                    $todo = $lib.utils.todo('getNexusChanges', (0), wait=(false))
+                    for $info in $lib.aha.callPeerGenr(cell..., $todo, skiprun=$skiprun) {
+                        $resps.append($info)
+                    }
+                    return($resps)
+                ''', opts={'vars': {'skiprun': cell00_rid}})
+                self.len(2, resp)
+
+                await self.asyncraises(s_exc.NoSuchName, core00.callStorm('''
+                    $todo = $lib.utils.todo('getTasks')
+                    $lib.aha.callPeerGenr(null, $todo)
+                '''))
+
+                # PeerApi
+                resp = await core00.callStorm('''
+                    $resps = $lib.list()
+                    $todo = $lib.utils.todo('getCellInfo')
+                    for ($name, $info) in $lib.aha.callPeerApi(cell..., $todo) {
+                        $resps.append(($name, $info))
+                    }
+                    return($resps)
+                ''')
+                self.len(2, resp)
+                self.eq(resp[0][0], '00.cell.synapse')
+                self.eq(resp[0][1][0], True)
+                self.isinstance(resp[0][1][1], dict)
+
+                resp = await core00.callStorm('''
+                    $resps = $lib.list()
+                    $todo = $lib.utils.todo(getCellInfo)
+                    for ($name, $info) in $lib.aha.callPeerApi(cell..., $todo) {
+                        $resps.append(($name, $info))
+                    }
+                    return($resps)
+                ''')
+                self.len(2, resp)
+
+                resp = await core00.callStorm('''
+                    $resps = $lib.list()
+                    $todo = $lib.utils.todo(getCellInfo)
+                    for ($name, $info) in $lib.aha.callPeerApi(cell..., $todo, skiprun=$skiprun) {
+                        $resps.append(($name, $info))
+                    }
+                    return($resps)
+                ''', opts={'vars': {'skiprun': cell00_rid}})
+                self.len(1, resp)
+
+                resp = await core00.callStorm('''
+                    $resps = $lib.list()
+                    $todo = $lib.utils.todo('getCellInfo')
+                    for ($name, $info) in $lib.aha.callPeerApi(cell..., $todo, timeout=(10)) {
+                        $resps.append(($name, $info))
+                    }
+                    return($resps)
+                ''')
+                self.len(2, resp)
+
+                await self.asyncraises(s_exc.NoSuchName, core00.callStorm('''
+                    $todo = $lib.utils.todo('getCellInfo')
+                    $lib.aha.callPeerApi(newp..., $todo)
+                '''))
+
+                await self.asyncraises(s_exc.NoSuchName, core00.callStorm('''
+                    $todo = $lib.utils.todo('getCellInfo')
+                    $lib.aha.callPeerApi(null, $todo)
+                '''))
+
+                await self.asyncraises(s_exc.NoSuchMeth, core00.callStorm('''
+                    $todo = $lib.utils.todo('bogusMethod')
+                    for $info in $lib.aha.callPeerApi(cell..., $todo) {
+                        ($ok, $info) = $info.1
+                        if (not $ok) {
+                            $lib.raise($info.err, $info.errmsg)
+                        }
+                    }
+                '''))
+
+                await aha.addAhaSvc('noiden.cell', info={'urlinfo': {'scheme': 'tcp',
+                                                                     'host': '0.0.0.0',
+                                                                     'port': '3030'}},
+                                  network='synapse')
+                await self.asyncraises(s_exc.NoSuchName, core00.callStorm('''
+                    $todo = $lib.utils.todo('getTasks')
+                    $lib.aha.callPeerGenr(noiden.cell..., $todo)
+                '''))
+                await self.asyncraises(s_exc.NoSuchName, core00.callStorm('''
+                    $todo = $lib.utils.todo('getCellInfo')
+                    $lib.aha.callPeerApi(noiden.cell..., $todo)
+                '''))
+
+                msgs = await core00.stormlist('aha.svc.mirror')
+                self.stormIsInPrint('Service Mirror Groups:', msgs)
+                self.stormIsInPrint('cell.synapse', msgs)
+                self.stormIsInPrint('00.cell.synapse', msgs)
+                self.stormIsInPrint('01.cell.synapse', msgs)
+                self.stormIsInPrint('Group Status: In Sync', msgs)
+
+                msgs = await core00.stormlist('aha.svc.mirror --timeout 30')
+                self.stormIsInPrint('Service Mirror Groups:', msgs)
+                self.stormIsInPrint('Group Status: In Sync', msgs)
+
+                async def mockCellInfo():
+                    return {
+                        'cell': {'ready': True, 'nexsindx': 10, 'uplink': None},
+                        'synapse': {'verstring': '2.190.0'},
+                    }
+
+                async def mockOutOfSyncCellInfo():
+                    return {
+                        'cell': {'ready': True, 'nexsindx': 5, 'uplink': cell00.iden},
+                        'synapse': {'verstring': '2.190.0'},
+                    }
+
+                with mock.patch.object(cell00, 'getCellInfo', mockCellInfo):
+                    with mock.patch.object(cell01, 'getCellInfo', mockOutOfSyncCellInfo):
+                        async def mock_call_aha(*args, **kwargs):
+                            todo = args[1]
+                            if todo[0] == 'waitNexsOffs':
+                                yield ('00.cell.synapse', (True, True))
+                                yield ('01.cell.synapse', (True, True))
+                            elif todo[0] == 'getCellInfo':
+                                if not hasattr(mock_call_aha, 'called'):
+                                    mock_call_aha.called = True
+                                    yield ('00.cell.synapse', (True, await mockCellInfo()))
+                                    yield ('01.cell.synapse', (True, await mockOutOfSyncCellInfo()))
+                                else:
+                                    yield ('00.cell.synapse', (True, await mockCellInfo()))
+                                    yield ('01.cell.synapse', (True, await mockCellInfo()))
+
+                        with mock.patch.object(aha, 'callAhaPeerApi', mock_call_aha):
+                            msgs = await core00.stormlist('aha.svc.mirror --wait')
+                            self.stormIsInPrint('Group Status: Out of Sync', msgs)
+                            self.stormIsInPrint('Updated status:', msgs)
+                            self.stormIsInPrint('Group Status: In Sync', msgs)
+
+                with mock.patch.object(cell00, 'getCellInfo', mockCellInfo):
+                    with mock.patch.object(cell01, 'getCellInfo', mockOutOfSyncCellInfo):
+                        msgs = await core00.stormlist('aha.svc.mirror --timeout 1')
+                        self.stormIsInPrint('Group Status: Out of Sync', msgs)
+
+                await aha.delAhaSvc('00.cell', network='synapse')
+                msgs = await core00.stormlist('aha.svc.mirror')
+                self.stormNotInPrint('Service Mirror Groups:', msgs)

synapse/tests/test_lib_stormlib_utils.py

@@ -0,0 +1,14 @@
+import synapse.exc as s_exc
+import synapse.tests.utils as s_test
+
+class UtilsTest(s_test.SynTest):
+
+    async def test_lib_stormlib_utils_todo(self):
+
+        async with self.getTestCore() as core:
+
+            valu = await core.callStorm('return($lib.utils.todo(foo))')
+            self.eq(valu, ('foo', (), {}))
+
+            valu = await core.callStorm('return($lib.utils.todo(fooName, arg1, arg2, keyword=bar, anotherkeyword=hehe))')
+            self.eq(valu, ('fooName', ('arg1', 'arg2'), {'keyword': 'bar', 'anotherkeyword': 'hehe'}))