synapse 2.195.1__py311-none-any.whl → 2.197.0__py311-none-any.whl

synapse/tests/test_axon.py

@@ -4,6 +4,7 @@ import csv
 import sys
 import base64
 import shutil
+import struct
 import asyncio
 import hashlib
 import logging
@@ -489,6 +490,28 @@ bar baz",vv
             with self.raises(s_exc.BadArg):
                 bytslist = [b async for b in axon.get(sha256, 0, size=0)]
 
+        # test unpack
+        intdata = struct.pack('>QQQ', 1, 2, 3)
+        size, sha256 = await axon.put(intdata)
+        self.eq((1,), await axon.unpack(sha256, '>Q'))
+        self.eq((2,), await axon.unpack(sha256, '>Q', offs=8))
+        self.eq((3,), await axon.unpack(sha256, '>Q', offs=16))
+        self.eq((2, 3), await axon.unpack(sha256, '>QQ', offs=8))
+
+        fmt = 'Q' * 150_000
+        with self.raises(s_exc.BadArg) as cm:
+            await axon.unpack(sha256, '>' + fmt)
+        self.isin('Struct format would read too much data', cm.exception.get('mesg'))
+
+        with self.raises(s_exc.BadArg):
+            await axon.unpack(sha256, 'not a valid format')
+
+        with self.raises(s_exc.BadArg):
+            await axon.unpack(sha256, 123)
+
+        with self.raises(s_exc.BadDataValu):
+            await axon.unpack(sha256, '>Q', offs=24)
+
     async def test_axon_base(self):
         async with self.getTestAxon() as axon:
             self.isin('axon', axon.dmon.shared)

synapse/tests/test_common.py

@@ -12,6 +12,34 @@ import synapse.tests.utils as s_t_utils
 logger = logging.getLogger(__name__)
 
 class CommonTest(s_t_utils.SynTest):
+
+    async def test_waitgenr(self):
+
+        async def genr():
+            yield 10
+            raise Exception('omg')
+
+        rets = [retn async for retn in s_common.waitgenr(genr(), 10)]
+
+        self.true(rets[0][0])
+        self.false(rets[1][0])
+
+        async def one():
+            yield 'item'
+
+        rets = [retn async for retn in s_common.waitgenr(one(), timeout=1.0)]
+        self.eq(rets, [(True, 'item')])
+
+        async def genr():
+            yield 1
+            await asyncio.sleep(60)
+            yield 2
+
+        rets = [retn async for retn in s_common.waitgenr(genr(), timeout=0.1)]
+        self.eq(rets[0], (True, 1))
+        self.false(rets[1][0])
+        self.eq(rets[1][1]['err'], 'TimeoutError')
+
     def test_tuplify(self):
         tv = ['node', [['test:str', 'test'],
                        {'tags': {

synapse/tests/test_datamodel.py

@@ -2,6 +2,7 @@ import synapse.exc as s_exc
 import synapse.datamodel as s_datamodel
 
 import synapse.lib.module as s_module
+import synapse.lib.schemas as s_schemas
 
 import synapse.cortex as s_cortex
 
@@ -334,3 +335,17 @@ class DataModelTest(s_t_utils.SynTest):
             self.none(core.model.edge(('meta:rule', 'matches', None)))
 
             core.model.delEdge(('meta:rule', 'matches', None))
+
+    async def test_datamodel_locked_subs(self):
+
+        async with self.getTestCore() as core:
+            await core.setDeprLock('it:prod:softver:semver:major', True)
+            nodes = await core.nodes('[ it:prod:softver=* :semver=3.1.0 ]')
+            self.none(nodes[0].get('semver:major'))
+            self.eq(1, nodes[0].get('semver:minor'))
+
+    def test_datamodel_schema_basetypes(self):
+        # N.B. This test is to keep synapse.lib.schemas.datamodel_basetypes const
+        # in sync with the default s_datamodel.Datamodel().types
+        basetypes = list(s_datamodel.Model().types)
+        self.eq(s_schemas.datamodel_basetypes, basetypes)

synapse/tests/test_lib_aha.py

@@ -828,6 +828,33 @@ class AhaTest(s_test.SynTest):
                         async with await s_telepath.openurl(url) as prox:
                             self.fail(f'Connected to an expired clone URL {url}')  # pragma: no cover
 
+    async def test_lib_aha_mirrors(self):
+
+        async with self.getTestAha() as aha:
+            async with await s_base.Base.anit() as base:
+                with self.getTestDir() as dirn:
+                    user = 'synuser'
+                    dirn00 = s_common.genpath(dirn, 'cell00')
+                    dirn01 = s_common.genpath(dirn, 'cell01')
+
+                    core00 = await base.enter_context(self.addSvcToAha(aha, '00.cortex', s_cortex.Cortex, dirn=dirn00,
+                                                                       provinfo={'conf': {'aha:user': user}}))
+                    self.eq(core00.conf.get('aha:user'), user)
+
+                    core01 = await base.enter_context(self.addSvcToAha(aha, '01.cortex', s_cortex.Cortex, dirn=dirn01,
+                                                                       conf={'axon': 'aha://cortex...'},
+                                                                       provinfo={'conf': {'aha:user': user}}))
+                    self.eq(core01.conf.get('aha:user'), user)
+
+                    async with aha.getLocalProxy() as ahaproxy:
+                        self.eq(None, await ahaproxy.getAhaSvcMirrors('99.bogus'))
+                        self.len(1, await ahaproxy.getAhaSvcMirrors('00.cortex.synapse'))
+                        self.nn(await ahaproxy.getAhaServer(host=aha._getDnsName(), port=aha.sockaddr[1]))
+
+                        todo = s_common.todo('getCellInfo')
+                        res = await asyncio.wait_for(await aha.callAhaSvcApi('00.cortex.synapse', todo, timeout=3), 3)
+                        self.nn(res)
+
     async def test_aha_httpapi(self):
 
         async with self.getTestAha() as aha:
@@ -905,6 +932,10 @@ class AhaTest(s_test.SynTest):
                     info = await resp.json()
                     self.eq(info.get('status'), 'err')
                     self.eq(info.get('code'), 'SchemaViolation')
+                async with sess.post(url, json={'name': 'doom' * 16}) as resp:
+                    info = await resp.json()
+                    self.eq(info.get('status'), 'err')
+                    self.eq(info.get('code'), 'BadArg')
 
             # Not an admin
             await aha.addUser('lowuser', passwd='lowuser')
@@ -1093,6 +1124,35 @@ class AhaTest(s_test.SynTest):
                     self.stormHasNoWarnErr(msgs)
                     self.stormIsInPrint('Removed AHA service pool: pool00.synapse', msgs)
 
+    async def test_aha_svc_api_exception(self):
+
+        async with self.getTestAha() as aha:
+
+            async def mockGetAhaSvcProxy(*args, **kwargs):
+                raise s_exc.SynErr(mesg='proxy error')
+
+            aha.getAhaSvcProxy = mockGetAhaSvcProxy
+            name = '00.cortex.synapse'
+            todo = ('bogus', (), {})
+            retn = await asyncio.wait_for(await aha.callAhaSvcApi(name, todo), 3)
+
+            self.false(retn[0])
+            self.eq('SynErr', retn[1].get('err'))
+            self.eq('proxy error', retn[1].get('errinfo').get('mesg'))
+
+            bad_info = {
+                'urlinfo': {
+                    'host': 'nonexistent.host',
+                    'port': 12345,
+                    'scheme': 'ssl'
+                }
+            }
+
+            await aha.addAhaSvc(name, bad_info)
+            async for ok, info in aha.callAhaPeerGenr(name, ('nonexistent.method', (), {})):
+                self.false(ok)
+                self.isin('err', info)
+
     async def test_aha_reprovision(self):
 
         with self.withNexusReplay() as stack:
@@ -1317,3 +1377,144 @@ class AhaTest(s_test.SynTest):
                 async with await s_telepath.openurl(provurl) as prox:
                     info = await prox.getUserInfo()
                     self.eq(info.get('aha:user'), 'bob.grey')
+
+    async def test_aha_gather(self):
+
+        async with self.getTestAha() as aha:
+
+            async with aha.waiter(3, 'aha:svcadd', timeout=10):
+
+                conf = {'aha:provision': await aha.addAhaSvcProv('00.cell')}
+                cell00 = await aha.enter_context(self.getTestCell(conf=conf))
+
+                conf = {'aha:provision': await aha.addAhaSvcProv('01.cell', {'mirror': 'cell'})}
+                cell01 = await aha.enter_context(self.getTestCell(conf=conf))
+
+            await cell01.sync()
+
+            async with cell01.getLocalProxy() as proxy:
+                self.true(proxy._hasTeleFeat('dynmirror'))
+                self.true(proxy._hasTeleMeth('getNexsIndx'))
+
+            nexsindx = await cell00.getNexsIndx()
+
+            # test the call endpoint
+            todo = s_common.todo('getCellInfo')
+            items = dict([item async for item in aha.callAhaPeerApi(cell00.iden, todo, timeout=3)])
+            self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse'))
+            self.true(all(item[0] for item in items.values()))
+            self.eq(cell00.runid, items['00.cell.synapse'][1]['cell']['run'])
+            self.eq(cell01.runid, items['01.cell.synapse'][1]['cell']['run'])
+
+            todo = s_common.todo('newp')
+            items = dict([item async for item in aha.callAhaPeerApi(cell00.iden, todo, timeout=3)])
+            self.false(any(item[0] for item in items.values()))
+            self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse'))
+
+            # test the genr endpoint
+            reals = [item async for item in cell00.getNexusChanges(0, wait=False)]
+            todo = s_common.todo('getNexusChanges', 0, wait=False)
+            items = [item async for item in aha.callAhaPeerGenr(cell00.iden, todo, timeout=3) if item[1]]
+            self.len(nexsindx * 2, items)
+
+            # ensure we handle down services correctly
+            async with aha.waiter(1, 'aha:svcdown', timeout=10):
+                await cell01.fini()
+
+            # test the call endpoint
+            todo = s_common.todo('getCellInfo')
+            items = dict([item async for item in aha.callAhaPeerApi(cell00.iden, todo, timeout=3)])
+            self.sorteq(items.keys(), ('00.cell.synapse',))
+            self.true(all(item[0] for item in items.values()))
+            self.eq(cell00.runid, items['00.cell.synapse'][1]['cell']['run'])
+
+            # test the genr endpoint
+            todo = s_common.todo('getNexusChanges', 0, wait=False)
+            items = [item async for item in aha.callAhaPeerGenr(cell00.iden, todo, timeout=3) if item[1]]
+            self.len(nexsindx, items)
+            self.true(all(item[1][0] for item in items))
+
+    async def test_lib_aha_peer_api(self):
+
+        async with self.getTestAha() as aha:
+
+            purl00 = await aha.addAhaSvcProv('0.cell')
+            purl01 = await aha.addAhaSvcProv('1.cell', provinfo={'mirror': '0.cell'})
+            purl02 = await aha.addAhaSvcProv('2.cell', provinfo={'mirror': '0.cell'})
+
+            cell00 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl00}))
+            cell01 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl01}))
+            cell02 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl02}))
+
+            await cell01.sync()
+            await cell02.sync()
+
+            todo = s_common.todo('getCellInfo')
+            items = [item async for item in cell00.callPeerApi(todo)]
+            self.len(2, items)
+
+    async def test_lib_aha_peer_genr(self):
+
+        async with self.getTestAha() as aha:
+
+            purl00 = await aha.addAhaSvcProv('0.cell')
+            purl01 = await aha.addAhaSvcProv('1.cell', provinfo={'mirror': '0.cell'})
+
+            cell00 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl00}))
+            cell01 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl01}))
+
+            await cell01.sync()
+
+            todo = s_common.todo('getNexusChanges', 0, wait=False)
+            items = dict([item async for item in cell00.callPeerGenr(todo)])
+            self.len(1, items)
+
+            todo = s_common.todo('getNexusChanges', 0, wait=False)
+            items = dict([item async for item in cell00.callPeerGenr(todo, timeout=2)])
+            self.len(1, items)
+
+    async def test_lib_aha_call_aha_peer_api_isactive(self):
+
+        async with self.getTestAha() as aha0:
+
+            async with aha0.waiter(3, 'aha:svcadd', timeout=10):
+
+                conf = {'aha:provision': await aha0.addAhaSvcProv('00.cell')}
+                cell00 = await aha0.enter_context(self.getTestCell(conf=conf))
+
+                conf = {'aha:provision': await aha0.addAhaSvcProv('01.cell', {'mirror': 'cell'})}
+                cell01 = await aha0.enter_context(self.getTestCell(conf=conf))
+
+            await cell01.sync()
+
+            # test active AHA peer
+            todo = s_common.todo('getCellInfo')
+            items = dict([item async for item in aha0.callAhaPeerApi(cell00.iden, todo, timeout=3)])
+            self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse'))
+
+            todo = s_common.todo('getNexusChanges', 0, wait=False)
+            items = dict([item async for item in aha0.callAhaPeerGenr(cell00.iden, todo, timeout=3)])
+            self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse'))
+
+            async with aha0.getLocalProxy() as proxy0:
+                purl = await proxy0.addAhaClone('01.aha.loop.vertex.link')
+
+            conf1 = {'clone': purl}
+            async with self.getTestAha(conf=conf1) as aha1:
+
+                await aha1.sync()
+
+                self.eq(aha0.iden, aha1.iden)
+                self.nn(aha1.conf.get('mirror'))
+
+                self.true(aha0.isactive)
+                self.false(aha1.isactive)
+
+                # test non-active AHA peer
+                todo = s_common.todo('getCellInfo')
+                items = dict([item async for item in aha1.callAhaPeerApi(cell00.iden, todo, timeout=3)])
+                self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse'))
+
+                todo = s_common.todo('getNexusChanges', 0, wait=False)
+                items = dict([item async for item in aha1.callAhaPeerGenr(cell00.iden, todo, timeout=3)])
+                self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse'))

synapse/tests/test_lib_boss.py

@@ -3,13 +3,21 @@ import asyncio
 import synapse.exc as s_exc
 import synapse.common as s_common
 import synapse.lib.boss as s_boss
+import synapse.lib.cell as s_cell
 import synapse.tests.utils as s_test
 
+class BossCell(s_cell.Cell):
+    async def initServiceRuntime(self):
+        self.cboss = await s_boss.Boss.anit()
+        self.onfini(self.cboss)
+
 class BossTest(s_test.SynTest):
 
     async def test_boss_base(self):
 
-        async with await s_boss.Boss.anit() as boss:
+        async with self.getTestCell(BossCell) as bcell:
+            boss = bcell.cboss
+            root = await bcell.auth.getUserByName('root')
 
             evnt = asyncio.Event()
 
@@ -20,18 +28,19 @@ class BossTest(s_test.SynTest):
 
             self.len(0, boss.ps())
 
-            synt = await boss.promote('test', None, info={'hehe': 'haha'})
+            synt = await boss.promote('test', root, info={'hehe': 'haha'})
 
             self.len(1, boss.ps())
 
             self.eq('test', synt.name)
             self.eq('haha', synt.info.get('hehe'))
+            self.eq(root.iden, synt.user.iden)
 
-            synt0 = await boss.execute(testloop(), 'testloop', None, info={'foo': 'bar'})
+            synt0 = await boss.execute(testloop(), 'testloop', root, info={'foo': 'bar'})
             iden = synt0.iden
 
             with self.raises(s_exc.BadArg):
-                _ = await boss.execute(asyncio.sleep(1), 'testsleep', None, iden=iden)
+                _ = await boss.execute(asyncio.sleep(1), 'testsleep', root, iden=iden)
 
             await evnt.wait()
 
@@ -47,8 +56,8 @@ class BossTest(s_test.SynTest):
                 iden = s_common.guid()
 
                 async def double_promote():
-                    await boss.promote(f'double', None, taskiden=iden)
-                    await boss.promote(f'double', None, taskiden=iden + iden)
+                    await boss.promote(f'double', root, taskiden=iden)
+                    await boss.promote(f'double', root, taskiden=iden + iden)
 
                 coro = boss.schedCoro(double_promote())
                 self.true(await stream.wait(timeout=6))

synapse/tests/test_lib_cell.py

@@ -784,6 +784,10 @@ class CellTest(s_t_utils.SynTest):
             cell.VERSION = (1, 2, 3)
             cell.VERSTRING = '1.2.3'
 
+            cell.features.update({
+                'testvalu': 2
+            })
+
             http_info = []
             host, port = await cell.addHttpsPort(0)
             http_info.append({'host': host, 'port': port})
@@ -807,6 +811,9 @@ class CellTest(s_t_utils.SynTest):
                 # A Cortex populated cellvers
                 self.isin('cortex:defaults', cnfo.get('cellvers', {}))
 
+                self.eq(info.get('features'), cell.features)
+                self.eq(info.get('features', {}).get('testvalu'), 2)
+
                 # Defaults aha data is
                 self.eq(cnfo.get('aha'), {'name': None, 'leader': None, 'network': None})
 
@@ -3294,6 +3301,51 @@ class CellTest(s_t_utils.SynTest):
                         await cell00.promote(graceful=True)
                     self.isin('02.cell is not the current leader', cm.exception.get('mesg'))
 
+    async def test_cell_get_aha_proxy(self):
+
+        async with self.getTestCell() as cell:
+
+            self.none(await cell.getAhaProxy())
+
+            class MockAhaClient:
+                def __init__(self, proxy=None):
+                    self._proxy = proxy
+
+                async def proxy(self, timeout=None):
+                    return self._proxy
+
+            with self.getAsyncLoggerStream('synapse.lib.cell', 'AHA client connection failed.') as stream:
+                cell.ahaclient = MockAhaClient()
+                self.none(await cell.getAhaProxy())
+                self.true(await stream.wait(timeout=1))
+
+            class MockProxyHasNot:
+                def _hasTeleFeat(self, name, vers):
+                    return False
+
+            cell.ahaclient = MockAhaClient(proxy=MockProxyHasNot())
+            self.none(await cell.getAhaProxy(feats=(('test', 1),)))
+
+            class MockProxyHas:
+                def _hasTeleFeat(self, name, vers):
+                    return True
+
+            mock_proxy = MockProxyHas()
+            cell.ahaclient = MockAhaClient(proxy=mock_proxy)
+            self.eq(await cell.getAhaProxy(), mock_proxy)
+            self.eq(await cell.getAhaProxy(feats=(('test', 1),)), mock_proxy)
+
+    async def test_lib_cell_sadaha(self):
+
+        async with self.getTestCell() as cell:
+
+            self.none(await cell.getAhaProxy())
+            cell.ahaclient = await s_telepath.Client.anit('cell:///tmp/newp')
+
+            # coverage for failure of aha client to connect
+            with self.raises(TimeoutError):
+                self.none(await cell.getAhaProxy(timeout=0.1))
+
     async def test_stream_backup_exception(self):
 
         with self.getTestDir() as dirn:
@@ -3356,3 +3408,55 @@ class CellTest(s_t_utils.SynTest):
                     with self.raises(s_exc.BackupAlreadyRunning):
                         async for _ in proxy.iterNewBackupArchive('newbackup', remove=True):
                             pass
+
+    async def test_cell_peer_noaha(self):
+
+        todo = s_common.todo('newp')
+        async with self.getTestCell() as cell:
+            async for item in cell.callPeerApi(todo):
+                pass
+            async for item in cell.callPeerGenr(todo):
+                pass
+
+    async def test_cell_task_apis(self):
+        async with self.getTestAha() as aha:
+
+            # test some of the gather API implementations...
+            purl00 = await aha.addAhaSvcProv('00.cell')
+            purl01 = await aha.addAhaSvcProv('01.cell', provinfo={'mirror': 'cell'})
+
+            cell00 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl00}))
+            cell01 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl01}))
+
+            await cell01.sync()
+
+            async def sleep99(cell):
+                await cell.boss.promote('sleep99', cell.auth.rootuser)
+                await cell00.fire('sleep99')
+                await asyncio.sleep(99)
+
+            async with cell00.waiter(2, 'sleep99', timeout=6):
+                task00 = cell00.schedCoro(sleep99(cell00))
+                task01 = cell01.schedCoro(sleep99(cell01))
+
+            tasks = [task async for task in cell00.getTasks(timeout=6)]
+
+            self.len(2, tasks)
+            self.eq(tasks[0]['service'], '00.cell.synapse')
+            self.eq(tasks[1]['service'], '01.cell.synapse')
+            self.eq(('sleep99', 'sleep99'), [task.get('name') for task in tasks])
+            self.eq(('root', 'root'), [task.get('username') for task in tasks])
+
+            self.eq(tasks[0], await cell00.getTask(tasks[0].get('iden')))
+            self.eq(tasks[1], await cell00.getTask(tasks[1].get('iden')))
+            self.none(await cell00.getTask(tasks[1].get('iden'), peers=False))
+
+            self.true(await cell00.killTask(tasks[0].get('iden')))
+
+            task01 = tasks[1].get('iden')
+            self.false(await cell00.killTask(task01, peers=False))
+
+            self.true(await cell00.killTask(task01))
+
+            self.none(await cell00.getTask(task01))
+            self.false(await cell00.killTask(task01))

synapse/tests/test_lib_jsonstor.py

@@ -116,6 +116,7 @@ class JsonStorTest(s_test.SynTest):
                     self.eq({'hehe': 'haha', 'zip': {}}, await prox.getPathObj('foo/bar'))
 
                     self.false(await prox.delPathObjProp('newp/newp', 'newp'))
+                    self.false(await prox.delPathObjProp('foo/bar', 'nested/newp'))
 
                     await prox.delPathObj('foo/bar')
                     self.none(await prox.getPathObj('foo/bar'))

synapse/tests/test_lib_modelrev.py

@@ -1766,3 +1766,9 @@ class ModelRevTest(s_tests.SynTest):
             self.eq('foo bar', nodes[0].get('model'))
             nodes = await core.nodes('transport:sea:vessel')
             self.eq('foo bar', nodes[0].get('model'))
+
+    async def test_modelrev_0_2_33(self):
+        async with self.getRegrCore('model-0.2.33') as core:
+            nodes = await core.nodes('entity:name')
+            self.len(1, nodes)
+            self.eq('foo bar', nodes[0].repr())