synapse 2.212.0__py311-none-any.whl → 2.214.0__py311-none-any.whl

synapse/tests/test_tools_cortex_layer.py

@@ -0,0 +1,419 @@
+import os
+
+import synapse.common as s_common
+
+import synapse.lib.msgpack as s_msgpack
+import synapse.lib.version as s_version
+
+import synapse.tests.utils as s_test
+
+import synapse.tools.cortex.layer.dump as s_t_dump
+import synapse.tools.cortex.layer.load as s_t_load
+
+class LayerTest(s_test.SynTest):
+
+    async def test_tools_layer_dump(self):
+
+        async with self.getTestCore() as core:
+
+            url = core.getLocalUrl()
+
+            layr00 = await core.addLayer()
+            layr00iden = layr00.get('iden')
+            view00 = await core.addView({'layers': [layr00iden]})
+
+            soffs = await core.getNexsIndx()
+
+            opts = {'view': view00.get('iden'), 'vars': {'n': 256}}
+            nodes = await core.nodes('for $i in $lib.range($n) { [test:int=$i] }', opts=opts)
+            self.len(256, nodes)
+
+            eoffs = soffs + 256
+
+            chunksize = 10
+
+            with self.getTestDir() as dirn:
+                argv = (
+                    '--url', url,
+                    '--chunksize', str(chunksize),
+                    '--offset', str(soffs),
+                    layr00iden,
+                    dirn,
+                )
+
+                outp = self.getTestOutp()
+                self.eq(0, await s_t_dump.main(argv, outp=outp))
+                outp.expect(f'Successfully exported layer {layr00iden}.')
+
+                for ii in range(soffs, eoffs - chunksize, chunksize):
+                    path = s_common.genpath(dirn, f'{core.iden}.{layr00iden}.{ii}-{ii + chunksize - 1}.nodeedits')
+                    self.true(os.path.exists(path))
+
+                # Open and inspect first file
+                filename = f'{core.iden}.{layr00iden}.{soffs}-{soffs + 9}.nodeedits'
+                msgs = list(s_msgpack.iterfile(s_common.genpath(dirn, filename)))
+                self.len(12, msgs)
+
+                self.eq(msgs[0][0], 'init')
+                self.eq(msgs[0][1].get('hdrvers'), 1)
+                self.eq(msgs[0][1].get('celliden'), core.iden)
+                self.eq(msgs[0][1].get('layriden'), layr00iden)
+                self.eq(msgs[0][1].get('offset'), soffs)
+                self.eq(msgs[0][1].get('chunksize'), chunksize)
+                self.nn(msgs[0][1].get('tick'))
+                self.isinstance(msgs[0][1].get('tick'), int)
+                self.eq(msgs[0][1].get('cellvers'), core.cellinfo.get('cell:version'))
+
+                for msg in msgs[1:-1]:
+                    self.len(2, msg)
+                    self.eq(msg[0], 'edit')
+                    self.len(3, msg[1])
+
+                self.eq(msgs[11][0], 'fini')
+                self.eq(msgs[11][1].get('offset'), soffs + 9)
+                self.nn(msgs[11][1].get('tock'))
+                self.isinstance(msgs[11][1].get('tock'), int)
+
+            # Test state tracking
+            with self.getTestDir() as dirn:
+                # Check state file is written
+                argv = (
+                    '--url', url,
+                    '--chunksize', str(chunksize),
+                    '--offset', str(soffs),
+                    layr00iden,
+                    dirn,
+                )
+
+                outp = self.getTestOutp()
+                self.eq(0, await s_t_dump.main(argv, outp=outp))
+                outp.expect(f'Successfully exported layer {layr00iden}.')
+
+                state = s_common.yamlload(dirn, f'{core.iden}.{layr00iden}.yaml')
+                self.eq(state, {'offset:next': eoffs})
+
+                # Check state file is read
+                opts = {'view': view00.get('iden'), 'vars': {'n': 256}}
+                nodes = await core.nodes('for $i in $lib.range($n) { [test:str=$i] }', opts=opts)
+                self.len(256, nodes)
+
+                state = {'offset:next': eoffs + 1}
+                statefile = s_common.genpath(dirn, 'state.yaml')
+                s_common.yamlsave(state, statefile)
+
+                argv = (
+                    '--url', url,
+                    '--statefile', statefile,
+                    layr00iden,
+                    s_common.genpath(dirn, 'next'),
+                )
+
+                outp = self.getTestOutp()
+                self.eq(0, await s_t_dump.main(argv, outp=outp))
+                outp.expect(f'Successfully exported layer {layr00iden}.')
+
+                state = s_common.yamlload(dirn, statefile)
+                self.eq(state, {'offset:next': eoffs + 256})
+
+    async def test_tools_layer_dump_errors(self):
+
+        # Non-cortex cell
+        async with self.getTestCell() as cell:
+            with self.getTestDir() as dirn:
+                with s_common.genfile(dirn, 'newp') as fd:
+                    filename = fd.name
+                    fd.write(s_msgpack.en(('init', {'offset': 0, 'cellvers': s_version.version})))
+
+                url = cell.getLocalUrl()
+                argv = ('--url', url, s_common.guid(), dirn)
+                outp = self.getTestOutp()
+                self.eq(1, await s_t_dump.main(argv, outp=outp))
+                outp.expect(f'ERROR: Layer dump tool only works on cortexes, not cell.')
+
+        async with self.getTestCore() as core:
+
+            url = core.getLocalUrl()
+
+            layr00 = await core.addLayer()
+            layr00iden = layr00.get('iden')
+            view00 = await core.addView({'layers': [layr00iden]})
+
+            soffs = await core.getNexsIndx()
+
+            opts = {'view': view00.get('iden'), 'vars': {'n': 256}}
+            nodes = await core.nodes('for $i in $lib.range($n) { [test:int=$i] }', opts=opts)
+            self.len(256, nodes)
+
+            eoffs = soffs + 256
+
+            chunksize = 10
+
+            with self.getTestDir() as dirn:
+                # Handle no edits from offset
+                argv = (
+                    '--url', url,
+                    '--chunksize', str(chunksize),
+                    '--offset', '9000',
+                    layr00iden,
+                    dirn,
+                )
+
+                outp = self.getTestOutp()
+                self.eq(1, await s_t_dump.main(argv, outp=outp))
+                outp.expect(f'ERROR: No edits to export starting from offset (9000).')
+
+                # Handle outdir being an existing file
+                filename = s_common.genpath(dirn, 'newp')
+                s_common.genfile(filename).close()
+
+                argv = (
+                    '--url', url,
+                    '--chunksize', str(chunksize),
+                    layr00iden,
+                    filename,
+                )
+
+                outp = self.getTestOutp()
+                self.eq(1, await s_t_dump.main(argv, outp=outp))
+                outp.expect(f'ERROR: Specified output directory {filename} exists but is not a directory.')
+
+                # Invalid layer iden
+                newpiden = s_common.guid()
+                argv = (
+                    '--url', url,
+                    '--chunksize', str(chunksize),
+                    newpiden,
+                    dirn,
+                )
+
+                outp = self.getTestOutp()
+                self.eq(1, await s_t_dump.main(argv, outp=outp))
+                outp.expect(f'ERROR: No such layer {newpiden}.')
+
+    async def test_tools_layer_load(self):
+        async with self.getTestCore() as core:
+
+            url = core.getLocalUrl()
+
+            layr00 = await core.addLayer()
+            layr00iden = layr00.get('iden')
+            view00 = await core.addView({'layers': [layr00iden]})
+
+            layr01 = await core.addLayer()
+            layr01iden = layr01.get('iden')
+            view01 = await core.addView({'layers': [layr01iden]})
+
+            soffs = await core.getNexsIndx()
+
+            opts = {'view': view00.get('iden'), 'vars': {'n': 256}}
+            nodes = await core.nodes('for $i in $lib.range($n) { [test:int=$i] }', opts=opts)
+            self.len(256, nodes)
+
+            eoffs = soffs + 256
+
+            with self.getTestDir() as dirn:
+                # Export layr00
+                argv = (
+                    '--url', url,
+                    layr00iden,
+                    dirn,
+                )
+
+                outp = self.getTestOutp()
+                self.eq(0, await s_t_dump.main(argv, outp=outp))
+                outp.expect(f'Successfully exported layer {layr00iden}.')
+
+                # Verify layr01 is empty
+                opts = {'view': view01.get('iden')}
+                nodes = await core.nodes('inet:ipv4', opts=opts)
+                self.len(0, nodes)
+
+                files = [os.path.join(dirn, k) for k in os.listdir(dirn) if k.endswith('.nodeedits')]
+                self.len(1, files)
+
+                # Import to layr01
+                argv = (
+                    '--url', url,
+                    '--dryrun',
+                    layr01iden,
+                    files[0],
+                )
+
+                outp = self.getTestOutp()
+                self.eq(0, await s_t_load.main(argv, outp=outp))
+                outp.expect('Processing the following nodeedits:')
+                outp.expect(f'{soffs:<16d} | {files[0]}')
+                outp.expect(f'Loading {files[0]}, offset={soffs}, tick=')
+                outp.expect(f'Successfully read {files[0]} as a dryrun test.')
+
+                # Verify layr01 is empty
+                opts = {'view': view01.get('iden')}
+                nodes = await core.nodes('inet:ipv4', opts=opts)
+                self.len(0, nodes)
+
+                # Import to layr01
+                argv = (
+                    '--url', url,
+                    layr01iden,
+                    files[0],
+                )
+
+                outp = self.getTestOutp()
+                self.eq(0, await s_t_load.main(argv, outp=outp))
+                outp.expect('Processing the following nodeedits:')
+                outp.expect(f'{soffs:<16d} | {files[0]}')
+                outp.expect(f'Loading {files[0]}, offset={soffs}, tick=')
+                outp.expect(f'Successfully loaded {files[0]} with {eoffs - soffs} edits ({soffs} - {eoffs - 1}).')
+
+                # Verify layr01 has data now
+                opts = {'view': view01.get('iden')}
+                nodes = await core.nodes('test:int', opts=opts)
+                self.len(256, nodes)
+
+    async def test_tools_layer_load_errors(self):
+
+        iden = s_common.guid()
+
+        # Non-cortex cell
+        with self.getTestDir() as dirn:
+            with s_common.genfile(dirn, 'newp') as fd:
+                filename = fd.name
+                fd.write(s_msgpack.en(('init', {'offset': 0, 'cellvers': s_version.version})))
+
+            async with self.getTestCell() as cell:
+                url = cell.getLocalUrl()
+                argv = ('--url', url, iden, filename)
+                outp = self.getTestOutp()
+                self.eq(1, await s_t_load.main(argv, outp=outp))
+                outp.expect(f'ERROR: Layer load tool only works on cortexes, not cell.')
+
+        # Non-existent file
+        argv = (iden, 'newp')
+        outp = self.getTestOutp()
+        self.eq(1, await s_t_load.main(argv, outp=outp))
+        outp.expect('Invalid input file specified: newp.')
+
+        # Input file is a directory
+        with self.getTestDir() as dirn:
+            argv = (iden, dirn)
+            outp = self.getTestOutp()
+            self.eq(1, await s_t_load.main(argv, outp=outp))
+            outp.expect(f'Invalid input file specified: {dirn}.')
+
+        # Input file doesn't have an init message first
+        with self.getTestDir() as dirn:
+            with s_common.genfile(dirn, 'noinit.nodeedits') as fd:
+                filename = fd.name
+                fd.write(s_msgpack.en(('newp', {})))
+
+            argv = (iden, filename)
+            outp = self.getTestOutp()
+            self.eq(1, await s_t_load.main(argv, outp=outp))
+            outp.expect(f'Invalid header in {filename}.')
+
+        # Invalid/too high cell version
+        with self.getTestDir() as dirn:
+            version = (99, 99, 0)
+            verstr = '.'.join(map(str, version))
+
+            with s_common.genfile(dirn, 'cellvers00.nodeedits') as fd:
+                file00 = fd.name
+                fd.write(s_msgpack.en(('init', {'offset': 0, 'cellvers': s_version.version})))
+
+            with s_common.genfile(dirn, 'cellvers01.nodeedits') as fd:
+                file01 = fd.name
+                fd.write(s_msgpack.en(('init', {'offset': 0, 'cellvers': version})))
+
+            async with self.getTestCore() as core:
+                url = core.getLocalUrl()
+                argv = ('--url', url, iden, file00, file01)
+                outp = self.getTestOutp()
+                self.eq(1, await s_t_load.main(argv, outp=outp))
+                outp.expect(f'Synapse version mismatch ({s_version.verstring} < {verstr}).')
+
+        # Invalid message type
+        with self.getTestDir() as dirn:
+            with s_common.genfile(dirn, 'badtype.nodeedits') as fd:
+                filename = fd.name
+                fd.write(s_msgpack.en((
+                    'init',
+                    {
+                        'hdrvers': 1,
+                        'celliden': s_common.guid(),
+                        'layriden': s_common.guid(),
+                        'offset': 0,
+                        'chunksize': 10000,
+                        'tick': s_common.now(),
+                        'cellvers': s_version.version,
+                    }
+                )))
+
+                fd.write(s_msgpack.en(('newp', {})))
+
+            async with self.getTestCore() as core:
+                url = core.getLocalUrl()
+                layriden = core.getView().layers[0].iden
+                argv = ('--url', url, layriden, filename)
+                outp = self.getTestOutp()
+                self.eq(1, await s_t_load.main(argv, outp=outp))
+                outp.expect('Unexpected message type: newp.')
+
+        # Missing fini
+        with self.getTestDir() as dirn:
+            with s_common.genfile(dirn, 'badtype.nodeedits') as fd:
+                filename = fd.name
+                fd.write(s_msgpack.en((
+                    'init',
+                    {
+                        'hdrvers': 1,
+                        'celliden': s_common.guid(),
+                        'layriden': s_common.guid(),
+                        'offset': 0,
+                        'chunksize': 10000,
+                        'tick': s_common.now(),
+                        'cellvers': s_version.version,
+                    }
+                )))
+
+            async with self.getTestCore() as core:
+                url = core.getLocalUrl()
+                layriden = core.getView().layers[0].iden
+                argv = ('--url', url, layriden, filename)
+                outp = self.getTestOutp()
+                self.eq(1, await s_t_load.main(argv, outp=outp))
+                outp.expect(f'Incomplete/corrupt export: {filename}.')
+
+        # Fini offset mismatch
+        with self.getTestDir() as dirn:
+            soffs = 0
+            eoffs = 1000
+            with s_common.genfile(dirn, 'badtype.nodeedits') as fd:
+                filename = fd.name
+                fd.write(s_msgpack.en((
+                    'init',
+                    {
+                        'hdrvers': 1,
+                        'celliden': s_common.guid(),
+                        'layriden': s_common.guid(),
+                        'offset': soffs,
+                        'chunksize': 10000,
+                        'tick': s_common.now(),
+                        'cellvers': s_version.version,
+                    }
+                )))
+
+                fd.write(s_msgpack.en((
+                    'fini',
+                    {
+                        'offset': eoffs,
+                        'tock': s_common.now(),
+                    }
+                )))
+
+            async with self.getTestCore() as core:
+                url = core.getLocalUrl()
+                layriden = core.getView().layers[0].iden
+                argv = ('--url', url, layriden, filename)
+                outp = self.getTestOutp()
+                self.eq(1, await s_t_load.main(argv, outp=outp))
+                outp.expect(f'Incomplete/corrupt export: {filename}. Expected offset {eoffs}, got {soffs}.')

synapse/tests/test_tools_demote.py

@@ -0,0 +1,114 @@
+import unittest.mock as mock
+
+import synapse.exc as s_exc
+import synapse.common as s_common
+
+import synapse.lib.cell as s_cell
+
+import synapse.tools.demote as s_tools_demote
+
+import synapse.tests.utils as s_test
+
+async def boom(*args, **kwargs):
+    raise s_exc.SynErr(mesg='BOOM')
+
+class DemoteToolTest(s_test.SynTest):
+
+    async def test_tool_demote_base(self):
+
+        async with self.getTestAha() as aha:
+
+            with self.getTestDir() as dirn:
+
+                dirn00 = s_common.genpath(dirn, '00.cell')
+                dirn01 = s_common.genpath(dirn, '01.cell')
+
+                cell00 = await aha.enter_context(self.addSvcToAha(aha, '00.cell', s_cell.Cell, dirn=dirn00))
+                cell01 = await aha.enter_context(self.addSvcToAha(aha, '01.cell', s_cell.Cell, dirn=dirn01,
+                                                                   provinfo={'mirror': 'cell'}))
+                self.true(cell00.isactive)
+                self.false(cell01.isactive)
+
+                await cell01.sync()
+
+                outp = self.getTestOutp()
+                argv = ['--url', cell00.getLocalUrl()]
+
+                self.eq(0, await s_tools_demote.main(argv, outp=outp))
+                outp.expect('Demoting leader: cell://')
+
+                self.false(cell00.isactive)
+                self.true(cell01.isactive)
+
+                await cell00.sync()
+
+                outp.clear()
+                self.eq(1, await s_tools_demote.main(argv, outp=outp))
+                outp.expect('Failed to demote service cell:')
+
+                # get some test coverage for the various levels of exception handlers...
+
+                with mock.patch.object(cell00, 'getNexsIndx', boom):
+                    with self.getLoggerStream('synapse') as stream:
+                        argv = ['--url', cell01.getLocalUrl(), '--timeout', '12']
+                        self.eq(1, await s_tools_demote.main(argv, outp=outp))
+                    stream.expect('...error retrieving nexus index for')
+
+                with mock.patch.object(cell00, 'promote', boom):
+                    with self.getLoggerStream('synapse') as stream:
+                        argv = ['--url', cell01.getLocalUrl(), '--timeout', '12']
+                        self.eq(1, await s_tools_demote.main(argv, outp=outp))
+                    stream.expect('...error promoting')
+
+                with mock.patch.object(cell01, '_getDemotePeers', boom):
+                    with self.getLoggerStream('synapse') as stream:
+                        argv = ['--url', cell01.getLocalUrl(), '--timeout', '12']
+                        outp.clear()
+                        self.eq(1, await s_tools_demote.main(argv, outp=outp))
+                        outp.expect('Error while demoting service')
+                    stream.expect('error during task: demote')
+
+                self.false(cell00.isactive)
+                self.true(cell01.isactive)
+
+                outp.clear()
+                self.eq(1, await s_tools_demote.main(['--url', 'newp://hehe'], outp=outp))
+                outp.expect('Error while demoting service newp://hehe')
+
+                self.true(await aha.schedCoro(cell01.shutdown(timeout=12)))
+                self.true(cell00.isactive)
+                self.false(cell01.isactive)
+
+                self.true(await cell01.waitfini(timeout=12))
+
+                # test demote with insufficient peers
+                with self.getLoggerStream('synapse') as stream:
+                    argv = ['--url', cell00.getLocalUrl(), '--timeout', '12']
+                    self.eq(1, await s_tools_demote.main(argv, outp=outp))
+                stream.expect('...no suitable services discovered.')
+
+    async def test_tool_demote_no_features(self):
+
+        async with self.getTestAha() as aha:
+            aha.features.pop('getAhaSvcsByIden')
+
+            with self.getTestDir() as dirn:
+
+                dirn00 = s_common.genpath(dirn, '00.cell')
+                dirn01 = s_common.genpath(dirn, '01.cell')
+
+                cell00 = await aha.enter_context(self.addSvcToAha(aha, '00.cell', s_cell.Cell, dirn=dirn00))
+                cell01 = await aha.enter_context(self.addSvcToAha(aha, '01.cell', s_cell.Cell, dirn=dirn01,
+                                                                   provinfo={'mirror': 'cell'}))
+                self.true(cell00.isactive)
+                self.false(cell01.isactive)
+
+                await cell01.sync()
+
+                outp = self.getTestOutp()
+                argv = ['--url', cell00.getLocalUrl()]
+                with self.getAsyncLoggerStream('synapse.daemon') as stream:
+                    self.eq(1, await s_tools_demote.main(argv, outp=outp))
+                stream.expect('AHA server does not support feature: getAhaSvcsByIden >= 1')
+                outp.expect('Error while demoting service')
+                outp.expect('AHA server does not support feature: getAhaSvcsByIden')

synapse/tests/test_tools_pkgs_gendocs.py

@@ -0,0 +1,100 @@
+import io
+import os
+import sys
+import json
+
+import synapse.exc as s_exc
+import synapse.common as s_common
+
+import synapse.tests.utils as s_t_utils
+
+import synapse.tools.pkgs.gendocs as s_t_gendocs
+import synapse.tools.pkgs.pandoc_filter as s_t_pandoc_filter
+
+class TestPkgBuildDocs(s_t_utils.SynTest):
+
+    def setUp(self):
+        if not s_t_gendocs.hasPandoc():
+            self.skip('pandoc is not available')
+        super().setUp()
+
+    async def test_pkg_builddocs(self):
+
+        with self.getTestDir(mirror='testpkg_build_docs') as dirn:
+            testpkgfp = os.path.join(dirn, 'testpkg.yaml')
+            self.true(os.path.isfile(testpkgfp))
+            argv = [testpkgfp, ]
+            r = await s_t_gendocs.main(argv)
+            self.eq(r, 0)
+
+            pkgdef = s_common.yamlload(testpkgfp)
+            efiles = set()
+            for dnfo in pkgdef.get('docs'):
+                bname = os.path.basename(dnfo.get('path'))
+                efiles.add(bname)
+                efiles.add(bname.rsplit('.', 1)[0] + '.rst')
+            builddir = os.path.join(dirn, 'docs', '_build')
+            self.eq(efiles, set(os.listdir(builddir)))
+
+            text = s_common.getbytes(os.path.join(builddir, 'bar.md')).decode()
+            self.isin('storm> [inet:asn=1]', text)
+            self.isin('inet:asn=1\n', text)
+            self.notin(':orphan:', text)
+            self.notin(':tocdepth:', text)
+
+            text = s_common.getbytes(os.path.join(builddir, 'stormpackage.md')).decode()
+            self.isin(':   baz (str): The baz.', text)
+            self.isin(':   Baz the bam:\n\n        yield $lib.import(apimod).search(bam)', text)
+
+        with self.getTestDir(mirror='testpkg_build_docs') as dirn:
+            testpkgfp = os.path.join(dirn, 'testpkg.yaml')
+            self.true(os.path.isfile(testpkgfp))
+            argv = [testpkgfp, '--rst-only']
+            r = await s_t_gendocs.main(argv)
+            self.eq(r, 0)
+
+            pkgdef = s_common.yamlload(testpkgfp)
+            efiles = set()
+            for dnfo in pkgdef.get('docs'):
+                bname = os.path.basename(dnfo.get('path'))
+                efiles.add(bname)
+                efiles.add(bname.rsplit('.', 1)[0] + '.rst')
+            builddir = os.path.join(dirn, 'docs', '_build')
+            self.eq(efiles, set(os.listdir(builddir)))
+
+            text = s_common.getbytes(os.path.join(builddir, 'bar.md')).decode()
+            self.eq('', text)
+            text = s_common.getbytes(os.path.join(builddir, 'stormpackage.md')).decode()
+            self.eq('', text)
+
+        with self.getTestDir(mirror='testpkg_build_docs') as dirn:
+
+            # Missing input files
+            testpkgfp = os.path.join(dirn, 'newp.yaml')
+            self.false(os.path.isfile(testpkgfp))
+            argv = [testpkgfp, ]
+            with self.raises(s_exc.BadArg) as cm:
+                await s_t_gendocs.main(argv)
+            self.isin('Package does not exist or does not contain yaml', cm.exception.get('mesg'))
+
+            # pandoc api version check coverage
+            old_stdin = sys.stdin
+            try:
+                sys.stdin = io.StringIO(json.dumps({'pandoc-api-version': [2, 0, 0]}))
+                with self.raises(Exception) as cm:
+                    s_t_pandoc_filter.main()
+                self.isin('does not match required version', str(cm.exception))
+            finally:
+                sys.stdin = old_stdin
+
+            # pandoc failure
+            outp = self.getTestOutp()
+            oldv = s_t_gendocs.PANDOC_FILTER
+            try:
+                s_t_gendocs.PANDOC_FILTER = os.path.join(dirn, 'newp.py')
+                with self.raises(s_exc.SynErr) as cm:
+                    await s_t_gendocs.main([os.path.join(dirn, 'testpkg.yaml'),], outp=outp)
+                self.isin('Error converting', cm.exception.get('mesg'))
+                outp.expect('ERR: Error running filter')
+            finally:
+                s_t_gendocs.PANDOC_FILTER = oldv