synapse 2.213.0__py311-none-any.whl → 2.215.0__py311-none-any.whl

synapse/tests/test_lib_stormtypes.py

@@ -2999,6 +2999,21 @@ class StormTypesTest(s_test.SynTest):
                 self.eq((5, 'baz'), await core.callStorm('return($lib.queue.get(poptest).pop())'))
                 self.none(await core.callStorm('return($lib.queue.get(poptest).pop())'))
 
+                # Coverage for the Cortex queue:del nexus handler
+                name = 'deleteme'
+                iden = s_common.guid()
+                await core.addCoreQueue(
+                    name,
+                    {'name': name,
+                     'creator': core.auth.rootuser.iden,
+                     'created': s_common.now(),
+                     'iden': iden}
+                )
+                await core.auth.delAuthGate(f'queue:{name}')
+                await core.delCoreQueue(name)
+                with self.raises(s_exc.NoSuchName):
+                    await core.getCoreQueue(name)
+
     async def test_storm_node_data(self):
 
         async with self.getTestCore() as core:

synapse/tests/test_lib_types.py

@@ -1,5 +1,6 @@
 import math
 import decimal
+import datetime
 import synapse.exc as s_exc
 import synapse.common as s_common
 import synapse.datamodel as s_datamodel
@@ -1320,7 +1321,10 @@ class TypesTest(s_t_utils.SynTest):
             nodes = await core.nodes('test:str:tick=201401*')
             self.eq({node.ndef[1] for node in nodes}, {'a'})
 
-            nodes = await core.nodes('test:str:tick*range=("-4200 days", now)')
+            utc = datetime.timezone.utc
+            delta = datetime.datetime.now(tz=utc) - datetime.datetime(2014, 1, 1, tzinfo=utc)
+            days = delta.days + 14
+            nodes = await core.nodes(f'test:str:tick*range=("-{days} days", now)')
             self.eq({node.ndef[1] for node in nodes}, {'a', 'b', 'c', 'd'})
 
             opts = {'vars': {'tick': tick, 'tock': tock}}

synapse/tests/test_model_inet.py

@@ -3002,17 +3002,20 @@ class InetModelTest(s_t_utils.SynTest):
                 :url="https://v.vtx.lk/slack"
                 :name="Synapse users slack"
                 :tenant={[ inet:service:tenant=({"id": "VS-31337"}) ]}
+                :app={[ inet:service:app=({"id": "app00"}) ]}
             ]
             '''
             nodes = await core.nodes(q)
             self.len(1, nodes)
             self.nn(nodes[0].get('tenant'))
+            self.nn(nodes[0].get('app'))
             self.eq(nodes[0].ndef, ('inet:service:instance', s_common.guid(('vertex', 'slack'))))
             self.eq(nodes[0].get('id'), 'T2XK1223Y')
             self.eq(nodes[0].get('platform'), platform.ndef[1])
             self.eq(nodes[0].get('url'), 'https://v.vtx.lk/slack')
             self.eq(nodes[0].get('name'), 'synapse users slack')
             platinst = nodes[0]
+            app00 = nodes[0].get('app')
 
             q = '''
             [
@@ -3023,6 +3026,7 @@ class InetModelTest(s_t_utils.SynTest):
                     :email=blackout@vertex.link
                     :profile={ gen.ps.contact.email vertex.employee blackout@vertex.link }
                     :tenant={[ inet:service:tenant=({"id": "VS-31337"}) ]}
+                    :app={[ inet:service:app=({"id": "a001"}) ]}
                 )
 
                 (inet:service:account=(visi, account, vertex, slack)
@@ -3037,6 +3041,7 @@ class InetModelTest(s_t_utils.SynTest):
             self.len(2, accounts)
 
             self.nn(accounts[0].get('tenant'))
+            self.nn(accounts[0].get('app'))
 
             profiles = await core.nodes('ps:contact')
             self.len(2, profiles)
@@ -3175,6 +3180,7 @@ class InetModelTest(s_t_utils.SynTest):
                 :platform=$platiden
                 :instance=$instiden
                 :topic=' My Topic   '
+                :app={ inet:service:app:id=app00 }
             ]
             '''
             opts = {'vars': {
@@ -3185,6 +3191,7 @@ class InetModelTest(s_t_utils.SynTest):
             nodes = await core.nodes(q, opts=opts)
             self.len(1, nodes)
             self.eq(nodes[0].ndef, ('inet:service:channel', s_common.guid(('general', 'channel', 'vertex', 'slack'))))
+            self.eq(nodes[0].get('app'), app00)
             self.eq(nodes[0].get('name'), 'general')
             self.eq(nodes[0].get('topic'), 'my topic')
             self.eq(nodes[0].get('period'), (1420070400000, 9223372036854775807))

synapse/tests/test_model_infotech.py

@@ -2438,3 +2438,34 @@ class InfotechModelTest(s_t_utils.SynTest):
             self.eq(2, nodes[0].get('assets:vulns:preexisting'))
 
             self.len(1, await core.nodes('it:sec:metrics -> ou:org +:name=vertex'))
+
+    async def test_infotech_windows(self):
+
+        async with self.getTestCore() as core:
+
+            nodes = await core.nodes('''
+                [ it:os:windows:service=*
+                    :name=Woot
+                    :host=*
+                    :type=(0x20)
+                    :start=(0x20)
+                    :errorcontrol=(0x20)
+                    :displayname="Foo Bar Baz"
+                    :imagepath=c:/windows/system32/woot.exe
+                    :description="Lorem ipsum dolor sit amet, consectetur adipiscing elit."
+                ]
+            ''')
+
+            self.len(1, nodes)
+            self.eq(nodes[0].get('name'), 'woot')
+            self.eq(nodes[0].get('type'), 0x20)
+            self.eq(nodes[0].get('start'), 0x20)
+            self.eq(nodes[0].get('errorcontrol'), 0x20)
+            self.eq(nodes[0].get('displayname'), 'foo bar baz')
+            self.eq(nodes[0].get('imagepath'), 'c:/windows/system32/woot.exe')
+            self.eq(nodes[0].get('description'), 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.')
+
+            self.len(1, await core.nodes('it:os:windows:service -> it:host'))
+            self.len(1, await core.nodes('it:os:windows:service -> file:path'))
+
+            self.len(1, await core.nodes('[ it:exec:proc=* :windows:service={ it:os:windows:service } ] -> it:os:windows:service'))

synapse/tests/test_telepath.py

@@ -110,7 +110,7 @@ class Foo:
 
     def raze(self):
         # test that SynErr makes it through
-        raise s_exc.NoSuchMeth(name='haha')
+        raise s_exc.SynErr(mesg='hehe', key='valu')
 
     async def corovalu(self, x, y):
         return x * 2 + y
@@ -305,11 +305,24 @@ class TeleTest(s_t_utils.SynTest):
             genr = prox.agenrboom()
             await self.asyncraises(s_exc.SynErr, genr.list())
 
-            await self.asyncraises(s_exc.NoSuchMeth, prox.raze())
+            with self.raises(s_exc.SynErr) as cm:
+                await prox.raze()
+            self.eq(cm.exception.get('mesg'), 'hehe')
+            self.eq(cm.exception.get('key'), 'valu')
 
-            await self.asyncraises(s_exc.NoSuchMeth, prox.fake())
+            with self.raises(s_exc.NoSuchMeth) as cm:
+                await prox.fake()
+            self.eq(cm.exception.get('mesg'), 'Foo has no method: fake.')
+            self.eq(cm.exception.get('name'), 'fake')
 
-            await self.asyncraises(s_exc.SynErr, prox.boom())
+            with self.raises(s_exc.NoSuchMeth) as cm:
+                await prox._fake()
+            self.eq(cm.exception.get('mesg'), 'Foo has no method: _fake.')
+            self.eq(cm.exception.get('name'), '_fake')
+
+            with self.raises(s_exc.NotMsgpackSafe) as cm:
+                await prox.boom()
+            self.isin("can not serialize 'Boom' object", cm.exception.get('mesg'))
 
         # Fini'ing a daemon fini's proxies connected to it.
         self.true(await s_coro.event_wait(evt, 2))
@@ -395,10 +408,12 @@ class TeleTest(s_t_utils.SynTest):
                 genr = prox.corogenr(3)
                 self.eq((0, 1, 2), await s_t_utils.alist(genr))
 
-                await self.asyncraises(s_exc.NoSuchMeth, prox.raze())
+                await self.asyncraises(s_exc.SynErr, prox.raze())
 
                 await self.asyncraises(s_exc.NoSuchMeth, prox.fake())
 
+                await self.asyncraises(s_exc.NoSuchMeth, prox._fake())
+
                 await self.asyncraises(s_exc.SynErr, prox.boom())
 
             # Fini'ing a daemon fini's proxies connected to it.
@@ -834,6 +849,18 @@ class TeleTest(s_t_utils.SynTest):
                             {'family': 'unix',
                              'addr': sockpath})
 
+    async def test_url_bad_cell_path(self):
+        with self.getTestDir() as dirn:
+            # Touch the sock path
+            s_common.genfile(dirn, 'newp', 'sock').close()
+            cell_url = f'cell://{dirn}/newp'
+            with self.raises(s_exc.LinkErr) as cm:
+                await s_telepath.openurl(cell_url)
+            self.isin('Cell path is not listening', cm.exception.get('mesg'))
+        with self.raises(s_exc.NoSuchPath) as cm:
+            await s_telepath.openurl(cell_url)
+        self.isin('Cell path does not exist', cm.exception.get('mesg'))
+
     async def test_ipv6(self):
         if s_common.envbool('CIRCLECI'):
             self.skip('ipv6 listener is not supported in circleci')

synapse/tests/test_tools_axon.py

@@ -0,0 +1,304 @@
+import io
+import os
+import tarfile
+
+from unittest import mock
+
+import synapse.common as s_common
+
+import synapse.tests.utils as s_t_utils
+
+import synapse.tools.axon.dump as axon_dump
+import synapse.tools.axon.load as axon_load
+
+class AxonToolsTest(s_t_utils.SynTest):
+
+    async def test_axon_dump_and_load_url_handling(self):
+        with self.getTestDir() as testdir:
+            argv = ['--url', 'cell:///definitelynotarealpath/axon', '--offset', '0', testdir]
+            outp = self.getTestOutp()
+            self.eq(1, await axon_dump.main(argv, outp=outp))
+            outp.expect('ERROR')
+            outp.expect('Cell path does not exist')
+
+        with self.getTestDir() as testdir:
+            async with self.getTestAxon(dirn=os.path.join(testdir, 'axon')) as axon:
+                tarfile = os.path.join(testdir, 'dummy.0-1.tar.gz')
+                with open(tarfile, 'wb') as fd:
+                    fd.write(b'dummy')
+                url = f'cell://baduser:badpass@/{axon.dirn}'
+                argv = ['--url', url, tarfile]
+                outp = self.getTestOutp()
+                self.eq(1, await axon_load.main(argv, outp=outp))
+                outp.expect('No such user')
+
+        with self.getTestDir() as testdir:
+            tarfile = os.path.join(testdir, 'dummy.0-1.tar.gz')
+            with open(tarfile, 'wb') as fd:
+                fd.write(b'dummy')
+            argv = ['--url', 'cell:///definitelynotarealpath/axon', tarfile]
+            outp = self.getTestOutp()
+            self.eq(1, await axon_load.main(argv, outp=outp))
+            outp.expect('ERROR')
+            outp.expect('Cell path does not exist')
+
+            async with self.getTestAxon(dirn=os.path.join(testdir, 'axon')) as axon:
+                dumpdir = os.path.join(testdir, 'dumpdir')
+                os.makedirs(dumpdir)
+                url = f'cell://baduser:badpass@/{axon.dirn}'
+                argv = ['--url', url, '--offset', '0', dumpdir]
+                outp = self.getTestOutp()
+                self.eq(1, await axon_dump.main(argv, outp=outp))
+                outp.expect('No such user')
+
+    async def test_axon_dump_and_load(self):
+        with self.getTestDir() as testdir:
+            async with self.getTestAxon(dirn=os.path.join(testdir, 'axon0')) as axon0:
+                blobs = [b'foo', b'bar', b'baz' * 1000]
+                sha2s = []
+                for blob in blobs:
+                    size, sha2 = await axon0.put(blob)
+                    sha2s.append((size, sha2, blob))
+                dumpdir = os.path.join(testdir, 'dumpdir')
+                os.makedirs(dumpdir)
+                argv = [
+                    '--url', f'cell:///{axon0.dirn}',
+                    '--offset', '0',
+                    dumpdir,
+                ]
+                self.eq(0, await axon_dump.main(argv))
+                files = os.listdir(dumpdir)
+                self.true(files, 'No dump file created')
+
+                tarfiles = sorted([os.path.join(dumpdir, f) for f in files if f.endswith('.tar.gz')])
+                async with self.getTestAxon(dirn=os.path.join(testdir, 'axon1')) as axon1:
+                    argv = ['--url', f'cell:///{axon1.dirn}'] + tarfiles
+                    self.eq(0, await axon_load.main(argv))
+                    for size, sha2, blob in sha2s:
+                        self.true(await axon1.has(sha2))
+                        out = b''
+                        async for byts in axon1.get(sha2):
+                            out += byts
+                        self.eq(out, blob)
+
+    async def test_dump_blob_size_mismatch(self):
+        with self.getTestDir() as testdir:
+            async with self.getTestAxon(dirn=os.path.join(testdir, 'axon')) as axon:
+                size, sha2 = await axon.put(b'hello world')
+                dumpdir = os.path.join(testdir, 'dumpdir')
+                os.makedirs(dumpdir)
+                orig_get = axon.get
+                async def bad_get(sha2arg, **kwargs):
+                    if sha2arg == sha2:
+                        yield b'hello' # corrupt it
+                    else:
+                        async for byts in orig_get(sha2arg):
+                            yield byts
+                axon.get = bad_get
+                argv = ['--url', f'cell:///{axon.dirn}', '--offset', '0', dumpdir]
+                outp = self.getTestOutp()
+                self.eq(1, await axon_dump.main(argv, outp=outp))
+                outp.expect('Blob size mismatch')
+
+    async def test_dump_not_axon_cell(self):
+        with self.getTestDir() as testdir:
+            async with self.getTestCore() as core:
+                curl = core.getLocalUrl()
+                argv = ['--url', curl, testdir]
+                outp = self.getTestOutp()
+                self.eq(1, await axon_dump.main(argv, outp=outp))
+                outp.expect('only works on axon')
+
+    async def test_dump_outdir_is_not_directory(self):
+        with self.getTestDir() as testdir:
+            outpath = os.path.join(testdir, 'notadir')
+            with open(outpath, 'w') as fd:
+                fd.write('not a directory')
+            async with self.getTestAxon(dirn=os.path.join(testdir, 'axon')) as axon:
+                argv = ['--url', f'cell:///{axon.dirn}', '--offset', '0', outpath]
+                outp = self.getTestOutp()
+                self.eq(1, await axon_dump.main(argv, outp=outp))
+                outp.expect('exists but is not a directory')
+
+    async def test_dump_rotate_size_and_load(self):
+        with self.getTestDir() as testdir:
+            async with self.getTestAxon(dirn=os.path.join(testdir, 'axon')) as axon:
+                blobs = [os.urandom(1024) for _ in range(10)]
+                sha2s = []
+                for blob in blobs:
+                    size, sha2 = await axon.put(blob)
+                    sha2s.append((size, sha2, blob))
+                dumpdir = os.path.join(testdir, 'dumpdir')
+                os.makedirs(dumpdir)
+                argv = ['--url', f'cell:///{axon.dirn}', '--offset', '0', '--rotate-size', '2048', dumpdir]
+                outp = self.getTestOutp()
+                self.eq(0, await axon_dump.main(argv, outp=outp))
+
+                tarfiles = sorted([os.path.join(dumpdir, f) for f in os.listdir(dumpdir) if f.endswith('.tar.gz')])
+                self.true(len(tarfiles) > 1)
+                async with self.getTestAxon(dirn=os.path.join(testdir, 'axon2')) as axon2:
+                    outp = self.getTestOutp()
+                    argv = ['--url', f'cell:///{axon2.dirn}'] + tarfiles
+                    self.eq(0, await axon_load.main(argv, outp=outp))
+                    for size, sha2, blob in sha2s:
+                        self.true(await axon2.has(sha2))
+                        out = b''
+                        async for byts in axon2.get(sha2):
+                            out += byts
+                        self.eq(out, blob)
+
+    async def test_dump_tempfile_rollover(self):
+        with self.getTestDir() as testdir:
+            async with self.getTestAxon(dirn=os.path.join(testdir, 'axon')) as axon:
+                n = 128
+                blobs = [os.urandom(n) for n in range(n)]
+                sha2s = []
+                for blob in blobs:
+                    size, sha2 = await axon.put(blob)
+                    sha2s.append((size, sha2, blob))
+                dumpdir = os.path.join(testdir, 'dumpdir')
+                os.makedirs(dumpdir)
+                argv = ['--url', f'cell:///{axon.dirn}', dumpdir]
+                outp = self.getTestOutp()
+
+                # Patch the SppoledTemporaryFile rollover size while dumping files.
+                with mock.patch('synapse.tools.axon.dump.MAX_SPOOL_SIZE', n / 2):
+                    self.eq(0, await axon_dump.main(argv, outp=outp))
+
+                tarfiles = sorted([os.path.join(dumpdir, f) for f in os.listdir(dumpdir) if f.endswith('.tar.gz')])
+                self.len(1, tarfiles)
+                async with self.getTestAxon(dirn=os.path.join(testdir, 'axon2')) as axon2:
+                    outp = self.getTestOutp()
+                    argv = ['--url', f'cell:///{axon2.dirn}'] + tarfiles
+                    self.eq(0, await axon_load.main(argv, outp=outp))
+                    for size, sha2, blob in sha2s:
+                        self.true(await axon2.has(sha2))
+                        out = b''
+                        async for byts in axon2.get(sha2):
+                            out += byts
+                        self.eq(out, blob)
+
+    async def test_dump_statefile(self):
+        with self.getTestDir() as testdir:
+            async with self.getTestAxon(dirn=testdir) as axon:
+                blob_data = b'blobdata'
+                await axon.put(blob_data)
+                outdir = os.path.join(testdir, 'out')
+                os.makedirs(outdir)
+                statefile = os.path.join(testdir, 'state.yaml')
+
+                argv = ['--url', axon.getLocalUrl(), '--statefile', statefile, outdir]
+                outp = self.getTestOutp()
+                self.eq(0, await axon_dump.main(argv, outp=outp))
+                self.true(os.path.isfile(statefile))
+                state = s_common.yamlload(statefile)
+                self.isin('offset:next', state)
+
+                s_common.yamlsave({'offset:next': 123}, statefile)
+                outp = self.getTestOutp()
+                self.eq(0, await axon_dump.main(argv, outp=outp))
+                self.true(os.path.isfile(statefile))
+                state = s_common.yamlload(statefile)
+                self.isin('offset:next', state)
+                self.ne(state['offset:next'], 123)
+
+                statedir = os.path.join(testdir, 'statedir')
+                os.makedirs(statedir)
+                argv = ['--url', axon.getLocalUrl(), '--statefile', statedir, outdir]
+                outp = self.getTestOutp()
+                self.eq(0, await axon_dump.main(argv, outp=outp))
+                cellinfo = await axon.getCellInfo()
+                celliden = cellinfo['cell']['iden']
+                statefile_path = os.path.join(statedir, f'{celliden}.yaml')
+                self.true(os.path.isfile(statefile_path))
+                state = s_common.yamlload(statefile_path)
+                self.isin('offset:next', state)
+
+    async def test_load_blobs_path_does_not_exist(self):
+        with self.getTestDir() as testdir:
+            missing = os.path.join(testdir, 'doesnotexist')
+            argv = ['--url', 'cell:///definitelynotarealpath/axon', missing]
+            outp = self.getTestOutp()
+            self.eq(1, await axon_load.main(argv, outp=outp))
+            outp.expect('ERROR: Cell path does not exist')
+
+    async def test_load_continue_and_invalid_blob(self):
+        with self.getTestDir() as testdir:
+            tarpath = os.path.join(testdir, 'test1.tar.gz')
+            with tarfile.open(tarpath, 'w:gz') as tar:
+                info = tarfile.TarInfo('notablob.txt')
+                data = b'not a blob'
+                info.size = len(data)
+                tar.addfile(info, io.BytesIO(data))
+                info2 = tarfile.TarInfo('invalidblobname.blob')
+                data2 = b'data'
+                info2.size = len(data2)
+                tar.addfile(info2, io.BytesIO(data2))
+
+            orig_uhex = s_common.uhex
+            def fake_uhex(val):
+                if val == 'invalidblobname':
+                    raise Exception('bad hex')
+                return orig_uhex(val)
+            s_common.uhex = fake_uhex
+
+            outp = self.getTestOutp()
+            async with self.getTestAxon(dirn=os.path.join(testdir, 'axon')) as axon:
+                argv = ['--url', axon.getLocalUrl(), tarpath]
+                self.eq(0, await axon_load.main(argv, outp=outp))
+            s_common.uhex = orig_uhex
+            outp.expect('Skipping invalid blob filename')
+
+    async def test_load_failed_to_extract(self):
+        with self.getTestDir() as testdir:
+            tarpath = os.path.join(testdir, 'test3.tar.gz')
+            with tarfile.open(tarpath, 'w:gz') as tar:
+                info = tarfile.TarInfo('deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef.blob')
+                info.type = tarfile.DIRTYPE
+                tar.addfile(info)
+            outp = self.getTestOutp()
+            async with self.getTestAxon(dirn=os.path.join(testdir, 'axon')) as axon:
+                argv = ['--url', axon.getLocalUrl(), tarpath]
+                self.eq(0, await axon_load.main(argv, outp=outp))
+            outp.expect('Failed to extract')
+
+    async def test_load_not_axon_cell(self):
+        with self.getTestDir() as testdir:
+            async with self.getTestCore() as core:
+                curl = core.getLocalUrl()
+                dumpdir = os.path.join(testdir, 'dumpdir')
+                os.makedirs(dumpdir)
+                argv = ['--url', curl, dumpdir]
+                outp = self.getTestOutp()
+                self.eq(1, await axon_load.main(argv, outp=outp))
+                outp.expect('only works on axon')
+
+    async def test_load_oserror_extracting_blob(self):
+        with self.getTestDir() as testdir:
+            tarpath = os.path.join(testdir, 'test_oserror.tar.gz')
+            with tarfile.open(tarpath, 'w:gz') as tar:
+                info = tarfile.TarInfo('deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef.blob')
+                data = b'12345'
+                info.size = len(data)
+                tar.addfile(info, io.BytesIO(data))
+            outp = self.getTestOutp()
+            async with self.getTestAxon(dirn=os.path.join(testdir, 'axon')) as axon:
+                with mock.patch('tarfile.TarFile.extractfile', side_effect=OSError("simulated extraction error")):
+                    argv = ['--url', axon.getLocalUrl(), tarpath]
+                    self.eq(0, await axon_load.main(argv, outp=outp))
+            outp.expect('WARNING: Error extracting')
+
+    async def test_load_skipping_existing_blob(self):
+        with self.getTestDir() as testdir:
+            async with self.getTestAxon(dirn=os.path.join(testdir, 'axon')) as axon:
+                blob = b'foo'
+                size, sha2 = await axon.put(blob)
+                tarpath = os.path.join(testdir, 'test2.tar.gz')
+                with tarfile.open(tarpath, 'w:gz') as tar:
+                    info = tarfile.TarInfo(f'{s_common.ehex(sha2)}.blob')
+                    info.size = len(blob)
+                    tar.addfile(info, io.BytesIO(blob))
+                outp = self.getTestOutp()
+                argv = ['--url', axon.getLocalUrl(), tarpath]
+                self.eq(0, await axon_load.main(argv, outp=outp))
+                outp.expect('Skipping existing blob')