synapse 2.173.1__py311-none-any.whl → 2.174.0__py311-none-any.whl

synapse/models/crypto.py

@@ -371,31 +371,53 @@ class CryptoModule(s_module.CoreModule):
                 ('crypto:algorithm', {}, ()),
 
                 ('crypto:key', {}, (
+
                     ('algorithm', ('crypto:algorithm', {}), {
                         'ex': 'aes256',
                         'doc': 'The cryptographic algorithm which uses the key material.'}),
+
                     ('mode', ('str', {'lower': True, 'onespace': True}), {
                         'doc': 'The algorithm specific mode in use.'}),
+
                     ('iv', ('hex', {}), {
                         'doc': 'The hex encoded initialization vector.'}),
+
+                    ('iv:text', ('it:dev:str', {}), {
+                        'doc': 'Set only if the :iv property decodes to ASCII.'}),
+
                     ('public', ('hex', {}), {
                         'doc': 'The hex encoded public key material if the algorithm has a public/private key pair.'}),
+
+                    ('public:text', ('it:dev:str', {}), {
+                        'doc': 'Set only if the :public property decodes to ASCII.'}),
+
                     ('public:md5', ('hash:md5', {}), {
                         'doc': 'The MD5 hash of the public key in raw binary form.'}),
+
                     ('public:sha1', ('hash:sha1', {}), {
                         'doc': 'The SHA1 hash of the public key in raw binary form.'}),
+
                     ('public:sha256', ('hash:sha256', {}), {
                         'doc': 'The SHA256 hash of the public key in raw binary form.'}),
+
                     ('private', ('hex', {}), {
                         'doc': 'The hex encoded private key material. All symmetric keys are private.'}),
+
+                    ('private:text', ('it:dev:str', {}), {
+                        'doc': 'Set only if the :private property decodes to ASCII.'}),
+
                     ('private:md5', ('hash:md5', {}), {
                         'doc': 'The MD5 hash of the private key in raw binary form.'}),
+
                     ('private:sha1', ('hash:sha1', {}), {
                         'doc': 'The SHA1 hash of the private key in raw binary form.'}),
+
                     ('private:sha256', ('hash:sha256', {}), {
                         'doc': 'The SHA256 hash of the private key in raw binary form.'}),
+
                     ('seed:passwd', ('inet:passwd', {}), {
                         'doc': 'The seed password used to generate the key material.'}),
+
                     ('seed:algorithm', ('crypto:algorithm', {}), {
                         'ex': 'pbkdf2',
                         'doc': 'The algorithm used to generate the key from the seed password.'})

synapse/models/economic.py

@@ -48,11 +48,11 @@ class EconModule(s_module.CoreModule):
                     'doc': 'An invoice issued requesting payment.'}),
 
                 ('econ:price', ('hugenum', {'norm': False}), {
-                    'doc': 'The amount of money expected, required, or given in payment for something',
+                    'doc': 'The amount of money expected, required, or given in payment for something.',
                     'ex': '2.20'}),
 
                 ('econ:currency', ('str', {'lower': True, 'strip': False}), {
-                    'doc': 'The name of a system of money in general use',
+                    'doc': 'The name of a system of money in general use.',
                     'ex': 'usd'}),
 
                 ('econ:fin:exchange', ('guid', {}), {
@@ -99,6 +99,7 @@ class EconModule(s_module.CoreModule):
 
             'forms': (
 
+                ('econ:currency', {}, ()),
                 ('econ:pay:iin', {}, (
 
                     ('org', ('ou:org', {}), {
@@ -205,6 +206,9 @@ class EconModule(s_module.CoreModule):
                     ('fee', ('econ:price', {}), {
                         'doc': 'The transaction fee paid by the recipient to the payment processor.'}),
 
+                    ('from:cash', ('bool', {}), {
+                        'doc': 'Set to true if the payment input was in cash.'}),
+
                     ('from:account', ('econ:bank:account', {}), {
                         'doc': 'The bank account which made the payment.'}),
 
@@ -220,6 +224,9 @@ class EconModule(s_module.CoreModule):
                     ('from:contact', ('ps:contact', {}), {
                         'doc': 'Contact information for the entity making the payment.'}),
 
+                    ('to:cash', ('bool', {}), {
+                        'doc': 'Set to true if the payment output was in cash.'}),
+
                     ('to:account', ('econ:bank:account', {}), {
                         'doc': 'The bank account which received the payment.'}),
 
@@ -256,6 +263,20 @@ class EconModule(s_module.CoreModule):
                     ('receipt', ('econ:acct:receipt', {}), {
                         'doc': 'The receipt that was issued for the payment.'}),
 
+                    ('place', ('geo:place', {}), {
+                        'doc': 'The place where the payment occurred.'}),
+
+                    ('place:name', ('geo:name', {}), {
+                        'doc': 'The name of the place where the payment occurred.'}),
+
+                    ('place:address', ('geo:address', {}), {
+                        'doc': 'The address of the place where the payment occurred.'}),
+
+                    ('place:loc', ('loc', {}), {
+                        'doc': 'The loc of the place where the payment occurred.'}),
+
+                    ('place:latlong', ('geo:latlong', {}), {
+                        'doc': 'The latlong where the payment occurred.'}),
                 )),
 
                 ('econ:acct:balance', {}, (

synapse/models/entity.py

@@ -0,0 +1,16 @@
+import synapse.lib.module as s_module
+
+class EntityModule(s_module.CoreModule):
+
+    def getModelDefs(self):
+        return (('entity', {
+
+            'types': (
+                ('entity:name', ('str', {'onespace': True, 'lower': True}), {
+                    'doc': 'A name used to refer to an entity.'}),
+            ),
+
+            'forms': (
+                ('entity:name', {}, ()),
+            ),
+        }),)

synapse/models/files.py

@@ -412,7 +412,10 @@ class FileModule(s_module.CoreModule):
                     'doc': 'The GUID of the metadata pulled from a Windows shortcut or LNK file.',
                 }),
             ),
-
+            'edges': (
+                (('file:bytes', 'refs', 'it:dev:str'), {
+                    'doc': 'The source file contains the target string.'}),
+            ),
             'forms': (
 
                 ('file:bytes', {}, (

synapse/models/geopol.py

@@ -88,6 +88,9 @@ class PolModule(s_module.CoreModule):
 
                         ('vitals', ('pol:vitals', {}), {
                             'doc': 'The most recent known vitals for the country.'}),
+
+                        ('currencies', ('array', {'type': 'econ:currency', 'sorted': True, 'uniq': True}), {
+                            'doc': 'The official currencies used in the country.'}),
                     )),
                     ('pol:immigration:status:type:taxonomy', {}, ()),
                     ('pol:immigration:status', {}, (

synapse/models/orgs.py

@@ -731,8 +731,7 @@ class OuModule(s_module.CoreModule):
                     ('contact', ('ps:contact', {}), {
                         'doc': 'The contact info for the person who holds the position.',
                     }),
-                    # TODO migrate to ou:jobtitle
-                    ('title', ('str', {'lower': True, 'onespace': True}), {
+                    ('title', ('ou:jobtitle', {}), {
                         'doc': 'The title of the position.',
                     }),
                     ('reports', ('array', {'type': 'ou:position', 'uniq': True, 'sorted': True}), {
@@ -1024,11 +1023,11 @@ class OuModule(s_module.CoreModule):
                     ('sponsors', ('array', {'type': 'ps:contact', 'uniq': True, 'sorted': True}), {
                         'doc': 'An array of contacts which sponsored the conference.',
                     }),
-                    ('name', ('str', {'lower': True}), {
+                    ('name', ('entity:name', {}), {
                         'doc': 'The full name of the conference.',
                         'ex': 'defcon 2017'}),
 
-                    ('names', ('array', {'type': 'str', 'typeopts': {'lower': True}, 'uniq': True, 'sorted': True}), {
+                    ('names', ('array', {'type': 'entity:name', 'uniq': True, 'sorted': True}), {
                         'doc': 'An array of alternate names for the conference.'}),
 
                     ('desc', ('str', {'lower': True}), {

synapse/tests/test_cortex.py

@@ -269,6 +269,19 @@ class CortexTest(s_t_utils.SynTest):
                     self.eq(await core00.getJsonObj('foo/bar'), 'zoinks')
                     self.eq(await core01.getJsonObj('foo/bar'), 'zoinks')
 
+        # Test startup sequencing. We must create the child cells prior to
+        # the nexus recover() call from occuring :)
+        with self.getTestDir() as dirn:
+            async with self.getTestCore(dirn=dirn) as core:
+                await core.callStorm('$lib.jsonstor.set((path,), hehe)')
+
+            with self.getAsyncLoggerStream('synapse.lib.nexus') as stream:
+                async with self.getTestCore(dirn=dirn) as core:
+                    q = 'return( $lib.jsonstor.get((path,)) )'
+                    self.eq('hehe', await core.callStorm(q))
+            stream.seek(0)
+            self.notin('Exception while replaying log', stream.read())
+
     async def test_cortex_layer_mirror(self):
 
         # test a layer mirror from a layer

synapse/tests/test_lib_agenda.py

@@ -331,7 +331,17 @@ class AgendaTest(s_t_utils.SynTest):
                 appt = await agenda.get(guid)
                 self.eq(appt.isrunning, True)
                 self.eq(core.view.iden, appt.task.info.get('view'))
-                await appt.task.kill()
+
+                self.true(await core._killCronTask(guid))
+
+                events = [
+                    {'event': 'cron:stop', 'info': {'iden': appt.iden}},
+                ]
+
+                task = core.schedCoro(s_t_utils.waitForBehold(core, events))
+                await asyncio.wait_for(task, timeout=5)
+
+                self.false(await core._killCronTask(guid))
 
                 appt = await agenda.get(guid)
                 self.eq(appt.isrunning, False)
@@ -798,3 +808,121 @@ class AgendaTest(s_t_utils.SynTest):
                 stream.seek(0)
                 data = stream.read()
                 self.isin("_Appt.edits() Invalid attribute received: invalid = 'newp'", data)
+
+    async def test_cron_kill(self):
+        async with self.getTestCore() as core:
+
+            data = []
+            evt = asyncio.Event()
+
+            async def task():
+                async for mesg in core.behold():
+                    data.append(mesg)
+                    if mesg.get('event') == 'cron:stop':
+                        evt.set()
+
+            core.schedCoro(task())
+
+            q = '$q=$lib.queue.gen(test) for $i in $lib.range(60) { $lib.time.sleep(0.1) $q.put($i) }'
+            guid = s_common.guid()
+            cdef = {
+                'creator': core.auth.rootuser.iden, 'iden': guid,
+                'storm': q,
+                'reqs': {'now': True}
+            }
+            await core.addCronJob(cdef)
+
+            q = '$q=$lib.queue.gen(test) for $valu in $q.get((0), wait=(true)) { return ($valu) }'
+            valu = await core.callStorm(q)
+            self.eq(valu, 0)
+
+            opts = {'vars': {'iden': guid}}
+            get_cron = 'return($lib.cron.get($iden).pack())'
+            cdef = await core.callStorm(get_cron, opts=opts)
+            self.true(cdef.get('isrunning'))
+
+            self.true(await core.callStorm('return($lib.cron.get($iden).kill())', opts=opts))
+
+            self.true(await asyncio.wait_for(evt.wait(), timeout=12))
+
+            cdef = await core.callStorm(get_cron, opts=opts)
+            self.false(cdef.get('isrunning'))
+
+    async def test_cron_kill_pool(self):
+
+        async with self.getTestAhaProv() as aha:
+
+            import synapse.cortex as s_cortex
+            import synapse.lib.base as s_base
+
+            async with await s_base.Base.anit() as base:
+
+                with self.getTestDir() as dirn:
+
+                    dirn00 = s_common.genpath(dirn, 'cell00')
+                    dirn01 = s_common.genpath(dirn, 'cell01')
+
+                    core00 = await base.enter_context(self.addSvcToAha(aha, '00.core', s_cortex.Cortex, dirn=dirn00))
+                    provinfo = {'mirror': '00.core'}
+                    core01 = await base.enter_context(self.addSvcToAha(aha, '01.core', s_cortex.Cortex, dirn=dirn01, provinfo=provinfo))
+
+                    self.len(1, await core00.nodes('[inet:asn=0]'))
+                    await core01.sync()
+                    self.len(1, await core01.nodes('inet:asn=0'))
+
+                    msgs = await core00.stormlist('aha.pool.add pool00...')
+                    self.stormHasNoWarnErr(msgs)
+                    self.stormIsInPrint('Created AHA service pool: pool00.loop.vertex.link', msgs)
+
+                    msgs = await core00.stormlist('aha.pool.svc.add pool00... 01.core...')
+                    self.stormHasNoWarnErr(msgs)
+                    self.stormIsInPrint('AHA service (01.core...) added to service pool (pool00.loop.vertex.link)', msgs)
+
+                    msgs = await core00.stormlist('cortex.storm.pool.set --connection-timeout 1 --sync-timeout 1 aha://pool00...')
+                    self.stormHasNoWarnErr(msgs)
+                    self.stormIsInPrint('Storm pool configuration set.', msgs)
+
+                    await core00.stormpool.waitready(timeout=12)
+
+                    data = []
+                    evt = asyncio.Event()
+
+                    async def task():
+                        async for mesg in core00.behold():
+                            data.append(mesg)
+                            if mesg.get('event') == 'cron:stop':
+                                evt.set()
+
+                    core00.schedCoro(task())
+
+                    q = '$q=$lib.queue.gen(test) for $i in $lib.range(60) { $lib.time.sleep(0.1) $q.put($i) }'
+                    guid = s_common.guid()
+                    cdef = {
+                        'creator': core00.auth.rootuser.iden, 'iden': guid,
+                        'storm': q,
+                        'reqs': {'NOW': True},
+                        'pool': True,
+                    }
+                    await core00.addCronJob(cdef)
+
+                    q = '$q=$lib.queue.gen(test) for $valu in $q.get((0), wait=(true)) { return ($valu) }'
+                    valu = await core00.callStorm(q, opts={'mirror': False})
+                    self.eq(valu, 0)
+
+                    opts = {'vars': {'iden': guid}, 'mirror': False}
+                    get_cron = 'return($lib.cron.get($iden).pack())'
+                    cdef = await core00.callStorm(get_cron, opts=opts)
+                    self.true(cdef.get('isrunning'))
+
+                    self.true(await core00.callStorm('return($lib.cron.get($iden).kill())', opts=opts))
+
+                    self.true(await asyncio.wait_for(evt.wait(), timeout=12))
+
+                    cdef00 = await core00.callStorm(get_cron, opts=opts)
+                    self.false(cdef00.get('isrunning'))
+
+                    cdef01 = await core01.callStorm(get_cron, opts=opts)
+                    self.false(cdef01.get('isrunning'))
+                    self.eq(cdef01.get('lastresult'), 'cancelled')
+                    self.gt(cdef00['laststarttime'], 0)
+                    self.eq(cdef00['laststarttime'], cdef01['laststarttime'])

synapse/tests/test_lib_ast.py

@@ -356,6 +356,27 @@ class AstTest(s_test.SynTest):
             self.len(1, nodes)
             self.none(nodes[0].get('.seen'))
 
+            # array var filter
+            q = '''
+                [(test:arrayprop=* :strs=(neato, burrito))
+                 (test:arrayprop=* :ints=(1,2,3,4,5))] |
+                 $pvar = "strs"
+                 +:$pvar*[=neato]
+            '''
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq(('neato', 'burrito'), nodes[0].get('strs'))
+
+            q = '$pvar=ints $avar=4 test:arrayprop +:$pvar*[=$avar]'
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq((1, 2, 3, 4, 5), nodes[0].get('ints'))
+
+            q = '$pvar=strs $avar=burr test:arrayprop +:$pvar*[^=$avar]'
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq(('neato', 'burrito'), nodes[0].get('strs'))
+
             # Sad paths
             q = '[test:str=newp -.newp]'
             await self.asyncraises(s_exc.NoSuchProp, core.nodes(q))

synapse/tests/test_lib_grammar.py

@@ -721,6 +721,8 @@ Queries = [
     '$foo=({"bar": null})',
     '$p="names" ps:contact:name=foo [ :$p?+=bar ]',
     '$p="names" ps:contact:name=foo [ :$p?-=bar ]',
+    '$pvar=stuff test:arrayprop +:$pvar*[=neato]',
+    '$pvar=ints test:arrayprop +:$pvar*[=$othervar]',
 ]
 
 # Generated with print_parse_list below
@@ -1346,6 +1348,8 @@ _ParseResults = [
     'Query: [SetVarOper: [Const: foo, DollarExpr: [ExprDict: [Const: bar, Const: None]]]]',
     'Query: [SetVarOper: [Const: p, Const: names], LiftPropBy: [Const: ps:contact:name, Const: =, Const: foo], EditPropSet: [RelProp: [VarValue: [Const: p]], Const: ?+=, Const: bar]]',
     'Query: [SetVarOper: [Const: p, Const: names], LiftPropBy: [Const: ps:contact:name, Const: =, Const: foo], EditPropSet: [RelProp: [VarValue: [Const: p]], Const: ?-=, Const: bar]]',
+    'Query: [SetVarOper: [Const: pvar, Const: stuff], LiftProp: [Const: test:arrayprop], FiltOper: [Const: +, ArrayCond: [RelProp: [VarValue: [Const: pvar]], Const: =, Const: neato]]]',
+    'Query: [SetVarOper: [Const: pvar, Const: ints], LiftProp: [Const: test:arrayprop], FiltOper: [Const: +, ArrayCond: [RelProp: [VarValue: [Const: pvar]], Const: =, VarValue: [Const: othervar]]]]',
 ]
 
 class GrammarTest(s_t_utils.SynTest):

synapse/tests/test_lib_httpapi.py

@@ -916,6 +916,7 @@ class HttpApiTest(s_tests.SynTest):
                     await core.callStorm('$c = $lib.cron.get($cron) $c.set("doc", "some docs")', opts=opts)
                     await core.callStorm('cron.del $cron', opts=opts)
 
+                    await core.addStormPkg(spkg)
                     await core.addStormPkg(spkg)
                     await core.addStormSvc(ssvc)
 

synapse/tests/test_lib_lmdbslab.py

@@ -1,4 +1,5 @@
 import os
+import json
 import asyncio
 import pathlib
 import multiprocessing
@@ -350,7 +351,7 @@ class LmdbSlabTest(s_t_utils.SynTest):
         with self.getTestDir() as dirn, patch('synapse.lib.lmdbslab.Slab.WARN_COMMIT_TIME_MS', 1), \
                 patch('synapse.common.now', self.simplenow):
             path = os.path.join(dirn, 'test.lmdb')
-            with self.getAsyncLoggerStream('synapse.lib.lmdbslab', 'Commit with') as stream:
+            with self.getStructuredAsyncLoggerStream('synapse.lib.lmdbslab', 'Commit with') as stream:
                 async with await s_lmdbslab.Slab.anit(path, map_size=100000) as slab:
                     foo = slab.initdb('foo', dupsort=True)
                     byts = b'\x00' * 256
@@ -358,6 +359,20 @@ class LmdbSlabTest(s_t_utils.SynTest):
                         slab.put(b'\xff\xff\xff\xff' + s_common.guid(i).encode('utf8'), byts, db=foo)
                 self.true(await stream.wait(timeout=1))
 
+            data = stream.getvalue()
+            msgs = [json.loads(m) for m in data.split('\\n') if m]
+            self.gt(len(msgs), 0)
+            self.nn(msgs[0].get('delta'))
+            self.nn(msgs[0].get('path'))
+            self.nn(msgs[0].get('xactopslen'))
+            self.sorteq([
+                'vm.swappiness',
+                'vm.dirty_expire_centisecs',
+                'vm.dirty_writeback_centisecs',
+                'vm.dirty_background_ratio',
+                'vm.dirty_ratio',
+            ], msgs[0].get('sysctls', {}).keys())
+
     async def test_lmdbslab_max_replay(self):
         with self.getTestDir() as dirn:
             path = os.path.join(dirn, 'test.lmdb')

synapse/tests/test_lib_modelrev.py

@@ -473,3 +473,60 @@ class ModelRevTest(s_tests.SynTest):
             nodes = await core.nodes('it:mitre:attack:technique=T0100')
             self.len(1, nodes)
             self.eq('lockpicking', nodes[0].get('name'))
+
+    async def test_modelrev_0_2_25(self):
+        async with self.getRegrCore('model-0.2.25') as core:
+
+            self.len(1, await core.nodes('econ:currency=usd'))
+
+            nodes = await core.nodes('ou:conference')
+            self.len(3, nodes)
+            names = [n.get('name') for n in nodes]
+            self.sorteq(names, (
+                'sleuthcon',
+                'defcon',
+                'recon',
+            ))
+
+            namess = [n.get('names') for n in nodes]
+            self.sorteq(namess, (
+                ('defcon 2024',),
+                ('recon 2024 conference',),
+                ('sleuthcon 2024',),
+            ))
+
+            connames = (
+                'sleuthcon', 'sleuthcon 2024',
+                'defcon', 'defcon 2024',
+                'recon', 'recon 2024 conference',
+            )
+
+            nodes = await core.nodes('entity:name')
+            self.len(6, nodes)
+            names = [n.ndef[1] for n in nodes]
+            self.sorteq(names, connames)
+
+            nodes = await core.nodes('ou:conference -> entity:name')
+            self.len(6, nodes)
+            names = [n.ndef[1] for n in nodes]
+            self.sorteq(names, connames)
+
+            positions = (
+                'president of the united states',
+                'vice president of the united states',
+            )
+
+            nodes = await core.nodes('ou:position')
+            self.len(2, nodes)
+            titles = [n.get('title') for n in nodes]
+            self.sorteq(titles, positions)
+
+            nodes = await core.nodes('ou:jobtitle')
+            self.len(2, nodes)
+            titles = [n.ndef[1] for n in nodes]
+            self.sorteq(titles, positions)
+
+            nodes = await core.nodes('ou:position -> ou:jobtitle')
+            self.len(2, nodes)
+            titles = [n.ndef[1] for n in nodes]
+            self.sorteq(titles, positions)

synapse/tests/test_lib_msgpack.py

@@ -17,6 +17,50 @@ class MsgPackTest(s_t_utils.SynTest):
         byts = s_msgpack._fallback_en(('hehe', 10))
         self.eq(byts, b'\x92\xa4hehe\n')
 
+    def test_msgpack_ext(self):
+        valu = 0xffffffffffffffffffffffffffffffff
+        item = ('woot', valu)
+        byts = s_msgpack.en(item)
+        self.eq(item, s_msgpack.un(byts))
+        self.eq(byts, s_msgpack._fallback_en(item))
+
+        unpk = s_msgpack.Unpk()
+        self.eq(((24, item),), unpk.feed(byts))
+        with self.raises(s_exc.SynErr):
+            s_msgpack._ext_un(99, b'red baloons')
+
+        # Negative number support as well.
+        negvalu = -1 * valu
+        negitem = ('woot', negvalu)
+        negbytes = s_msgpack.en(negitem)
+        self.eq(negitem, s_msgpack.un(negbytes))
+        self.eq(negbytes, s_msgpack._fallback_en(negitem))
+
+        # Check across item.bit_length() boundaries
+        v = 0xffffffffffffffff
+        for i in (1, 0xffffffffffffffff + 1, 0xffffffffffffffff + 2):
+            nv = v + i
+            buf = s_msgpack.en(nv)
+            self.eq(nv, s_msgpack.un(buf))
+        v = -0x8000000000000000
+        for i in (1, 0x7fffffffffffffff, 0x7fffffffffffffff + 1):
+            nv = v - i
+            buf = s_msgpack.en(nv)
+            self.eq(nv, s_msgpack.un(buf))
+
+        # We can also support values > 128 bits in width
+        valu = 0xfffffffffffffffffffffffffffffffff
+        item = ('woot', valu)
+        byts = s_msgpack.en(item)
+        self.eq(item, s_msgpack.un(byts))
+        self.eq(byts, s_msgpack._fallback_en(item))
+
+        negvalu = -1 * valu
+        negitem = ('woot', negvalu)
+        negbytes = s_msgpack.en(negitem)
+        self.eq(negitem, s_msgpack.un(negbytes))
+        self.eq(negbytes, s_msgpack._fallback_en(negitem))
+
     def test_msgpack_un(self):
         item = s_msgpack.un(b'\x92\xa4hehe\n')
         self.eq(item, ('hehe', 10))
@@ -106,8 +150,8 @@ class MsgPackTest(s_t_utils.SynTest):
         self.checkLoadfile(s_msgpack._fallback_en)
 
     def checkTypes(self, enfunc):
-        # This is a future-proofing test for msgpack to ensure that
-        buf = b'\x92\xa4hehe\x85\xa3str\xa41234\xa3int\xcd\x04\xd2\xa5float\xcb@(\xae\x14z\xe1G\xae\xa3bin\xc4\x041234\xa9realworld\xac\xc7\x8b\xef\xbf\xbd\xed\xa1\x82\xef\xbf\xbd\x12'
+        # This is a future-proofing test for msgpack to ensure that we have stability with msgpack-python
+        buf = b'\x92\xa4hehe\x8b\xa3str\xa41234\xa3int\xcd\x04\xd2\xa5float\xcb@(\xae\x14z\xe1G\xae\xa3bin\xc4\x041234\xa9realworld\xac\xc7\x8b\xef\xbf\xbd\xed\xa1\x82\xef\xbf\xbd\x12\xabalmostlarge\xcf\xff\xff\xff\xff\xff\xff\xff\xfe\xb1extlargeThreshold\xcf\xff\xff\xff\xff\xff\xff\xff\xff\xa8extlarge\xc7\t\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\xabalmostsmall\xd3\x80\x00\x00\x00\x00\x00\x00\x01\xb4almostsmallThreshold\xd3\x80\x00\x00\x00\x00\x00\x00\x00\xa8extsmall\xc7\t\x01\xff\x7f\xff\xff\xff\xff\xff\xff\xff'
         struct = (
             'hehe',
             {
@@ -115,7 +159,15 @@ class MsgPackTest(s_t_utils.SynTest):
                 'int': 1234,
                 'float': 12.34,
                 'bin': b'1234',
-                'realworld': '\u01cb\ufffd\ud842\ufffd\u0012'
+                'realworld': '\u01cb\ufffd\ud842\ufffd\u0012',
+                'almostlarge': 0xffffffffffffffff - 1,
+                'extlargeThreshold': 0xffffffffffffffff,
+                # extlarge is handled with our custom extension type
+                'extlarge': 0xffffffffffffffff + 1,
+                'almostsmall': -0x8000000000000000 + 1,
+                'almostsmallThreshold': -0x8000000000000000,
+                # extsmall is handled with our custom extension type
+                'extsmall': -0x8000000000000000 - 1,
             }
         )
         unode = s_msgpack.un(buf)
@@ -135,7 +187,7 @@ class MsgPackTest(s_t_utils.SynTest):
         unpk = s_msgpack.Unpk()
         objs = unpk.feed(buf)
         self.len(1, objs)
-        self.eq(objs[0], (71, struct))
+        self.eq(objs[0], (212, struct))
 
         # Generic isok helper
         self.true(s_msgpack.isok(1))
@@ -148,6 +200,8 @@ class MsgPackTest(s_t_utils.SynTest):
         self.true(s_msgpack.isok([1]))
         self.true(s_msgpack.isok((1,)))
         self.true(s_msgpack.isok({1: 1}))
+        self.true(s_msgpack.isok(0xffffffffffffffff + 1))
+        self.true(s_msgpack.isok(-0x8000000000000000 - 1))
         # unpackage types
         self.false(s_msgpack.isok({1, 2}))  # set
         self.false(s_msgpack.isok(print))  # function
@@ -198,10 +252,6 @@ class MsgPackTest(s_t_utils.SynTest):
         self.raises(s_exc.NotMsgpackSafe, enfunc, {1, 2})
         self.raises(s_exc.NotMsgpackSafe, enfunc, Exception())
         self.raises(s_exc.NotMsgpackSafe, enfunc, s_msgpack.en)
-        # too long
-        with self.raises(s_exc.NotMsgpackSafe) as cm:
-            enfunc({'longlong': 45234928034723904723906})
-        self.isin('OverflowError', cm.exception.get('mesg'))
 
     def test_msgpack_bad_types(self):
         self.checkBadTypes(s_msgpack.en)

synapse/tests/test_lib_nexus.py

@@ -3,7 +3,6 @@ from unittest import mock
 
 import synapse.exc as s_exc
 import synapse.common as s_common
-import synapse.cortex as s_cortex
 
 import synapse.lib.cell as s_cell
 import synapse.lib.nexus as s_nexus
@@ -302,3 +301,47 @@ class NexusTest(s_t_utils.SynTest):
                     with self.raises(s_exc.IsReadOnly):
                         await cell01.sync()
                     self.isin(s_nexus.leaderversion, cell01.nexsroot.writeholds)
+
+    async def test_mirror_nexus_loop_failure(self):
+        with self.getTestDir() as dirn:
+
+            s_common.yamlsave({'nexslog:en': True}, dirn, 'cell.yaml')
+            async with await s_cell.Cell.anit(dirn=dirn) as cell00:
+
+                await cell00.runBackup(name='cell01')
+
+                path = s_common.genpath(dirn, 'backups', 'cell01')
+
+                conf = s_common.yamlload(path, 'cell.yaml')
+                conf['mirror'] = f'cell://{dirn}'
+                s_common.yamlsave(conf, path, 'cell.yaml')
+
+                seen = False
+                restarted = False
+                orig = s_nexus.NexsRoot.delWriteHold
+
+                # Patch NexsRoot.delWriteHold so we can cause an exception in
+                # the setup part of the nexus loop (NexsRoot.runMirrorLoop). The
+                # exception should only happen one time so we can check that the
+                # proxy and the nexus loop were both restarted
+                async def delWriteHold(self, reason):
+                    nonlocal seen
+                    nonlocal restarted
+                    if not seen:
+                        seen = True
+                        raise Exception('Knock over the nexus setup.')
+
+                    restarted = True
+                    return await orig(self, reason)
+
+                with mock.patch('synapse.lib.nexus.NexsRoot.delWriteHold', delWriteHold):
+                    with self.getLoggerStream('synapse.lib.nexus') as stream:
+                        async with await s_cell.Cell.anit(dirn=path) as cell01:
+                            await cell01.sync()
+
+                    stream.seek(0)
+                    data = stream.read()
+                    mesg = 'Unknown error during mirror loop startup: Knock over the nexus setup.'
+                    self.isin(mesg, data)
+
+                self.true(restarted)