synapse 2.193.0__py311-none-any.whl → 2.194.0__py311-none-any.whl

synapse/models/person.py

@@ -254,6 +254,7 @@ class PsModule(s_module.CoreModule):
                         'doc': 'The most recent known vitals for the person.',
                     }),
                     ('name', ('ps:name', {}), {
+                        'alts': ('names',),
                         'doc': 'The localized name for the person.',
                     }),
                     ('name:sur', ('ps:tokn', {}), {
@@ -350,12 +351,17 @@ class PsModule(s_module.CoreModule):
                         'doc': 'The most recent known vitals for the contact.',
                     }),
                     ('name', ('ps:name', {}), {
-                        'doc': 'The person name listed for the contact.',
-                    }),
+                        'alts': ('names',),
+                        'doc': 'The person name listed for the contact.'}),
+
+                    ('bio', ('str', {}), {
+                        'doc': 'A brief bio provided for the contact.'}),
+
                     ('desc', ('str', {}), {
-                        'doc': 'A description of this contact.',
-                    }),
+                        'doc': 'A description of this contact.'}),
+
                     ('title', ('ou:jobtitle', {}), {
+                        'alts': ('titles',),
                         'doc': 'The job/org title listed for this contact.'}),
 
                     ('titles', ('array', {'type': 'ou:jobtitle', 'sorted': True, 'uniq': True}), {
@@ -365,12 +371,14 @@ class PsModule(s_module.CoreModule):
                         'doc': 'The photo listed for this contact.',
                     }),
                     ('orgname', ('ou:name', {}), {
+                        'alts': ('orgnames',),
                         'doc': 'The listed org/company name for this contact.',
                     }),
                     ('orgfqdn', ('inet:fqdn', {}), {
                         'doc': 'The listed org/company FQDN for this contact.',
                     }),
                     ('user', ('inet:user', {}), {
+                        'alts': ('users',),
                         'doc': 'The username or handle for this contact.'}),
 
                     ('service:accounts', ('array', {'type': 'inet:service:account', 'sorted': True, 'uniq': True}), {
@@ -412,6 +420,7 @@ class PsModule(s_module.CoreModule):
                         'doc': 'The home or main site for this contact.',
                     }),
                     ('email', ('inet:email', {}), {
+                        'alts': ('emails',),
                         'doc': 'The main email address for this contact.',
                     }),
                     ('email:work', ('inet:email', {}), {
@@ -440,6 +449,7 @@ class PsModule(s_module.CoreModule):
                         'doc': 'The work phone number for this contact.',
                     }),
                     ('id:number', ('ou:id:number', {}), {
+                        'alts': ('id:numbers',),
                         'doc': 'An ID number issued by an org and associated with this contact.',
                     }),
                     ('adid', ('it:adid', {}), {
@@ -479,6 +489,7 @@ class PsModule(s_module.CoreModule):
                     }),
 
                     ('lang', ('lang:language', {}), {
+                        'alts': ('langs',),
                         'doc': 'The language specified for the contact.'}),
 
                     ('langs', ('array', {'type': 'lang:language'}), {

synapse/models/risk.py

@@ -310,6 +310,7 @@ class RiskModule(s_module.CoreModule):
                         'doc': "The reporting organization's assessed location of the threat cluster."}),
 
                     ('org:name', ('ou:name', {}), {
+                        'alts': ('org:names',),
                         'ex': 'apt1',
                         'doc': "The reporting organization's name for the threat cluster."}),
 
@@ -383,6 +384,7 @@ class RiskModule(s_module.CoreModule):
                         'doc': 'The authoritative software family for the tool.'}),
 
                     ('soft:name', ('it:prod:softname', {}), {
+                        'alts': ('soft:names',),
                         'doc': 'The reporting organization\'s name for the tool.'}),
 
                     ('soft:names', ('array', {'type': 'it:prod:softname', 'uniq': True, 'sorted': True}), {
@@ -441,6 +443,7 @@ class RiskModule(s_module.CoreModule):
                 ('risk:vuln', {}, (
 
                     ('name', ('risk:vulnname', {}), {
+                        'alts': ('names',),
                         'doc': 'A user specified name for the vulnerability.'}),
 
                     ('names', ('array', {'type': 'risk:vulnname', 'sorted': True, 'uniq': True}), {

synapse/models/telco.py

@@ -369,9 +369,16 @@ class TelcoModule(s_module.CoreModule):
                     ('wifi:bssid', ('inet:mac', {}), {}),
 
                     # host specific data
-                    ('adid', ('it:adid', {}), {}),
-                    ('aaid', ('it:os:android:aaid', {}), {}),
-                    ('idfa', ('it:os:ios:idfa', {}), {}),
+                    ('adid', ('it:adid', {}), {
+                        'doc': 'The advertising ID of the mobile telemetry sample.'}),
+
+                    ('aaid', ('it:os:android:aaid', {}), {
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use :adid.'}),
+
+                    ('idfa', ('it:os:ios:idfa', {}), {
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use :adid.'}),
 
                     # User related data
                     ('name', ('ps:name', {}), {}),

synapse/tests/test_axon.py

@@ -990,12 +990,12 @@ bar baz",vv
             self.isinstance(resp.get('err'), tuple)
 
             q = f'''
-            $fields = $lib.list(
-                ({{'name':'file', 'sha256':$sha256, 'filename':'file'}}),
-                ({{'name':'zip_password', 'value':'test'}}),
-                ({{'name':'dict', 'value':({{'foo':'bar'}}) }}),
-                ({{'name':'bytes', 'value':$bytes}})
-            )
+            $fields = ([
+                {{'name':'file', 'sha256':$sha256, 'filename':'file'}},
+                {{'name':'zip_password', 'value':'test'}},
+                {{'name':'dict', 'value':{{'foo':'bar'}} }},
+                {{'name':'bytes', 'value':$bytes}}
+            ])
             $resp = $lib.inet.http.post("https://127.0.0.1:{port}/api/v1/pushfile",
                                         fields=$fields, ssl_verify=(0))
             return($resp)

synapse/tests/test_cortex.py

@@ -440,7 +440,7 @@ class CortexTest(s_t_utils.SynTest):
                  'interfaces': ['lookup'],
                  'storm': '''
                      function lookup(tokens) {
-                        $looks = $lib.list()
+                        $looks = ()
                         for $token in $tokens { $looks.append( (inet:fqdn, $token) ) }
                         return($looks)
                      }
@@ -679,7 +679,7 @@ class CortexTest(s_t_utils.SynTest):
             # functions that don't return a generator
             storm = '''
             function x() {
-                $lst = $lib.list()
+                $lst = ()
                 [ ou:org=* ou:org=* +#cat ]
                 $lst.append($node)
                 fini { return($lst) }
@@ -692,7 +692,7 @@ class CortexTest(s_t_utils.SynTest):
 
             storm = '''
             function x() {
-                $lst = $lib.list()
+                $lst = ()
                 [ ou:org=* ou:org=* +#dog ]
                 $lst.append($node)
                 fini { return($lst) }
@@ -974,7 +974,7 @@ class CortexTest(s_t_utils.SynTest):
                 await core.nodes('media:news -(refs)> $(10)')
 
             self.eq(1, await core.callStorm('''
-                $list = $lib.list()
+                $list = ()
                 for $edge in $lib.view.get().getEdges() { $list.append($edge) }
                 return($list.size())
             '''))
@@ -982,7 +982,7 @@ class CortexTest(s_t_utils.SynTest):
             # check that auto-deleting a node's edges works
             await core.nodes('media:news | delnode')
             self.eq(0, await core.callStorm('''
-                $list = $lib.list()
+                $list = ()
                 for $edge in $lib.view.get().getEdges() { $list.append($edge) }
                 return($list.size())
             '''))
@@ -1017,7 +1017,7 @@ class CortexTest(s_t_utils.SynTest):
                 self.eq(('1', '2'), retn)
 
                 with self.raises(s_exc.StormRuntimeError):
-                    q = '$foo=$lib.list() $bar=$foo.index(10) return ( $bar )'
+                    q = '$foo=() $bar=$foo.index(10) return ( $bar )'
                     await proxy.callStorm(q)
 
                 with self.raises(s_exc.SynErr) as cm:
@@ -1033,6 +1033,26 @@ class CortexTest(s_t_utils.SynTest):
                 self.eq(cm.exception.get('hehe'), 'haha')
                 self.eq(cm.exception.get('key'), 1)
 
+                # We convert StormLoopCtrl and StormGenrCtrl into StormRuntimeError
+                opts = {'vars': {'i': 2}}
+                q = 'if ($i = 2) { break }'
+                with self.raises(s_exc.StormRuntimeError) as cm:
+                    await core.callStorm(q, opts=opts)
+                self.eq(cm.exception.get('mesg'),
+                        'Loop control statement "break" used outside of a loop.')
+
+                q = 'if ($i = 2) { continue }'
+                with self.raises(s_exc.StormRuntimeError) as cm:
+                    await core.callStorm(q, opts=opts)
+                self.eq(cm.exception.get('mesg'),
+                        'Loop control statement "continue" used outside of a loop.')
+
+                q = 'if ($i = 2) { stop }'
+                with self.raises(s_exc.StormRuntimeError) as cm:
+                    await core.callStorm(q, opts=opts)
+                self.eq(cm.exception.get('mesg'),
+                        'Generator control statement "stop" used outside of a generator function.')
+
             with self.getAsyncLoggerStream('synapse.lib.view', 'callStorm cancelled') as stream:
                 async with core.getLocalProxy() as proxy:
 
@@ -1069,7 +1089,7 @@ class CortexTest(s_t_utils.SynTest):
                     self.eq('ok', retn.get('status'))
                     self.eq('asdf', retn['result'])
 
-                body = {'query': '$foo=$lib.list() $bar=$foo.index(10) return ( $bar )'}
+                body = {'query': '$foo=() $bar=$foo.index(10) return ( $bar )'}
                 async with sess.get(f'https://localhost:{port}/api/v1/storm/call', json=body) as resp:
                     retn = await resp.json()
                     self.eq('err', retn.get('status'))
@@ -1081,7 +1101,7 @@ class CortexTest(s_t_utils.SynTest):
                     retn = await resp.json()
                     self.eq('err', retn.get('status'))
                     self.eq('StormExit', retn.get('code'))
-                    self.eq('', retn.get('mesg'))
+                    self.eq('StormExit: ', retn.get('mesg'))
 
                 # No body
                 async with sess.get(f'https://localhost:{port}/api/v1/storm/call') as resp:
@@ -1760,7 +1780,7 @@ class CortexTest(s_t_utils.SynTest):
             self.len(1, nodes)
             self.eq(set(nodes[0].tags.keys()), {'foo', 'bar', 'bar.baz'})
 
-            nodes = await wcore.nodes('$foo=$lib.list("foo", "bar.baz") [test:int=4 +#$foo]')
+            nodes = await wcore.nodes('$foo=(["foo", "bar.baz"]) [test:int=4 +#$foo]')
             self.len(1, nodes)
             self.eq(set(nodes[0].tags.keys()), {'foo', 'bar', 'bar.baz'})
 
@@ -3499,6 +3519,97 @@ class CortexBasicTest(s_t_utils.SynTest):
             for node in nodes:
                 self.nn(node.getTag('hehe'))
 
+            # Break and Continue cannot cross function boundaries and will instead raise a catchable StormRuntimeError
+            keywords = ('break', 'continue')
+            base_func_q = '''
+            function inner(v) {
+                if ( $v = 2 ) {
+                    KEYWORD
+                }
+                return ( $v )
+            }
+            $N = (5)
+
+            for $valu in $lib.range($N) {
+                $lib.print(`{$inner($valu)}/{$N}`)
+            }
+            '''
+            func_catch_q = '''
+            function inner(v) {
+                if ( $v = 2 ) {
+                    KEYWORD
+                }
+                return ( $v )
+            }
+            $N = (5)
+            try {
+                for $valu in $lib.range($N) {
+                    $lib.print(`{$inner($valu)}/{$N}`)
+                }
+            } catch StormRuntimeError as err {
+                $lib.print(`caught: {$err.mesg}`)
+            }
+            '''
+            for keyword in keywords:
+                q = base_func_q.replace('KEYWORD', keyword)
+                msgs = await core.stormlist(q)
+                self.stormIsInPrint('1/5', msgs)
+                self.stormNotInPrint('2/5', msgs)
+                self.stormIsInErr(f'function inner - Loop control statement "{keyword}" used outside of a loop.',
+                                  msgs)
+
+                q = func_catch_q.replace('KEYWORD', keyword)
+                msgs = await core.stormlist(q)
+                self.stormIsInPrint('1/5', msgs)
+                self.stormNotInPrint('2/5', msgs)
+                self.stormIsInPrint(f'function inner - Loop control statement "{keyword}" used outside of a loop.',
+                                    msgs)
+
+            # The toplevel use of the keywords will convert them into StormRuntimeError in the message stream
+            # but prevent them from being caught.
+            base_top_q = '''
+            $N = (5)
+            for $j in $lib.range($N) {
+                if ($j = 2) { break }
+                $lib.print(`{$j}/{$N}`)
+            }
+            if ($j = 2) {
+                KEYWORD
+            }
+            '''
+            top_catch_q = '''
+            $N = (5)
+            for $j in $lib.range($N) {
+                if ($j = 2) { break }
+                $lib.print(`{$j}/{$N}`)
+            }
+            try {
+                if ($j = 2) {
+                    KEYWORD
+                }
+            } catch StormRuntimeError as err {
+                $lib.print(`caught: {$err.mesg}`)
+            }
+            '''
+            for keyword in keywords:
+                q = base_top_q.replace('KEYWORD', keyword)
+                msgs = await core.stormlist(q)
+                self.stormIsInPrint('1/5', msgs)
+                self.stormNotInPrint('2/5', msgs)
+                self.stormIsInErr(f'Loop control statement "{keyword}" used outside of a loop.',
+                                  msgs)
+                errname = [m[1][0] for m in msgs if m[0] == 'err'][0]
+                self.eq(errname, 'StormRuntimeError')
+
+                q = top_catch_q.replace('KEYWORD', keyword)
+                msgs = await core.stormlist(q)
+                self.stormIsInPrint('1/5', msgs)
+                self.stormNotInPrint('2/5', msgs)
+                self.stormIsInErr(f'Loop control statement "{keyword}" used outside of a loop.',
+                                    msgs)
+                errname = [m[1][0] for m in msgs if m[0] == 'err'][0]
+                self.eq(errname, 'StormRuntimeError')
+
     async def test_storm_varcall(self):
 
         async with self.getTestCore() as core:
@@ -5119,6 +5230,14 @@ class CortexBasicTest(s_t_utils.SynTest):
             self.eq(('inet:fqdn', 'nest.com'), nodes[0].ndef)
             self.eq(('inet:fqdn', 'nest.com'), nodes[1].ndef)
 
+            with self.raises(s_exc.StormRuntimeError) as err:
+                await core.nodes('[ it:dev:int=1 ] for $n in $node.value() { }')
+            self.isin("'int' object is not iterable: 1", err.exception.errinfo.get('mesg'))
+
+            with self.raises(s_exc.StormRuntimeError) as err:
+                await core.nodes('for $n in { .created return($node) } { }')
+            self.isin("'node' object is not iterable", err.exception.errinfo.get('mesg'))
+
     async def test_storm_whileloop(self):
 
         async with self.getTestCore() as core:
@@ -5460,7 +5579,7 @@ class CortexBasicTest(s_t_utils.SynTest):
             self.len(1, nodes)
 
     async def test_storm_order(self):
-        q = '''[test:str=foo :hehe=bar] $tvar=$lib.text() $tvar.add(1) $tvar.add(:hehe) $lib.print($tvar.str()) '''
+        q = '''[test:str=foo :hehe=bar] $tvar=() $tvar.append(1) $tvar.append(:hehe) $lib.print($lib.str.join('', $tvar)) '''
         async with self.getTestCore() as core:
             mesgs = await core.stormlist(q)
             self.stormIsInPrint('1bar', mesgs)
@@ -5684,7 +5803,7 @@ class CortexBasicTest(s_t_utils.SynTest):
 
                         url02 = core02.getLocalUrl()
                         opts = {'vars': {'url01': url01, 'url02': url02}}
-                        strim = 'return($lib.cell.trimNexsLog(consumers=$lib.list($url01, $url02), timeout=$lib.null))'
+                        strim = 'return($lib.cell.trimNexsLog(consumers=($url01, $url02), timeout=$lib.null))'
 
                         await core00.nodes('[ inet:ipv4=11.0.0.0/28 ]')
                         ips00 = await core00.count('inet:ipv4')

synapse/tests/test_exc.py

@@ -16,6 +16,7 @@ class ExcTest(s_t_utils.SynTest):
     def test_basic(self):
         e = s_exc.SynErr(mesg='words', foo='bar')
         self.eq(e.get('foo'), 'bar')
+        self.eq(e.items(), {'mesg': 'words', 'foo': 'bar'})
         self.eq("SynErr: foo='bar' mesg='words'", str(e))
         e.set('hehe', 1234)
         e.set('foo', 'words')

synapse/tests/test_lib_agenda.py

@@ -441,7 +441,7 @@ class AgendaTest(s_t_utils.SynTest):
                 await self.asyncraises(s_exc.DupIden, core.addCronJob(cdef))
                 await core.delCronJob(viewiden)
 
-                self.nn(core.getAuthGate(viewiden))
+                self.nn(await core.getAuthGate(viewiden))
 
     async def test_agenda_persistence(self):
         ''' Test we can make/change/delete appointments and they are persisted to storage '''
@@ -1097,3 +1097,127 @@ class AgendaTest(s_t_utils.SynTest):
                     self.eq(cdef01.get('lastresult'), 'cancelled')
                     self.gt(cdef00['laststarttime'], 0)
                     self.eq(cdef00['laststarttime'], cdef01['laststarttime'])
+
+    async def test_agenda_graceful_promotion_with_running_cron(self):
+
+        async with self.getTestAha() as aha:
+
+            conf00 = {
+                'aha:provision': await aha.addAhaSvcProv('00.cortex')
+            }
+
+            async with self.getTestCore(conf=conf00) as core00:
+                self.false(core00.conf.get('mirror'))
+
+                q = '''
+                while((true)) {
+                    $lib.log.error('I AM A ERROR LOG MESSAGE')
+                    $lib.time.sleep(6)
+                }
+                '''
+                msgs = await core00.stormlist('cron.at --now $q', opts={'vars': {'q': q}})
+                self.stormHasNoWarnErr(msgs)
+
+                crons00 = await core00.callStorm('return($lib.cron.list())')
+                self.len(1, crons00)
+
+                prov01 = {'mirror': '00.cortex'}
+                conf01 = {
+                    'aha:provision': await aha.addAhaSvcProv('01.cortex', provinfo=prov01),
+                }
+
+                async with self.getTestCore(conf=conf01) as core01:
+
+                    with self.getAsyncLoggerStream('synapse.storm.log', 'I AM A ERROR LOG MESSAGE') as stream:
+                        self.true(await stream.wait(timeout=6))
+
+                    cron = await core00.callStorm('return($lib.cron.list())')
+                    self.len(1, cron)
+                    self.true(cron[0].get('isrunning'))
+
+                    await core01.promote(graceful=True)
+
+                    self.false(core00.isactive)
+                    self.true(core01.isactive)
+
+                    await core00.sync()
+
+                    cron00 = await core00.callStorm('return($lib.cron.list())')
+                    self.len(1, cron00)
+                    self.false(cron00[0].get('isrunning'))
+                    self.eq(cron00[0].get('lasterrs')[0], 'aborted')
+
+                    cron01 = await core01.callStorm('return($lib.cron.list())')
+                    self.len(1, cron01)
+                    self.false(cron01[0].get('isrunning'))
+                    self.eq(cron01[0].get('lasterrs')[0], 'aborted')
+
+    async def test_agenda_force_promotion_with_running_cron(self):
+
+        async with self.getTestAha() as aha:
+
+            conf00 = {
+                'aha:provision': await aha.addAhaSvcProv('00.cortex')
+            }
+
+            async with self.getTestCore(conf=conf00) as core00:
+                self.false(core00.conf.get('mirror'))
+
+                q = '''
+                while((true)) {
+                    $lib.log.error('I AM A ERROR LOG MESSAGE')
+                    $lib.time.sleep(6)
+                }
+                '''
+                msgs = await core00.stormlist('cron.at --now $q', opts={'vars': {'q': q}})
+                self.stormHasNoWarnErr(msgs)
+
+                crons00 = await core00.callStorm('return($lib.cron.list())')
+                self.len(1, crons00)
+
+                prov01 = {'mirror': '00.cortex'}
+                conf01 = {
+                    'aha:provision': await aha.addAhaSvcProv('01.cortex', provinfo=prov01),
+                }
+
+                async with self.getTestCore(conf=conf01) as core01:
+
+                    cron = await core00.callStorm('return($lib.cron.list())')
+                    self.len(1, cron)
+                    self.true(cron[0].get('isrunning'))
+
+                    await core01.promote(graceful=False)
+
+                    self.true(core00.isactive)
+                    self.true(core01.isactive)
+
+                    cron01 = await core01.callStorm('return($lib.cron.list())')
+                    self.len(1, cron01)
+                    self.false(cron01[0].get('isrunning'))
+                    self.eq(cron01[0].get('lasterrs')[0], 'aborted')
+
+    async def test_agenda_clear_running_none_nexttime(self):
+
+        async with self.getTestCore() as core:
+
+            cdef = {
+                'creator': core.auth.rootuser.iden,
+                'iden': s_common.guid(),
+                'storm': '$lib.log.info("test")',
+                'reqs': {},
+                'incunit': 'minute',
+                'incvals': 1
+            }
+            await core.addCronJob(cdef)
+
+            appt = core.agenda.appts[cdef['iden']]
+            self.true(appt in core.agenda.apptheap)
+
+            appt.isrunning = True
+            appt.nexttime = None
+
+            await core.agenda.clearRunningStatus()
+            self.false(appt in core.agenda.apptheap)
+
+            crons = await core.callStorm('return($lib.cron.list())')
+            self.len(1, crons)

synapse/tests/test_lib_aha.py

@@ -59,7 +59,7 @@ class AhaTest(s_test.SynTest):
                     self.len(ahacount, await proxy0.getAhaUrls())
                     self.len(ahacount, await proxy0.getAhaServers())
 
-                    purl = await proxy0.addAhaClone(zoinks)
+                    purl = await proxy0.addAhaClone(zoinks, port=0)
 
                 conf1 = {'clone': purl}
                 async with self.getTestAha(conf=conf1, dirn=dir1) as aha1:
@@ -574,7 +574,7 @@ class AhaTest(s_test.SynTest):
                 }
                 s_common.yamlsave(axonconf, axonpath, 'cell.yaml')
 
-                argv = (axonpath, '--auth-passwd', 'rootbeer')
+                argv = (axonpath, '--auth-passwd', 'rootbeer', '--https', '0')
                 async with await s_axon.Axon.initFromArgv(argv) as axon:
 
                     # opts were copied through successfully
@@ -1161,7 +1161,8 @@ class AhaTest(s_test.SynTest):
                     aconf = {
                         'aha:name': 'aha',
                         'aha:network': networkname,
-                        'provision:listen': f'ssl://aha.{networkname}:0'
+                        'dmon:listen': f'ssl://aha.{networkname}:0',
+                        'provision:listen': f'ssl://aha.{networkname}:0',
                     }
                     name = aconf.get('aha:name')
                     netw = aconf.get('aha:network')
@@ -1300,11 +1301,17 @@ class AhaTest(s_test.SynTest):
         conf = {
             'aha:network': 'synapse',
             'dns:name': 'here.loop.vertex.link',
+            'dmon:listen': 'ssl://0.0.0.0:0?hostname=here.loop.vertex.link&ca=synapse',
         }
-        mesg = 'provision listening: ssl://0.0.0.0:27272?hostname=here.loop.vertex.link'
-        with self.getAsyncLoggerStream('synapse.lib.aha', mesg) as stream:
+
+        orig = s_aha.AhaCell._getProvListen
+        def _getProvListen(_self):
+            ret = orig(_self)
+            self.eq(ret, 'ssl://0.0.0.0:27272?hostname=here.loop.vertex.link')
+            return 'ssl://0.0.0.0:0?hostname=here.loop.vertex.link'
+
+        with mock.patch('synapse.lib.aha.AhaCell._getProvListen', _getProvListen):
             async with self.getTestCell(s_aha.AhaCell, conf=conf) as aha:
-                self.true(await stream.wait(timeout=6))
                 # And the URL works with our listener :)
                 provurl = await aha.addAhaUserEnroll('bob.grey')
                 async with await s_telepath.openurl(provurl) as prox: