synapse 2.165.0__py311-none-any.whl → 2.166.0__py311-none-any.whl

synapse/tests/test_lib_stormlib_spooled.py

@@ -0,0 +1,190 @@
+import synapse.exc as s_exc
+import synapse.lib.stormtypes as s_stormtypes
+
+import synapse.tests.utils as s_test
+
+class StormlibSpooledTest(s_test.SynTest):
+    async def test_lib_spooled_set(self):
+        async with self.getTestCore() as core:
+            await core.nodes('[inet:ipv4=1.2.3.4 :asn=20]')
+            await core.nodes('[inet:ipv4=5.6.7.8 :asn=30]')
+
+            q = '''
+                $set = $lib.spooled.set()
+                $set.add(1, 2, 3, 4)
+                return($set)
+            '''
+            valu = await core.callStorm(q)
+            self.eq({'1', '2', '3', '4'}, valu)
+
+            q = '''
+                $set = $lib.spooled.set()
+                $set.adds($lib.list(1, 2, 3, 4))
+                return($set)
+            '''
+            valu = await core.callStorm(q)
+            self.eq({'1', '2', '3', '4'}, valu)
+
+            q = '''
+                $set = $lib.spooled.set()
+                inet:ipv4 $set.add(:asn)
+                $set.rems((:asn,:asn))
+                [ graph:node="*" ] +graph:node [ :data=$set.list() ]
+            '''
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq(nodes[0].get('data'), ())
+
+            q = '''
+                $set = $lib.spooled.set()
+                $set.add($foo)
+                $set.add($bar)
+                $set.add($biz)
+                return(($set.has($foo), $set.has(lolnop)))
+            '''
+            valu = await core.callStorm(q, opts={'vars': {'foo': b'foo', 'bar': b'bar', 'biz': b'biz'}})
+            self.eq(True, valu[0])
+            self.eq(False, valu[1])
+
+            q = '''
+                $set = $lib.spooled.set()
+                $set.adds(('foo', 'bar', 'baz', 'biz', 'biz', 'biz', 'beep', 'boop'))
+                return($set.size())
+            '''
+            valu = await core.callStorm(q)
+            self.eq(6, valu)
+
+            q = '''
+                $set = $lib.spooled.set()
+                $set.adds(('foo', 'bar', 'baz', 'biz', 'biz', 'biz', 'beep', 'boop'))
+
+                $set.rems(('baz', 'beep', 'bar'))
+                return($set)
+
+            '''
+            valu = await core.callStorm(q)
+            self.eq({'foo', 'boop', 'biz'}, valu)
+
+            q = '''
+                $set = $lib.spooled.set(1, 2, 3, 4 ,5)
+                $set.add(1, 2, 3, 4)
+                return($set.list())
+            '''
+            valu = await core.callStorm(q)
+            self.isinstance(valu, tuple)
+            self.len(5, valu)
+            self.isin('1', valu)
+            self.isin('2', valu)
+            self.isin('3', valu)
+            self.isin('4', valu)
+            self.isin('5', valu)
+
+            q = '''
+                $set = $lib.spooled.set()
+                $set.add($lib.true)
+                $set.add($lib.false)
+                $set.add($lib.true)
+                $set.add($lib.false)
+                $set.add('more stuff')
+
+                $dict = ({
+                    "foo": "bar",
+                    "biz": "baz",
+                })
+                $set.adds($dict)
+                return($set)
+            '''
+            valu = await core.callStorm(q)
+            self.len(5, valu)
+            self.isin(False, valu)
+            self.isin(True, valu)
+            self.isin('more stuff', valu)
+            self.isin(('biz', 'baz'), valu)
+            self.isin(('foo', 'bar'), valu)
+
+            q = '''
+                $set = $lib.spooled.set()
+                $set.adds($items)
+                for $x in $set {
+                    $lib.print(`{$x} exists in the set`)
+                }
+                return()
+            '''
+            msgs = await core.stormlist(q, opts={'vars': {'items': [True, 'neato', False, 9001]}})
+            self.len(7, msgs)
+            self.stormIsInPrint('false exists in the set', msgs)
+            self.stormIsInPrint('true exists in the set', msgs)
+            self.stormIsInPrint('neato exists in the set', msgs)
+            self.stormIsInPrint('9001 exists in the set', msgs)
+
+            q = '''
+                $set = $lib.spooled.set(neato, neato, neato, neato)
+                $lib.print(`The set is {$set}`)
+            '''
+            msgs = await core.stormlist(q, opts={'vars': {'items': [True, 'neato', False, 9001]}})
+            self.stormIsInPrint("The set is {'neato'}", msgs)
+
+            # force a fallback
+            q = '''
+                $set = $lib.spooled.set()
+                $set.adds($lib.range(1500))
+                return($set.size())
+            '''
+            valu = await core.callStorm(q)
+            self.eq(1500, valu)
+
+            # sad paths
+            # too complex
+            q = '''
+                $set = $lib.spooled.set()
+                $set.add($stormnode)
+                return($set)
+            '''
+            stormnode = s_stormtypes.Node(nodes[0])
+            await self.asyncraises(s_exc.StormRuntimeError, core.callStorm(q, {'vars': {'stormnode': stormnode}}))
+
+            # mutable failures
+            q = '''
+                $set = $lib.spooled.set()
+                $set.add(({'neato': 'burrito'}))
+                return($set)
+            '''
+            await self.asyncraises(s_exc.StormRuntimeError, core.callStorm(q))
+
+            q = '''
+                $set = $lib.spooled.set()
+                $dict = ({'neato': 'burrito'})
+                $set.adds(($dict, $dict))
+                return($set)
+            '''
+            await self.asyncraises(s_exc.StormRuntimeError, core.callStorm(q))
+
+            q = '''
+                $set = $lib.spooled.set()
+                $form = $lib.model.form('inet:ipv4')
+                $set.adds(($stormnode, $form, $form))
+                return($set)
+            '''
+            # it'll blow up on the first
+            await self.asyncraises(s_exc.StormRuntimeError, core.callStorm(q, {'vars': {'stormnode': stormnode}}))
+
+            q = '''
+                $dict = ({'foo': 'bar'})
+                $set = $lib.spooled.set($dict)
+                return($set)
+            '''
+            await self.asyncraises(s_exc.StormRuntimeError, core.callStorm(q))
+
+            # type not msgpack-able
+            q = '''
+                $set = $lib.spooled.set()
+                $set.add($lib.model.form("inet:ipv4"))
+                return($set)
+            '''
+            await self.asyncraises(s_exc.StormRuntimeError, core.callStorm(q))
+
+            q = '''
+                $set = $lib.spooled.set($stormnode)
+                return($set)
+            '''
+            await self.asyncraises(s_exc.StormRuntimeError, core.callStorm(q, {'vars': {'stormnode': stormnode}}))

synapse/tests/test_lib_stormtypes.py

@@ -20,7 +20,6 @@ import synapse.lib.storm as s_storm
 import synapse.lib.hashset as s_hashset
 import synapse.lib.httpapi as s_httpapi
 import synapse.lib.modelrev as s_modelrev
-import synapse.lib.provenance as s_provenance
 import synapse.lib.stormtypes as s_stormtypes
 
 import synapse.tests.utils as s_test
@@ -4512,7 +4511,6 @@ class StormTypesTest(s_test.SynTest):
 
         MONO_DELT = 1543827303.0
         unixtime = datetime.datetime(year=2018, month=12, day=5, hour=7, minute=0, tzinfo=tz.utc).timestamp()
-        s_provenance.reset()
 
         def timetime():
             return unixtime
@@ -4846,6 +4844,7 @@ class StormTypesTest(s_test.SynTest):
 
                     self.stormIsInPrint('last result:     finished successfully with 0 nodes', mesgs)
                     self.stormIsInPrint('entries:         <None>', mesgs)
+                    self.stormIsInPrint('pool:            false', mesgs)
 
                     # Test 'stat' command
                     mesgs = await core.stormlist('cron.stat xxx')
@@ -4919,7 +4918,7 @@ class StormTypesTest(s_test.SynTest):
                 self.stormIsInErr('data.iden must match pattern', msgs)
 
                 opts = {'vars': {'iden': 'cd263bd133a5dafa1e1c5e9a01d9d486'}}
-                q = "cron.add --iden $iden --day +1 --minute 14 {[test:guid=$lib.guid()]}"
+                q = "cron.add --pool --iden $iden --day +1 --minute 14 {[test:guid=$lib.guid()]}"
                 msgs = await core.stormlist(q, opts=opts)
                 self.stormIsInPrint('Created cron job: cd263bd133a5dafa1e1c5e9a01d9d486', msgs)
 
@@ -6465,6 +6464,10 @@ words\tword\twrd'''
             self.eq(merge['creator'], core.auth.rootuser.iden)
             self.none(merge.get('updated'))
 
+            merging = 'return($lib.view.get().getMergingViews()) '
+
+            self.eq([fork00], await core.callStorm(merging))
+
             with self.raises(s_exc.AuthDeny):
                 core.getView(fork00).reqValidVoter(root.iden)
 
@@ -6540,8 +6543,13 @@ words\tword\twrd'''
             self.none(core.getView(fork00))
             nodes = await core.nodes('inet:fqdn')
             self.len(2, nodes)
+
+            # previously successful merges
             self.len(1, await core.callStorm('$list = ([]) for $merge in $lib.view.get().getMerges() { $list.append($merge) } fini { return($list) }'))
 
+            # current open merge requests
+            self.eq([], await core.callStorm(merging))
+
             # test out the delMergeRequest logic / cleanup
             forkdef = await core.getView().fork()
 
@@ -6549,6 +6557,7 @@ words\tword\twrd'''
 
             opts = {'view': fork.iden}
             self.nn(await core.callStorm('return($lib.view.get().setMergeRequest())', opts=opts))
+            self.eq([fork.iden], await core.callStorm(merging))
 
             # confirm that you may not re-parent to a view with a merge request
             layr = await core.addLayer()
@@ -6564,6 +6573,8 @@ words\tword\twrd'''
             self.nn(await core.callStorm('return($lib.view.get().delMergeRequest())', opts=opts))
             self.len(0, [vote async for vote in fork.getMergeVotes()])
 
+            self.eq([], await core.callStorm(merging))
+
             # test coverage for beholder progress events...
             forkdef = await core.getView().fork()
             fork = core.getView(forkdef.get('iden'))
@@ -6572,10 +6583,12 @@ words\tword\twrd'''
             await core.stormlist('[ inet:ipv4=1.2.3.0/20 ]', opts=opts)
             await core.callStorm('return($lib.view.get().setMergeRequest())', opts=opts)
 
+            self.eq([fork.iden], await core.callStorm(merging))
+
             nevents = 8
             if s_common.envbool('SYNDEV_NEXUS_REPLAY'):
                 # view:merge:vote:set fires twice
-                nevents = nevents + 1
+                nevents += 1
             waiter = core.waiter(nevents, 'cell:beholder')
 
             opts = {'view': fork.iden, 'user': visi.iden}
@@ -6598,6 +6611,8 @@ words\tword\twrd'''
             self.eq(msgs[-1][1]['info']['merge']['creator'], core.auth.rootuser.iden)
             self.eq(msgs[-1][1]['info']['votes'][0]['user'], visi.iden)
 
+            self.eq([], await core.callStorm(merging))
+
             # test coverage for bad state for merge request
             fork00 = await core.getView().fork()
             fork01 = await core.getView(fork00['iden']).fork()
@@ -6606,8 +6621,11 @@ words\tword\twrd'''
             with self.raises(s_exc.BadState):
                 await core.callStorm('return($lib.view.get().setMergeRequest())', opts=opts)
 
+            self.eq([], await core.callStorm(merging))
+
             await core.delView(fork01['iden'])
             await core.callStorm('return($lib.view.get().setMergeRequest())', opts=opts)
+            self.eq([fork00['iden']], await core.callStorm(merging))
 
             core.getView().layers[0].readonly = True
             with self.raises(s_exc.BadState):
@@ -6626,6 +6644,8 @@ words\tword\twrd'''
             await core.stormlist('[ inet:ipv4=5.5.5.5 ]', opts=opts)
             await core.callStorm('return($lib.view.get().setMergeRequest())', opts=opts)
 
+            self.eq(set([fork.iden, fork00['iden']]), set(await core.callStorm(merging)))
+
             # hamstring the runViewMerge method on the new view
             async def fake():
                 return
@@ -6660,3 +6680,6 @@ words\tword\twrd'''
 
             msgs = await core.stormlist('$lib.view.get().set(quorum, $lib.null)')
             self.stormHasNoWarnErr(msgs)
+
+            with self.raises(s_exc.BadState):
+                await core.callStorm(merging)

synapse/tests/test_model_inet.py

@@ -2767,3 +2767,70 @@ class InetModelTest(s_t_utils.SynTest):
             self.eq(nodes[0].get('client'), 'tcp://1.2.3.4')
             self.eq(nodes[0].get('client:ipv4'), 0x01020304)
             self.eq(nodes[0].get('client:ipv6'), '::1')
+
+    async def test_model_inet_tls_handshake(self):
+
+        async with self.getTestCore() as core:
+            props = {
+                'ja3': '1' * 32,
+                'ja3s': '2' * 32,
+                'client': 'tcp://1.2.3.4:8888',
+                'server': 'tcp://5.6.7.8:9999'
+            }
+
+            nodes = await core.nodes('''
+                [
+                    inet:tls:handshake=*
+                        :time=now
+                        :flow=*
+                        :server=$server
+                        :server:cert=*
+                        :server:fingerprint:ja3=$ja3s
+                        :client=$client
+                        :client:cert=*
+                        :client:fingerprint:ja3=$ja3
+                ]
+            ''', opts={'vars': props})
+            self.len(1, nodes)
+            self.nn(nodes[0].get('time'))
+            self.nn(nodes[0].get('flow'))
+            self.nn(nodes[0].get('server:cert'))
+            self.nn(nodes[0].get('client:cert'))
+
+            self.eq(props['ja3'], nodes[0].get('client:fingerprint:ja3'))
+            self.eq(props['ja3s'], nodes[0].get('server:fingerprint:ja3'))
+
+            self.eq(props['client'], nodes[0].get('client'))
+            self.eq(props['server'], nodes[0].get('server'))
+
+    async def test_model_inet_ja3(self):
+
+        async with self.getTestCore() as core:
+
+            ja3 = '76e7b0cb0994d60a4b3f360a088fac39'
+            nodes = await core.nodes('[ inet:tls:ja3:sample=(tcp://1.2.3.4, $md5) ]', opts={'vars': {'md5': ja3}})
+            self.len(1, nodes)
+            self.eq(nodes[0].get('client'), 'tcp://1.2.3.4')
+            self.eq(nodes[0].get('ja3'), ja3)
+
+            ja3 = '4769ad08107979c719d86270e706fed5'
+            nodes = await core.nodes('[ inet:tls:ja3s:sample=(tcp://2.2.2.2, $md5) ]', opts={'vars': {'md5': ja3}})
+            self.len(1, nodes)
+            self.eq(nodes[0].get('server'), 'tcp://2.2.2.2')
+            self.eq(nodes[0].get('ja3s'), ja3)
+
+    async def test_model_inet_tls_certs(self):
+
+        async with self.getTestCore() as core:
+
+            server = 'e4f6db65dbaa7a4598f7379f75dcd5f5'
+            client = 'df8d1f7e04f7c4a322e04b0b252e2851'
+            nodes = await core.nodes('[inet:tls:servercert=(tcp://1.2.3.4:1234, $server)]', opts={'vars': {'server': server}})
+            self.len(1, nodes)
+            self.eq(nodes[0].get('server'), 'tcp://1.2.3.4:1234')
+            self.eq(nodes[0].get('cert'), server)
+
+            nodes = await core.nodes('[inet:tls:clientcert=(tcp://5.6.7.8:5678, $client)]', opts={'vars': {'client': client}})
+            self.len(1, nodes)
+            self.eq(nodes[0].get('client'), 'tcp://5.6.7.8:5678')
+            self.eq(nodes[0].get('cert'), client)

synapse/tests/test_model_risk.py

@@ -435,6 +435,8 @@ class RiskModelTest(s_t_utils.SynTest):
                 :public:url=https://wikileaks.org/acme
                 :reporter={ gen.ou.org vertex }
                 :reporter:name=vertex
+                :size:bytes=99
+                :extortion=*
             ]''')
             self.len(1, nodes)
             self.eq('wikileaks acme leak', nodes[0].get('name'))
@@ -442,9 +444,11 @@ class RiskModelTest(s_t_utils.SynTest):
             self.eq(1698883200000, nodes[0].get('disclosed'))
             self.eq('public.', nodes[0].get('type'))
             self.eq(1, nodes[0].get('public'))
+            self.eq(99, nodes[0].get('size:bytes'))
             self.eq('https://wikileaks.org/acme', nodes[0].get('public:url'))
             self.eq('vertex', nodes[0].get('reporter:name'))
 
+            self.len(1, await core.nodes('risk:leak -> risk:extortion'))
             self.len(1, await core.nodes('risk:leak -> risk:leak:type:taxonomy'))
             self.len(1, await core.nodes('risk:leak :owner -> ps:contact +:orgname=acme'))
             self.len(1, await core.nodes('risk:leak :leaker -> ps:contact +:orgname=wikileaks'))
@@ -454,6 +458,7 @@ class RiskModelTest(s_t_utils.SynTest):
 
             nodes = await core.nodes('''[ risk:extortion=*
                 :demanded=20231102
+                :deadline=20240329
                 :name="APT99 Extorted     ACME"
                 :desc="APT99 extorted ACME for a zillion vertex coins."
                 :type=fingain
@@ -474,6 +479,7 @@ class RiskModelTest(s_t_utils.SynTest):
             self.eq('apt99 extorted acme', nodes[0].get('name'))
             self.eq('APT99 extorted ACME for a zillion vertex coins.', nodes[0].get('desc'))
             self.eq(1698883200000, nodes[0].get('demanded'))
+            self.eq(1711670400000, nodes[0].get('deadline'))
             self.eq('fingain.', nodes[0].get('type'))
             self.eq(1, nodes[0].get('public'))
             self.eq(1, nodes[0].get('success'))

synapse/tests/test_telepath.py

@@ -831,15 +831,17 @@ class TeleTest(s_t_utils.SynTest):
 
             await targ.waitready()
 
-            self.eq(110, await targ.dostuff(100))
+            prox00 = await targ.proxy(timeout=12)
+            self.eq(110, await prox00.dostuff(100))
+
             self.eq(1, fail0.count)
             self.eq(0, fail1.count)
 
-            _prox = await targ.proxy()
             await dmon0.fini()
-            self.true(await _prox.waitfini(10))
+            self.true(await prox00.waitfini(10))
 
-            self.eq(110, await targ.dostuff(100))
+            prox01 = await targ.proxy(timeout=12)
+            self.eq(110, await prox01.dostuff(100))
             self.eq(1, fail0.count)
             self.eq(1, fail1.count)
 
@@ -855,15 +857,36 @@ class TeleTest(s_t_utils.SynTest):
                 mesgs = stream.read()
                 self.notin('password', mesgs)
 
-            self.eq(110, await targ.dostuff(100))
+            prox00 = await targ.proxy(timeout=12)
+            self.eq(110, await prox00.dostuff(100))
 
             self.eq(1, fail0.count)
             self.eq(2, fail1.count)
 
         async with await s_telepath.open(url1) as targ:
-            await targ.waitready()
-            self.eq(110, await targ.dostuff(100))
+            await targ.waitready(timeout=12)
+            prox00 = await targ.proxy(timeout=12)
+            self.eq(110, await prox00.dostuff(100))
+
+        async def onlink(proxy, urlinfo):
+            self.eq(110, await proxy.dostuff(100))
+            _url = s_telepath.zipurl(urlinfo)
+            logger.info(f'Connected to url={_url}')
+
+        with self.getAsyncLoggerStream('synapse.tests.test_telepath',
+                                       f'Connected to url=tcp://127.0.0.1:{addr1[1]}/foo') as stream:
+            async with await s_telepath.open(url1, onlink=onlink) as targ:
+                self.true(await stream.wait(timeout=12))
+
+        # Coverage
+        async def badonlink(proxy, urlinfo):
+            raise ValueError('oopsie')
+
+        with self.getAsyncLoggerStream('synapse.telepath', 'onlink: ') as stream:
+            async with await s_telepath.open(url1, onlink=badonlink) as targ:
+                self.true(await stream.wait(timeout=12))
 
+        await dmon0.fini()
         await dmon1.fini()
 
     async def test_telepath_poolsize(self):

synapse/tests/test_tools_modrole.py

@@ -2,6 +2,26 @@ import synapse.lib.output as s_output
 import synapse.tests.utils as s_test
 import synapse.tools.modrole as s_t_modrole
 
+rolelist = s_test.deguidify('''
+Roles:
+  ce6928dfe88cace918c405bbef51e72f - all
+  0f316b3f3e6ec970fcdb9a085fcd5b77 - visi
+'''.strip())
+
+roleinfo = s_test.deguidify('''
+Role: visi (145c3321a0cd0cd06de19174415a7aeb)
+
+  Rules:
+    [0  ] - !foo.bar.baz
+    [1  ] - foo.bar
+
+  Gates:
+    3ad191c63a201df3246c6d6ff81763ad
+      Admin: False
+      [0  ] - !bar.baz.faz
+      [1  ] - bar.baz
+'''.strip())
+
 class ModRoleTest(s_test.SynTest):
 
     async def test_tools_modrole(self):
@@ -40,6 +60,60 @@ class ModRoleTest(s_test.SynTest):
             self.true(bool(visi.allowed('foo.bar.gaz'.split('.'))))
             self.false(bool(visi.allowed('foo.bar.baz'.split('.'))))
 
+            gateiden = core.getLayer().iden
+            argv = (
+                '--svcurl', svcurl,
+                'visi',
+                '--allow', 'bar.baz',
+                '--deny', 'bar.baz.faz',
+                '--gate', gateiden,
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_modrole.main(argv, outp=outp))
+            self.isin(f'...adding allow rule: bar.baz on gate {gateiden}', str(outp))
+            self.isin(f'...adding deny rule: bar.baz.faz on gate {gateiden}', str(outp))
+
+            gate = await core.getAuthGate(gateiden)
+            for role in gate['roles']:
+                if role['iden'] == visi.iden:
+                    self.isin((True, ('bar', 'baz')), role['rules'])
+                    self.isin((False, ('bar', 'baz', 'faz')), role['rules'])
+
+            argv = (
+                '--svcurl', svcurl,
+                '--list',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_modrole.main(argv, outp=outp))
+            self.isin(rolelist, s_test.deguidify(str(outp)))
+
+            argv = (
+                '--svcurl', svcurl,
+                '--list',
+                'visi',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(0, await s_t_modrole.main(argv, outp=outp))
+            self.isin(roleinfo, s_test.deguidify(str(outp)))
+
+            argv = (
+                '--svcurl', svcurl,
+                '--list',
+                'newprole',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(1, await s_t_modrole.main(argv, outp=outp))
+            self.isin('ERROR: Role not found: newprole', str(outp))
+
+            argv = (
+                '--svcurl', svcurl,
+                'visi',
+                '--gate', 'newp',
+            )
+            outp = s_output.OutPutStr()
+            self.eq(1, await s_t_modrole.main(argv, outp=outp))
+            self.isin('ERROR: No auth gate found with iden: newp', str(outp))
+
             argv = (
                 '--svcurl', svcurl,
                 '--add',
@@ -59,3 +133,10 @@ class ModRoleTest(s_test.SynTest):
             self.eq(0, await s_t_modrole.main(argv, outp=outp))
             self.isin('...deleting role: visi', str(outp))
             self.none(await core.auth.getRoleByName('visi'))
+
+            argv = (
+                '--svcurl', svcurl,
+            )
+            outp = s_output.OutPutStr()
+            self.eq(1, await s_t_modrole.main(argv, outp=outp))
+            self.isin('ERROR: A rolename argument is required when --list is not specified.', str(outp))