synapse 2.152.0__py311-none-any.whl → 2.154.0__py311-none-any.whl

synapse/tests/test_lib_stormlib_iters.py

@@ -0,0 +1,116 @@
+from unittest import mock
+
+import synapse.exc as s_exc
+
+import synapse.tests.utils as s_test
+
+import synapse.lib.stormlib.iters as s_stormlib_iters
+
+class StormLibItersTest(s_test.SynTest):
+
+    async def test_stormlib_iters_zip(self):
+
+        async with self.getTestCore() as core:
+
+            async with s_test.matchContexts(self):
+
+                q = '''
+                for $item in $lib.iters.zip((1,2,3), (4,5,6), (7,8,9)) {
+                    $lib.print($item)
+                }
+                '''
+                msgs = await core.stormlist(q)
+                self.stormIsInPrint("['1', '4', '7']", msgs)
+                self.stormIsInPrint("['2', '5', '8']", msgs)
+                self.stormIsInPrint("['3', '6', '9']", msgs)
+
+                q = '''
+                for $item in $lib.iters.zip((1,2,3), (4,5,6), (7,8)) {
+                    $lib.print($item)
+                }
+                '''
+                msgs = await core.stormlist(q)
+                self.stormIsInPrint("['1', '4', '7']", msgs)
+                self.stormIsInPrint("['2', '5', '8']", msgs)
+                self.stormNotInPrint("['3', '6']", msgs)
+
+                q = '''
+                function nodes() {
+                    [ it:dev:str=4 it:dev:str=5 it:dev:str=6 ]
+                }
+                function emitter() {
+                    emit 7
+                    emit 8
+                    emit 9
+                }
+                for ($a, $b, $c) in $lib.iters.zip((1,2,3), $nodes(), $emitter()) {
+                    $lib.print(($a, $b.0.repr(), $c))
+                }
+                '''
+                msgs = await core.stormlist(q)
+                self.stormIsInPrint("['1', '4', '7']", msgs)
+                self.stormIsInPrint("['2', '5', '8']", msgs)
+                self.stormIsInPrint("['3', '6', '9']", msgs)
+
+                q = '''
+                function nodes() {
+                    [ it:dev:str=4 it:dev:str=5 it:dev:str=6 ]
+                }
+                function emitter() {
+                    emit 7
+                    emit 8
+                    emit 9
+                }
+                for ($a, $b, $c) in $lib.iters.zip((1,2,3), $nodes(), $emitter()) {
+                    $lib.print(($a, $b.0.repr(), $c))
+                    $lib.raise(foo, bar)
+                }
+                '''
+                msgs = await core.stormlist(q)
+                self.stormIsInPrint("['1', '4', '7']", msgs)
+                self.stormNotInPrint("['2', '5', '8']", msgs)
+                self.stormNotInPrint("['3', '6', '9']", msgs)
+
+                q = '''
+                function nodes() {
+                    [ it:dev:str=4  it:dev:str=5 it:dev:str=6]
+                }
+                function emitter() {
+                    emit 7
+                    $lib.raise(foo, bar)
+                    emit 8
+                    emit 9
+                }
+                for ($a, $b, $c) in $lib.iters.zip((1,2,3), $nodes(), $emitter()) {
+                    $lib.print(($a, $b.0.repr(), $c))
+                }
+                '''
+                msgs = await core.stormlist(q)
+                self.stormIsInPrint("['1', '4', '7']", msgs)
+                self.stormNotInPrint("['2', '5', '8']", msgs)
+                self.stormNotInPrint("['3', '6', '9']", msgs)
+
+                err = "$lib.iters.zip() encountered errors in 1 iterators during iteration: (foo: bar)"
+                self.stormIsInErr(err, msgs)
+
+                async def boom(self, genr):
+                    print(newp)
+                    yield True
+
+                with mock.patch.object(s_stormlib_iters.LibIters, 'enum', boom):
+                    q = '''
+                    function e1() {
+                        $lib.raise(foo, bar)
+                    }
+                    function e2() {
+                        $lib.print($newp)
+                    }
+                    for ($a, $b, $c) in $lib.iters.zip($e1(), $e2(), $lib.iters.enum($e2())) {
+                    }
+                    '''
+                    msgs = await core.stormlist(q)
+                    err = "$lib.iters.zip() encountered errors in 3 iterators during iteration: " \
+                          "(foo: bar), " \
+                          "(NoSuchVar: Missing variable: newp), " \
+                          "(NameError: name 'newp' is not defined)"
+                    self.stormIsInErr(err, msgs)

synapse/tests/test_lib_stormlib_stats.py

@@ -0,0 +1,187 @@
+import synapse.exc as s_exc
+import synapse.tests.utils as s_test
+
+import synapse.lib.stormlib.stats as s_stormlib_stats
+
+chartnorm = '''
+40 | 5 | ##################################################
+30 | 4 | ########################################
+20 | 3 | ##############################
+10 | 2 | ####################
+ 0 | 1 | ##########
+'''.strip()
+
+chartrev = '''
+ 0 | 1 | ##########
+10 | 2 | ####################
+20 | 3 | ##############################
+30 | 4 | ########################################
+40 | 5 | ##################################################
+'''.strip()
+
+chartsize = '''
+40 | 5 | ##################################################
+30 | 4 | ########################################
+20 | 3 | ##############################
+'''.strip()
+
+chartsizerev = '''
+ 0 | 1 | ##########
+10 | 2 | ####################
+20 | 3 | ##############################
+'''.strip()
+
+chartwidth = '''
+40 | 5 | ##########
+30 | 4 | ########
+20 | 3 | ######
+10 | 2 | ####
+ 0 | 1 | ##
+'''.strip()
+
+chartlabelwidth = '''
+4 | 5 | ##################################################
+3 | 4 | ########################################
+2 | 3 | ##############################
+1 | 2 | ####################
+0 | 1 | ##########
+'''.strip()
+
+chartchar = '''
+40 | 5 | ++++++++++++++++++++++++++++++++++++++++++++++++++
+30 | 4 | ++++++++++++++++++++++++++++++++++++++++
+20 | 3 | ++++++++++++++++++++++++++++++
+10 | 2 | ++++++++++++++++++++
+ 0 | 1 | ++++++++++
+'''.strip()
+
+
+class StatsTest(s_test.SynTest):
+
+    async def test_stormlib_stats_countby(self):
+
+        async with self.getTestCore() as core:
+
+            q = '''
+            $i = (0)
+            for $x in $lib.range(5) {
+                for $y in $lib.range(($x + 1)) {
+                    [ inet:ipv4=$i :asn=($x * 10) ]
+                    $i = ($i + 1)
+                }
+            }
+            '''
+            await core.nodes(q)
+
+            msgs = await core.stormlist('inet:ipv4 | stats.countby :asn')
+            self.stormIsInPrint(chartnorm, msgs)
+
+            msgs = await core.stormlist('inet:ipv4 -> inet:asn | stats.countby')
+            self.stormIsInPrint(chartnorm, msgs)
+
+            msgs = await core.stormlist('inet:ipv4 | stats.countby :asn --reverse')
+            self.stormIsInPrint(chartrev, msgs)
+
+            msgs = await core.stormlist('inet:ipv4 | stats.countby :asn --size 3')
+            self.stormIsInPrint(chartsize, msgs)
+
+            msgs = await core.stormlist('inet:ipv4 | stats.countby :asn --size 3 --reverse')
+            self.stormIsInPrint(chartsizerev, msgs)
+
+            msgs = await core.stormlist(f'inet:ipv4 | stats.countby :asn --bar-width 10')
+            self.stormIsInPrint(chartwidth, msgs)
+
+            msgs = await core.stormlist(f'inet:ipv4 | stats.countby :asn --label-max-width 1')
+            self.stormIsInPrint(chartlabelwidth, msgs)
+
+            msgs = await core.stormlist('inet:ipv4 | stats.countby :asn --char "+"')
+            self.stormIsInPrint(chartchar, msgs)
+
+            msgs = await core.stormlist('stats.countby foo')
+            self.stormIsInPrint('No values to display!', msgs)
+
+            self.len(0, await core.nodes('inet:ipv4 | stats.countby :asn'))
+            self.len(15, await core.nodes('inet:ipv4 | stats.countby :asn --yield'))
+
+            with self.raises(s_exc.BadArg):
+                self.len(15, await core.nodes('inet:ipv4 | stats.countby :asn --label-max-width "-1"'))
+
+            with self.raises(s_exc.BadArg):
+                self.len(15, await core.nodes('inet:ipv4 | stats.countby :asn --bar-width "-1"'))
+
+            with self.raises(s_exc.BadArg):
+                self.len(15, await core.nodes('inet:ipv4 | stats.countby ({})'))
+
+    async def test_stormlib_stats_tally(self):
+
+        async with self.getTestCore() as core:
+
+            q = '''
+                $tally = $lib.stats.tally()
+
+                $tally.inc(foo)
+                $tally.inc(foo)
+
+                $tally.inc(bar)
+                $tally.inc(bar, 3)
+
+                for ($name, $valu) in $tally {
+                    [ test:comp=($valu, $name) ]
+                }
+
+                $lib.print('tally: foo={foo} baz={baz}', foo=$tally.get(foo), baz=$tally.get(baz))
+                $lib.print('tally.len()={v}', v=$lib.len($tally))
+            '''
+            mesgs = await core.stormlist(q)
+            nodes = [m[1] for m in mesgs if m[0] == 'node']
+            self.len(2, nodes)
+            self.eq(nodes[0][0], ('test:comp', (2, 'foo')))
+            self.eq(nodes[1][0], ('test:comp', (4, 'bar')))
+            self.stormIsInPrint('tally: foo=2 baz=0', mesgs)
+            self.stormIsInPrint('tally.len()=2', mesgs)
+
+            q = '''
+                $tally = $lib.stats.tally()
+                $tally.inc(foo, 1)
+                $tally.inc(bar, 2)
+                $tally.inc(baz, 3)
+                return($tally.sorted())
+            '''
+            vals = await core.callStorm(q)
+            self.eq(vals, [('foo', 1), ('bar', 2), ('baz', 3)])
+
+            q = '''
+                $tally = $lib.stats.tally()
+                $tally.inc(foo, 1)
+                $tally.inc(bar, 2)
+                $tally.inc(baz, 3)
+                return($tally.sorted(reverse=$lib.true))
+            '''
+            vals = await core.callStorm(q)
+            self.eq(vals, [('baz', 3), ('bar', 2), ('foo', 1)])
+
+            q = '''
+                $tally = $lib.stats.tally()
+                $tally.inc(foo, 1)
+                $tally.inc(bar, 2)
+                $tally.inc(baz, 3)
+                return($tally.sorted(byname=$lib.true))
+            '''
+            vals = await core.callStorm(q)
+            self.eq(vals, [('bar', 2), ('baz', 3), ('foo', 1)])
+
+            q = '''
+                $tally = $lib.stats.tally()
+                $tally.inc(foo, 1)
+                $tally.inc(bar, 2)
+                $tally.inc(baz, 3)
+                return($tally.sorted(byname=$lib.true, reverse=$lib.true))
+            '''
+            vals = await core.callStorm(q)
+            self.eq(vals, [('foo', 1), ('baz', 3), ('bar', 2)])
+
+            tally = s_stormlib_stats.StatTally()
+            await tally.inc('foo')
+
+            async for (name, valu) in tally:
+                self.eq((name, valu), ('foo', 1))

synapse/tests/test_lib_stormlib_storm.py

@@ -28,3 +28,11 @@ class LibStormTest(s_test.SynTest):
 
             # for coverage of forked call...
             self.nn(s_parser.parseEval('woot'))
+
+            # Readonly functionality is sane
+            msgs = await core.stormlist('$lib.print($lib.storm.eval( "{$lib.print(wow)}" ))')
+            self.stormIsInPrint('wow', msgs)
+            self.stormIsInPrint('$lib.null', msgs)
+
+            with self.raises(s_exc.IsReadOnly):
+                await core.callStorm('$lib.storm.eval( "{$lib.auth.users.add(readonly)}" )', opts={'readonly': True})

synapse/tests/test_lib_stormsvc.py

@@ -317,7 +317,30 @@ class StormvarService(s_cell.CellApi, s_stormsvc.StormSvc):
                     $lib.print('my foo var is {f}', f=$fooz)
                     ''',
                 },
-            )
+            ),
+            'modules': (
+                {
+                    'name': 'testmod',
+                    'storm': '',
+                },
+                {
+                    'name': 'apimod',
+                    'storm': 'function status() { return($lib.true) }',
+                    'apidefs': (
+                        {
+                            'name': 'status',
+                            'desc': 'Status of the foo.',
+                            'type': {
+                                'type': 'function',
+                                'returns': {
+                                    'type': 'boolean',
+                                    'desc': 'Where foo is ok',
+                                },
+                            },
+                        },
+                    ),
+                },
+            ),
         },
     )
 

synapse/tests/test_lib_stormtypes.py

@@ -613,6 +613,14 @@ class StormTypesTest(s_test.SynTest):
                     'name': 'test',
                     'storm': '$valu=$modconf.valu function getvalu() { return($valu) }',
                     'modconf': {'valu': 'foo'},
+                },
+                {
+                    'name': 'test.danger',
+                    'storm': '''
+                        init { $src=$lib.null }
+                        function genSrc() { if $src { return ($src) } [meta:source=(s1,)] $src=$node return ($node) }
+                        $genSrc()
+                        '''
                 }
             ],
             'commands': [
@@ -735,9 +743,13 @@ class StormTypesTest(s_test.SynTest):
 
             await core.callStorm('$test = $lib.import(test) $test.modconf.valu=bar')
             self.eq('foo', await core.callStorm('return($lib.import(test).getvalu())'))
+            self.eq('foo', await core.callStorm('return($lib.import(test).getvalu())', opts={'readonly': True}))
+
+            with self.raises(s_exc.IsReadOnly):
+                await core.callStorm('return($lib.import(test.danger).src)', opts={'readonly': True})
 
             mods = await core.getStormMods()
-            self.len(1, mods)
+            self.len(2, mods)
             mods['test']['modconf']['valu'] = 'bar'
             mods = await core.getStormMods()
             self.eq('foo', mods['test']['modconf']['valu'])
@@ -1181,6 +1193,24 @@ class StormTypesTest(s_test.SynTest):
             '''
             self.eq(2, await core.callStorm(q))
 
+            q = '''
+            $q=${ return( $lib.auth.users.byname(root).name ) }
+            return ( $q.exec() )
+            '''
+            self.eq('root', await core.callStorm(q, opts={'readonly': True}))
+
+            q = '''
+            $q=${ test:int=1 }
+            return ( $q.size() )
+            '''
+            self.eq(1, await core.callStorm(q, opts={'readonly': True}))
+
+            with self.raises(s_exc.IsReadOnly):
+                await core.callStorm('$foo=${ [test:str=readonly] } return ( $foo.exec() )', opts={'readonly': True})
+
+            with self.raises(s_exc.IsReadOnly):
+                await core.callStorm('$foo=${ [test:str=readonly] } return( $foo.size() )', opts={'readonly': True})
+
     async def test_storm_lib_node(self):
         async with self.getTestCore() as core:
             nodes = await core.nodes('[ test:str=woot :tick=2001] [ test:int=$node.isform(test:str) ] +test:int')
@@ -1514,6 +1544,72 @@ class StormTypesTest(s_test.SynTest):
                 q = '$foo=$lib.text(foo) $bar=$lib.text(bar) $v=($foo, aString, $bar,) $v.sort() return ($v)'
                 await core.callStorm(q)
 
+            q = '$l = (1, 2, (3), 4, 1, (3), 3, asdf) return ( $l.unique() )'
+            self.eq(['1', '2', 3, '4', '3', 'asdf'], await core.callStorm(q))
+
+            q = '$a=$lib.text(hehe) $b=$lib.text(haha) $c=$lib.text(hehe) $foo=($a, $b, $c) return ($foo.unique())'
+            self.eq(['hehe', 'haha'], await core.callStorm(q))
+
+            await core.addUser('lowuser1')
+            await core.addUser('lowuser2')
+            q = '''
+            $a=$lib.auth.users.byname(lowuser1) $b=$lib.auth.users.byname(lowuser2) $r=$lib.auth.users.byname(root)
+            $l = ($a, $r, $b, $b, $a ) $l2 = $l.unique() $l3 = ()
+            for $user in $l2 {
+                $l3.append($user.name)
+            }
+            return ( $l3 )'''
+            self.eq(['lowuser1', 'root', 'lowuser2'], await core.callStorm(q))
+
+            q = '$l=(1, 2, 3, 3, $lib.queue) return ( $lib.len($l.unique()) )'
+            self.eq(4, await core.callStorm(q))
+
+            # funcs are different class instances here
+            q = '$l = (1, 2, 2, $lib.inet.http.get, $lib.inet.http.get) return ($lib.len($l.unique()))'
+            self.eq(4, await core.callStorm(q))
+
+            # funcs are the same class instance
+            q = '$hehe = $lib.inet.http $l = (1, 2, 2, $hehe.get, $hehe.get) return ($lib.len($l.unique()))'
+            self.eq(3, await core.callStorm(q))
+
+            q = '''
+            function foo() {}
+            function bar() {}
+            $l = ($foo, $bar)
+            return ($lib.len($l.unique()))
+            '''
+            self.eq(2, await core.callStorm(q))
+
+            q = '''
+            function foo() {}
+            function bar() {}
+            $l = ($bar, $foo, $bar)
+            return ($lib.len($l.unique()))
+            '''
+            self.eq(2, await core.callStorm(q))
+
+            q = '''
+            $q1 = $lib.queue.gen(hehe)
+            $q2 = $lib.queue.get(hehe)
+            $l = ($q1, $q2)
+            return ( $lib.len($l.unique()) ) '''
+            self.eq(1, await core.callStorm(q))
+
+            q = '''
+            $q1 = $lib.queue.gen(hehe)
+            $q2 = $lib.queue.get(hehe)
+            $l = ($q1, $q2, $q2)
+            return ( $lib.len($l.unique()) ) '''
+            self.eq(1, await core.callStorm(q))
+
+            q = '''
+            $q1 = $lib.queue.gen(hehe)
+            $q2 = $lib.queue.get(hehe)
+            $q3 = $lib.queue.get(hehe)
+            $l = ($q1, $q2, $q3)
+            return ( $lib.len($l.unique()) ) '''
+            self.eq(1, await core.callStorm(q))
+
             # Python Tuples can be treated like a List object for accessing via data inside of.
             q = '[ test:comp=(10,lol) ] $x=$node.ndef().index(1).index(1) [ test:str=$x ]'
             nodes = await core.nodes(q)
@@ -3265,74 +3361,6 @@ class StormTypesTest(s_test.SynTest):
             errs = [m for m in msgs if m[0] == 'err']
             self.len(0, errs)
 
-    async def test_lib_stormtypes_stats(self):
-
-        async with self.getTestCore() as core:
-
-            q = '''
-                $tally = $lib.stats.tally()
-
-                $tally.inc(foo)
-                $tally.inc(foo)
-
-                $tally.inc(bar)
-                $tally.inc(bar, 3)
-
-                for ($name, $valu) in $tally {
-                    [ test:comp=($valu, $name) ]
-                }
-
-                $lib.print('tally: foo={foo} baz={baz}', foo=$tally.get(foo), baz=$tally.get(baz))
-                $lib.print('tally.len()={v}', v=$lib.len($tally))
-            '''
-            mesgs = await core.stormlist(q)
-            nodes = [m[1] for m in mesgs if m[0] == 'node']
-            self.len(2, nodes)
-            self.eq(nodes[0][0], ('test:comp', (2, 'foo')))
-            self.eq(nodes[1][0], ('test:comp', (4, 'bar')))
-            self.stormIsInPrint('tally: foo=2 baz=0', mesgs)
-            self.stormIsInPrint('tally.len()=2', mesgs)
-
-            q = '''
-                $tally = $lib.stats.tally()
-                $tally.inc(foo, 1)
-                $tally.inc(bar, 2)
-                $tally.inc(baz, 3)
-                return($tally.sorted())
-            '''
-            vals = await core.callStorm(q)
-            self.eq(vals, [('foo', 1), ('bar', 2), ('baz', 3)])
-
-            q = '''
-                $tally = $lib.stats.tally()
-                $tally.inc(foo, 1)
-                $tally.inc(bar, 2)
-                $tally.inc(baz, 3)
-                return($tally.sorted(reverse=$lib.true))
-            '''
-            vals = await core.callStorm(q)
-            self.eq(vals, [('baz', 3), ('bar', 2), ('foo', 1)])
-
-            q = '''
-                $tally = $lib.stats.tally()
-                $tally.inc(foo, 1)
-                $tally.inc(bar, 2)
-                $tally.inc(baz, 3)
-                return($tally.sorted(byname=$lib.true))
-            '''
-            vals = await core.callStorm(q)
-            self.eq(vals, [('bar', 2), ('baz', 3), ('foo', 1)])
-
-            q = '''
-                $tally = $lib.stats.tally()
-                $tally.inc(foo, 1)
-                $tally.inc(bar, 2)
-                $tally.inc(baz, 3)
-                return($tally.sorted(byname=$lib.true, reverse=$lib.true))
-            '''
-            vals = await core.callStorm(q)
-            self.eq(vals, [('foo', 1), ('baz', 3), ('bar', 2)])
-
     async def test_storm_lib_layer(self):
 
         async with self.getTestCoreAndProxy() as (core, prox):
@@ -3533,10 +3561,18 @@ class StormTypesTest(s_test.SynTest):
             layrs0 = (layr0.iden, ldef1.get('iden'))
             layrs1 = (ldef1.get('iden'), layr0.iden)
 
+            # fork the default view to test editing the root view layers
+            fork00 = await core.callStorm('return($lib.view.get().fork().iden)')
+            self.eq(2, await core.callStorm('return($lib.view.get().layers.size())', opts={'view': fork00}))
+
             await core.callStorm('$lib.view.get().set(layers, $layers)', opts={'vars': {'layers': layrs0}})
             ldefs = await core.callStorm('return($lib.view.get().get(layers))')
             self.eq(layrs0, [x.get('iden') for x in ldefs])
 
+            layers = await core.callStorm('return($lib.view.get().layers)', opts={'view': fork00})
+            self.eq(layrs0, [layr['iden'] for layr in layers][-2:])
+            self.len(3, layers)
+
             await core.callStorm('$lib.view.get().set(layers, $layers)', opts={'vars': {'layers': layrs1}})
             ldefs = await core.callStorm('return($lib.view.get().get(layers))')
             self.eq(layrs1, [x.get('iden') for x in ldefs])
@@ -6289,6 +6325,25 @@ words\tword\twrd'''
                 'size:bytes': 651,
             }, await core.callStorm('return($lib.axon.metrics())'))
 
+            bin_buf = b'\xbb/$\xc0A\xf1\xbf\xbc\x00_\x82v4\xf6\xbd\x1b'
+            binsize, bin256 = await core.axon.put(bin_buf)
+
+            opts = {'vars': {'sha256': s_common.ehex(bin256)}}
+            with self.raises(s_exc.BadDataValu):
+                self.eq('', await core.callStorm('''
+                    $items = $lib.list()
+                    for $item in $lib.axon.readlines($sha256, errors=$lib.null) {
+                        $items.append($item)
+                    }
+                    return($items)
+                ''', opts=opts))
+
+            self.eq(('/$A\x00_v4\x1b',), await core.callStorm('''
+                $items = $lib.list()
+                for $item in $lib.axon.readlines($sha256, errors=ignore) { $items.append($item) }
+                return($items)
+            ''', opts=opts))
+
     async def test_storm_lib_axon_perms(self):
 
         async with self.getTestCore() as core: