synapse 2.168.0__py311-none-any.whl → 2.170.0__py311-none-any.whl

synapse/tests/test_lib_scrape.py

@@ -508,6 +508,77 @@ bad_uncs = [
 
 unc_paths = '\n'.join(good_uncs + bad_uncs)
 
+cpedata = r'''GOOD DATA
+cpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:other
+cpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:tspace non matched word
+cpe:2.3:a:*:*:*:*:*:*:*:*:*:*
+cpe:2.3:h:*:*:*:*:*:*:*:*:*:*
+cpe:2.3:o:*:*:*:*:*:*:*:*:*:*
+cpe:2.3:-:*:*:*:*:*:*:*:*:*:*
+cpe:2.3:*:*:*:*:*:*:*:*:*:*:*
+cpe:2.3:*:-:na:*:*:*:*:*:*:*:*
+cpe:2.3:*:.:dot:*:*:*:*:*:*:*:*
+cpe:2.3:*:_:underscore:*:*:*:*:*:*:*:*
+
+A few quoted characters
+cpe:2.3:*:\!:quoted:*:*:*:*:*:*:*:*
+cpe:2.3:*:\?:quoted:*:*:*:*:*:*:*:*
+cpe:2.3:*:\*:quoted:*:*:*:*:*:*:*:*
+cpe:2.3:*:\\:escapeescape:*:*:*:*:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:-:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:*:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:en:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:usa:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:usa-en:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:usa-123:*:*:*:*
+
+A few examples
+cpe:2.3:a:ntp:ntp:4.2.8:p3:*:*:*:*:*:*
+cpe:2.3:o:microsoft:windows_7:-:sp2:*:*:*:*:*:*
+cpe:2.3:a:hp:insight:7.4.0.1570:-:*:*:online:win2003:x64:*
+cpe:2.3:a:foo\\bar:big\$money_2010:*:*:*:*:special:ipod_touch:80gb:*
+cpe:2.3:a:hp:openview_network_manager:7.51:*:*:*:*:linux:*:*
+cpe:2.3:a:apple:swiftnio_http\/2:1.19.1:*:*:*:*:swift:*:*
+
+Some quoted examples
+cpe:2.3:a:fooo:bar_baz\:_beep_bpp_sys:1.1:*:*:*:*:ios:*:*
+cpe:2.3:a:lemonldap-ng:apache\:\:session\:\:browsable:0.9:*:*:*:*:perl:*:*
+cpe:2.3:a:daemon-ng:hurray\:\::0.x:*:*:*:*:*:*:*
+cpe:2.3:a:microsoft:intern\^et_explorer:8.0.6001:beta:*:*:*:*:*:*
+
+TEXT examples
+Example double quoted cpe value "cpe:2.3:a:vertex:synapse:*:*:*:*:*:*:*:dquotes".
+Example double quoted cpe value "cpe:2.3:a:vertex:synapse:*:*:*:*:*:*:*:MiXeDcAsE".
+Example single quoted cpe value "cpe:2.3:a:vertex:synapse:*:*:*:*:*:*:*:squotes".
+A CPE at the end of a sentence like this captures the period... cpe:2.3:a:*:*:*:*:*:*:*:*:*:hasperiod.
+Some CPE are exciting! Like this cpe:2.3:a:*:*:*:*:*:*:*:*:*:noexclaim!
+Some CPE are boring! Like this cpe:2.3:a:*:*:*:*:*:*:*:*:*:noslash\
+Unicode endings are omitted cpe:2.3:a:*:*:*:*:*:*:*:*:*:unicodeend0ॐ
+Unicode quotes “cpe:2.3:a:*:*:*:*:*:*:*:*:*:smartquotes”
+cpe:2.3:*:?why??:*:*:*:*:*:*:*:*:*
+cpe:2.3:*:*why*:*:*:*:*:*:*:*:*:*
+
+EMBEDDED TEXT
+wordscpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:otherxxx:newp
+wordscpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:otherzzz:
+
+BAD values
+cpe:2.3:*:?:spec1:*:*:*:*:*:*:*:*
+cpe:2.3:a:vertex:synapse:*:*:*:NEWP:*:*:*:*
+cpe:2.3:a::::::::::
+cpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:ॐ:other
+cpe:2.3:a:vendor:product:version:update:edition
+cpe:2.3:a:opps:bad_quote\\/2:1.19.1:*:*:*:*:swift:*:*
+
+# Bad languages
+cpe:2.3:*:langtest:*:*:*:*:a:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:aaaa:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:usa-o:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:usa-omn:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:usa-12:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:usa-1234:*:*:*:*
+'''
+
 class ScrapeTest(s_t_utils.SynTest):
 
     def test_scrape_basic(self):
@@ -1050,77 +1121,7 @@ class ScrapeTest(s_t_utils.SynTest):
             self.eq(erv, fv)
 
     def test_scrape_cpe(self):
-        cpedata = r'''
-        GOOD DATA
-        cpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:other
-        cpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:tspace non matched word
-        cpe:2.3:a:*:*:*:*:*:*:*:*:*:*
-        cpe:2.3:h:*:*:*:*:*:*:*:*:*:*
-        cpe:2.3:o:*:*:*:*:*:*:*:*:*:*
-        cpe:2.3:-:*:*:*:*:*:*:*:*:*:*
-        cpe:2.3:*:*:*:*:*:*:*:*:*:*:*
-        cpe:2.3:*:-:na:*:*:*:*:*:*:*:*
-        cpe:2.3:*:.:dot:*:*:*:*:*:*:*:*
-        cpe:2.3:*:_:underscore:*:*:*:*:*:*:*:*
-
-        A few quoted characters
-        cpe:2.3:*:\!:quoted:*:*:*:*:*:*:*:*
-        cpe:2.3:*:\?:quoted:*:*:*:*:*:*:*:*
-        cpe:2.3:*:\*:quoted:*:*:*:*:*:*:*:*
-        cpe:2.3:*:\\:escapeescape:*:*:*:*:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:-:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:*:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:en:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:usa:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:usa-en:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:usa-123:*:*:*:*
-
-        A few examples
-        cpe:2.3:a:ntp:ntp:4.2.8:p3:*:*:*:*:*:*
-        cpe:2.3:o:microsoft:windows_7:-:sp2:*:*:*:*:*:*
-        cpe:2.3:a:hp:insight:7.4.0.1570:-:*:*:online:win2003:x64:*
-        cpe:2.3:a:foo\\bar:big\$money_2010:*:*:*:*:special:ipod_touch:80gb:*
-        cpe:2.3:a:hp:openview_network_manager:7.51:*:*:*:*:linux:*:*
-        cpe:2.3:a:apple:swiftnio_http\/2:1.19.1:*:*:*:*:swift:*:*
-
-        Some quoted examples
-        cpe:2.3:a:fooo:bar_baz\:_beep_bpp_sys:1.1:*:*:*:*:ios:*:*
-        cpe:2.3:a:lemonldap-ng:apache\:\:session\:\:browsable:0.9:*:*:*:*:perl:*:*
-        cpe:2.3:a:daemon-ng:hurray\:\::0.x:*:*:*:*:*:*:*
-        cpe:2.3:a:microsoft:intern\^et_explorer:8.0.6001:beta:*:*:*:*:*:*
-
-        TEXT examples
-        Example double quoted cpe value "cpe:2.3:a:vertex:synapse:*:*:*:*:*:*:*:dquotes".
-        Example double quoted cpe value "cpe:2.3:a:vertex:synapse:*:*:*:*:*:*:*:MiXeDcAsE".
-        Example single quoted cpe value "cpe:2.3:a:vertex:synapse:*:*:*:*:*:*:*:squotes".
-        A CPE at the end of a sentence like this captures the period... cpe:2.3:a:*:*:*:*:*:*:*:*:*:hasperiod.
-        Some CPE are exciting! Like this cpe:2.3:a:*:*:*:*:*:*:*:*:*:noexclaim!
-        Some CPE are boring! Like this cpe:2.3:a:*:*:*:*:*:*:*:*:*:noslash\
-        Unicode endings are omitted cpe:2.3:a:*:*:*:*:*:*:*:*:*:unicodeend0ॐ
-        Unicode quotes “cpe:2.3:a:*:*:*:*:*:*:*:*:*:smartquotes”
-        cpe:2.3:*:?why??:*:*:*:*:*:*:*:*:*
-        cpe:2.3:*:*why*:*:*:*:*:*:*:*:*:*
-
-        EMBEDDED TEXT
-        wordscpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:otherxxx:newp
-        wordscpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:otherzzz:
-
-        BAD values
-        cpe:2.3:*:?:spec1:*:*:*:*:*:*:*:*
-        cpe:2.3:a:vertex:synapse:*:*:*:NEWP:*:*:*:*
-        cpe:2.3:a::::::::::
-        cpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:ॐ:other
-        cpe:2.3:a:vendor:product:version:update:edition
-        cpe:2.3:a:opps:bad_quote\\/2:1.19.1:*:*:*:*:swift:*:*
-
-        # Bad languages
-        cpe:2.3:*:langtest:*:*:*:*:a:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:aaaa:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:usa-o:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:usa-omn:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:usa-12:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:usa-1234:*:*:*:*
-        '''
+
         nodes = sorted(set(s_scrape.scrape(cpedata, ptype='it:sec:cpe')))
         nodes.remove(('it:sec:cpe', 'cpe:2.3:*:*:*:*:*:*:*:*:*:*:*'))
         nodes.remove(('it:sec:cpe', 'cpe:2.3:*:-:na:*:*:*:*:*:*:*:*'))

synapse/tests/test_lib_snap.py

@@ -584,7 +584,10 @@ class SnapTest(s_t_utils.SynTest):
     async def test_snap_editor(self):
 
         async with self.getTestCore() as core:
-            nodes = await core.nodes('[ media:news=63381924986159aff183f0c85bd8ebad +(refs)> {[ inet:fqdn=vertex.link ]} ]')
+
+            await core.nodes('$lib.model.ext.addTagProp(test, (str, ({})), ({}))')
+            await core.nodes('[ media:news=63381924986159aff183f0c85bd8ebad +(refs)> {[ inet:fqdn=vertex.link ]} +#foo ]')
+
             root = core.auth.rootuser
             async with await core.view.snap(user=root) as snap:
                 async with snap.getEditor() as editor:
@@ -608,6 +611,14 @@ class SnapTest(s_t_utils.SynTest):
                     self.len(1, nodeedits)
                     self.len(1, nodeedits[0][2])
 
+                    self.false(await news.hasData('foo'))
+                    await news.setData('foo', 'bar')
+                    self.true(await news.hasData('foo'))
+
+                    self.false(news.hasTagProp('foo', 'test'))
+                    await news.setTagProp('foo', 'test', 'bar')
+                    self.true(news.hasTagProp('foo', 'test'))
+
                 async with snap.getEditor() as editor:
                     news = await editor.addNode('media:news', '63381924986159aff183f0c85bd8ebad')
 
@@ -629,6 +640,10 @@ class SnapTest(s_t_utils.SynTest):
                     self.false(await news.delEdge('pwns', 1))
                     self.false(await news.delEdge('pwns', 'bar'))
 
+                    self.true(await news.hasData('foo'))
+
+                    self.true(news.hasTagProp('foo', 'test'))
+
             self.len(1, await core.nodes('media:news -(pwns)> *'))
 
             self.len(1, await core.nodes('[ test:ro=foo :writeable=hehe :readable=haha ]'))

synapse/tests/test_lib_storm.py

@@ -19,6 +19,8 @@ import synapse.lib.version as s_version
 import synapse.tests.utils as s_t_utils
 from synapse.tests.utils import alist
 
+import synapse.tools.backup as s_tools_backup
+
 class StormTest(s_t_utils.SynTest):
 
     async def test_lib_storm_jsonexpr(self):
@@ -2041,6 +2043,90 @@ class StormTest(s_t_utils.SynTest):
 
             self.eq((1, 6), await core.callStorm('return($lib.queue.gen(foo).get(1))'))
 
+    async def test_storm_dmon_query_state(self):
+        with self.getTestDir() as dirn:
+            dirn00 = s_common.gendir(dirn, 'core00')
+            dirn01 = s_common.gendir(dirn, 'core01')
+            dirn02 = s_common.gendir(dirn, 'core02')
+
+            async with self.getTestCore(dirn=dirn00) as core00:
+
+                msgs = await core00.stormlist('[ inet:ipv4=1.2.3.4 ]')
+                self.stormHasNoWarnErr(msgs)
+
+            s_tools_backup.backup(dirn00, dirn01)
+            s_tools_backup.backup(dirn00, dirn02)
+
+            async with self.getTestCore(dirn=dirn00) as core00:
+                conf01 = {'mirror': core00.getLocalUrl()}
+
+                async with self.getTestCore(dirn=dirn01, conf=conf01) as core01:
+
+                    conf02 = {'mirror': core01.getLocalUrl()}
+
+                    async with self.getTestCore(dirn=dirn02, conf=conf02) as core02:
+
+                        await core02.sync()
+
+                        nodes = await core01.nodes('inet:ipv4')
+                        self.len(1, nodes)
+                        self.eq(nodes[0].ndef, ('inet:ipv4', 16909060))
+
+                        q = '''
+                        $lib.queue.gen(dmonloop)
+                        return(
+                            $lib.dmon.add(${
+                                $queue = $lib.queue.get(dmonloop)
+                                while $lib.true {
+                                    ($offs, $mesg) = $queue.get()
+
+                                    switch $mesg.0 {
+                                        "print": { $lib.print($mesg.1) }
+                                        "warn": { $lib.warn($mesg.1) }
+                                        "leave": {
+                                            $lib.print(leaving)
+                                            break
+                                        }
+                                        *: { continue }
+                                    }
+
+                                    $queue.cull($offs)
+                                }
+                            }, name=dmonloop)
+                        )
+                        '''
+                        ddef = await core02.callStorm(q)
+                        self.nn(ddef['iden'])
+
+                        dmons = await core02.getStormDmons()
+                        self.len(1, dmons)
+                        self.eq(dmons[0]['iden'], ddef['iden'])
+
+                        info = await core02.getStormDmon(ddef['iden'])
+                        self.eq(info['iden'], ddef['iden'])
+                        self.eq(info['name'], 'dmonloop')
+                        self.eq(info['status'], 'running')
+
+                        await core02.callStorm('$lib.queue.get(dmonloop).put((print, printfoo))')
+                        await core02.callStorm('$lib.queue.get(dmonloop).put((warn, warnfoo))')
+
+                        info = await core02.getStormDmon(ddef['iden'])
+                        self.eq(info['status'], 'running')
+
+                        logs = await core02.getStormDmonLog(ddef['iden'])
+                        msgs = [k[1] for k in logs]
+                        self.stormIsInPrint('printfoo', msgs)
+                        self.stormIsInWarn('warnfoo', msgs)
+
+                        await core02.callStorm('$lib.queue.get(dmonloop).put((leave,))')
+
+                        info = await core02.getStormDmon(ddef['iden'])
+                        self.eq(info['status'], 'sleeping')
+
+                        logs = await core02.getStormDmonLog(ddef['iden'])
+                        msgs = [k[1] for k in logs]
+                        self.stormIsInPrint('leaving', msgs)
+
     async def test_storm_pipe(self):
 
         async with self.getTestCore() as core:
@@ -3098,6 +3184,38 @@ class StormTest(s_t_utils.SynTest):
             nodes = await core.nodes(q, opts={'view': fork, 'vars': {'view': view}})
             self.len(1, nodes)
 
+    async def test_storm_viewexec(self):
+
+        async with self.getTestCore() as core:
+
+            view = await core.callStorm('return( $lib.view.get().iden )')
+            fork = await core.callStorm('return( $lib.view.get().fork().iden )')
+
+            await core.addStormPkg({
+                'name': 'testpkg',
+                'version': (0, 0, 1),
+                'modules': (
+                    {'name': 'priv.exec',
+                     'asroot:perms': [['power-ups', 'testpkg']],
+                     'modconf': {'viewiden': fork},
+                     'storm': '''
+                        function asroot () {
+                            view.exec $modconf.viewiden { $foo=bar } | return($foo)
+                        }
+                     '''},
+                ),
+            })
+
+            visi = await core.auth.addUser('visi')
+            asvisi = {'user': visi.iden}
+
+            await core.stormlist('auth.user.addrule visi power-ups.testpkg')
+
+            with self.raises(s_exc.AuthDeny):
+                await core.callStorm('return(woot)', opts={'user': visi.iden, 'view': fork})
+
+            self.eq('bar', await core.callStorm('return($lib.import(priv.exec).asroot())', opts=asvisi))
+
     async def test_storm_argv_parser(self):
 
         pars = s_storm.Parser(prog='hehe')

synapse/tests/test_lib_stormlib_cortex.py

@@ -281,6 +281,18 @@ $request.reply(206, headers=$headers, body=({"no":"body"}))
                 self.eq(data.get('json'), 'err')
                 self.eq(data.get('path'), 'echo/words/wOw')
 
+                # Storm query logging includes the httpapi iden in structlog data
+                core.stormlog = True
+                with self.getStructuredAsyncLoggerStream('synapse.storm', 'Executing storm query') as stream:
+                    resp = await sess.get(url)
+                    self.eq(resp.status, 200)
+                    self.true(await stream.wait(timeout=12))
+                data = stream.getvalue()
+                raw_mesgs = [m for m in data.split('\n') if m]
+                msgs = [json.loads(m) for m in raw_mesgs]
+                self.eq(msgs[0].get('httpapi'), echoiden)
+                core.stormlog = False
+
                 # Sad paths on the $request methods
                 q = '''$api = $lib.cortex.httpapi.add(testpath02)
                 $api.methods.get = ${ $request.sendcode(200) $request.sendheaders('beep beep') }
@@ -1268,6 +1280,12 @@ for $i in $values {
             return ( ($api.iden) )'''
             iden06 = await core.callStorm(q)
 
+            # attempt to JSON decode non-JSON request
+            q = '''$api = $lib.cortex.httpapi.add(bad07)
+            $api.methods.get = ${ $foo = $request.json() }
+            return ( ($api.iden) )'''
+            iden07 = await core.callStorm(q)
+
             async with self.getHttpSess(auth=('root', 'root'), port=hport) as sess:
                 resp = await sess.get(f'https://localhost:{hport}/api/ext/bad00')
                 self.eq(resp.status, 500)
@@ -1317,6 +1335,15 @@ for $i in $values {
                     self.eq(resp.status, 201)
                     self.eq(await resp.json(), {})
 
+                with self.getAsyncLoggerStream('synapse.lib.httpapi',
+                                               f'Error executing Extended HTTP API {iden07}: StormRuntimeError') as stream:
+                    resp = await sess.get(f'https://localhost:{hport}/api/ext/bad07')
+                    self.true(await stream.wait(timeout=6))
+                    self.eq(resp.status, 500)
+                    data = await resp.json()
+                    self.eq(data.get('code'), 'StormRuntimeError')
+                    self.isin('Failed to decode request body as JSON: Expecting value', data.get('mesg'))
+
     async def test_cortex_httpapi_dynamic(self):
 
         # API endpoints can be dynamic. Use at your own risk.