synapse 2.186.0__py311-none-any.whl → 2.188.0__py311-none-any.whl

synapse/tests/test_lib_nexus.py

@@ -310,6 +310,32 @@ class NexusTest(s_t_utils.SynTest):
                         await cell01.sync()
                     self.isin(s_nexus.leaderversion, cell01.nexsroot.writeholds)
 
+                cell00.getCellInfo = getCellInfo
+
+                # test case where a mirror which is updated first may push events
+                # the leader does not yet have handlers for
+                async with await s_cell.Cell.anit(dirn=path) as cell01:
+                    cell01.nexsiden = 'newp'
+                    with self.raises(s_exc.NoSuchIden) as cm:
+                        await cell01.sync()
+                    self.eq(cm.exception.get('mesg'), 'No Nexus Pusher with iden newp.')
+
+                    self.none(await cell00.nexsroot.nexslog.last())
+                    self.none(await cell01.nexsroot.nexslog.last())
+
+                    cell01.nexsiden = cell00.nexsiden
+                    await cell01.sync()
+
+                    self.eq(0, (await cell00.nexsroot.nexslog.last())[0])
+                    self.eq(0, (await cell01.nexsroot.nexslog.last())[0])
+
+                    with self.raises(s_exc.NoSuchName) as cm:
+                        await cell01._push('newp')
+                    self.eq(cm.exception.get('mesg'), 'No Nexus handler for event newp.')
+
+                    self.eq(0, (await cell00.nexsroot.nexslog.last())[0])
+                    self.eq(0, (await cell01.nexsroot.nexslog.last())[0])
+
     async def test_mirror_nexus_loop_failure(self):
         with self.getTestDir() as dirn:
 

synapse/tests/test_lib_scrape.py

@@ -1,5 +1,7 @@
 from unittest import mock
 
+import regex
+
 import synapse.exc as s_exc
 
 import synapse.lib.scrape as s_scrape
@@ -889,14 +891,20 @@ class ScrapeTest(s_t_utils.SynTest):
         nodes.remove(('inet:url', 'smb://1:2:3:4:5:6:7:8/share'))
 
     async def test_scrape_async(self):
+        text = 'log4j vuln CVE-2021-44228 is pervasive'
+        ndefs = await s_t_utils.alist(s_scrape.scrapeAsync(text))
+        self.eq(ndefs, (('it:sec:cve', 'CVE-2021-44228'),))
 
-        with mock.patch('synapse.lib.scrape.SCRAPE_SPAWN_LENGTH', 0):
-            ndefs = await s_t_utils.alist(s_scrape.scrapeAsync('log4j vuln CVE-2021-44228 is pervasive'))
-            self.eq(ndefs, (('it:sec:cve', 'CVE-2021-44228'),))
+        regx = regex.compile('(?P<valu>CVE-[0-9]{4}-[0-9]{4,})(?:[^a-z0-9]|$)')
+        infos = s_scrape._genMatchList(text, regx, {})
+        self.eq(infos, [{'match': 'CVE-2021-44228', 'offset': 11, 'valu': 'CVE-2021-44228'}])
 
-            text = 'endashs are a vulnerability  CVE\u20132022\u20131138 '
-            infos = await s_t_utils.alist(s_scrape.contextScrapeAsync(text))
-            self.eq(infos, [{'match': 'CVE–2022–1138', 'offset': 29, 'valu': 'CVE-2022-1138', 'form': 'it:sec:cve'}])
+        text = 'endashs are a vulnerability  CVE\u20132022\u20131138 '
+        infos = await s_t_utils.alist(s_scrape.contextScrapeAsync(text))
+        self.eq(infos, [{'match': 'CVE–2022–1138', 'offset': 29, 'valu': 'CVE-2022-1138', 'form': 'it:sec:cve'}])
+
+        infos = s_scrape._contextScrapeList(text)
+        self.eq(infos, [{'match': 'CVE–2022–1138', 'offset': 29, 'valu': 'CVE-2022-1138', 'form': 'it:sec:cve'}])
 
     def test_scrape_sequential(self):
         md5 = ('a' * 32, 'b' * 32,)

synapse/tests/test_lib_spooled.py

@@ -28,11 +28,34 @@ class SpooledTest(s_test.SynTest):
 
             await sset.add(10)
 
+            newset = await sset.copy()
+            self.len(1, newset)
+            self.true(10 in newset)
+            self.false(newset.fallback)
+
             # Trigger fallback
             await sset.add(20)
             await sset.add(30)
             await sset.add(None)
 
+            newset = await sset.copy()
+            self.true(10 in newset)
+            self.true(20 in newset)
+            self.true(30 in newset)
+            self.true(None in newset)
+            self.len(4, newset)
+
+            await newset.clear()
+            self.false(10 in newset)
+            self.false(20 in newset)
+            self.false(30 in newset)
+            self.false(None in newset)
+            self.len(0, newset)
+
+            self.true(os.path.isdir(newset.slab.path))
+            await newset.fini()
+            self.false(os.path.isdir(newset.slab.path))
+
             self.len(4, sset)
 
             await sset.add(20)
@@ -68,6 +91,17 @@ class SpooledTest(s_test.SynTest):
                 self.true(os.path.isdir(sset.slab.path))
                 self.true(os.path.abspath(sset.slab.path).startswith(dirn))
 
+                newset = await sset.copy()
+                self.true(os.path.isdir(newset.slab.path))
+                self.true(os.path.abspath(newset.slab.path).startswith(dirn))
+
+            # Slabs should get removed on fini
+            self.false(os.path.isdir(sset.slab.path))
+
+            self.true(os.path.isdir(newset.slab.path))
+            await newset.fini()
+            self.false(os.path.isdir(newset.slab.path))
+
     async def test_spooled_dict(self):
 
         async def runtest(x):

synapse/tests/test_lib_storm.py

@@ -638,6 +638,8 @@ class StormTest(s_t_utils.SynTest):
             with self.raises(s_exc.NoSuchVar):
                 await core.nodes('background { $lib.print($foo) }')
 
+            await core.nodes('background ${ $foo=test $lib.print($foo) }')
+
             await core.nodes('background { $lib.time.sleep(4) }')
             task = await core.callStorm('for $t in $lib.ps.list() { if $t.info.background { return($t) } }')
             self.nn(task)
@@ -667,6 +669,9 @@ class StormTest(s_t_utils.SynTest):
             q = '[ inet:fqdn=www.vertex.link ] $q=:domain | parallel $q'
             await self.asyncraises(s_exc.StormRuntimeError, core.nodes(q))
 
+            nodes = await core.nodes('ou:org | parallel ${ $foo=bar [ :name=$foo ]}')
+            self.true(all([n.get('name') == 'bar' for n in nodes]))
+
             # test $lib.exit() and the StormExit handlers
             msgs = [m async for m in core.view.storm('$lib.exit()')]
             self.eq(msgs[-1][0], 'fini')
@@ -2137,6 +2142,49 @@ class StormTest(s_t_utils.SynTest):
             self.nn(bot.get('country::flag::md5'))
             self.eq(bot['country::flag::md5'][0], 'fa818a259cbed7ce8bc2a22d35a464fc')
 
+            await core.nodes('''
+                [( risk:vulnerable=*
+                    :mitigated=true
+                    :node={ [ it:prod:hardware=* :name=foohw ] return($node.ndef()) }
+                    :vuln={[ risk:vuln=* :name=barvuln ]}
+                    +#test
+                )]
+                [( inet:service:rule=*
+                    :object={ risk:vulnerable#test return($node.ndef()) }
+                    :grantee={ [ inet:service:account=* :id=foocon ] return($node.ndef()) }
+                    +#test
+                )]
+            ''')
+
+            opts = {
+                'embeds': {
+                    'risk:vulnerable': {
+                        'vuln': ['name'],
+                        'node': ['name'],
+                    },
+                    'inet:service:rule': {
+                        'object': ['mitigated', 'newp'],
+                        'object::node': ['name', 'newp'],
+                        'grantee': ['id', 'newp'],
+                    }
+                }
+            }
+            msgs = await core.stormlist('inet:service:rule#test :object -+> risk:vulnerable', opts=opts)
+            nodes = sorted([m[1] for m in msgs if m[0] == 'node'], key=lambda p: p[0][0])
+            self.eq(['inet:service:rule', 'risk:vulnerable'], [n[0][0] for n in nodes])
+
+            embeds = nodes[0][1]['embeds']
+            self.eq(1, embeds['object']['mitigated'])
+            self.eq(None, embeds['object']['newp'])
+            self.eq('foohw', embeds['object::node']['name'])
+            self.eq(None, embeds['object::node']['newp'])
+            self.eq('foocon', embeds['grantee']['id'])
+            self.eq(None, embeds['grantee']['newp'])
+
+            embeds = nodes[1][1]['embeds']
+            self.eq('barvuln', embeds['vuln']['name'])
+            self.eq('foohw', embeds['node']['name'])
+
     async def test_storm_wget(self):
 
         async def _getRespFromSha(core, mesgs):

synapse/tests/test_lib_stormlib_model.py

@@ -403,24 +403,7 @@ class StormlibModelTest(s_test.SynTest):
             q = 'test:str=src $n=$node -> { test:str=deny $lib.model.migration.copyTags($n, $node) }'
             await self.asyncraises(s_exc.AuthDeny, core.nodes(q, opts=aslow))
 
-            with self.raises(s_exc.NoSuchProp) as exc:
-                await core.callStorm('$lib.model.migration.liftByPropValuNoNorm(formname, propname, valu)')
-            self.eq(exc.exception.get('mesg'), 'Could not find prop: formname:propname')
-
-            with self.raises(s_exc.AuthDeny) as exc:
-                await core.callStorm('$lib.model.migration.liftByPropValuNoNorm(it:prod:soft, cpe, valu)')
-            self.eq(exc.exception.get('mesg'), '$lib.model.migration.liftByPropValuNoNorm() is restricted to model migrations only.')
-
-            with self.raises(s_exc.BadArg) as exc:
-                await core.callStorm('$lib.model.migration.setNodePropValuNoNorm(notanode, propname, valu)')
-            self.eq(exc.exception.get('mesg'), '$lib.model.migration.setNodePropValuNoNorm() argument must be a node.')
-
-            with self.raises(s_exc.AuthDeny) as exc:
-                await core.callStorm('test:str $lib.model.migration.setNodePropValuNoNorm($node, propname, valu)')
-            self.eq(exc.exception.get('mesg'), '$lib.model.migration.setNodePropValuNoNorm() is restricted to model migrations only.')
-
             # copy extended properties
-
             await self.asyncraises(s_exc.BadArg, core.nodes('test:str=src $lib.model.migration.copyExtProps($node, newp)'))
             await self.asyncraises(s_exc.BadArg, core.nodes('test:str=dst $lib.model.migration.copyExtProps(newp, $node)'))
 
@@ -442,259 +425,6 @@ class StormlibModelTest(s_test.SynTest):
             self.len(1, nodes)
             self.eq(nodes[0].get('_foo'), 'foobarbaz')
 
-    async def test_model_migration_s_itSecCpe_2_170_0(self):
-
-        async with self.getRegrCore('itSecCpe_2_170_0', maxvers=(0, 2, 27)) as core:
-            # Migrate it:sec:cpe nodes with a valid CPE2.3, valid CPE2.2
-            q = 'it:sec:cpe +#test.cpe.23valid +#test.cpe.22valid'
-            nodes = await core.nodes(q)
-            self.len(2, nodes)
-            self.eq(
-                [
-                    ('it:sec:cpe', 'cpe:2.3:a:abine:donottrackme_-_mobile_privacy:1.1.8:*:*:*:*:android:*:*'),
-                    ('it:sec:cpe', 'cpe:2.3:a:01generator:pireospay:-:*:*:*:*:prestashop:*:*')
-                ],
-                [node.ndef for node in nodes]
-            )
-
-            q = '''
-            it:sec:cpe +#test.cpe.23valid +#test.cpe.22valid
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node)
-            $node.data.load(migration.s.itSecCpe_2_170_0)
-            '''
-            nodes = await core.nodes(q)
-
-            data = nodes[0].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.none(data.get('reason'))
-
-            data = nodes[1].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.none(data.get('reason'))
-
-        async with self.getRegrCore('itSecCpe_2_170_0', maxvers=(0, 2, 27)) as core:
-            # Migrate it:sec:cpe nodes with a valid CPE2.3, invalid CPE2.2
-            q = '''
-            it:sec:cpe +#test.cpe.23valid +#test.cpe.22invalid
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node)
-            '''
-            nodes = await core.nodes(q)
-            self.len(3, nodes)
-            self.eq(
-                [
-                    ('it:sec:cpe', 'cpe:2.3:a:1c:1c\\:enterprise:-:*:*:*:*:*:*:*'),
-                    ('it:sec:cpe', 'cpe:2.3:o:zyxel:nas542_firmware:5.21\\%28aazf.15\\%29co:*:*:*:*:*:*:*'),
-                    ('it:sec:cpe', 'cpe:2.3:a:abinitio:control\\>center:-:*:*:*:*:*:*:*'),
-                ],
-                [node.ndef for node in nodes]
-            )
-
-            q = '''
-            it:sec:cpe +#test.cpe.23valid +#test.cpe.22invalid
-            $node.data.load(migration.s.itSecCpe_2_170_0)
-            '''
-            nodes = await core.nodes(q)
-            self.len(3, nodes)
-
-            data = nodes[0].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['v2_2', 'product'])
-            self.eq(nodes[0].get('v2_2'), 'cpe:/a:1c:1c%3aenterprise:-')
-            self.eq(nodes[0].get('product'), '1c:enterprise')
-
-            data = nodes[1].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.none(data.get('valu'))
-            self.eq(data['updated'], ['v2_2', 'version'])
-            self.eq(nodes[1].get('v2_2'), 'cpe:/o:zyxel:nas542_firmware:5.21%2528aazf.15%2529co')
-            self.eq(nodes[1].get('version'), '5.21%28aazf.15%29co')
-
-            data = nodes[2].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['v2_2', 'product'])
-            self.eq(nodes[2].get('v2_2'), 'cpe:/a:abinitio:control%3ecenter:-')
-            self.eq(nodes[2].get('product'), 'control>center')
-
-            # The migration of this node was not correct because the CPE2.3 string (primary property) is valid but was
-            # not created correctly due to a bad CPE2.2 input value. Now we update :v2_2 to be correct, and re-run the
-            # migration. This time, we specify `prefer_v22=True` and `force=True` so the migration will use the updated
-            # :v2_2 prop for reparsing the strings. Force will cause the migration to continue past the check where both
-            # the primary property and :v2_2 are valid.
-            q = '''
-            it:sec:cpe:product=nas542_firmware [ :v2_2="cpe:/o:zyxel:nas542_firmware:5.21%28aazf.15%29co" ]
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node, prefer_v22=$lib.true, force=$lib.true)
-            '''
-            nodes = await core.nodes(q)
-            self.len(1, nodes)
-
-            # Lift the updated node and check the migration did what was expected.
-            q = '''
-            it:sec:cpe:product=nas542_firmware
-            $node.data.load(migration.s.itSecCpe_2_170_0)
-            '''
-            nodes = await core.nodes(q)
-            self.len(1, nodes)
-
-            data = nodes[0].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['version'])
-            self.eq(data['valu'], 'cpe:2.3:o:zyxel:nas542_firmware:5.21\\(aazf.15\\)co:*:*:*:*:*:*:*')
-            self.eq(nodes[0].get('v2_2'), 'cpe:/o:zyxel:nas542_firmware:5.21%28aazf.15%29co')
-            self.eq(nodes[0].get('version'), '5.21(aazf.15)co')
-
-        async with self.getRegrCore('itSecCpe_2_170_0', maxvers=(0, 2, 27)) as core:
-            # Migrate it:sec:cpe nodes with a invalid CPE2.3, valid CPE2.2
-            q = '''
-            it:sec:cpe +#test.cpe.23invalid +#test.cpe.22valid
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node)
-            '''
-            nodes = await core.nodes(q)
-            self.len(4, nodes)
-            self.eq(
-                [
-                    ('it:sec:cpe', 'cpe:2.3:h:d\\-link:dir\\-850l:*:*:*:*:*:*:*:*'),
-                    ('it:sec:cpe', 'cpe:2.3:a:acurax:under_construction_%2f_maintenance_mode:-::~~~wordpress~~:*:*:*:*:*'),
-                    ('it:sec:cpe', 'cpe:2.3:a:10web:social_feed_for_instagram:1.0.0::~~premium~wordpress~~:*:*:*:*:*'),
-                    ('it:sec:cpe', 'cpe:2.3:o:zyxel:nas326_firmware:5.21%28aazf.14%29c0:*:*:*:*:*:*:*'),
-                ],
-                [node.ndef for node in nodes]
-            )
-
-            q = '''
-            it:sec:cpe +#test.cpe.23invalid +#test.cpe.22valid
-            $node.data.load(migration.s.itSecCpe_2_170_0)
-            '''
-            nodes = await core.nodes(q)
-            self.len(4, nodes)
-
-            data = nodes[0].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['vendor', 'product'])
-            self.eq(data['valu'], 'cpe:2.3:h:d-link:dir-850l:*:*:*:*:*:*:*:*')
-            self.eq(nodes[0].get('vendor'), 'd-link')
-            self.eq(nodes[0].get('product'), 'dir-850l')
-
-            data = nodes[1].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['product', 'update', 'edition', 'target_sw'])
-            self.eq(data['valu'], 'cpe:2.3:a:acurax:under_construction_\\/_maintenance_mode:-:*:*:*:*:wordpress:*:*')
-            self.eq(nodes[1].get('product'), 'under_construction_/_maintenance_mode')
-            self.eq(nodes[1].get('update'), '*')
-            self.eq(nodes[1].get('edition'), '*')
-            self.eq(nodes[1].get('target_sw'), 'wordpress')
-
-            data = nodes[2].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['update', 'edition', 'sw_edition', 'target_sw'])
-            self.eq(data['valu'], 'cpe:2.3:a:10web:social_feed_for_instagram:1.0.0:*:*:*:premium:wordpress:*:*')
-            self.eq(nodes[2].get('update'), '*')
-            self.eq(nodes[2].get('edition'), '*')
-            self.eq(nodes[2].get('sw_edition'), 'premium')
-            self.eq(nodes[2].get('target_sw'), 'wordpress')
-
-            data = nodes[3].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['version'])
-            self.eq(data['valu'], 'cpe:2.3:o:zyxel:nas326_firmware:5.21\\(aazf.14\\)c0:*:*:*:*:*:*:*')
-            self.eq(nodes[3].get('version'), '5.21(aazf.14)c0')
-
-        async with self.getRegrCore('itSecCpe_2_170_0', maxvers=(0, 2, 27)) as core:
-            # Migrate it:sec:cpe nodes with a invalid CPE2.3, invalid CPE2.2
-            q = '''
-            it:sec:cpe +#test.cpe.23invalid +#test.cpe.22invalid
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node)
-            '''
-            msgs = await core.stormlist(q)
-            mesg = 'itSecCpe_2_170_0(it:sec:cpe=cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*): '
-            mesg += 'Unable to migrate due to invalid data. Primary property and :v2_2 are both invalid.'
-            self.stormIsInWarn(mesg, msgs)
-
-            ndefs = [m[1][0] for m in msgs if m[0] == 'node']
-            self.eq(
-                [
-                    ('it:sec:cpe', 'cpe:2.3:a:openbsd:openssh:7.4\r\n:*:*:*:*:*:*:*'),
-                    ('it:sec:cpe', 'cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*')
-                ],
-                ndefs
-            )
-
-            q = '''
-            it:sec:cpe +#test.cpe.23invalid +#test.cpe.22invalid
-            $node.data.load(migration.s.itSecCpe_2_170_0)
-            '''
-            nodes = await core.nodes(q)
-            self.len(2, nodes)
-
-            for node in nodes:
-                data = node.nodedata['migration.s.itSecCpe_2_170_0']
-                self.eq(data, {
-                    'status': 'failed',
-                    'reason': 'Unable to migrate due to invalid data. Primary property and :v2_2 are both invalid.',
-                })
-
-            # Now update the :v2_2 on one of the nodes and migrate again
-            q = '''
-            it:sec:cpe:version^=8.2p1 [ :v2_2="cpe:/a:openbsd:openssh:8.2p1_ubuntu-4ubuntu0.2" ]
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node)
-            '''
-            msgs = await core.stormlist(q)
-            self.stormHasNoWarnErr(msgs)
-
-            q = '''
-            it:sec:cpe:version^=8.2p1
-            $node.data.load(migration.s.itSecCpe_2_170_0)
-            '''
-            nodes = await core.nodes(q)
-            self.len(1, nodes)
-
-            data = nodes[0].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['version'])
-            self.eq(data['valu'], 'cpe:2.3:a:openbsd:openssh:8.2p1_ubuntu-4ubuntu0.2:*:*:*:*:*:*:*')
-            self.eq(nodes[0].get('version'), '8.2p1_ubuntu-4ubuntu0.2')
-
-            # Run the migration again to make sure we identify already migrated
-            # nodes correctly and bail early.
-            q = '''
-            it:sec:cpe:version^=8.2p1
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node)
-            '''
-            msgs = await core.stormlist(q)
-            self.stormIsInPrint('DEBUG: itSecCpe_2_170_0(it:sec:cpe=cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*): Node already migrated.', msgs)
-
-            q = '''
-            it:sec:cpe:version^=8.2p1
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node, force=$lib.true)
-            '''
-            msgs = await core.stormlist(q)
-            self.stormIsInPrint('DEBUG: itSecCpe_2_170_0(it:sec:cpe=cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*): No property updates required.', msgs)
-
-        async with self.getTestCore() as core:
-            with self.raises(s_exc.BadArg):
-                await core.callStorm('$lib.model.migration.s.itSecCpe_2_170_0(newp)')
-
-            with self.raises(s_exc.BadArg):
-                await core.callStorm('[ inet:fqdn=vertex.link ] $lib.model.migration.s.itSecCpe_2_170_0($node)')
-
     async def test_stormlib_model_migrations_risk_hasvuln_vulnerable(self):
 
         async with self.getTestCore() as core:

synapse/tests/test_lib_stormlib_modelext.py

@@ -25,6 +25,13 @@ class StormtypesModelextTest(s_test.SynTest):
 
                 $edgeinfo = ({"doc": "A test edge."})
                 $lib.model.ext.addEdge(inet:user, _copies, *, $edgeinfo)
+
+                $typeopts = ({"lower": true, "onespace": true})
+                $typeinfo = ({"doc": "A test type doc."})
+                $forminfo = ({"doc": "A test type form doc."})
+                $lib.model.ext.addType(_test:type, str, $typeopts, $typeinfo)
+                $lib.model.ext.addForm(_test:typeform, _test:type, ({}), $forminfo)
+                $lib.model.ext.addForm(_test:typearry, array, ({"type": "_test:type"}), $forminfo)
             ''')
 
             nodes = await core.nodes('[ _visi:int=10 :tick=20210101 ._woot=30 +#lol:score=99 ]')
@@ -39,6 +46,18 @@ class StormtypesModelextTest(s_test.SynTest):
             self.eq(nodes[0].ndef, ('test:int', 1234))
             self.eq(nodes[0].get('_tick'), 1609459200000)
 
+            nodes = await core.nodes('[_test:typeform="  FoO BaR  "]')
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef, ('_test:typeform', 'foo bar'))
+
+            with self.raises(s_exc.DupTypeName):
+                q = '$lib.model.ext.addType(_test:type, str, ({}), ({}))'
+                await core.callStorm(q)
+
+            with self.raises(s_exc.DupTypeName):
+                q = '$lib.model.ext.addForm(_test:type, str, ({}), ({}))'
+                await core.callStorm(q)
+
             with self.raises(s_exc.DupPropName):
                 q = '''$lib.model.ext.addFormProp(_visi:int, tick, (time, ({})), ({}))'''
                 await core.callStorm(q)
@@ -69,7 +88,7 @@ class StormtypesModelextTest(s_test.SynTest):
             await core._delAllTagProp('score', {})
             self.len(0, await core.nodes('#lol:score'))
 
-            await core.callStorm('_visi:int=10 test:int=1234 | delnode')
+            await core.callStorm('_visi:int=10 test:int=1234 _test:typeform | delnode')
             await core.callStorm('''
                 $lib.model.ext.delTagProp(score, force=(true))
                 $lib.model.ext.delUnivProp(_woot, force=(true))
@@ -79,6 +98,22 @@ class StormtypesModelextTest(s_test.SynTest):
                 $lib.model.ext.delEdge(inet:user, _copies, *)
             ''')
 
+            with self.raises(s_exc.CantDelType) as cm:
+                await core.callStorm('$lib.model.ext.delType(_test:type)')
+            self.isin('still in use by other types', cm.exception.get('mesg'))
+
+            await core.callStorm('$lib.model.ext.delForm(_test:typeform)')
+
+            with self.raises(s_exc.CantDelType) as cm:
+                await core.callStorm('$lib.model.ext.delType(_test:type)')
+            self.isin('still in use by array types', cm.exception.get('mesg'))
+
+            await core.callStorm('$lib.model.ext.delForm(_test:typearry)')
+            await core.callStorm('$lib.model.ext.delType(_test:type)')
+
+            self.none(core.model.type('_test:type'))
+            self.none(core.model.form('_test:typeform'))
+            self.none(core.model.form('_test:typearry'))
             self.none(core.model.form('_visi:int'))
             self.none(core.model.prop('._woot'))
             self.none(core.model.prop('_visi:int:tick'))
@@ -100,6 +135,14 @@ class StormtypesModelextTest(s_test.SynTest):
             q = '''$lib.model.ext.addTagProp(some:_score, (int, ({})), ({}))'''
             self.none(await core.callStorm(q))
 
+            with self.raises(s_exc.BadTypeDef):
+                q = '$lib.model.ext.addType(test:type, str, ({}), ({}))'
+                await core.callStorm(q)
+
+            with self.raises(s_exc.BadTypeDef):
+                q = '$lib.model.ext.delType(test:type)'
+                await core.callStorm(q)
+
             with self.raises(s_exc.BadPropDef):
                 q = '''$l =$lib.list('str', ({})) $d=({"doc": "Foo"})
                 $lib.model.ext.addFormProp('test:str', '_test:_my^prop', $l, $d)
@@ -158,6 +201,18 @@ class StormtypesModelextTest(s_test.SynTest):
                     $lib.model.ext.addForm(_visi:int, int, $typeinfo, $forminfo)
                 ''', opts=opts)
 
+            with self.raises(s_exc.AuthDeny) as cm:
+                await core.callStorm('''
+                    $lib.model.ext.addType(_test:type, str, ({}), ({}))
+                ''', opts=opts)
+            self.isin('permission model.type.add._test:type', cm.exception.get('mesg'))
+
+            with self.raises(s_exc.AuthDeny) as cm:
+                await core.callStorm('''
+                    $lib.model.ext.delType(_test:type)
+                ''', opts=opts)
+            self.isin('permission model.type.del._test:type', cm.exception.get('mesg'))
+
             with self.raises(s_exc.AuthDeny):
                 await core.callStorm('''
                     $propinfo = ({"doc": "A test prop doc."})
@@ -206,6 +261,10 @@ class StormtypesModelextTest(s_test.SynTest):
         # Add props which conflict with what was previously dumped
         async with self.getTestCore() as core:
             await core.callStorm('''
+                $typeopts = ({"lower": true})
+                $typeinfo = ({"doc": "A test type doc."})
+                $lib.model.ext.addType(_test:type, str, $typeopts, $typeinfo)
+
                 $typeinfo = ({})
                 $forminfo = ({"doc": "NEWP"})
                 $lib.model.ext.addForm(_visi:int, int, $typeinfo, $forminfo)
@@ -226,6 +285,11 @@ class StormtypesModelextTest(s_test.SynTest):
                 $lib.model.ext.addEdge(inet:user, _copies, *, $edgeinfo)
             ''')
 
+            q = '''return ($lib.model.ext.addExtModel($model_defs))'''
+            with self.raises(s_exc.BadTypeDef) as cm:
+                opts = {'vars': {'model_defs': {'types': model_defs['types']}}}
+                await core.callStorm(q, opts)
+
             q = '''return ($lib.model.ext.addExtModel($model_defs))'''
             with self.raises(s_exc.BadFormDef) as cm:
                 opts = {'vars': {'model_defs': {'forms': model_defs['forms']}}}
@@ -255,6 +319,9 @@ class StormtypesModelextTest(s_test.SynTest):
         async with self.getTestCore() as core:
             opts = {'vars': {'model_defs': model_defs}}
             q = '''
+            for ($name, $type, $opts, $info) in $model_defs.types {
+                $lib.model.ext.addType($name, $type, $opts, $info)
+            }
             for ($name, $type, $opts, $info) in $model_defs.forms {
                 $lib.model.ext.addForm($name, $type, $opts, $info)
             }
@@ -473,6 +540,14 @@ class StormtypesModelextTest(s_test.SynTest):
                 '$lib.model.ext.addForm(inet:fqdn, _foo:bar, ({}), ())',
                 'Form type info should be a dict.'
             ),
+            (
+                '$lib.model.ext.addType(_test:type, str, (guid, ()), ())',
+                'Type options should be a dict.'
+            ),
+            (
+                '$lib.model.ext.addType(_test:type, str, ({}), ())',
+                'Type info should be a dict.'
+            ),
             (
                 '$lib.model.ext.addFormProp(inet:fqdn, _foo:bar, ({}), ())',
                 'Form property type definitions should be a tuple.'

synapse/tests/test_lib_stormlib_scrape.py

@@ -105,14 +105,6 @@ class StormScrapeTest(s_test.SynTest):
             self.stormIsInPrint('inet:url=https://giggles.com/mallory.html', msgs)
             self.stormIsInPrint("'match': 'hzzps[:]\\\\giggles.com/mallory.html'", msgs)
 
-            with mock.patch('synapse.lib.scrape.SCRAPE_SPAWN_LENGTH', 0):
-                msgs = await core.stormlist(q, opts={'vars': {'text': text}})
-                self.stormIsInPrint('ps:name=alice', msgs)
-                self.stormIsInPrint('inet:fqdn=foo.bar.com', msgs)
-                self.stormIsInPrint('inet:url=https://1.2.3.4/alice.html', msgs)
-                self.stormIsInPrint('inet:url=https://giggles.com/mallory.html', msgs)
-                self.stormIsInPrint("'match': 'hzzps[:]\\\\giggles.com/mallory.html'", msgs)
-
             cq = '''$ret=$lib.list()
             for ($form, $valu) in $lib.scrape.ndefs($text) {
                 $ret.append(($form, $valu))

synapse/tests/test_lib_stormtypes.py

@@ -907,6 +907,17 @@ class StormTypesTest(s_test.SynTest):
                        'refs',
                        '20153b758f9d5eaaa38e4f4a65c36da797c3e59e549620fa7c4895e1a920991f'), edges)
 
+            data = await core.callStorm('''
+                $data = ({})
+                inet:user=visi
+                for ($name, $valu) in $lib.layer.get().getNodeData($node.iden()) { $data.$name = $valu }
+                return($data)
+            ''')
+            foo = data.get('foo')
+            self.nn(foo)
+            self.nn(foo.get('asof'))
+            self.eq('bar', foo.get('data'))
+
             msgs = await core.stormlist('$lib.print($lib.null)')
             self.stormIsInPrint('$lib.null', msgs)
             self.stormNotInPrint('None', msgs)
@@ -2331,7 +2342,7 @@ class StormTypesTest(s_test.SynTest):
                 path = pode[1]['path']
                 self.len(1, path)
                 key = list(path.keys())[0]
-                self.true(key.startswith("Node{(('inet:fqdn', 'vertex.link'), {'iden':"))
+                self.true(key.startswith('vertex.link'))
                 self.eq(('foo', 'bar'), path[key])
 
                 q = '''