synapse 2.169.0__py311-none-any.whl → 2.171.0__py311-none-any.whl

synapse/tests/test_cortex.py

@@ -2430,6 +2430,41 @@ class CortexTest(s_t_utils.SynTest):
             msgs = await core.stormlist('test:int :loc -> test:newp')
             self.stormIsInErr('No property named test:newp', msgs)
 
+            # ndef pivots
+            await core.nodes('''
+                [
+                    ( test:str=ndefpivdst )
+                    ( test:str=ndefpivsrc :bar=(test:str, ndefpivdst) )
+                    ( test:str=ndefpivprp :bar=(test:str, ndefpivdst) )
+                ]
+            ''')
+
+            nodes = await core.nodes('test:str=ndefpivsrc -> test:str')
+            self.eq(['ndefpivdst'], [n.ndef[1] for n in nodes])
+
+            nodes = await core.nodes('test:str=ndefpivsrc -> test:str:bar')
+            self.len(0, nodes)
+
+            nodes = await core.nodes('test:str=ndefpivdst -> test:str:bar')
+            self.sorteq(['ndefpivprp', 'ndefpivsrc'], [n.ndef[1] for n in nodes])
+
+            nodes = await core.nodes('test:str=ndefpivsrc :bar -> * +test:str')
+            self.eq(['ndefpivdst'], [n.ndef[1] for n in nodes])
+
+            nodes = await core.nodes('test:str=ndefpivsrc :bar -> test:str:bar')
+            self.sorteq(['ndefpivprp', 'ndefpivsrc'], [n.ndef[1] for n in nodes])
+
+            nodes = await core.nodes('test:str=ndefpivsrc :bar -> test:str')
+            self.eq(['ndefpivdst'], [n.ndef[1] for n in nodes])
+
+            nodes = await core.nodes('test:str=ndefpivsrc :bar -> test:int')
+            self.len(0, nodes)
+
+            await core.nodes('test:str=ndefpivdst delnode')
+            msgs = await core.stormlist('test:str=ndefpivsrc :bar -> test:str')
+            self.len(0, [m for m in msgs if m[0] == 'node'])
+            self.stormIsInWarn("Missing node corresponding to ndef ('test:str', 'ndefpivdst')", msgs)
+
             # Bad pivot syntax go here
             for q in ['test:pivcomp :lulz <- *',
                       'test:pivcomp :lulz <+- *',
@@ -6257,12 +6292,14 @@ class CortexBasicTest(s_t_utils.SynTest):
 
             # Can't delete the default view
             await self.asyncraises(s_exc.SynErr, core.delView(core.view.iden))
+            await self.asyncraises(s_exc.SynErr, core._delViewWithLayer(core.view.iden, None, None))
 
             # Can't delete a layer in a view
             await self.asyncraises(s_exc.SynErr, core.delLayer(core.view.layers[0].iden))
 
             # Can't delete a nonexistent view
             await self.asyncraises(s_exc.NoSuchView, core.delView('XXX'))
+            await self.asyncraises(s_exc.NoSuchView, core.delViewWithLayer('XXX'))
 
             # Can't delete a nonexistent layer
             await self.asyncraises(s_exc.NoSuchLayer, core.delLayer('XXX'))
@@ -6275,6 +6312,77 @@ class CortexBasicTest(s_t_utils.SynTest):
             await core.delView(view2_iden)
             await self.asyncraises(s_exc.NoSuchView, core.delView(view2_iden))
 
+            layr = await core.addLayer()
+            layriden = layr['iden']
+            vdef3 = {'layers': (layriden,)}
+            view3_iden = (await core.addView(vdef3)).get('iden')
+
+            opts = {'view': view3_iden}
+            await core.callStorm('$lib.view.get().set(protected, $lib.true)', opts=opts)
+
+            await self.asyncraises(s_exc.CantDelView, core.delViewWithLayer(view3_iden))
+
+            await core.callStorm('$lib.view.get().set(protected, $lib.false)', opts=opts)
+
+            view3 = core.getView(view3_iden)
+            vdef4 = await view3.fork()
+
+            deadlayr = view3.layers[0].iden
+            view4_iden = vdef4.get('iden')
+            view4 = core.getView(view4_iden)
+
+            self.eq(view4.parent, view3)
+            self.len(2, view4.layers)
+
+            await core.auth.rootuser.setPasswd('secret')
+            host, port = await core.dmon.listen('tcp://127.0.0.1:0/')
+            layr2 = await core.callStorm('$layer=$lib.layer.add() return($layer)')
+            varz = {'iden': layriden, 'tgt': layr2.get('iden'), 'port': port}
+            opts = {'vars': varz, 'view': view3_iden}
+
+            pullq = '$layer=$lib.layer.get($iden).addPull(`tcp://root:secret@127.0.0.1:{$port}/*/layer/{$tgt}`)'
+            pushq = '$layer=$lib.layer.get($iden).addPush(`tcp://root:secret@127.0.0.1:{$port}/*/layer/{$tgt}`)'
+            msgs = await core.stormlist(pullq, opts=opts)
+            self.stormHasNoWarnErr(msgs)
+
+            msgs = await core.stormlist(pushq, opts=opts)
+            self.stormHasNoWarnErr(msgs)
+
+            coros = len(core.activecoros)
+
+            layridens = [lyr.iden for lyr in view4.layers if lyr.iden != view3.layers[0].iden]
+            events = [
+                {'event': 'view:setlayers', 'info': {'iden': view4.iden, 'layers': layridens}},
+                {'event': 'view:set', 'info': {'iden': view4.iden, 'name': 'parent', 'valu': None}}
+            ]
+            task = core.schedCoro(s_t_utils.waitForBehold(core, events))
+
+            await core.delViewWithLayer(view3_iden)
+
+            await asyncio.wait_for(task, timeout=1)
+
+            # push/pull activecoros have been deleted
+            self.len(coros - 2, core.activecoros)
+
+            self.none(view4.parent)
+            self.len(1, view4.layers)
+            self.none(core.getLayer(deadlayr))
+
+            vdef5 = await view4.fork()
+            view5 = core.getView(vdef5.get('iden'))
+
+            usedlayr = view4.layers[0].iden
+            vdef6 = {'layers': (usedlayr,)}
+            view6 = core.getView((await core.addView(vdef6)).get('iden'))
+
+            await core.delViewWithLayer(view4_iden)
+
+            self.none(view5.parent)
+            self.len(1, view5.layers)
+
+            self.nn(core.getLayer(usedlayr))
+            self.eq([usedlayr], [lyr.iden for lyr in view6.layers])
+
     async def test_cortex_view_opts(self):
         '''
         Test that the view opts work

synapse/tests/test_lib_ast.py

@@ -670,6 +670,28 @@ class AstTest(s_test.SynTest):
             self.len(1, nodes)
             self.eq('geo:nloc', nodes[0].ndef[0])
 
+            await core.nodes('[ test:str=ndefs :ndefs=((it:dev:int, 1), (it:dev:int, 2)) ]')
+            await core.nodes('test:str=ndefs [ :ndefs += (inet:fqdn, woot.com) ]')
+            self.len(1, nodes)
+
+            nodes = await core.nodes('it:dev:int=1 -> test:str:ndefs')
+            self.len(1, nodes)
+            self.eq('ndefs', nodes[0].ndef[1])
+            self.eq(nodes[0].getNodeRefs(), [
+                ('ndefs', ('it:dev:int', 1)),
+                ('ndefs', ('it:dev:int', 2)),
+                ('ndefs', ('inet:fqdn', 'woot.com'))
+            ])
+
+            nodes = await core.nodes('[ test:str = norefs ]')
+            self.eq(nodes[0].getNodeRefs(), [])
+
+            self.len(1, await core.nodes('it:dev:int=1 -> test:str'))
+            self.len(3, await core.nodes('test:str=ndefs -> *'))
+            self.len(2, await core.nodes('test:str=ndefs -> it:dev:int'))
+            self.len(3, await core.nodes('test:str=ndefs :ndefs -> *'))
+            self.len(2, await core.nodes('test:str=ndefs :ndefs -> it:dev:int'))
+
     async def test_ast_pivot(self):
         # a general purpose pivot test. come on in!
         async with self.getTestCore() as core:
@@ -2761,6 +2783,50 @@ class AstTest(s_test.SynTest):
                     self.stormHasNoWarnErr(msgs)
                     self.len(0, calls)
 
+    async def test_ast_tag_optimization(self):
+        calls = []
+        origtag = s_snap.Snap.nodesByTag
+
+        async def checkTag(self, tag, form=None, reverse=False):
+            calls.append(('tag', tag, form))
+            async for node in origtag(self, tag, form=form, reverse=reverse):
+                yield node
+
+        with mock.patch('synapse.lib.snap.Snap.nodesByTag', checkTag):
+            async with self.getTestCore() as core:
+                self.len(1, await core.nodes('[inet:asn=200 :name=visi]'))
+                self.len(1, await core.nodes('[test:int=12 +#visi]'))
+                self.len(1, await core.nodes('[test:int=99 +#visi]'))
+
+                nodes = await core.nodes('test:int +#$x', opts={'vars': {'x': 'visi'}})
+                self.len(2, nodes)
+                self.len(1, calls)
+                self.eq(('tag', 'visi', 'test:int'), calls[0])
+
+                calls = []
+                # not for non-runtsafe
+                nodes = await core.nodes('inet:asn:name $valu=:name test:int +#$valu')
+                self.len(2, nodes)
+                self.len(0, calls)
+
+                nodes = await core.nodes('''
+                    $tag = 'yeyeyeyeyeyeyeyeye'
+                    if $x {
+                        $tag = 'visi'
+                    } else {
+                        $tag = 'lolnope'
+                    }
+                    test:int +#$tag
+                ''', opts={'vars': {'x': True}})
+                self.len(2, nodes)
+                self.len(1, calls)
+                self.eq(('tag', 'visi', 'test:int'), calls[0])
+
+                calls = []
+                nodes = await core.nodes('test:int +#$x', opts={'vars': {'x': 'v*'}})
+                self.len(2, nodes)
+                self.len(0, calls)
+
     async def test_ast_cmdoper(self):
 
         async with self.getTestCore() as core:

synapse/tests/test_lib_cell.py

@@ -1,6 +1,7 @@
 import os
 import ssl
 import sys
+import json
 import time
 import base64
 import signal
@@ -33,6 +34,7 @@ import synapse.lib.version as s_version
 import synapse.lib.hiveauth as s_hiveauth
 import synapse.lib.lmdbslab as s_lmdbslab
 import synapse.lib.crypto.passwd as s_passwd
+import synapse.lib.platforms.linux as s_linux
 
 import synapse.tools.backup as s_tools_backup
 
@@ -2611,3 +2613,113 @@ class CellTest(s_t_utils.SynTest):
 
             with self.raises(s_exc.NoSuchIden):
                 await cell.delUserApiKey(newp)
+
+    async def test_cell_check_sysctl(self):
+        sysctls = s_linux.getSysctls()
+
+        sysvals = s_cell.Cell.SYSCTL_VALS.copy()
+        sysvals['vm.dirty_expire_centisecs'] += 1
+        sysvals['vm.dirty_writeback_centisecs'] += 1
+
+        # Detect and report incorrect values
+        with self.getStructuredAsyncLoggerStream('synapse.lib.cell') as stream:
+            with mock.patch.object(s_cell.Cell, 'SYSCTL_VALS', sysvals):
+                async with self.getTestCore(conf={'health:sysctl:checks': True}):
+                    pass
+
+        stream.seek(0)
+        data = stream.getvalue()
+        raw_mesgs = [m for m in data.split('\\n') if m]
+        msgs = [json.loads(m) for m in raw_mesgs]
+
+        self.len(1, msgs)
+
+        mesg = f'Sysctl values different than expected: {", ".join(sysvals)}. '
+        mesg += 'See https://synapse.docs.vertex.link/en/latest/synapse/devopsguide.html#performance-tuning '
+        mesg += 'for information about these sysctl parameters.'
+        self.eq(msgs[0]['message'], mesg)
+        self.eq(msgs[0]['sysctls'], [
+            {'name': 'vm.dirty_expire_centisecs', 'expected': 21, 'actual': sysctls['vm.dirty_expire_centisecs']},
+            {'name': 'vm.dirty_writeback_centisecs', 'expected': 21, 'actual': sysctls['vm.dirty_writeback_centisecs']},
+        ])
+
+        # Copy the current sysctl valus to the cell so the check passes
+        sysvals = {
+            'vm.dirty_expire_centisecs': sysctls['vm.dirty_expire_centisecs'],
+            'vm.dirty_writeback_centisecs': sysctls['vm.dirty_writeback_centisecs'],
+        }
+
+        # Detect correct values and stop the task
+        with self.getLoggerStream('synapse.lib.cell') as stream:
+            with mock.patch.object(s_cell.Cell, 'SYSCTL_VALS', sysvals):
+                async with self.getTestCore():
+                    pass
+
+        stream.seek(0)
+        data = stream.read()
+        self.len(0, data)
+
+        # Disable the sysctl check and don't check at all
+        with self.getLoggerStream('synapse.lib.cell') as stream:
+            conf = {'health:sysctl:checks': False}
+            async with self.getTestCore(conf=conf):
+                pass
+
+        stream.seek(0)
+        data = stream.read()
+        self.len(0, data, msg=data)
+
+    async def test_cell_version_regression(self):
+        oldver = (0, 1, 0)
+        newver = (0, 2, 0)
+
+        class TestCell(s_cell.Cell):
+            VERSION = newver
+
+        with self.getTestDir() as dirn:
+            async with self.getTestCell(TestCell, dirn=dirn):
+                pass
+
+            with self.raises(s_exc.BadVersion) as exc:
+                with mock.patch.object(TestCell, 'VERSION', oldver):
+                    with self.getLoggerStream('synapse.lib.cell') as stream:
+                        async with self.getTestCell(TestCell, dirn=dirn):
+                            pass
+
+            mesg = f'Cell version regression (testcell) is not allowed! Stored version: {newver}, current version: {oldver}.'
+            self.eq(exc.exception.get('mesg'), mesg)
+            self.eq(exc.exception.get('currver'), oldver)
+            self.eq(exc.exception.get('lastver'), newver)
+
+            stream.seek(0)
+            data = stream.read()
+            self.isin(mesg, data)
+
+            async with self.getTestCell(TestCell, dirn=dirn):
+                pass
+
+        with self.getTestDir() as dirn:
+            async with self.getTestCell(s_cell.Cell, dirn=dirn):
+                pass
+
+            synver = list(s_version.version)
+            synver[1] -= 1
+            synver = tuple(synver)
+
+            with self.raises(s_exc.BadVersion) as exc:
+                with mock.patch.object(s_version, 'version', synver):
+                    with self.getLoggerStream('synapse.lib.cell') as stream:
+                        async with self.getTestCell(s_cell.Cell, dirn=dirn):
+                            pass
+
+            mesg = f'Synapse version regression (cell) is not allowed! Stored version: {s_version.version}, current version: {synver}.'
+            self.eq(exc.exception.get('mesg'), mesg)
+            self.eq(exc.exception.get('currver'), synver)
+            self.eq(exc.exception.get('lastver'), s_version.version)
+
+            stream.seek(0)
+            data = stream.read()
+            self.isin(mesg, data)
+
+            async with self.getTestCell(s_cell.Cell, dirn=dirn):
+                pass

synapse/tests/test_lib_layer.py

@@ -26,7 +26,7 @@ class LayerTest(s_t_utils.SynTest):
 
     def checkLayrvers(self, core):
         for layr in core.layers.values():
-            self.eq(layr.layrvers, 10)
+            self.eq(layr.layrvers, 11)
 
     async def test_layer_verify(self):
 
@@ -1530,6 +1530,57 @@ class LayerTest(s_t_utils.SynTest):
             verbs = [verb async for verb in nodes0[0].iterEdgeVerbs(buid2)]
             self.len(0, verbs)
 
+    async def test_layer_v11(self):
+
+        try:
+
+            oldv = s_layer.MIGR_COMMIT_SIZE
+            s_layer.MIGR_COMMIT_SIZE = 1
+
+            async with self.getRegrCore('layer-v11') as core:
+
+                wlyrs_byview = await core.callStorm('''
+                    $wlyrs = ({})
+                    for $view in $lib.view.list() {
+                        $wlyrs.($view.get(name)) = $view.layers.0.iden
+                    }
+                    return($wlyrs)
+                ''')
+                self.len(8, wlyrs_byview)
+
+                layr = core.getLayer(iden=wlyrs_byview['default'])
+                await self.agenlen(2, layr.getStorNodesByForm('test:str'))
+                await self.agenlen(1, layr.getStorNodesByForm('syn:tag'))
+
+                layr = core.getLayer(iden=wlyrs_byview['prop'])
+                await self.agenlen(1, layr.getStorNodesByForm('test:str'))
+
+                layr = core.getLayer(iden=wlyrs_byview['tags'])
+                await self.agenlen(1, layr.getStorNodesByForm('test:str'))
+                await self.agenlen(1, layr.getStorNodesByForm('syn:tag'))
+
+                layr = core.getLayer(iden=wlyrs_byview['tagp'])
+                await self.agenlen(1, layr.getStorNodesByForm('test:str'))
+                await self.agenlen(0, layr.getStorNodesByForm('syn:tag'))
+
+                layr = core.getLayer(iden=wlyrs_byview['n1eg'])
+                await self.agenlen(1, layr.getStorNodesByForm('test:str'))
+                await self.agenlen(1, layr.getStorNodesByForm('test:int'))
+
+                layr = core.getLayer(iden=wlyrs_byview['n2eg'])
+                await self.agenlen(0, layr.getStorNodesByForm('test:str'))
+                await self.agenlen(1, layr.getStorNodesByForm('test:int'))
+
+                layr = core.getLayer(iden=wlyrs_byview['data'])
+                await self.agenlen(1, layr.getStorNodesByForm('test:str'))
+
+                layr = core.getLayer(iden=wlyrs_byview['noop'])
+                await self.agenlen(0, layr.getStorNodes())
+                await self.agenlen(0, layr.getStorNodesByForm('test:str'))
+
+        finally:
+            s_layer.MIGR_COMMIT_SIZE = oldv
+
     async def test_layer_logedits_default(self):
         async with self.getTestCore() as core:
             self.true(core.getLayer().logedits)

synapse/tests/test_lib_lmdbslab.py

@@ -649,6 +649,42 @@ class LmdbSlabTest(s_t_utils.SynTest):
                 self.eq((b'1', b'1'), next(it))
                 self.raises(StopIteration, next, it)
 
+    async def test_lmdbslab_scanback(self):
+
+        with self.getTestDir() as dirn:
+
+            path = os.path.join(dirn, 'test.lmdb')
+
+            async with await s_lmdbslab.Slab.anit(path, map_size=100000, growsize=10000) as slab:
+
+                foodup = slab.initdb('foodup', dupsort=True)
+                foonodup = slab.initdb('foonodup', dupsort=False)
+
+                for db in (foodup, foonodup):
+                    slab.put(b'\x01', b'foo', db=db)
+                    slab.put(b'\x01\x01', b'bar', db=db)
+                    slab.put(b'\x01\x03', b'baz', db=db)
+                    slab.put(b'\x02', b'faz', db=db)
+
+                items = list(slab.scanByPrefBack(b'\x01', db=foonodup))
+                self.eq(items, (
+                    (b'\x01\x03', b'baz'),
+                    (b'\x01\x01', b'bar'),
+                    (b'\x01', b'foo')
+                ))
+
+                self.eq((), list(slab.scanByPrefBack(b'\x00', db=foonodup)))
+
+                slab.put(b'\x01\x03', b'waz', db=foodup)
+
+                items = list(slab.scanByPrefBack(b'\x01', db=foodup))
+                self.eq(items, (
+                    (b'\x01\x03', b'waz'),
+                    (b'\x01\x03', b'baz'),
+                    (b'\x01\x01', b'bar'),
+                    (b'\x01', b'foo')
+                ))
+
     async def test_lmdbslab_count_empty(self):
 
         with self.getTestDir() as dirn:

synapse/tests/test_lib_scrape.py

@@ -508,6 +508,77 @@ bad_uncs = [
 
 unc_paths = '\n'.join(good_uncs + bad_uncs)
 
+cpedata = r'''GOOD DATA
+cpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:other
+cpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:tspace non matched word
+cpe:2.3:a:*:*:*:*:*:*:*:*:*:*
+cpe:2.3:h:*:*:*:*:*:*:*:*:*:*
+cpe:2.3:o:*:*:*:*:*:*:*:*:*:*
+cpe:2.3:-:*:*:*:*:*:*:*:*:*:*
+cpe:2.3:*:*:*:*:*:*:*:*:*:*:*
+cpe:2.3:*:-:na:*:*:*:*:*:*:*:*
+cpe:2.3:*:.:dot:*:*:*:*:*:*:*:*
+cpe:2.3:*:_:underscore:*:*:*:*:*:*:*:*
+
+A few quoted characters
+cpe:2.3:*:\!:quoted:*:*:*:*:*:*:*:*
+cpe:2.3:*:\?:quoted:*:*:*:*:*:*:*:*
+cpe:2.3:*:\*:quoted:*:*:*:*:*:*:*:*
+cpe:2.3:*:\\:escapeescape:*:*:*:*:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:-:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:*:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:en:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:usa:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:usa-en:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:usa-123:*:*:*:*
+
+A few examples
+cpe:2.3:a:ntp:ntp:4.2.8:p3:*:*:*:*:*:*
+cpe:2.3:o:microsoft:windows_7:-:sp2:*:*:*:*:*:*
+cpe:2.3:a:hp:insight:7.4.0.1570:-:*:*:online:win2003:x64:*
+cpe:2.3:a:foo\\bar:big\$money_2010:*:*:*:*:special:ipod_touch:80gb:*
+cpe:2.3:a:hp:openview_network_manager:7.51:*:*:*:*:linux:*:*
+cpe:2.3:a:apple:swiftnio_http\/2:1.19.1:*:*:*:*:swift:*:*
+
+Some quoted examples
+cpe:2.3:a:fooo:bar_baz\:_beep_bpp_sys:1.1:*:*:*:*:ios:*:*
+cpe:2.3:a:lemonldap-ng:apache\:\:session\:\:browsable:0.9:*:*:*:*:perl:*:*
+cpe:2.3:a:daemon-ng:hurray\:\::0.x:*:*:*:*:*:*:*
+cpe:2.3:a:microsoft:intern\^et_explorer:8.0.6001:beta:*:*:*:*:*:*
+
+TEXT examples
+Example double quoted cpe value "cpe:2.3:a:vertex:synapse:*:*:*:*:*:*:*:dquotes".
+Example double quoted cpe value "cpe:2.3:a:vertex:synapse:*:*:*:*:*:*:*:MiXeDcAsE".
+Example single quoted cpe value "cpe:2.3:a:vertex:synapse:*:*:*:*:*:*:*:squotes".
+A CPE at the end of a sentence like this captures the period... cpe:2.3:a:*:*:*:*:*:*:*:*:*:hasperiod.
+Some CPE are exciting! Like this cpe:2.3:a:*:*:*:*:*:*:*:*:*:noexclaim!
+Some CPE are boring! Like this cpe:2.3:a:*:*:*:*:*:*:*:*:*:noslash\
+Unicode endings are omitted cpe:2.3:a:*:*:*:*:*:*:*:*:*:unicodeend0ॐ
+Unicode quotes “cpe:2.3:a:*:*:*:*:*:*:*:*:*:smartquotes”
+cpe:2.3:*:?why??:*:*:*:*:*:*:*:*:*
+cpe:2.3:*:*why*:*:*:*:*:*:*:*:*:*
+
+EMBEDDED TEXT
+wordscpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:otherxxx:newp
+wordscpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:otherzzz:
+
+BAD values
+cpe:2.3:*:?:spec1:*:*:*:*:*:*:*:*
+cpe:2.3:a:vertex:synapse:*:*:*:NEWP:*:*:*:*
+cpe:2.3:a::::::::::
+cpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:ॐ:other
+cpe:2.3:a:vendor:product:version:update:edition
+cpe:2.3:a:opps:bad_quote\\/2:1.19.1:*:*:*:*:swift:*:*
+
+# Bad languages
+cpe:2.3:*:langtest:*:*:*:*:a:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:aaaa:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:usa-o:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:usa-omn:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:usa-12:*:*:*:*
+cpe:2.3:*:langtest:*:*:*:*:usa-1234:*:*:*:*
+'''
+
 class ScrapeTest(s_t_utils.SynTest):
 
     def test_scrape_basic(self):
@@ -1050,77 +1121,7 @@ class ScrapeTest(s_t_utils.SynTest):
             self.eq(erv, fv)
 
     def test_scrape_cpe(self):
-        cpedata = r'''
-        GOOD DATA
-        cpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:other
-        cpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:tspace non matched word
-        cpe:2.3:a:*:*:*:*:*:*:*:*:*:*
-        cpe:2.3:h:*:*:*:*:*:*:*:*:*:*
-        cpe:2.3:o:*:*:*:*:*:*:*:*:*:*
-        cpe:2.3:-:*:*:*:*:*:*:*:*:*:*
-        cpe:2.3:*:*:*:*:*:*:*:*:*:*:*
-        cpe:2.3:*:-:na:*:*:*:*:*:*:*:*
-        cpe:2.3:*:.:dot:*:*:*:*:*:*:*:*
-        cpe:2.3:*:_:underscore:*:*:*:*:*:*:*:*
-
-        A few quoted characters
-        cpe:2.3:*:\!:quoted:*:*:*:*:*:*:*:*
-        cpe:2.3:*:\?:quoted:*:*:*:*:*:*:*:*
-        cpe:2.3:*:\*:quoted:*:*:*:*:*:*:*:*
-        cpe:2.3:*:\\:escapeescape:*:*:*:*:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:-:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:*:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:en:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:usa:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:usa-en:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:usa-123:*:*:*:*
-
-        A few examples
-        cpe:2.3:a:ntp:ntp:4.2.8:p3:*:*:*:*:*:*
-        cpe:2.3:o:microsoft:windows_7:-:sp2:*:*:*:*:*:*
-        cpe:2.3:a:hp:insight:7.4.0.1570:-:*:*:online:win2003:x64:*
-        cpe:2.3:a:foo\\bar:big\$money_2010:*:*:*:*:special:ipod_touch:80gb:*
-        cpe:2.3:a:hp:openview_network_manager:7.51:*:*:*:*:linux:*:*
-        cpe:2.3:a:apple:swiftnio_http\/2:1.19.1:*:*:*:*:swift:*:*
-
-        Some quoted examples
-        cpe:2.3:a:fooo:bar_baz\:_beep_bpp_sys:1.1:*:*:*:*:ios:*:*
-        cpe:2.3:a:lemonldap-ng:apache\:\:session\:\:browsable:0.9:*:*:*:*:perl:*:*
-        cpe:2.3:a:daemon-ng:hurray\:\::0.x:*:*:*:*:*:*:*
-        cpe:2.3:a:microsoft:intern\^et_explorer:8.0.6001:beta:*:*:*:*:*:*
-
-        TEXT examples
-        Example double quoted cpe value "cpe:2.3:a:vertex:synapse:*:*:*:*:*:*:*:dquotes".
-        Example double quoted cpe value "cpe:2.3:a:vertex:synapse:*:*:*:*:*:*:*:MiXeDcAsE".
-        Example single quoted cpe value "cpe:2.3:a:vertex:synapse:*:*:*:*:*:*:*:squotes".
-        A CPE at the end of a sentence like this captures the period... cpe:2.3:a:*:*:*:*:*:*:*:*:*:hasperiod.
-        Some CPE are exciting! Like this cpe:2.3:a:*:*:*:*:*:*:*:*:*:noexclaim!
-        Some CPE are boring! Like this cpe:2.3:a:*:*:*:*:*:*:*:*:*:noslash\
-        Unicode endings are omitted cpe:2.3:a:*:*:*:*:*:*:*:*:*:unicodeend0ॐ
-        Unicode quotes “cpe:2.3:a:*:*:*:*:*:*:*:*:*:smartquotes”
-        cpe:2.3:*:?why??:*:*:*:*:*:*:*:*:*
-        cpe:2.3:*:*why*:*:*:*:*:*:*:*:*:*
-
-        EMBEDDED TEXT
-        wordscpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:otherxxx:newp
-        wordscpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:target_hw:otherzzz:
-
-        BAD values
-        cpe:2.3:*:?:spec1:*:*:*:*:*:*:*:*
-        cpe:2.3:a:vertex:synapse:*:*:*:NEWP:*:*:*:*
-        cpe:2.3:a::::::::::
-        cpe:2.3:a:vendor:product:version:update:edition:lng:sw_edition:target_sw:ॐ:other
-        cpe:2.3:a:vendor:product:version:update:edition
-        cpe:2.3:a:opps:bad_quote\\/2:1.19.1:*:*:*:*:swift:*:*
-
-        # Bad languages
-        cpe:2.3:*:langtest:*:*:*:*:a:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:aaaa:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:usa-o:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:usa-omn:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:usa-12:*:*:*:*
-        cpe:2.3:*:langtest:*:*:*:*:usa-1234:*:*:*:*
-        '''
+
         nodes = sorted(set(s_scrape.scrape(cpedata, ptype='it:sec:cpe')))
         nodes.remove(('it:sec:cpe', 'cpe:2.3:*:*:*:*:*:*:*:*:*:*:*'))
         nodes.remove(('it:sec:cpe', 'cpe:2.3:*:-:na:*:*:*:*:*:*:*:*'))

synapse/tests/test_lib_snap.py

@@ -584,7 +584,10 @@ class SnapTest(s_t_utils.SynTest):
     async def test_snap_editor(self):
 
         async with self.getTestCore() as core:
-            nodes = await core.nodes('[ media:news=63381924986159aff183f0c85bd8ebad +(refs)> {[ inet:fqdn=vertex.link ]} ]')
+
+            await core.nodes('$lib.model.ext.addTagProp(test, (str, ({})), ({}))')
+            await core.nodes('[ media:news=63381924986159aff183f0c85bd8ebad +(refs)> {[ inet:fqdn=vertex.link ]} +#foo ]')
+
             root = core.auth.rootuser
             async with await core.view.snap(user=root) as snap:
                 async with snap.getEditor() as editor:
@@ -608,6 +611,14 @@ class SnapTest(s_t_utils.SynTest):
                     self.len(1, nodeedits)
                     self.len(1, nodeedits[0][2])
 
+                    self.false(await news.hasData('foo'))
+                    await news.setData('foo', 'bar')
+                    self.true(await news.hasData('foo'))
+
+                    self.false(news.hasTagProp('foo', 'test'))
+                    await news.setTagProp('foo', 'test', 'bar')
+                    self.true(news.hasTagProp('foo', 'test'))
+
                 async with snap.getEditor() as editor:
                     news = await editor.addNode('media:news', '63381924986159aff183f0c85bd8ebad')
 
@@ -629,6 +640,10 @@ class SnapTest(s_t_utils.SynTest):
                     self.false(await news.delEdge('pwns', 1))
                     self.false(await news.delEdge('pwns', 'bar'))
 
+                    self.true(await news.hasData('foo'))
+
+                    self.true(news.hasTagProp('foo', 'test'))
+
             self.len(1, await core.nodes('media:news -(pwns)> *'))
 
             self.len(1, await core.nodes('[ test:ro=foo :writeable=hehe :readable=haha ]'))