synapse 2.178.0__py311-none-any.whl → 2.180.0__py311-none-any.whl

synapse/tests/test_lib_ast.py

@@ -4100,3 +4100,234 @@ class AstTest(s_test.SynTest):
             q = 'media:news=(m0,) [-:published]'
             msgs = await core.stormlist(q, opts=aslow)
             self.stormIsInErr('must have permission node.prop.del.media:news.published', msgs)
+
+    async def test_ast_path_links(self):
+
+        async with self.getTestCore() as core:  # type: s_cortex.Cortex
+            guid = s_common.guid()
+            opts = {'vars': {'guid': guid}}
+
+            burr = (await core.nodes('[test:comp=(1234, burrito)]'))[0]
+            guid = (await core.nodes('[test:guid=$guid :size=176 :tick=now]', opts=opts))[0]
+            edge = (await core.nodes('[test:edge=(("test:guid", $guid), ("test:str", abcd))]', opts=opts))[0]
+            comp = (await core.nodes('[test:complexcomp=(1234, STUFF) +#foo.bar]'))[0]
+            tstr = (await core.nodes('[test:str=foobar :bar=(test:ro, "ackbar") :ndefs=((test:guid, $guid), (test:auto, "auto"))]', opts=opts))[0]
+            arry = (await core.nodes('[test:arrayprop=* :ints=(3245, 678) :strs=("foo", "bar", "foobar")]'))[0]
+            ostr = (await core.nodes('test:str=foo [ :bar=(test:ro, "ackbar") :ndefs=((test:int, 176), )]'))[0]
+            pstr = (await core.nodes('test:str=bar [ :ndefs=((test:guid, $guid), (test:auto, "auto"), (test:ro, "ackbar"))]', opts=opts))[0]
+            (await core.nodes('[test:arrayform=(1234, 176)]'))[0]
+            (await core.nodes('[test:arrayform=(3245, 678)]'))[0]
+
+            await core.nodes('test:int=176 [ <(seen)+ { test:guid } ]')
+            await core.nodes('test:int=176 [ <(someedge)+ { test:guid } ]')
+            await core.nodes('test:complexcomp [ <(refs)+ { test:arrayprop } ]')
+            await core.nodes('test:complexcomp [ +(concerns)> { test:ro } ]')
+            await core.nodes('test:edge [ <(seen)+ { test:guid } ]')
+
+            small = (await core.nodes('test:int=176'))[0]
+            large = (await core.nodes('test:int=1234'))[0]
+            auto = (await core.nodes('test:auto'))[0]
+            basetag = (await core.nodes('syn:tag=foo'))[0]
+            tag = (await core.nodes('syn:tag=foo.bar'))[0]
+            ro = (await core.nodes('test:ro'))[0]
+
+            def _assert_edge(msgs, src, edge, nidx=0, eidx=0):
+                nodes = [m[1] for m in msgs if m[0] == 'node']
+                links = nodes[nidx][1].get('links')
+                self.nn(links)
+                self.lt(eidx, len(links))
+                self.eq(links[eidx], (src.iden(), edge))
+
+            opts = {'links': True}
+
+            # non-runtsafe lift could be anything
+            msgs = await core.stormlist('test:str=foobar $newform=$node.props.bar.0 *$newform', opts={'links': True, 'vars': {'form': 'inet:ipv4'}})
+            _assert_edge(msgs, tstr, {'type': 'runtime'}, nidx=1)
+
+            # FormPivot
+            # -> baz:ndef
+            msgs = await core.stormlist('test:guid -> test:edge:n1', opts=opts)
+            _assert_edge(msgs, guid, {'type': 'prop', 'prop': 'n1', 'reverse': True})
+
+            # plain old pivot
+            msgs = await core.stormlist('test:int=176 -> test:guid:size', opts=opts)
+            _assert_edge(msgs, small, {'type': 'prop', 'prop': 'size', 'reverse': True})
+
+            # graph edge dest form uses n1 automagically
+            msgs = await core.stormlist('test:guid -> test:edge', opts=opts)
+            _assert_edge(msgs, guid, {'type': 'prop', 'prop': 'n1', 'reverse': True})
+
+            # <syn:tag> -> <form>
+            msgs = await core.stormlist('syn:tag=foo.bar -> test:complexcomp', opts=opts)
+            _assert_edge(msgs, tag, {'type': 'tag', 'tag': 'foo.bar', 'reverse': True})
+
+            # source node is a graph edge, use n2
+            msgs = await core.stormlist('test:edge -> test:str', opts=opts)
+            _assert_edge(msgs, edge, {'type': 'prop', 'prop': 'n2'})
+
+            # refs out - prop
+            msgs = await core.stormlist('test:complexcomp -> test:int', opts=opts)
+            _assert_edge(msgs, comp, {'type': 'prop', 'prop': 'foo'})
+
+            # refs out - array
+            msgs = await core.stormlist('test:arrayprop -> test:int', opts=opts)
+            _assert_edge(msgs, arry, {'type': 'prop', 'prop': 'ints'})
+            _assert_edge(msgs, arry, {'type': 'prop', 'prop': 'ints'}, nidx=1)
+
+            # refs out - ndef
+            msgs = await core.stormlist('test:str -> test:ro', opts=opts)
+            _assert_edge(msgs, pstr, {'type': 'prop', 'prop': 'ndefs'})
+            _assert_edge(msgs, ostr, {'type': 'prop', 'prop': 'bar'}, nidx=1)
+            _assert_edge(msgs, tstr, {'type': 'prop', 'prop': 'bar'}, nidx=2)
+
+            # refs out - ndefarray
+            msgs = await core.stormlist('test:str -> test:auto', opts=opts)
+            _assert_edge(msgs, pstr, {'type': 'prop', 'prop': 'ndefs'})
+            _assert_edge(msgs, tstr, {'type': 'prop', 'prop': 'ndefs'}, nidx=1)
+
+            # reverse prop refs
+            msgs = await core.stormlist('test:int -> test:complexcomp', opts=opts)
+            _assert_edge(msgs, large, {'type': 'prop', 'prop': 'foo', 'reverse': True})
+
+            # reverse array refs
+            msgs = await core.stormlist('test:int -> test:arrayprop', opts=opts)
+            sixer = (await core.nodes('test:int=678'))[0]
+            thou = (await core.nodes('test:int=3245'))[0]
+            _assert_edge(msgs, sixer, {'type': 'prop', 'prop': 'ints', 'reverse': True})
+            _assert_edge(msgs, thou, {'type': 'prop', 'prop': 'ints', 'reverse': True}, nidx=1)
+
+            # reverse ndef refs
+            msgs = await core.stormlist('test:ro -> test:str', opts=opts)
+            _assert_edge(msgs, ro, {'type': 'prop', 'prop': 'bar', 'reverse': True})
+
+            # reverse ndefarray refs
+            msgs = await core.stormlist('test:auto -> test:str', opts=opts)
+            _assert_edge(msgs, auto, {'type': 'prop', 'prop': 'ndefs', 'reverse': True})
+
+            # PivotOut syn:tag
+            msgs = await core.stormlist('syn:tag -> *', opts=opts)
+            _assert_edge(msgs, basetag, {'type': 'tag', 'tag': 'foo', 'reverse': True})
+            _assert_edge(msgs, tag, {'type': 'tag', 'tag': 'foo.bar', 'reverse': True}, nidx=1)
+
+            # PivotOut edge uses n2 automatically
+            msgs = await core.stormlist('test:edge -> *', opts=opts)
+            _assert_edge(msgs, edge, {'type': 'prop', 'prop': 'n2'})
+
+            # PivotOut prop
+            msgs = await core.stormlist('test:guid -> *', opts=opts)
+            _assert_edge(msgs, guid, {'type': 'prop', 'prop': 'size'})
+
+            # PivotOut prop array
+            msgs = await core.stormlist('test:arrayprop -> *', opts=opts)
+            _assert_edge(msgs, arry, {'type': 'prop', 'prop': 'ints'})
+            _assert_edge(msgs, arry, {'type': 'prop', 'prop': 'ints'}, nidx=1)
+            _assert_edge(msgs, arry, {'type': 'prop', 'prop': 'strs'}, nidx=2)
+            _assert_edge(msgs, arry, {'type': 'prop', 'prop': 'strs'}, nidx=3)
+            _assert_edge(msgs, arry, {'type': 'prop', 'prop': 'strs'}, nidx=4)
+
+            # PivotOut prop ndef and ndef array
+            msgs = await core.stormlist('test:str=foobar -> *', opts=opts)
+            _assert_edge(msgs, tstr, {'type': 'prop', 'prop': 'bar'})
+            _assert_edge(msgs, tstr, {'type': 'prop', 'prop': 'ndefs'}, nidx=1)
+            _assert_edge(msgs, tstr, {'type': 'prop', 'prop': 'ndefs'}, nidx=2)
+
+            # PivotToTags
+            msgs = await core.stormlist('test:complexcomp -> #', opts=opts)
+            _assert_edge(msgs, comp, {'type': 'tag', 'tag': 'foo.bar'})
+
+            # PivotIn prop
+            msgs = await core.stormlist('test:int=176 <- *', opts=opts)
+            _assert_edge(msgs, small, {'type': 'prop', 'prop': 'size', 'reverse': True})
+
+            # PivotIn array prop
+            msgs = await core.stormlist('test:str=foobar <- *', opts=opts)
+            _assert_edge(msgs, tstr, {'type': 'prop', 'prop': 'strs', 'reverse': True})
+
+            # PivotIn edge uses n1 automatically
+            msgs = await core.stormlist('test:edge <- *', opts=opts)
+            _assert_edge(msgs, edge, {'type': 'prop', 'prop': 'n1', 'reverse': True})
+
+            # PivotIn ndef
+            msgs = await core.stormlist('test:ro <- *', opts=opts)
+            _assert_edge(msgs, ro, {'type': 'prop', 'prop': 'bar', 'reverse': True})
+
+            # PivotIn array ndef
+            msgs = await core.stormlist('test:auto <- *', opts=opts)
+            _assert_edge(msgs, auto, {'type': 'prop', 'prop': 'ndefs', 'reverse': True})
+
+            # PivotInFrom "<- edge"
+            abcd = (await core.nodes('test:str=abcd'))[0]
+            msgs = await core.stormlist('test:str <- test:edge', opts=opts)
+            _assert_edge(msgs, abcd, {'type': 'prop', 'prop': 'n2', 'reverse': True})
+
+            # PivotInFrom "edge <- form"
+            msgs = await core.stormlist('test:edge <- test:guid', opts=opts)
+            _assert_edge(msgs, edge, {'type': 'prop', 'prop': 'n1', 'reverse': True})
+
+            # PropPivotOut prop
+            msgs = await core.stormlist('test:guid :size -> *', opts=opts)
+            _assert_edge(msgs, guid, {'type': 'prop', 'prop': 'size'})
+
+            # PropPivotOut ndef
+            msgs = await core.stormlist('test:str=foobar :bar -> *', opts=opts)
+            _assert_edge(msgs, tstr, {'type': 'prop', 'prop': 'bar'})
+
+            # PropPivotOut array
+            msgs = await core.stormlist('test:arrayprop :ints -> *', opts=opts)
+            _assert_edge(msgs, arry, {'type': 'prop', 'prop': 'ints'})
+            _assert_edge(msgs, arry, {'type': 'prop', 'prop': 'ints'}, nidx=1)
+
+            # PropPivotOut array ndef
+            msgs = await core.stormlist('test:str=foobar :ndefs -> *', opts=opts)
+            _assert_edge(msgs, tstr, {'type': 'prop', 'prop': 'ndefs'})
+            _assert_edge(msgs, tstr, {'type': 'prop', 'prop': 'ndefs'}, nidx=1)
+
+            # PropPivot prop to form
+            msgs = await core.stormlist('test:guid :size -> test:int', opts=opts)
+            _assert_edge(msgs, guid, {'type': 'prop', 'prop': 'size'})
+
+            # PropPivot ndef prop
+            msgs = await core.stormlist('test:str :bar -> test:ro', opts=opts)
+            _assert_edge(msgs, ostr, {'type': 'prop', 'prop': 'bar'})
+            _assert_edge(msgs, tstr, {'type': 'prop', 'prop': 'bar'}, nidx=1)
+
+            # PropPivot array
+            msgs = await core.stormlist('test:arrayprop :ints -> test:int', opts=opts)
+            _assert_edge(msgs, arry, {'type': 'prop', 'prop': 'ints'})
+            _assert_edge(msgs, arry, {'type': 'prop', 'prop': 'ints'}, nidx=1)
+
+            # PropPivot dst array primary prop
+            msgs = await core.stormlist('test:guid :size -> test:arrayform', opts=opts)
+            _assert_edge(msgs, guid, {'type': 'prop', 'prop': 'size'})
+
+            # PropPivot oops all arrays
+            msgs = await core.stormlist('test:arrayprop :ints -> test:arrayform', opts=opts)
+            _assert_edge(msgs, arry, {'type': 'prop', 'prop': 'ints'})
+
+            # PropPivot src ndef array
+            msgs = await core.stormlist('test:str=foobar :ndefs -> test:guid', opts=opts)
+            _assert_edge(msgs, tstr, {'type': 'prop', 'prop': 'ndefs'})
+
+            # prop to prop
+            msgs = await core.stormlist('test:comp :hehe -> test:complexcomp:foo', opts=opts)
+            _assert_edge(msgs, burr, {'type': 'prop', 'prop': 'hehe', 'dest': 'test:complexcomp:foo'})
+
+            # N1Walk
+            msgs = await core.stormlist('test:arrayprop -(*)> *', opts=opts)
+            _assert_edge(msgs, arry, {'type': 'edge', 'verb': 'refs'})
+
+            # N2Walk
+            msgs = await core.stormlist('test:edge <(*)- *', opts=opts)
+            _assert_edge(msgs, edge, {'type': 'edge', 'verb': 'seen', 'reverse': True})
+
+            # N1WalkNPivo
+            msgs = await core.stormlist('test:complexcomp --> *', opts=opts)
+            _assert_edge(msgs, comp, {'type': 'prop', 'prop': 'foo'})
+            _assert_edge(msgs, comp, {'type': 'edge', 'verb': 'concerns'}, nidx=1)
+
+            # N2WalNkPivo
+            msgs = await core.stormlist('test:int=176 <-- *', opts=opts)
+            _assert_edge(msgs, small, {'type': 'prop', 'prop': 'size', 'reverse': True})
+            _assert_edge(msgs, small, {'type': 'prop', 'prop': 'ndefs', 'reverse': True}, nidx=1)
+            _assert_edge(msgs, small, {'type': 'edge', 'verb': 'seen', 'reverse': True}, nidx=2)
+            _assert_edge(msgs, small, {'type': 'edge', 'verb': 'someedge', 'reverse': True}, nidx=3)

synapse/tests/test_lib_cell.py

@@ -28,6 +28,7 @@ import synapse.lib.base as s_base
 import synapse.lib.cell as s_cell
 import synapse.lib.coro as s_coro
 import synapse.lib.link as s_link
+import synapse.lib.drive as s_drive
 import synapse.lib.nexus as s_nexus
 import synapse.lib.certdir as s_certdir
 import synapse.lib.msgpack as s_msgpack
@@ -155,8 +156,217 @@ async def altAuthCtor(cell):
     cell.onfini(auth.fini)
     return auth
 
+testDataSchema_v0 = {
+    'type': 'object',
+    'properties': {
+        'type': {'type': 'string'},
+        'size': {'type': 'number'},
+    },
+    'required': ['type', 'size'],
+    'additionalProperties': False,
+}
+
+testDataSchema_v1 = {
+    'type': 'object',
+    'properties': {
+        'type': {'type': 'string'},
+        'size': {'type': 'number'},
+        'woot': {'type': 'string'},
+    },
+    'required': ['type', 'size', 'woot'],
+    'additionalProperties': False,
+}
+
 class CellTest(s_t_utils.SynTest):
 
+    async def test_cell_drive(self):
+
+        async with self.getTestCell() as cell:
+
+            with self.raises(s_exc.BadName):
+                s_drive.reqValidName('A' * 512)
+
+            info = {'name': 'users'}
+            pathinfo = await cell.addDriveItem(info)
+
+            info = {'name': 'root'}
+            pathinfo = await cell.addDriveItem(info, path='users')
+
+            with self.raises(s_exc.DupIden):
+                await cell.drive.addItemInfo(pathinfo[-1], path='users')
+
+            rootdir = pathinfo[-1].get('iden')
+            self.eq(0, pathinfo[-1].get('kids'))
+
+            info = {'name': 'win32k.sys', 'type': 'hehe'}
+            with self.raises(s_exc.NoSuchType):
+                info = await cell.addDriveItem(info, reldir=rootdir)
+
+            infos = [i async for i in cell.getDriveKids(s_drive.rootdir)]
+            self.len(1, infos)
+            self.eq(1, infos[0].get('kids'))
+            self.eq('users', infos[0].get('name'))
+
+            # TODO how to handle iden match with additional property mismatch
+
+            await cell.drive.setTypeSchema('woot', testDataSchema_v0)
+
+            info = {'name': 'win32k.sys', 'type': 'woot'}
+            info = await cell.addDriveItem(info, reldir=rootdir)
+
+            iden = info[-1].get('iden')
+
+            tick = s_common.now()
+            rootuser = cell.auth.rootuser.iden
+
+            with self.raises(s_exc.SchemaViolation):
+                versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser}
+                await cell.setDriveData(iden, versinfo, {'newp': 'newp'})
+
+            versinfo = {'version': (1, 1, 0), 'updated': tick + 10, 'updater': rootuser}
+            info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'haha', 'size': 20})
+            self.eq(info.get('version'), (1, 1, 0))
+            self.eq(versinfo.get('version'), (1, 1, 0))
+
+            versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser}
+            info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'hehe', 'size': 0})
+            self.eq(info.get('version'), (1, 1, 0))
+            self.eq(versinfo.get('version'), (1, 0, 0))
+
+            versinfo10, data10 = await cell.getDriveData(iden, vers=(1, 0, 0))
+            self.eq(versinfo10.get('updated'), tick)
+            self.eq(versinfo10.get('updater'), rootuser)
+            self.eq(versinfo10.get('version'), (1, 0, 0))
+
+            versinfo11, data11 = await cell.getDriveData(iden, vers=(1, 1, 0))
+            self.eq(versinfo11.get('updated'), tick + 10)
+            self.eq(versinfo11.get('updater'), rootuser)
+            self.eq(versinfo11.get('version'), (1, 1, 0))
+
+            versions = [vers async for vers in cell.getDriveDataVersions(iden)]
+            self.len(2, versions)
+            self.eq(versions[0], versinfo11)
+            self.eq(versions[1], versinfo10)
+
+            info = await cell.delDriveData(iden, vers=(0, 0, 0))
+
+            versions = [vers async for vers in cell.getDriveDataVersions(iden)]
+            self.len(2, versions)
+            self.eq(versions[0], versinfo11)
+            self.eq(versions[1], versinfo10)
+
+            info = await cell.delDriveData(iden, vers=(1, 1, 0))
+            self.eq(info.get('updated'), tick)
+            self.eq(info.get('version'), (1, 0, 0))
+
+            info = await cell.delDriveData(iden, vers=(1, 0, 0))
+            self.eq(info.get('size'), 0)
+            self.eq(info.get('version'), (0, 0, 0))
+            self.none(info.get('updated'))
+            self.none(info.get('updater'))
+
+            # repopulate a couple data versions to test migration and delete
+            versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser}
+            info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'hehe', 'size': 0})
+            versinfo = {'version': (1, 1, 0), 'updated': tick + 10, 'updater': rootuser}
+            info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'haha', 'size': 17})
+            self.eq(versinfo, (await cell.getDriveData(iden))[0])
+
+            # This will be done by the cell in a cell storage version migration...
+            async def migrate_v1(info, versinfo, data):
+                data['woot'] = 'woot'
+                return data
+
+            await cell.drive.setTypeSchema('woot', testDataSchema_v1, migrate_v1)
+
+            versinfo, data = await cell.getDriveData(iden, vers=(1, 0, 0))
+            self.eq('woot', data.get('woot'))
+
+            versinfo, data = await cell.getDriveData(iden, vers=(1, 1, 0))
+            self.eq('woot', data.get('woot'))
+
+            self.nn(await cell.getDriveInfo(iden))
+            self.len(2, [vers async for vers in cell.getDriveDataVersions(iden)])
+
+            await cell.delDriveData(iden)
+            self.len(1, [vers async for vers in cell.getDriveDataVersions(iden)])
+
+            await cell.delDriveInfo(iden)
+
+            self.none(await cell.getDriveInfo(iden))
+            self.len(0, [vers async for vers in cell.getDriveDataVersions(iden)])
+
+            with self.raises(s_exc.NoSuchPath):
+                await cell.getDrivePath('users/root/win32k.sys')
+
+            pathinfo = await cell.addDrivePath('foo/bar/baz')
+            self.len(3, pathinfo)
+            self.eq('foo', pathinfo[0].get('name'))
+            self.eq(1, pathinfo[0].get('kids'))
+            self.eq('bar', pathinfo[1].get('name'))
+            self.eq(1, pathinfo[1].get('kids'))
+            self.eq('baz', pathinfo[2].get('name'))
+            self.eq(0, pathinfo[2].get('kids'))
+
+            self.eq(pathinfo, await cell.addDrivePath('foo/bar/baz'))
+
+            baziden = pathinfo[2].get('iden')
+            self.eq(pathinfo, await cell.drive.getItemPath(baziden))
+
+            info = await cell.setDriveInfoPerm(baziden, {'users': {rootuser: 3}, 'roles': {}})
+            self.eq(3, info['perm']['users'][rootuser])
+
+            with self.raises(s_exc.NoSuchIden):
+                # s_drive.rootdir is all 00s... ;)
+                await cell.setDriveInfoPerm(s_drive.rootdir, {'users': {}, 'roles': {}})
+
+            await cell.addDrivePath('hehe/haha')
+            pathinfo = await cell.setDriveInfoPath(baziden, 'hehe/haha/hoho')
+
+            self.eq('hoho', pathinfo[-1].get('name'))
+            self.eq(baziden, pathinfo[-1].get('iden'))
+
+            self.true(await cell.drive.hasPathInfo('hehe/haha/hoho'))
+            self.false(await cell.drive.hasPathInfo('foo/bar/baz'))
+
+            pathinfo = await cell.getDrivePath('foo/bar')
+            self.eq(0, pathinfo[-1].get('kids'))
+
+            pathinfo = await cell.getDrivePath('hehe/haha')
+            self.eq(1, pathinfo[-1].get('kids'))
+
+            with self.raises(s_exc.DupName):
+                iden = pathinfo[-2].get('iden')
+                name = pathinfo[-1].get('name')
+                cell.drive.reqFreeStep(iden, name)
+
+            walks = [item async for item in cell.drive.walkPathInfo('hehe')]
+            self.len(3, walks)
+            # confirm walked paths are yielded depth first...
+            self.eq('hoho', walks[0].get('name'))
+            self.eq('haha', walks[1].get('name'))
+            self.eq('hehe', walks[2].get('name'))
+
+            iden = walks[2].get('iden')
+            walks = [item async for item in cell.drive.walkItemInfo(iden)]
+            self.len(3, walks)
+            self.eq('hoho', walks[0].get('name'))
+            self.eq('haha', walks[1].get('name'))
+            self.eq('hehe', walks[2].get('name'))
+
+            self.none(cell.drive.getTypeSchema('newp'))
+
+            cell.drive.validators.pop('woot')
+            self.nn(cell.drive.getTypeValidator('woot'))
+
+            # move to root dir
+            pathinfo = await cell.setDriveInfoPath(baziden, 'zipzop')
+            self.len(1, pathinfo)
+            self.eq(s_drive.rootdir, pathinfo[-1].get('parent'))
+
+            pathinfo = await cell.setDriveInfoPath(baziden, 'hehe/haha/hoho')
+            self.len(3, pathinfo)
+
     async def test_cell_auth(self):
 
         with self.getTestDir() as dirn:
@@ -860,6 +1070,7 @@ class CellTest(s_t_utils.SynTest):
                 self.nn(slab['xactops'])
                 self.nn(slab['mapsize'])
                 self.nn(slab['readonly'])
+                self.nn(slab['readahead'])
                 self.nn(slab['lockmemory'])
                 self.nn(slab['recovering'])
 
@@ -2914,3 +3125,17 @@ class CellTest(s_t_utils.SynTest):
 
             async with self.getTestCell(s_cell.Cell, dirn=dirn):
                 pass
+
+    async def test_cell_initslab_fini(self):
+        class SlabCell(s_cell.Cell):
+            async def initServiceStorage(self):
+                self.long_lived_slab = await self._initSlabFile(os.path.join(self.dirn, 'slabs', 'long.lmdb'))
+                short_slab = await self._initSlabFile(os.path.join(self.dirn, 'slabs', 'short.lmdb'), ephemeral=True)
+                self.short_slab_path = short_slab.lenv.path()
+                await short_slab.fini()
+
+        async with self.getTestCell(SlabCell) as cell:
+            self.true(os.path.isdir(cell.short_slab_path))
+            self.isin(cell.long_lived_slab.fini, cell._fini_funcs)
+            slabs = [s for s in cell.tofini if isinstance(s, s_lmdbslab.Slab) and s.lenv.path() == cell.short_slab_path]
+            self.len(0, slabs)

synapse/tests/test_lib_coro.py

@@ -41,6 +41,18 @@ def nopickle():
 
 class CoroTest(s_t_utils.SynTest):
 
+    async def test_coro_chunks(self):
+        async def agen():
+            for i in range(101):
+                yield i
+
+        chunks = []
+        async for chunk in s_coro.chunks(agen()):
+            chunks.append(chunk)
+
+        self.len(1, chunks[1])
+        self.len(100, chunks[0])
+
     async def test_coro_event(self):
 
         evnt = s_coro.Event()

synapse/tests/test_lib_layer.py

@@ -2216,3 +2216,25 @@ class LayerTest(s_t_utils.SynTest):
 
             self.notin('pulls', dst_tree)
             self.notin('pushs', dst_tree)
+
+    async def test_layer_readahead(self):
+
+        async with self.getTestCore() as core:
+
+            layr = core.getLayer()
+            self.true(layr.layrslab.readahead)
+            self.true(layr.layrslab.lenv.flags()['readahead'])
+            self.false(layr.nodeeditslab.readahead)
+            self.false(layr.nodeeditslab.lenv.flags()['readahead'])
+            self.false(layr.dataslab.readahead)
+            self.false(layr.dataslab.lenv.flags()['readahead'])
+
+            with self.setTstEnvars(SYNDEV_CORTEX_LAYER_READAHEAD='false'):
+                iden = await core.callStorm('return($lib.layer.add().iden)')
+                layr = core.getLayer(iden)
+                self.false(layr.layrslab.readahead)
+                self.false(layr.layrslab.lenv.flags()['readahead'])
+                self.false(layr.nodeeditslab.readahead)
+                self.false(layr.nodeeditslab.lenv.flags()['readahead'])
+                self.false(layr.dataslab.readahead)
+                self.false(layr.dataslab.lenv.flags()['readahead'])

synapse/tests/test_lib_modelrev.py

@@ -559,3 +559,10 @@ class ModelRevTest(s_tests.SynTest):
             self.len(2, rnodes)
 
             self.eq([node.ndef[0] for node in nodes], [node.ndef[0] for node in reversed(rnodes)])
+
+    async def test_modelrev_0_2_27(self):
+
+        async with self.getRegrCore('model-0.2.27') as core:
+            nodes = await core.nodes('it:dev:repo:commit:id=" Foo "')
+            self.len(1, nodes)
+            self.eq('Foo', nodes[0].get('id'))

synapse/tests/test_lib_node.py

@@ -243,7 +243,8 @@ class NodeTest(s_t_utils.SynTest):
                     nodepaths = await alist(node.storm(runt, '-> test:int [:loc=$foo]', opts={'vars': {'foo': 'us'}}))
                     self.eq(nodepaths[0][0].props.get('loc'), 'us')
 
-                    path = nodepaths[0][1].fork(node)  # type: s_node.Path
+                    link = {'type': 'runtime'}
+                    path = nodepaths[0][1].fork(node, link)  # type: s_node.Path
                     path.vars['zed'] = 'ca'
 
                     # Path present, opts not present
@@ -251,12 +252,18 @@ class NodeTest(s_t_utils.SynTest):
                     self.eq(nodes[0][0].props.get('loc'), 'ca')
                     # path is not updated due to frame scope
                     self.none(path.vars.get('bar'), 'us')
+                    self.len(2, path.links)
+                    self.eq({'type': 'prop', 'prop': 'hehe'}, path.links[0][1])
+                    self.eq(link, path.links[1][1])
 
                     # Path present, opts present but no opts['vars']
                     nodes = await alist(node.storm(runt, '-> test:int [:loc=$zed] $bar=$foo', opts={}, path=path))
                     self.eq(nodes[0][0].props.get('loc'), 'ca')
                     # path is not updated due to frame scope
                     self.none(path.vars.get('bar'))
+                    self.len(2, path.links)
+                    self.eq({'type': 'prop', 'prop': 'hehe'}, path.links[0][1])
+                    self.eq(link, path.links[1][1])
 
                     # Path present, opts present with vars
                     nodes = await alist(node.storm(runt, '-> test:int [:loc=$zed] $bar=$baz',
@@ -294,6 +301,10 @@ class NodeTest(s_t_utils.SynTest):
                     self.eq(len(pcln.nodes), len(path.nodes))
                     pcln.nodes.pop(-1)
                     self.ne(len(pcln.nodes), len(path.nodes))
+                    # Ensure the link elements are independent
+                    pcln.links.append({'type': 'edge', 'verb': 'seen'})
+                    self.len(3, pcln.links)
+                    self.len(2, path.links)
 
                     # push a frame and clone it - ensure clone mods do not
                     # modify the original path

synapse/tests/test_lib_storm.py

@@ -3059,12 +3059,29 @@ class StormTest(s_t_utils.SynTest):
             self.len(4, nodes)
 
             q = 'inet:ipv4=1.2.3.4 | tee --join { -> * } { <- * }'
-            nodes = await core.nodes(q)
+            msgs = await core.stormlist(q, opts={'links': True})
+            nodes = [m[1] for m in msgs if m[0] == 'node']
             self.len(4, nodes)
-            self.eq(nodes[0].ndef, ('inet:asn', 0))
-            self.eq(nodes[1].ndef[0], ('inet:dns:a'))
-            self.eq(nodes[2].ndef[0], ('edge:refs'))
-            self.eq(nodes[3].ndef, ('inet:ipv4', 0x01020304))
+
+            self.eq(nodes[0][0], ('inet:asn', 0))
+            links = nodes[0][1]['links']
+            self.len(1, links)
+            self.eq({'type': 'prop', 'prop': 'asn'}, links[0][1])
+
+            self.eq(nodes[1][0][0], ('inet:dns:a'))
+            links = nodes[1][1]['links']
+            self.len(1, links)
+            self.eq({'type': 'prop', 'prop': 'ipv4', 'reverse': True}, links[0][1])
+
+            self.eq(nodes[2][0][0], ('edge:refs'))
+            links = nodes[2][1]['links']
+            self.len(1, links)
+            self.eq({'type': 'prop', 'prop': 'n2', 'reverse': True}, links[0][1])
+
+            self.eq(nodes[3][0], ('inet:ipv4', 0x01020304))
+            links = nodes[2][1]['links']
+            self.len(1, links)
+            self.eq({'type': 'prop', 'prop': 'n2', 'reverse': True}, links[0][1])
 
             q = 'inet:ipv4=1.2.3.4 | tee --join { -> * } { <- * } { -> edge:refs:n2 :n1 -> * }'
             nodes = await core.nodes(q)
@@ -3834,10 +3851,18 @@ class StormTest(s_t_utils.SynTest):
             self.eq(sorted([n.ndef[1] for n in nodes]), ['test1', 'test2'])
 
             q = '[(test:str=refs) (test:str=foo)] $v=$node.value() | lift.byverb $v'
-            nodes = await core.nodes(q)
+            msgs = await core.stormlist(q, opts={'links': True})
+            nodes = [n[1] for n in msgs if n[0] == 'node']
             self.len(4, nodes)
-            self.eq({n.ndef[1] for n in nodes},
+            self.eq({n[0][1] for n in nodes},
                     {'test1', 'test2', 'refs', 'foo'})
+            links = nodes[1][1]['links']
+            self.len(1, links)
+            self.eq({'type': 'runtime'}, links[0][1])
+
+            links = nodes[2][1]['links']
+            self.len(1, links)
+            self.eq({'type': 'runtime'}, links[0][1])
 
     async def test_storm_nested_root(self):
         async with self.getTestCore() as core: