synapse 2.197.0__py311-none-any.whl → 2.199.0__py311-none-any.whl

synapse/tests/test_cortex.py

@@ -1001,6 +1001,114 @@ class CortexTest(s_t_utils.SynTest):
             self.eq(cm.exception.get('mesg'),
                     'walk operation expected a string or list.  got: 0.')
 
+            await core.nodes('[media:news=*]')
+
+            nodes = await core.nodes('$n = {[it:dev:str=foo]} media:news [ +(refs)> $n ]')
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef[0], 'media:news')
+
+            nodes = await core.nodes('media:news -(refs)> it:dev:str')
+            self.len(1, nodes)
+
+            q = '''
+            function foo() {
+                for $x in $lib.range(5) {
+                    [ it:dev:int=$x ]
+                    emit $node
+                }
+            }
+            media:news [ +(refs)> $foo() ]
+            '''
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef[0], 'media:news')
+
+            nodes = await core.nodes('media:news -(refs)> it:dev:int')
+            self.len(5, nodes)
+
+            nodes = await core.nodes('$n = {[it:dev:str=foo]} media:news [ -(refs)> $n ]')
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef[0], 'media:news')
+
+            nodes = await core.nodes('media:news -(refs)> it:dev:str')
+            self.len(0, nodes)
+
+            q = '''
+            function foo() {
+                for $x in $lib.range(5) {
+                    [ it:dev:int=$x ]
+                    emit $node
+                }
+            }
+            media:news [ -(refs)> $foo() ]
+            '''
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef[0], 'media:news')
+
+            nodes = await core.nodes('media:news -(refs)> it:dev:int')
+            self.len(0, nodes)
+
+            nodes = await core.nodes('$n = {[it:dev:str=foo]} media:news [ <(refs)+ $n ]')
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef[0], 'media:news')
+
+            nodes = await core.nodes('media:news <(refs)- it:dev:str')
+            self.len(1, nodes)
+
+            q = '''
+            function foo() {
+                for $x in $lib.range(5) {
+                    [ it:dev:int=$x ]
+                    emit $node
+                }
+            }
+            media:news [ <(refs)+ $foo() ]
+            '''
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef[0], 'media:news')
+
+            nodes = await core.nodes('media:news <(refs)- it:dev:int')
+            self.len(5, nodes)
+
+            nodes = await core.nodes('$n = {[it:dev:str=foo]} media:news [ <(refs)- $n ]')
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef[0], 'media:news')
+
+            nodes = await core.nodes('media:news <(refs)- it:dev:str')
+            self.len(0, nodes)
+
+            q = '''
+            function foo() {
+                for $x in $lib.range(5) {
+                    [ it:dev:int=$x ]
+                    emit $node
+                }
+            }
+            media:news [ <(refs)- $foo() ]
+            '''
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef[0], 'media:news')
+
+            nodes = await core.nodes('media:news <(refs)- it:dev:int')
+            self.len(0, nodes)
+
+            await core.nodes('[media:news=*]')
+
+            nodes = await core.nodes('$n = {[it:dev:str=foo]} $edge=refs media:news [ +($edge)> $n ]')
+            self.len(2, nodes)
+
+            nodes = await core.nodes('media:news -(refs)> it:dev:str')
+            self.len(2, nodes)
+
+            nodes = await core.nodes('$n = {[it:dev:str=foo]} $edge=refs media:news [ -($edge)> $n ]')
+            self.len(2, nodes)
+
+            nodes = await core.nodes('media:news -(refs)> it:dev:str')
+            self.len(0, nodes)
+
     async def test_cortex_callstorm(self):
 
         async with self.getTestCore(conf={'auth:passwd': 'root'}) as core:

synapse/tests/test_datamodel.py

@@ -338,11 +338,33 @@ class DataModelTest(s_t_utils.SynTest):
 
     async def test_datamodel_locked_subs(self):
 
-        async with self.getTestCore() as core:
-            await core.setDeprLock('it:prod:softver:semver:major', True)
-            nodes = await core.nodes('[ it:prod:softver=* :semver=3.1.0 ]')
-            self.none(nodes[0].get('semver:major'))
-            self.eq(1, nodes[0].get('semver:minor'))
+        conf = {'modules': [('synapse.tests.utils.DeprModule', {})]}
+        async with self.getTestCore(conf=conf) as core:
+
+            msgs = await core.stormlist('[ test:deprsub=bar :range=(1, 5) ]')
+            self.stormHasNoWarnErr(msgs)
+
+            msgs = await core.stormlist('[ test:deprsub2=(foo, (2, 6)) ]')
+            self.stormHasNoWarnErr(msgs)
+
+            nodes = await core.nodes('test:deprsub=bar')
+            self.eq(1, nodes[0].get('range:min'))
+            self.eq(5, nodes[0].get('range:max'))
+
+            nodes = await core.nodes('test:deprsub2=(foo, (2, 6))')
+            self.eq(2, nodes[0].get('range:min'))
+            self.eq(6, nodes[0].get('range:max'))
+
+            await core.setDeprLock('test:deprsub:range:min', True)
+            nodes = await core.nodes('[ test:deprsub=foo :range=(1, 5) ]')
+            self.none(nodes[0].get('range:min'))
+            self.eq(5, nodes[0].get('range:max'))
+
+            await core.nodes('test:deprsub2 | delnode')
+            await core.setDeprLock('test:deprsub2:range:max', True)
+            nodes = await core.nodes('[ test:deprsub2=(foo, (2, 6)) ]')
+            self.none(nodes[0].get('range:max'))
+            self.eq(2, nodes[0].get('range:min'))
 
     def test_datamodel_schema_basetypes(self):
         # N.B. This test is to keep synapse.lib.schemas.datamodel_basetypes const

synapse/tests/test_lib_aha.py

@@ -133,13 +133,18 @@ class AhaTest(s_test.SynTest):
         with self.getTestDir() as dirn:
             cryo0_dirn = s_common.gendir(dirn, 'cryo0')
             async with self.getTestAha(dirn=dirn) as aha:
+
+                replaymult = 1
+                if s_common.envbool('SYNDEV_NEXUS_REPLAY'):
+                    replaymult = 2
+
                 purl = await aha.addAhaSvcProv('0.cryo')
 
-                wait00 = aha.waiter(1, 'aha:svcadd')
+                wait00 = aha.waiter(1 * replaymult, 'aha:svcadd')
 
                 conf = {'aha:provision': purl}
                 async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo:
-                    self.isin(len(await wait00.wait(timeout=6)), (1, 2))
+                    self.len(1 * replaymult, await wait00.wait(timeout=6))
 
                     svc = await aha.getAhaSvc('0.cryo...')
                     linkiden = svc.get('svcinfo', {}).get('online')
@@ -147,16 +152,16 @@ class AhaTest(s_test.SynTest):
                     # Tear down the Aha cell.
                     await aha.__aexit__(None, None, None)
 
-            async with self.getTestAha(dirn=dirn) as aha:
-                wait01 = aha.waiter(1, 'aha:svcdown')
-                await wait01.wait(timeout=6)
-                svc = await aha.getAhaSvc('0.cryo...')
-                self.notin('online', svc.get('svcinfo'))
+            with self.getAsyncLoggerStream('synapse.lib.aha', f'Set [0.cryo.synapse] offline.') as stream:
+                async with self.getTestAha(dirn=dirn) as aha:
+                    self.true(await asyncio.wait_for(stream.wait(), timeout=12))
+                    svc = await aha.getAhaSvc('0.cryo...')
+                    self.notin('online', svc.get('svcinfo'))
 
-                # Try setting something down a second time
-                await aha.setAhaSvcDown('0.cryo', linkiden, network='synapse')
-                svc = await aha.getAhaSvc('0.cryo...')
-                self.notin('online', svc.get('svcinfo'))
+                    # Try setting something down a second time
+                    await aha.setAhaSvcDown('0.cryo', linkiden, network='synapse')
+                    svc = await aha.getAhaSvc('0.cryo...')
+                    self.notin('online', svc.get('svcinfo'))
 
     async def test_lib_aha_basics(self):
 
@@ -444,8 +449,13 @@ class AhaTest(s_test.SynTest):
 
             async with self.getTestAha() as aha:
 
+                replaymult = 1
+                if s_common.envbool('SYNDEV_NEXUS_REPLAY'):
+                    replaymult = 2
+
                 aha.testerr = True
                 wait00 = aha.waiter(1, 'aha:svcadd')
+
                 conf = {'aha:provision': await aha.addAhaSvcProv('0.cryo')}
                 async with self.getTestCryo(conf=conf) as cryo:
 
@@ -454,7 +464,7 @@ class AhaTest(s_test.SynTest):
                     svc = await aha.getAhaSvc('0.cryo...')
                     self.none(svc)
 
-                    wait01 = aha.waiter(1, 'aha:svcadd')
+                    wait01 = aha.waiter(1 * replaymult, 'aha:svcadd')
                     aha.testerr = False
 
                     self.nn(await wait01.wait(timeout=2))

synapse/tests/test_lib_ast.py

@@ -492,6 +492,63 @@ class AstTest(s_test.SynTest):
                 q = '$foo=newp [test:str=foo :hehe*$foo=heval]'
                 nodes = await core.nodes(q)
 
+    async def test_ast_setmultioper(self):
+        async with self.getTestCore() as core:
+
+            nodes = await core.nodes('[ test:arrayprop="*" :ints=(1,) ]')
+            self.eq(nodes[0].get('ints'), (1,))
+
+            nodes = await core.nodes('test:arrayprop [ :ints++=([3, 4]) ]')
+            self.eq(nodes[0].get('ints'), (1, 3, 4))
+
+            nodes = await core.nodes('test:arrayprop [ :ints++=(null) ]')
+            self.eq(nodes[0].get('ints'), (1, 3, 4))
+
+            nodes = await core.nodes('test:arrayprop [ :ints--=(null) ]')
+            self.eq(nodes[0].get('ints'), (1, 3, 4))
+
+            nodes = await core.nodes('test:arrayprop [ :strs++=(foo, bar, baz) ]')
+            self.eq(nodes[0].get('strs'), ('foo', 'bar', 'baz'))
+
+            with self.raises(s_exc.BadTypeValu):
+                await core.nodes('test:arrayprop [ :ints++=(["newp", 5, 6]) ]')
+
+            nodes = await core.nodes('test:arrayprop [ :ints?++=(["newp", 5, 6]) ]')
+            self.eq(nodes[0].get('ints'), (1, 3, 4, 5, 6))
+
+            with self.raises(s_exc.BadTypeValu):
+                await core.nodes('test:arrayprop [ :ints--=(["newp", 5, 6]) ]')
+
+            nodes = await core.nodes('test:arrayprop [ :ints?--=(["newp", 5, 6, 7]) ]')
+            self.eq(nodes[0].get('ints'), (1, 3, 4))
+
+            nodes = await core.nodes('[ test:str=foo :ndefs++={[ test:str=bar ]} ]')
+            self.eq(nodes[0].get('ndefs'), (('test:str', 'bar'),))
+
+            nodes = await core.nodes('test:str=foo  [ :ndefs++={[ test:str=baz test:str=faz ]} ]')
+            self.eq(nodes[0].get('ndefs'), (('test:str', 'bar'), ('test:str', 'baz'), ('test:str', 'faz')))
+
+            nodes = await core.nodes('test:str=foo  [ :ndefs--={ test:str=baz test:str=faz } ]')
+            self.eq(nodes[0].get('ndefs'), (('test:str', 'bar'),))
+
+            with self.raises(s_exc.NoSuchProp):
+                await core.nodes('test:arrayprop [ :newp++=(["newp", 5, 6]) ]')
+
+            badq = [
+                'test:str [ :hehe++=([3, 4]) ]',
+                'test:str [ :hehe?++=([3, 4]) ]',
+                'test:str [ :hehe--=([3, 4]) ]',
+                'test:str [ :hehe?--=([3, 4]) ]',
+                'test:arrayprop [ :ints++=(3) ]',
+                'test:arrayprop [ :ints?++=(3) ]',
+                'test:arrayprop [ :ints--=(3) ]',
+                'test:arrayprop [ :ints?--=(3) ]',
+            ]
+
+            for q in badq:
+                with self.raises(s_exc.StormRuntimeError):
+                    await core.nodes(q)
+
     async def test_ast_editparens(self):
 
         async with self.getTestCore() as core:

synapse/tests/test_lib_auth.py

@@ -1,9 +1,15 @@
 import string
 import pathlib
 
+from unittest import mock
+
 import synapse.exc as s_exc
+import synapse.common as s_common
 import synapse.telepath as s_telepath
-import synapse.lib.hiveauth as s_hiveauth
+
+import synapse.lib.auth as s_auth
+import synapse.lib.cell as s_cell
+import synapse.lib.lmdbslab as s_lmdbslab
 
 import synapse.tests.utils as s_test
 
@@ -426,6 +432,126 @@ class AuthTest(s_test.SynTest):
             with self.raises(s_exc.SchemaViolation):
                 await core.auth.allrole.setRules([(True, )])
 
+    async def test_auth_archived_locked_interaction(self):
+
+        # Check that we can't unlock an archived user
+        async with self.getTestCore() as core:
+            lowuser = await core.addUser('lowuser')
+            useriden = lowuser.get('iden')
+
+            await core.setUserArchived(useriden, True)
+
+            udef = await core.getUserDef(useriden)
+            self.true(udef.get('archived'))
+            self.true(udef.get('locked'))
+
+            # Unlocking an archived user is invalid
+            with self.raises(s_exc.BadArg) as exc:
+                await core.setUserLocked(useriden, False)
+            self.eq(exc.exception.get('mesg'), 'Cannot unlock archived user.')
+            self.eq(exc.exception.get('user'), useriden)
+            self.eq(exc.exception.get('username'), 'lowuser')
+
+        # Check our cell migration that locks archived users
+        async with self.getRegrCore('unlocked-archived-users') as core:
+            for ii in range(10):
+                user = await core.getUserDefByName(f'lowuser{ii:02d}')
+                self.nn(user)
+                self.true(user.get('archived'))
+                self.true(user.get('locked'))
+
+        # Check behavior of upgraded mirrors and non-upgraded leader
+        async with self.getTestAha() as aha:
+
+            with self.getTestDir() as dirn:
+                path00 = s_common.gendir(dirn, 'cell00')
+                path01 = s_common.gendir(dirn, 'cell01')
+
+                with mock.patch('synapse.lib.cell.NEXUS_VERSION', (2, 177)):
+                    async with self.addSvcToAha(aha, '00.cell', s_cell.Cell, dirn=path00) as cell00:
+                        lowuser = await cell00.addUser('lowuser')
+                        useriden = lowuser.get('iden')
+                        await cell00.setUserArchived(useriden, True)
+
+                        with mock.patch('synapse.lib.cell.NEXUS_VERSION', (2, 198)):
+                            async with self.addSvcToAha(aha, '01.cell', s_cell.Cell, dirn=path01, provinfo={'mirror': 'cell'}) as cell01:
+                                await cell01.sync()
+                                udef = await cell01.getUserDef(useriden)
+                                self.true(udef.get('locked'))
+                                self.true(udef.get('archived'))
+
+                                # Simulate a call to cell00.setUserLocked(useriden, False) to bypass
+                                # the check in cell00.auth.setUserInfo()
+                                await cell00.auth._push('user:info', useriden, 'locked', False)
+                                await cell01.sync()
+
+                                udef00 = await cell00.getUserDef(useriden)
+                                self.true(udef00.get('archived'))
+                                self.false(udef00.get('locked'))
+
+                                udef01 = await cell01.getUserDef(useriden)
+                                self.true(udef01.get('archived'))
+                                self.false(udef01.get('locked'))
+
+        # Check that we don't blowup/schism if an upgraded mirror is behind a leader with a pending
+        # user:info event that unlocks an archived user
+        async with self.getTestAha() as aha:
+
+            with self.getTestDir() as dirn:
+                path00 = s_common.gendir(dirn, 'cell00')
+                path01 = s_common.gendir(dirn, 'cell01')
+
+                async with self.addSvcToAha(aha, '00.cell', s_cell.Cell, dirn=path00) as cell00:
+                    lowuser = await cell00.addUser('lowuser')
+                    useriden = lowuser.get('iden')
+                    await cell00.setUserLocked(useriden, True)
+
+                    async with self.addSvcToAha(aha, '01.cell', s_cell.Cell, dirn=path01, provinfo={'mirror': 'cell'}) as cell01:
+                        await cell01.sync()
+                        udef = await cell01.getUserDef(useriden)
+                        self.true(udef.get('locked'))
+                        self.false(udef.get('archived'))
+
+                    # Set user locked while cell01 is offline so it will get the edit when it comes
+                    # back
+                    await cell00.setUserLocked(useriden, False)
+                    await cell00.sync()
+
+                # Edit the slabs on both cells directly to archive the user
+                lmdb00 = s_common.genpath(path00, 'slabs', 'cell.lmdb')
+                lmdb01 = s_common.genpath(path01, 'slabs', 'cell.lmdb')
+
+                slab00 = await s_lmdbslab.Slab.anit(lmdb00, map_size=s_cell.SLAB_MAP_SIZE)
+                slab01 = await s_lmdbslab.Slab.anit(lmdb01, map_size=s_cell.SLAB_MAP_SIZE)
+
+                # Simulate the cell migration which locks archived users
+                for slab in (slab00, slab01):
+                    authkv = slab.getSafeKeyVal('auth')
+                    userkv = authkv.getSubKeyVal('user:info:')
+
+                    info = userkv.get(useriden)
+                    info['archived'] = True
+                    info['locked'] = True
+                    userkv.set(useriden, info)
+
+                await slab00.fini()
+                await slab01.fini()
+
+                # Spin the cells back up and wait for the edit to sync to cell01
+                async with self.addSvcToAha(aha, '00.cell', s_cell.Cell, dirn=path00) as cell00:
+                    udef = await cell00.getUserDef(useriden)
+                    self.true(udef.get('archived'))
+                    self.true(udef.get('locked'))
+
+                    async with self.addSvcToAha(aha, '01.cell', s_cell.Cell, dirn=path01, provinfo={'mirror': 'cell'}) as cell01:
+                        await cell01.sync()
+                        udef = await cell01.getUserDef(useriden)
+                        self.true(udef.get('archived'))
+                        self.true(udef.get('locked'))
+
+                        self.ge(cell00.nexsvers, (2, 198))
+                        self.ge(cell01.nexsvers, (2, 198))
+
     async def test_auth_password_policy(self):
         policy = {
             'complexity': {
@@ -446,6 +572,21 @@ class AuthTest(s_test.SynTest):
         pass3 = 'ZXN-pyv7ber-kzq2kgh'
 
         conf = {'auth:passwd:policy': policy}
+        async with self.getTestCore(conf=conf) as core:
+
+            user = await core.auth.addUser('blackout@vertex.link')
+
+            self.none(user.info.get('policy:previous'))
+            await user.setPasswd(pass1, nexs=False)
+            await user.setPasswd(pass2, nexs=False)
+            await user.setPasswd(pass3, nexs=False)
+            self.len(2, user.info.get('policy:previous'))
+
+            await user.tryPasswd('newp')
+            self.eq(1, user.info.get('policy:attempts'))
+            await user.setLocked(False, logged=False)
+            self.eq(0, user.info.get('policy:attempts'))
+
         async with self.getTestCore(conf=conf) as core:
             auth = core.auth
             self.nn(auth.policy)
@@ -708,7 +849,7 @@ class AuthTest(s_test.SynTest):
             await core.callStorm('auth.role.addrule ninjas --gate $gate another.rule',
                                  opts={'vars': {'gate': fork}})
 
-            user = await core.auth.getUserByName('lowuser')  # type: s_hiveauth.HiveUser
+            user = await core.auth.getUserByName('lowuser')  # type: s_auth.User
             self.false(user.allowed(('hehe',)))
             self.false(user.allowed(('hehe',), deepdeny=True))
             self.true(user.allowed(('hehe', 'haha')))

synapse/tests/test_lib_grammar.py

@@ -735,6 +735,17 @@ Queries = [
     '[test:str=foo :$foo*$bar.baz=heval]',
     '[test:str=foo :$foo*$bar.("baz")=heval]',
     '[test:str=foo :$foo*$bar.baz()=heval]',
+    '[test:str=foo +(refs)> $n]',
+    '[test:str=foo +(refs)> $n.baz()]',
+    '[test:str=foo -(refs)> $n]',
+    '[test:str=foo <(refs)+ $n]',
+    '[test:str=foo <(refs)+ $n.baz()]',
+    '[test:str=foo <(refs)- $n]',
+    '[test:str=foo :bar++=([1, 2])]',
+    '[test:str=foo :$foo++=([1, 2])]',
+    '[test:str=foo :bar--=(foo, bar)]',
+    '[test:str=foo :bar?++=$baz]',
+    '[test:str=foo :bar?--={[it:dev:str=foo]}]',
 ]
 
 # Generated with print_parse_list below
@@ -778,7 +789,7 @@ _ParseResults = [
     'Query: [TryCatch: [Query: [LiftPropBy: [Const: inet:ipv4, Const: =, Const: asdf]], CatchBlock: [Const: TypeError, Const: err, Query: []]]]',
     'Query: [TryCatch: [Query: [LiftPropBy: [Const: inet:ipv4, Const: =, Const: asdf]], CatchBlock: [Const: FooBar, Const: err, Query: []], CatchBlock: [Const: *, Const: err, Query: []]]]',
     'Query: [LiftByArray: [Const: test:array, Const: =, Const: 1.2.3.4]]',
-    'Query: [CmdOper: [Const: macro.set, List: [Const: hehe, EmbedQuery: inet:ipv4]]]',
+    'Query: [CmdOper: [Const: macro.set, List: [Const: hehe, EmbedQuery:  inet:ipv4 ]]]',
     'Query: [SetVarOper: [Const: q, EmbedQuery: #foo.bar]]',
     'Query: [CmdOper: [Const: metrics.edits.byprop, List: [Const: inet:fqdn:domain, Const: --newv, VarDeref: [VarValue: [Const: lib], Const: null]]]]',
     'Query: [CmdOper: [Const: tee, Const: ()]]',
@@ -1364,7 +1375,7 @@ _ParseResults = [
     'Query: [SetVarOper: [Const: p, Const: names], LiftPropBy: [Const: ps:contact:name, Const: =, Const: foo], EditPropSet: [RelProp: [VarValue: [Const: p]], Const: ?-=, Const: bar]]',
     'Query: [SetVarOper: [Const: pvar, Const: stuff], LiftProp: [Const: test:arrayprop], FiltOper: [Const: +, ArrayCond: [RelProp: [VarValue: [Const: pvar]], Const: =, Const: neato]]]',
     'Query: [SetVarOper: [Const: pvar, Const: ints], LiftProp: [Const: test:arrayprop], FiltOper: [Const: +, ArrayCond: [RelProp: [VarValue: [Const: pvar]], Const: =, VarValue: [Const: othervar]]]]',
-    'Query: [SetVarOper: [Const: foo, DollarExpr: [ExprDict: [Const: foo, EmbedQuery: inet:fqdn]]]]',
+    'Query: [SetVarOper: [Const: foo, DollarExpr: [ExprDict: [Const: foo, EmbedQuery:  inet:fqdn ]]]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [Const: hehe], CondSetOper: [Const: unset], Const: heval]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [Const: hehe], CondSetOper: [VarValue: [Const: foo]], Const: heval]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [Const: unset], Const: heval]]',
@@ -1372,6 +1383,17 @@ _ParseResults = [
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [VarDeref: [VarValue: [Const: bar], Const: baz]], Const: heval]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [VarDeref: [VarValue: [Const: bar], DollarExpr: [Const: baz]]], Const: heval]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [FuncCall: [VarDeref: [VarValue: [Const: bar], Const: baz], CallArgs: [], CallKwargs: []]], Const: heval]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditEdgeAdd: [Const: refs, VarValue: [Const: n]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditEdgeAdd: [Const: refs, FuncCall: [VarDeref: [VarValue: [Const: n], Const: baz], CallArgs: [], CallKwargs: []]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditEdgeDel: [Const: refs, VarValue: [Const: n]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditEdgeAdd: [Const: refs, VarValue: [Const: n]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditEdgeAdd: [Const: refs, FuncCall: [VarDeref: [VarValue: [Const: n], Const: baz], CallArgs: [], CallKwargs: []]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditEdgeDel: [Const: refs, VarValue: [Const: n]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditPropSetMulti: [RelProp: [Const: bar], Const: ++=, DollarExpr: [ExprList: [Const: 1, Const: 2]]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditPropSetMulti: [RelProp: [VarValue: [Const: foo]], Const: ++=, DollarExpr: [ExprList: [Const: 1, Const: 2]]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditPropSetMulti: [RelProp: [Const: bar], Const: --=, List: [Const: foo, Const: bar]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditPropSetMulti: [RelProp: [Const: bar], Const: ?++=, VarValue: [Const: baz]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditPropSetMulti: [RelProp: [Const: bar], Const: ?--=, SubQuery: [Query: [EditNodeAdd: [FormName: [Const: it:dev:str], Const: =, Const: foo]]]]]',
 ]
 
 class GrammarTest(s_t_utils.SynTest):
@@ -1663,6 +1685,27 @@ class GrammarTest(s_t_utils.SynTest):
         errinfo = cm.exception.errinfo
         self.eq(1, errinfo.get('mesg').count('#'))
 
+        query = '$q = ${ /* secret comment */ $lib.print([hello) } $lib.macro.set(hehe, $q)'
+        parser = s_parser.Parser(query)
+        with self.raises(s_exc.BadSyntax) as cm:
+            _ = parser.query()
+        info = cm.exception.errinfo.get('highlight')
+        self.eq((40, 41), info['offsets'])
+        self.eq((1, 1), info['lines'])
+        self.eq((41, 42), info['columns'])
+
+        query = """function test(hello) {
+                    +'''asdf
+                    asdfasdf'''
+        }"""
+        parser = s_parser.Parser(query)
+        with self.raises(s_exc.BadSyntax) as cm:
+            _ = parser.query()
+        info = cm.exception.errinfo.get('highlight')
+        self.eq((44, 83), info['offsets'])
+        self.eq((2, 3), info['lines'])
+        self.eq((22, 31), info['columns'])
+
     async def test_quotes(self):
 
         # Test vectors
@@ -1766,6 +1809,15 @@ class GrammarTest(s_t_utils.SynTest):
         self.false(s_grammar.isPropName('.hehe'))
         self.false(s_grammar.isPropName('testcmd'))
 
+    async def test_embed_offsets(self):
+
+        embq = ' /* secret comment */ $lib.print(hello) /* haha */ $lib.print(goodbye) /*foo */ '
+        query = f'$q = ${{{embq}}} $lib.print($q)'
+        parser = s_parser.Parser(query)
+        q = parser.query()
+        embed = q.kids[0].kids[1]
+        self.eq(embq, embed.getAstText())
+
 def gen_parse_list():
     '''
     Prints out the Asts for a list of queries in order to compare ASTs between versions of parsers

synapse/tests/test_lib_lmdbslab.py

@@ -371,6 +371,30 @@ class LmdbSlabTest(s_t_utils.SynTest):
                 'vm.dirty_ratio',
             ], msgs[0].get('sysctls', {}).keys())
 
+    async def test_lmdbslab_commit_over_max_xactops(self):
+
+        # Make sure that we don't confuse the periodic commit with the max replay log commit
+        with (self.getTestDir() as dirn,
+              patch('synapse.lib.lmdbslab.Slab.WARN_COMMIT_TIME_MS', 1),
+              patch('synapse.lib.lmdbslab.Slab.COMMIT_PERIOD', 100)
+              ):
+            path = os.path.join(dirn, 'test.lmdb')
+
+            async with await s_lmdbslab.Slab.anit(path, max_replay_log=100, map_size=100_000_000) as slab:
+                foo = slab.initdb('foo', dupsort=True)
+
+                byts = b'\x00' * 256
+                for i in range(1000):
+                    slab.put(b'\xff\xff\xff\xff' + s_common.guid(i).encode('utf8'), byts, db=foo)
+                    await asyncio.sleep(0)
+
+            # Let the slab close and then grab its stats
+            stats = slab.statinfo()
+            commitstats = stats.get('commitstats', ())
+            self.gt(len(commitstats), 0)
+            commitstats = [x[1] for x in commitstats if x[1] != 0]
+            self.eq(commitstats, (100, 100, 100, 100, 100, 100, 100, 100, 100, 100))
+
     async def test_lmdbslab_max_replay(self):
         with self.getTestDir() as dirn:
             path = os.path.join(dirn, 'test.lmdb')

synapse/tests/test_lib_storm.py

@@ -143,6 +143,26 @@ class StormTest(s_t_utils.SynTest):
             self.eq(props.get('name'), 'org name 77')
             self.eq(props.get('desc'), 'an org desc')
 
+            nodes = await core.nodes('ou:org=({"name": "the vertex project", "type": "lulz"})')
+            self.len(1, nodes)
+            orgn = nodes[0].ndef
+            self.eq(orgn, nodes11[0].ndef)
+
+            q = '[ ps:contact=* :org={ ou:org=({"name": "the vertex project", "type": "lulz"}) } ]'
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            cont = nodes[0]
+            self.eq(cont.get('org'), orgn[1])
+
+            nodes = await core.nodes('ps:contact:org=({"name": "the vertex project", "type": "lulz"})')
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef, cont.ndef)
+
+            self.len(0, await core.nodes('ps:contact:org=({"name": "vertex", "type": "newp"})'))
+
+            with self.raises(s_exc.BadTypeValu):
+                await core.nodes('inet:flow:from=({"name": "vertex", "type": "newp"})')
+
     async def test_lib_storm_jsonexpr(self):
         async with self.getTestCore() as core:
 
@@ -1737,6 +1757,10 @@ class StormTest(s_t_utils.SynTest):
             msgs = await core.stormlist('test:ro | merge', opts=altview)
             self.stormIsInWarn("Cannot merge read only property with conflicting value", msgs)
 
+            await core.nodes('[ test:str=foo +(refs)> { for $i in $lib.range(1001) {[ test:int=$i ]}}]', opts=altview)
+            await core.nodes('test:str=foo -(refs)+> * merge --apply', opts=altview)
+            self.len(1001, await core.nodes('test:str=foo -(refs)> *'))
+
     async def test_storm_merge_stricterr(self):
 
         conf = {'modules': [('synapse.tests.utils.DeprModule', {})]}

synapse/tests/test_lib_stormlib_macro.py

@@ -89,7 +89,7 @@ class MacroTest(s_test.SynTest):
             name = 'v' * 491
             q = '$lib.macro.set($name, ${ help }) return ( $lib.macro.get($name) )'
             mdef = await core.callStorm(q, opts={'vars': {'name': name}})
-            self.eq(mdef.get('storm'), 'help')
+            self.eq(mdef.get('storm'), ' help ')
 
             badname = 'v' * 492
             with self.raises(s_exc.BadArg):
@@ -381,7 +381,7 @@ class MacroTest(s_test.SynTest):
                     self.eq('storm:macro:add', addmesg['data']['event'])
                     macro = addmesg['data']['info']['macro']
                     self.eq(macro['name'], 'foobar')
-                    self.eq(macro['storm'], 'file:bytes | [+#neato]')
+                    self.eq(macro['storm'], ' file:bytes | [+#neato] ')
                     self.ne(visi.iden, macro['user'])
                     self.ne(visi.iden, macro['creator'])
                     self.nn(macro['iden'])
@@ -390,7 +390,7 @@ class MacroTest(s_test.SynTest):
                     self.eq('storm:macro:mod', setmesg['data']['event'])
                     event = setmesg['data']['info']
                     self.nn(event['macro'])
-                    self.eq(event['info']['storm'], 'inet:ipv4 | [+#burrito]')
+                    self.eq(event['info']['storm'], ' inet:ipv4 | [+#burrito] ')
                     self.nn(event['info']['updated'])
 
                     modmesg = await sock.receive_json()