synapse 2.197.0__py311-none-any.whl → 2.198.0__py311-none-any.whl

synapse/tests/test_lib_grammar.py

@@ -735,6 +735,17 @@ Queries = [
     '[test:str=foo :$foo*$bar.baz=heval]',
     '[test:str=foo :$foo*$bar.("baz")=heval]',
     '[test:str=foo :$foo*$bar.baz()=heval]',
+    '[test:str=foo +(refs)> $n]',
+    '[test:str=foo +(refs)> $n.baz()]',
+    '[test:str=foo -(refs)> $n]',
+    '[test:str=foo <(refs)+ $n]',
+    '[test:str=foo <(refs)+ $n.baz()]',
+    '[test:str=foo <(refs)- $n]',
+    '[test:str=foo :bar++=([1, 2])]',
+    '[test:str=foo :$foo++=([1, 2])]',
+    '[test:str=foo :bar--=(foo, bar)]',
+    '[test:str=foo :bar?++=$baz]',
+    '[test:str=foo :bar?--={[it:dev:str=foo]}]',
 ]
 
 # Generated with print_parse_list below
@@ -778,7 +789,7 @@ _ParseResults = [
     'Query: [TryCatch: [Query: [LiftPropBy: [Const: inet:ipv4, Const: =, Const: asdf]], CatchBlock: [Const: TypeError, Const: err, Query: []]]]',
     'Query: [TryCatch: [Query: [LiftPropBy: [Const: inet:ipv4, Const: =, Const: asdf]], CatchBlock: [Const: FooBar, Const: err, Query: []], CatchBlock: [Const: *, Const: err, Query: []]]]',
     'Query: [LiftByArray: [Const: test:array, Const: =, Const: 1.2.3.4]]',
-    'Query: [CmdOper: [Const: macro.set, List: [Const: hehe, EmbedQuery: inet:ipv4]]]',
+    'Query: [CmdOper: [Const: macro.set, List: [Const: hehe, EmbedQuery:  inet:ipv4 ]]]',
     'Query: [SetVarOper: [Const: q, EmbedQuery: #foo.bar]]',
     'Query: [CmdOper: [Const: metrics.edits.byprop, List: [Const: inet:fqdn:domain, Const: --newv, VarDeref: [VarValue: [Const: lib], Const: null]]]]',
     'Query: [CmdOper: [Const: tee, Const: ()]]',
@@ -1364,7 +1375,7 @@ _ParseResults = [
     'Query: [SetVarOper: [Const: p, Const: names], LiftPropBy: [Const: ps:contact:name, Const: =, Const: foo], EditPropSet: [RelProp: [VarValue: [Const: p]], Const: ?-=, Const: bar]]',
     'Query: [SetVarOper: [Const: pvar, Const: stuff], LiftProp: [Const: test:arrayprop], FiltOper: [Const: +, ArrayCond: [RelProp: [VarValue: [Const: pvar]], Const: =, Const: neato]]]',
     'Query: [SetVarOper: [Const: pvar, Const: ints], LiftProp: [Const: test:arrayprop], FiltOper: [Const: +, ArrayCond: [RelProp: [VarValue: [Const: pvar]], Const: =, VarValue: [Const: othervar]]]]',
-    'Query: [SetVarOper: [Const: foo, DollarExpr: [ExprDict: [Const: foo, EmbedQuery: inet:fqdn]]]]',
+    'Query: [SetVarOper: [Const: foo, DollarExpr: [ExprDict: [Const: foo, EmbedQuery:  inet:fqdn ]]]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [Const: hehe], CondSetOper: [Const: unset], Const: heval]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [Const: hehe], CondSetOper: [VarValue: [Const: foo]], Const: heval]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [Const: unset], Const: heval]]',
@@ -1372,6 +1383,17 @@ _ParseResults = [
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [VarDeref: [VarValue: [Const: bar], Const: baz]], Const: heval]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [VarDeref: [VarValue: [Const: bar], DollarExpr: [Const: baz]]], Const: heval]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [FuncCall: [VarDeref: [VarValue: [Const: bar], Const: baz], CallArgs: [], CallKwargs: []]], Const: heval]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditEdgeAdd: [Const: refs, VarValue: [Const: n]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditEdgeAdd: [Const: refs, FuncCall: [VarDeref: [VarValue: [Const: n], Const: baz], CallArgs: [], CallKwargs: []]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditEdgeDel: [Const: refs, VarValue: [Const: n]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditEdgeAdd: [Const: refs, VarValue: [Const: n]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditEdgeAdd: [Const: refs, FuncCall: [VarDeref: [VarValue: [Const: n], Const: baz], CallArgs: [], CallKwargs: []]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditEdgeDel: [Const: refs, VarValue: [Const: n]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditPropSetMulti: [RelProp: [Const: bar], Const: ++=, DollarExpr: [ExprList: [Const: 1, Const: 2]]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditPropSetMulti: [RelProp: [VarValue: [Const: foo]], Const: ++=, DollarExpr: [ExprList: [Const: 1, Const: 2]]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditPropSetMulti: [RelProp: [Const: bar], Const: --=, List: [Const: foo, Const: bar]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditPropSetMulti: [RelProp: [Const: bar], Const: ?++=, VarValue: [Const: baz]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditPropSetMulti: [RelProp: [Const: bar], Const: ?--=, SubQuery: [Query: [EditNodeAdd: [FormName: [Const: it:dev:str], Const: =, Const: foo]]]]]',
 ]
 
 class GrammarTest(s_t_utils.SynTest):
@@ -1663,6 +1685,27 @@ class GrammarTest(s_t_utils.SynTest):
         errinfo = cm.exception.errinfo
         self.eq(1, errinfo.get('mesg').count('#'))
 
+        query = '$q = ${ /* secret comment */ $lib.print([hello) } $lib.macro.set(hehe, $q)'
+        parser = s_parser.Parser(query)
+        with self.raises(s_exc.BadSyntax) as cm:
+            _ = parser.query()
+        info = cm.exception.errinfo.get('highlight')
+        self.eq((40, 41), info['offsets'])
+        self.eq((1, 1), info['lines'])
+        self.eq((41, 42), info['columns'])
+
+        query = """function test(hello) {
+                    +'''asdf
+                    asdfasdf'''
+        }"""
+        parser = s_parser.Parser(query)
+        with self.raises(s_exc.BadSyntax) as cm:
+            _ = parser.query()
+        info = cm.exception.errinfo.get('highlight')
+        self.eq((44, 83), info['offsets'])
+        self.eq((2, 3), info['lines'])
+        self.eq((22, 31), info['columns'])
+
     async def test_quotes(self):
 
         # Test vectors
@@ -1766,6 +1809,15 @@ class GrammarTest(s_t_utils.SynTest):
         self.false(s_grammar.isPropName('.hehe'))
         self.false(s_grammar.isPropName('testcmd'))
 
+    async def test_embed_offsets(self):
+
+        embq = ' /* secret comment */ $lib.print(hello) /* haha */ $lib.print(goodbye) /*foo */ '
+        query = f'$q = ${{{embq}}} $lib.print($q)'
+        parser = s_parser.Parser(query)
+        q = parser.query()
+        embed = q.kids[0].kids[1]
+        self.eq(embq, embed.getAstText())
+
 def gen_parse_list():
     '''
     Prints out the Asts for a list of queries in order to compare ASTs between versions of parsers

synapse/tests/test_lib_lmdbslab.py

@@ -371,6 +371,30 @@ class LmdbSlabTest(s_t_utils.SynTest):
                 'vm.dirty_ratio',
             ], msgs[0].get('sysctls', {}).keys())
 
+    async def test_lmdbslab_commit_over_max_xactops(self):
+
+        # Make sure that we don't confuse the periodic commit with the max replay log commit
+        with (self.getTestDir() as dirn,
+              patch('synapse.lib.lmdbslab.Slab.WARN_COMMIT_TIME_MS', 1),
+              patch('synapse.lib.lmdbslab.Slab.COMMIT_PERIOD', 100)
+              ):
+            path = os.path.join(dirn, 'test.lmdb')
+
+            async with await s_lmdbslab.Slab.anit(path, max_replay_log=100, map_size=100_000_000) as slab:
+                foo = slab.initdb('foo', dupsort=True)
+
+                byts = b'\x00' * 256
+                for i in range(1000):
+                    slab.put(b'\xff\xff\xff\xff' + s_common.guid(i).encode('utf8'), byts, db=foo)
+                    await asyncio.sleep(0)
+
+            # Let the slab close and then grab its stats
+            stats = slab.statinfo()
+            commitstats = stats.get('commitstats', ())
+            self.gt(len(commitstats), 0)
+            commitstats = [x[1] for x in commitstats if x[1] != 0]
+            self.eq(commitstats, (100, 100, 100, 100, 100, 100, 100, 100, 100, 100))
+
     async def test_lmdbslab_max_replay(self):
         with self.getTestDir() as dirn:
             path = os.path.join(dirn, 'test.lmdb')

synapse/tests/test_lib_storm.py

@@ -143,6 +143,26 @@ class StormTest(s_t_utils.SynTest):
             self.eq(props.get('name'), 'org name 77')
             self.eq(props.get('desc'), 'an org desc')
 
+            nodes = await core.nodes('ou:org=({"name": "the vertex project", "type": "lulz"})')
+            self.len(1, nodes)
+            orgn = nodes[0].ndef
+            self.eq(orgn, nodes11[0].ndef)
+
+            q = '[ ps:contact=* :org={ ou:org=({"name": "the vertex project", "type": "lulz"}) } ]'
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            cont = nodes[0]
+            self.eq(cont.get('org'), orgn[1])
+
+            nodes = await core.nodes('ps:contact:org=({"name": "the vertex project", "type": "lulz"})')
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef, cont.ndef)
+
+            self.len(0, await core.nodes('ps:contact:org=({"name": "vertex", "type": "newp"})'))
+
+            with self.raises(s_exc.BadTypeValu):
+                await core.nodes('inet:flow:from=({"name": "vertex", "type": "newp"})')
+
     async def test_lib_storm_jsonexpr(self):
         async with self.getTestCore() as core:
 

synapse/tests/test_lib_stormlib_macro.py

@@ -89,7 +89,7 @@ class MacroTest(s_test.SynTest):
             name = 'v' * 491
             q = '$lib.macro.set($name, ${ help }) return ( $lib.macro.get($name) )'
             mdef = await core.callStorm(q, opts={'vars': {'name': name}})
-            self.eq(mdef.get('storm'), 'help')
+            self.eq(mdef.get('storm'), ' help ')
 
             badname = 'v' * 492
             with self.raises(s_exc.BadArg):
@@ -381,7 +381,7 @@ class MacroTest(s_test.SynTest):
                     self.eq('storm:macro:add', addmesg['data']['event'])
                     macro = addmesg['data']['info']['macro']
                     self.eq(macro['name'], 'foobar')
-                    self.eq(macro['storm'], 'file:bytes | [+#neato]')
+                    self.eq(macro['storm'], ' file:bytes | [+#neato] ')
                     self.ne(visi.iden, macro['user'])
                     self.ne(visi.iden, macro['creator'])
                     self.nn(macro['iden'])
@@ -390,7 +390,7 @@ class MacroTest(s_test.SynTest):
                     self.eq('storm:macro:mod', setmesg['data']['event'])
                     event = setmesg['data']['info']
                     self.nn(event['macro'])
-                    self.eq(event['info']['storm'], 'inet:ipv4 | [+#burrito]')
+                    self.eq(event['info']['storm'], ' inet:ipv4 | [+#burrito] ')
                     self.nn(event['info']['updated'])
 
                     modmesg = await sock.receive_json()

synapse/tests/test_lib_stormtypes.py

@@ -749,7 +749,7 @@ class StormTypesTest(s_test.SynTest):
             self.stormIsInPrint("['1', 2, '3']", mesgs)
 
             mesgs = await core.stormlist('$lib.print(${ $foo=bar })')
-            self.stormIsInPrint('storm:query: "$foo=bar"', mesgs)
+            self.stormIsInPrint('storm:query: " $foo=bar "', mesgs)
 
             mesgs = await core.stormlist('$lib.print($lib.set(1,2,3))')
             self.stormIsInPrint("'1'", mesgs)
@@ -1168,7 +1168,7 @@ class StormTypesTest(s_test.SynTest):
             fires = [m for m in msgs if m[0] == 'storm:fire']
             self.len(1, fires)
             self.eq(fires[0][1].get('data').get('q'),
-                    "$lib.print('fire in the hole')")
+                    " $lib.print('fire in the hole') ")
 
             q = '''
             $q=${ [test:int=1 test:int=2] }
@@ -1409,6 +1409,18 @@ class StormTypesTest(s_test.SynTest):
             with self.raises(s_exc.BadJsonText):
                 await core.callStorm('return(("foo").json())')
 
+            with self.raises(s_exc.BadArg):
+                await core.nodes("$lib.regex.search('?id=([0-9]+)', 'foo')")
+
+            with self.raises(s_exc.BadArg):
+                await core.nodes("$lib.regex.search('(?au)\\w', 'foo')")
+
+            with self.raises(s_exc.BadArg):
+                await core.nodes("$lib.regex.replace('(?P<a>x)', '\\g<ab>', 'xx')")
+
+            with self.raises(s_exc.BadArg):
+                await core.nodes("$lib.regex.replace('(?P<a>x)', '(?au)\\w', 'xx')")
+
     async def test_storm_lib_bytes_gzip(self):
         async with self.getTestCore() as core:
             hstr = 'ohhai'

synapse/tests/test_model_infotech.py

@@ -767,6 +767,9 @@ class InfotechModelTest(s_t_utils.SynTest):
 
                     :host={it:host | limit 1}
                     :sandbox:file=*
+                    :service:platform=*
+                    :service:instance=*
+                    :service:account=*
             ]''')
             self.len(1, nodes)
             self.eq(10, nodes[0].get('severity'))
@@ -775,6 +778,9 @@ class InfotechModelTest(s_t_utils.SynTest):
             # check that the host activity model was inherited
             self.nn(nodes[0].get('host'))
             self.len(1, await core.nodes('it:log:event :sandbox:file -> file:bytes'))
+            self.len(1, await core.nodes('it:log:event :service:account -> inet:service:account'))
+            self.len(1, await core.nodes('it:log:event :service:platform -> inet:service:platform'))
+            self.len(1, await core.nodes('it:log:event :service:instance -> inet:service:instance'))
 
             nodes = await core.nodes('it:host | limit 1 | [ :keyboard:layout=qwerty :keyboard:language=$lib.gen.langByCode(en.us) ]')
             self.len(1, nodes)
@@ -2034,6 +2040,9 @@ class InfotechModelTest(s_t_utils.SynTest):
                     :offset=99
                     :synuser=$root
                     // we can assume the rest of the interface props work
+                    :service:platform = *
+                    :service:instance = *
+                    :service:account = *
                 ]
             ''', opts=opts)
             self.eq(1658275200000, nodes[0].get('time'))
@@ -2044,6 +2053,10 @@ class InfotechModelTest(s_t_utils.SynTest):
             self.eq(core.auth.rootuser.iden, nodes[0].get('synuser'))
             self.len(1, await core.nodes('it:exec:query -> it:query +it:query="SELECT * FROM threats"'))
 
+            self.len(1, await core.nodes('it:exec:query :service:account -> inet:service:account'))
+            self.len(1, await core.nodes('it:exec:query :service:platform -> inet:service:platform'))
+            self.len(1, await core.nodes('it:exec:query :service:instance -> inet:service:instance'))
+
     async def test_infotech_softid(self):
 
         async with self.getTestCore() as core:

synapse/tests/test_model_risk.py

@@ -299,6 +299,9 @@ class RiskModelTest(s_t_utils.SynTest):
                     :host=*
                     :priority=high
                     :severity=highest
+                    :service:platform=*
+                    :service:instance=*
+                    :service:account=*
                 ]
             ''')
             self.len(1, nodes)
@@ -318,6 +321,9 @@ class RiskModelTest(s_t_utils.SynTest):
             self.len(1, await core.nodes('risk:alert -> risk:vuln'))
             self.len(1, await core.nodes('risk:alert -> risk:attack'))
             self.len(1, await core.nodes('risk:alert :engine -> it:prod:softver'))
+            self.len(1, await core.nodes('risk:alert :service:account -> inet:service:account'))
+            self.len(1, await core.nodes('risk:alert :service:platform -> inet:service:platform'))
+            self.len(1, await core.nodes('risk:alert :service:instance -> inet:service:instance'))
 
             nodes = await core.nodes('''[
                     risk:compromise=*

synapse/tests/test_model_syn.py

@@ -63,6 +63,18 @@ class SynModelTest(s_t_utils.SynTest):
 
             self.eq('root', await core.callStorm(f'return($lib.repr(syn:user, {iden}))'))
 
+            with self.raises(s_exc.BadTypeValu) as exc:
+                await core.callStorm('return($lib.cast(syn:user, newp))')
+            self.eq(exc.exception.get('mesg'), 'No user named newp and value is not a guid.')
+            self.eq(exc.exception.get('valu'), 'newp')
+            self.eq(exc.exception.get('name'), 'syn:user')
+
+            with self.raises(s_exc.BadTypeValu) as exc:
+                await core.callStorm('[ it:exec:query=* :synuser=* ]')
+            self.isin('syn:user values must be a valid username or a guid.', exc.exception.get('mesg'))
+            self.eq(exc.exception.get('valu'), '*')
+            self.eq(exc.exception.get('name'), 'syn:user')
+
             (ok, iden) = await core.callStorm('return($lib.trycast(syn:role, all))')
             self.true(ok)
             self.eq(iden, core.auth.allrole.iden)
@@ -74,6 +86,18 @@ class SynModelTest(s_t_utils.SynTest):
 
             self.eq('all', await core.callStorm(f'return($lib.repr(syn:role, {iden}))'))
 
+            with self.raises(s_exc.BadTypeValu) as exc:
+                await core.callStorm('return($lib.cast(syn:role, newp))')
+            self.eq(exc.exception.get('mesg'), 'No role named newp and value is not a guid.')
+            self.eq(exc.exception.get('valu'), 'newp')
+            self.eq(exc.exception.get('name'), 'syn:role')
+
+            with self.raises(s_exc.BadTypeValu) as exc:
+                await core.callStorm('$lib.cast(syn:role, *)')
+            self.eq(exc.exception.get('mesg'), 'syn:role values must be a valid rolename or a guid.')
+            self.eq(exc.exception.get('valu'), '*')
+            self.eq(exc.exception.get('name'), 'syn:role')
+
             # coverage for DataModel without a cortex reference
             iden = s_common.guid()
 
@@ -89,6 +113,40 @@ class SynModelTest(s_t_utils.SynTest):
             self.eq(iden, synuser.norm(iden)[0])
             self.eq(iden, synrole.norm(iden)[0])
 
+    async def test_synuser_merge_failure(self):
+        async with self.getTestCore() as core:
+
+            visi = await core.addUser('visi')
+            view = await core.callStorm('return($lib.view.get().fork().iden)')
+
+            q = '[proj:project=(p1,) :creator=visi ]'
+            msgs = await core.stormlist(q, opts={'view': view})
+            self.stormHasNoWarnErr(msgs)
+
+            q = 'proj:project=(p1,)'
+            msgs = await core.stormlist(q, opts={'view': view, 'repr': True})
+            self.stormHasNoWarnErr(msgs)
+
+            await core.delUser(visi.get('iden'))
+
+            q = 'proj:project=(p1,)'
+            msgs = await core.stormlist(q, opts={'view': view, 'repr': True})
+            self.stormHasNoWarnErr(msgs)
+
+            # this works
+            q = '$lib.view.get($view).merge()'
+            msgs = await core.stormlist(q, opts={'vars': {'view': view}})
+            self.stormHasNoWarnErr(msgs)
+
+            # this fails
+            q = 'proj:project | merge --apply'
+            msgs = await core.stormlist(q, opts={'view': view, 'repr': True})
+            self.stormHasNoWarnErr(msgs)
+
+            q = 'proj:project=(p1,)'
+            msgs = await core.stormlist(q, opts={'vars': {'view': view}})
+            self.stormHasNoWarnErr(msgs)
+
     async def test_syn_tag(self):
 
         async with self.getTestCore() as core:

synapse/tests/test_tools_genpkg.py

@@ -73,6 +73,10 @@ class GenPkgTest(s_test.SynTest):
             ymlpath = s_common.genpath(dirname, 'files', 'stormpkg', 'badapidef.yaml')
             await s_genpkg.main((ymlpath,))
 
+        with self.raises(s_exc.SchemaViolation):
+            ymlpath = s_common.genpath(dirname, 'files', 'stormpkg', 'badendpoints.yaml')
+            await s_genpkg.main((ymlpath,))
+
         ymlpath = s_common.genpath(dirname, 'files', 'stormpkg', 'testpkg.yaml')
         async with self.getTestCore() as core:
 
@@ -111,6 +115,12 @@ class GenPkgTest(s_test.SynTest):
             self.eq(pdef['commands'][0]['name'], 'testpkgcmd')
             self.eq(pdef['commands'][0]['storm'], 'inet:ipv6\n')
 
+            self.eq(pdef['commands'][0]['endpoints'], [
+                {'path': '/v1/test/one'},
+                {'path': '/v1/test/two', 'host': 'vertex.link'},
+                {'path': '/v1/test/three', 'desc': 'endpoint three'},
+            ])
+
             self.eq(pdef['perms'][0]['perm'], ['power-ups', 'testpkg', 'user'])
             self.eq(pdef['perms'][0]['gate'], 'cortex')
             self.eq(pdef['perms'][0]['desc'], 'Controls user access to testpkg.')

synapse/tools/hive/load.py

@@ -47,6 +47,7 @@ async def main(argv, outp=s_output.stdout):
             try:
                 s_version.reqVersion(hive._getSynVers(), reqver)
                 if 'synapse.lib.hive.HiveApi' in classes:
+                    s_common.deprecated('Connecting directly to a Hive via Telepath', curv='2.198.0', eolv='2.199.0')
                     await hive.loadHiveTree(tree, path=path, trim=opts.trim)
                 else:
                     todo = s_common.todo('loadHiveTree', tree, path=path, trim=opts.trim)

{synapse-2.197.0.dist-info → synapse-2.198.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: synapse
-Version: 2.197.0
+Version: 2.198.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License: Apache License 2.0