synapse 2.206.0__py311-none-any.whl → 2.208.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/axon.py +4 -4
- synapse/cortex.py +12 -3
- synapse/lib/boss.py +5 -1
- synapse/lib/modelrev.py +27 -4
- synapse/lib/storm.lark +2 -2
- synapse/lib/storm.py +5 -5
- synapse/lib/stormhttp.py +6 -6
- synapse/lib/stormlib/auth.py +1 -1
- synapse/lib/stormlib/cortex.py +1 -1
- synapse/lib/stormlib/vault.py +1 -1
- synapse/lib/stormsvc.py +1 -1
- synapse/lib/stormtypes.py +64 -37
- synapse/lib/task.py +7 -1
- synapse/lib/time.py +8 -2
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +6 -1
- synapse/tests/test_axon.py +10 -0
- synapse/tests/test_cortex.py +4 -4
- synapse/tests/test_datamodel.py +1 -1
- synapse/tests/test_lib_ast.py +5 -5
- synapse/tests/test_lib_boss.py +2 -2
- synapse/tests/test_lib_nexus.py +5 -11
- synapse/tests/test_lib_storm.py +83 -6
- synapse/tests/test_lib_stormhttp.py +21 -8
- synapse/tests/test_lib_stormlib_cache.py +2 -2
- synapse/tests/test_lib_stormlib_scrape.py +1 -1
- synapse/tests/test_lib_stormlib_storm.py +63 -0
- synapse/tests/test_lib_stormsvc.py +3 -0
- synapse/tests/test_lib_stormtypes.py +44 -9
- synapse/tests/test_lib_trigger.py +3 -4
- synapse/tests/test_lib_view.py +3 -3
- synapse/tests/test_tools_csvtool.py +1 -1
- {synapse-2.206.0.dist-info → synapse-2.208.0.dist-info}/METADATA +6 -6
- {synapse-2.206.0.dist-info → synapse-2.208.0.dist-info}/RECORD +37 -37
- {synapse-2.206.0.dist-info → synapse-2.208.0.dist-info}/WHEEL +0 -0
- {synapse-2.206.0.dist-info → synapse-2.208.0.dist-info}/licenses/LICENSE +0 -0
- {synapse-2.206.0.dist-info → synapse-2.208.0.dist-info}/top_level.txt +0 -0
synapse/tests/test_axon.py
CHANGED
|
@@ -19,6 +19,7 @@ import synapse.common as s_common
|
|
|
19
19
|
import synapse.telepath as s_telepath
|
|
20
20
|
|
|
21
21
|
import synapse.lib.coro as s_coro
|
|
22
|
+
import synapse.lib.json as s_json
|
|
22
23
|
import synapse.lib.certdir as s_certdir
|
|
23
24
|
import synapse.lib.httpapi as s_httpapi
|
|
24
25
|
import synapse.lib.msgpack as s_msgpack
|
|
@@ -1005,6 +1006,15 @@ bar baz",vv
|
|
|
1005
1006
|
self.eq(True, resp['ok'])
|
|
1006
1007
|
self.eq(200, resp['code'])
|
|
1007
1008
|
|
|
1009
|
+
jsonq = f'''$resp = $lib.axon.wput($sha256, "https://127.0.0.1:{port}/api/v1/pushfile", ssl=(0))
|
|
1010
|
+
return ( $lib.json.save($resp) )
|
|
1011
|
+
'''
|
|
1012
|
+
resp = await core.callStorm(jsonq, opts=opts)
|
|
1013
|
+
self.isinstance(resp, str)
|
|
1014
|
+
resp = s_json.loads(resp)
|
|
1015
|
+
self.eq(True, resp['ok'])
|
|
1016
|
+
self.eq(200, resp['code'])
|
|
1017
|
+
|
|
1008
1018
|
opts = {'vars': {'sha256': s_common.ehex(s_common.buid())}}
|
|
1009
1019
|
resp = await core.callStorm(q, opts=opts)
|
|
1010
1020
|
self.eq(False, resp['ok'])
|
synapse/tests/test_cortex.py
CHANGED
|
@@ -1240,7 +1240,7 @@ class CortexTest(s_t_utils.SynTest):
|
|
|
1240
1240
|
$ddef = $lib.dmon.add(${
|
|
1241
1241
|
$lib.print(hi)
|
|
1242
1242
|
$lib.warn(omg)
|
|
1243
|
-
$s =
|
|
1243
|
+
$s = `Running {$auto.type} {$auto.iden}`
|
|
1244
1244
|
$lib.log.info($s, ({"iden": $auto.iden}))
|
|
1245
1245
|
$que = $lib.queue.get(foo)
|
|
1246
1246
|
$que.put(done)
|
|
@@ -5224,7 +5224,7 @@ class CortexBasicTest(s_t_utils.SynTest):
|
|
|
5224
5224
|
self.len(1, nodes)
|
|
5225
5225
|
self.nn(nodes[0].getTag('bar'))
|
|
5226
5226
|
|
|
5227
|
-
q = '[test:str=yop +#$
|
|
5227
|
+
q = '$t="{first}.{last}" [test:str=yop +#$t.format(first=foo, last=bar)]'
|
|
5228
5228
|
nodes = await core.nodes(q)
|
|
5229
5229
|
self.len(1, nodes)
|
|
5230
5230
|
self.nn(nodes[0].getTag('foo.bar'))
|
|
@@ -5695,7 +5695,7 @@ class CortexBasicTest(s_t_utils.SynTest):
|
|
|
5695
5695
|
self.len(1, nodes)
|
|
5696
5696
|
|
|
5697
5697
|
async def test_storm_order(self):
|
|
5698
|
-
q = '''[test:str=foo :hehe=bar] $tvar=() $tvar.append(1) $tvar.append(:hehe) $lib.print(
|
|
5698
|
+
q = '''[test:str=foo :hehe=bar] $tvar=() $tvar.append(1) $tvar.append(:hehe) $lib.print(('').join($tvar)) '''
|
|
5699
5699
|
async with self.getTestCore() as core:
|
|
5700
5700
|
mesgs = await core.stormlist(q)
|
|
5701
5701
|
self.stormIsInPrint('1bar', mesgs)
|
|
@@ -7731,7 +7731,7 @@ class CortexBasicTest(s_t_utils.SynTest):
|
|
|
7731
7731
|
$ddef = $lib.dmon.add(${
|
|
7732
7732
|
$lib.print(hi)
|
|
7733
7733
|
$lib.warn(omg)
|
|
7734
|
-
$s =
|
|
7734
|
+
$s = `Running {$auto.type} {$auto.iden}`
|
|
7735
7735
|
$lib.log.info($s, ({"iden": $auto.iden}))
|
|
7736
7736
|
})
|
|
7737
7737
|
''')
|
synapse/tests/test_datamodel.py
CHANGED
|
@@ -299,7 +299,7 @@ class DataModelTest(s_t_utils.SynTest):
|
|
|
299
299
|
syn:type:subof=comp $opts=:opts
|
|
300
300
|
-> syn:form:type $valu=$node.value()
|
|
301
301
|
for ($name, $thing) in $opts.fields {
|
|
302
|
-
$v
|
|
302
|
+
$v=`{$valu}:{$name}` syn:prop=$v
|
|
303
303
|
}
|
|
304
304
|
+syn:prop
|
|
305
305
|
-:ro=1
|
synapse/tests/test_lib_ast.py
CHANGED
|
@@ -74,9 +74,9 @@ foo_stormpkg = {
|
|
|
74
74
|
}
|
|
75
75
|
|
|
76
76
|
function outer(arg1, add) {
|
|
77
|
-
$strbase =
|
|
77
|
+
$strbase = `(Run: {$counter}) we got back `
|
|
78
78
|
$reti = $inner($arg1, $add)
|
|
79
|
-
$mesg = $
|
|
79
|
+
$mesg = `{$strbase}{$reti}`
|
|
80
80
|
$counter = $( $counter + $add )
|
|
81
81
|
$lib.print("foobar is {foobar}", foobar=$foobar)
|
|
82
82
|
return ($mesg)
|
|
@@ -296,7 +296,7 @@ class AstTest(s_test.SynTest):
|
|
|
296
296
|
$newvar=:hehe
|
|
297
297
|
-.created
|
|
298
298
|
$s.append("yar {x}", x=$newvar)
|
|
299
|
-
$lib.print(
|
|
299
|
+
$lib.print(('').join($s))
|
|
300
300
|
'''
|
|
301
301
|
mesgs = await core.stormlist(q)
|
|
302
302
|
prints = [m[1]['mesg'] for m in mesgs if m[0] == 'print']
|
|
@@ -2029,7 +2029,7 @@ class AstTest(s_test.SynTest):
|
|
|
2029
2029
|
q = '$val=$lib.base64.decode("dmlzaQ==") function x(parm1=$val) { return($parm1) }'
|
|
2030
2030
|
self.len(0, await core.nodes(q))
|
|
2031
2031
|
|
|
2032
|
-
self.eq('foo', await core.callStorm('
|
|
2032
|
+
self.eq('foo', await core.callStorm('$template="{func}" return($template.format(func=foo))'))
|
|
2033
2033
|
|
|
2034
2034
|
msgs = await core.stormlist('$lib.null()')
|
|
2035
2035
|
erfo = [m for m in msgs if m[0] == 'err'][0]
|
|
@@ -3580,7 +3580,7 @@ class AstTest(s_test.SynTest):
|
|
|
3580
3580
|
|
|
3581
3581
|
q = '''
|
|
3582
3582
|
init { $foo = bar }
|
|
3583
|
-
init { $baz =
|
|
3583
|
+
init { $baz = `foo={$foo}` }
|
|
3584
3584
|
$lib.print($baz)
|
|
3585
3585
|
'''
|
|
3586
3586
|
msgs = await core.stormlist(q)
|
synapse/tests/test_lib_boss.py
CHANGED
|
@@ -56,8 +56,8 @@ class BossTest(s_test.SynTest):
|
|
|
56
56
|
iden = s_common.guid()
|
|
57
57
|
|
|
58
58
|
async def double_promote():
|
|
59
|
-
await boss.promote(
|
|
60
|
-
await boss.promote(
|
|
59
|
+
await boss.promote('double', root, taskiden=iden)
|
|
60
|
+
await boss.promote('double', root, taskiden=iden + iden)
|
|
61
61
|
|
|
62
62
|
coro = boss.schedCoro(double_promote())
|
|
63
63
|
self.true(await stream.wait(timeout=6))
|
synapse/tests/test_lib_nexus.py
CHANGED
|
@@ -385,19 +385,13 @@ class NexusTest(s_t_utils.SynTest):
|
|
|
385
385
|
evnt = asyncio.Event()
|
|
386
386
|
|
|
387
387
|
async with self.getTestCore() as core:
|
|
388
|
-
orig = core._nexshands['view:delwithlayer'][0]
|
|
389
|
-
|
|
390
|
-
async def holdlock(self, viewiden, layriden, nexsitem, newparent=None):
|
|
391
|
-
evnt.set()
|
|
392
|
-
await asyncio.sleep(1)
|
|
393
|
-
await orig(self, viewiden, layriden, nexsitem, newparent=newparent)
|
|
394
|
-
|
|
395
|
-
core._nexshands['view:delwithlayer'] = (holdlock, True)
|
|
396
|
-
|
|
397
388
|
forkiden = await core.callStorm('return($lib.view.get().fork().iden)')
|
|
398
389
|
|
|
399
|
-
|
|
400
|
-
|
|
390
|
+
# Remove the nexus handler for the fork's write layer to simulate a view delete/edit race
|
|
391
|
+
layriden = core.getView(forkiden).layers[0].iden
|
|
392
|
+
layr = core.nexsroot._nexskids.pop(layriden)
|
|
401
393
|
|
|
402
394
|
with self.raises(s_exc.NoSuchIden):
|
|
403
395
|
await core.nodes('[ it:dev:str=foo ]', opts={'view': forkiden})
|
|
396
|
+
|
|
397
|
+
core.nexsroot._nexskids[layriden] = layr
|
synapse/tests/test_lib_storm.py
CHANGED
|
@@ -308,7 +308,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
308
308
|
''')
|
|
309
309
|
self.eq('valu=12', retn)
|
|
310
310
|
|
|
311
|
-
q = "$hehe=({'k': 'v'}) $fs
|
|
311
|
+
q = "$hehe=({'k': 'v'}) $fs=`{$hehe}56` return((`{$hehe}56`, $fs))"
|
|
312
312
|
retn = await core.callStorm(q)
|
|
313
313
|
self.eq("{'k': 'v'}56", retn[0])
|
|
314
314
|
self.eq(retn[0], retn[1])
|
|
@@ -322,6 +322,14 @@ class StormTest(s_t_utils.SynTest):
|
|
|
322
322
|
|
|
323
323
|
self.eq("foo 'bar'", await core.callStorm("$foo=bar return(`foo '{$foo}'`)"))
|
|
324
324
|
self.eq(r"\'''''bar'''", await core.callStorm(r"$foo=bar return(`\\'\''''{$foo}'''`)"))
|
|
325
|
+
self.eq(r"\bar", await core.callStorm(r"$foo=bar return(`\\{$foo}`)"))
|
|
326
|
+
self.eq(r"\`bar", await core.callStorm(r"$foo=bar return(`\\\`{$foo}`)"))
|
|
327
|
+
self.eq(r"\{bar", await core.callStorm(r"$foo=bar return(`\\\{{$foo}`)"))
|
|
328
|
+
self.eq(r"foo\bar", await core.callStorm(r"$foo=foo $bar=bar return(`{$foo}\\{$bar}`)"))
|
|
329
|
+
self.eq(r"foo \bar", await core.callStorm(r"$foo=foo $bar=bar return(`{$foo} \\{$bar}`)"))
|
|
330
|
+
|
|
331
|
+
with self.raises(s_exc.BadSyntax):
|
|
332
|
+
await core.callStorm(r"$foo=bar return(`\\{{$foo}`)")
|
|
325
333
|
|
|
326
334
|
async def test_lib_storm_emit(self):
|
|
327
335
|
async with self.getTestCore() as core:
|
|
@@ -2179,7 +2187,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
2179
2187
|
[ ou:org=(cov,) ]
|
|
2180
2188
|
|
|
2181
2189
|
{ for $i in $lib.range(1001) {
|
|
2182
|
-
$prop =
|
|
2190
|
+
$prop = `_test{$i}`
|
|
2183
2191
|
[ :$prop = $i
|
|
2184
2192
|
+#$prop:score = $i
|
|
2185
2193
|
+($i)> { ou:org=(cov,) }
|
|
@@ -2826,6 +2834,75 @@ class StormTest(s_t_utils.SynTest):
|
|
|
2826
2834
|
]
|
|
2827
2835
|
self.eq(exp, evnts)
|
|
2828
2836
|
|
|
2837
|
+
async def test_storm_pkg_onload_active(self):
|
|
2838
|
+
pkg = {
|
|
2839
|
+
'name': 'testload',
|
|
2840
|
+
'version': '0.3.0',
|
|
2841
|
+
'modules': (
|
|
2842
|
+
{
|
|
2843
|
+
'name': 'testload',
|
|
2844
|
+
'storm': 'function x() { return((0)) }',
|
|
2845
|
+
},
|
|
2846
|
+
),
|
|
2847
|
+
'onload': '''
|
|
2848
|
+
$lib.print(testprint)
|
|
2849
|
+
$lib.warn(testwarn)
|
|
2850
|
+
|
|
2851
|
+
$queue = $lib.queue.gen(onload:test)
|
|
2852
|
+
|
|
2853
|
+
$vers = $lib.globals.get(testload:version, (0))
|
|
2854
|
+
$vers = ($vers + 1)
|
|
2855
|
+
$lib.globals.set(testload:version, $vers)
|
|
2856
|
+
$queue.put($vers)
|
|
2857
|
+
'''
|
|
2858
|
+
}
|
|
2859
|
+
|
|
2860
|
+
with self.getTestDir() as dirn:
|
|
2861
|
+
dirn00 = s_common.gendir(dirn, 'core00')
|
|
2862
|
+
dirn01 = s_common.gendir(dirn, 'core01')
|
|
2863
|
+
|
|
2864
|
+
async with self.getTestCore(dirn=dirn00) as core00:
|
|
2865
|
+
|
|
2866
|
+
waiter = core00.waiter(2, 'core:pkg:onload:start', 'core:pkg:onload:complete')
|
|
2867
|
+
|
|
2868
|
+
await core00.addStormPkg(pkg)
|
|
2869
|
+
|
|
2870
|
+
events = await waiter.wait(timeout=10)
|
|
2871
|
+
self.eq(events, [
|
|
2872
|
+
('core:pkg:onload:start', {'pkg': 'testload'}),
|
|
2873
|
+
('core:pkg:onload:complete', {'pkg': 'testload'}),
|
|
2874
|
+
])
|
|
2875
|
+
|
|
2876
|
+
self.eq((0, 1), await core00.callStorm('return($lib.queue.gen(onload:test).get((0), cull=(false)))'))
|
|
2877
|
+
|
|
2878
|
+
s_tools_backup.backup(dirn00, dirn01)
|
|
2879
|
+
|
|
2880
|
+
async with self.getTestCore(dirn=dirn00) as core00:
|
|
2881
|
+
|
|
2882
|
+
self.eq((1, 2), await core00.callStorm('return($lib.queue.gen(onload:test).get((1), cull=(false)))'))
|
|
2883
|
+
|
|
2884
|
+
conf01 = {'mirror': core00.getLocalUrl()}
|
|
2885
|
+
|
|
2886
|
+
async with self.getTestCore(dirn=dirn01, conf=conf01) as core01:
|
|
2887
|
+
|
|
2888
|
+
await core01.sync()
|
|
2889
|
+
|
|
2890
|
+
waiter = core01.waiter(2, 'core:pkg:onload:start', 'core:pkg:onload:complete')
|
|
2891
|
+
|
|
2892
|
+
await core01.promote()
|
|
2893
|
+
|
|
2894
|
+
events = await waiter.wait(timeout=10)
|
|
2895
|
+
self.eq(events, [
|
|
2896
|
+
('core:pkg:onload:start', {'pkg': 'testload'}),
|
|
2897
|
+
('core:pkg:onload:complete', {'pkg': 'testload'}),
|
|
2898
|
+
])
|
|
2899
|
+
|
|
2900
|
+
self.eq((2, 3), await core01.callStorm('return($lib.queue.gen(onload:test).get((2), cull=(false)))'))
|
|
2901
|
+
|
|
2902
|
+
await core01.waitfini()
|
|
2903
|
+
|
|
2904
|
+
await core00.waitfini()
|
|
2905
|
+
|
|
2829
2906
|
async def test_storm_tree(self):
|
|
2830
2907
|
|
|
2831
2908
|
async with self.getTestCore() as core:
|
|
@@ -5083,7 +5160,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
5083
5160
|
batch $lib.true --size 5 ${
|
|
5084
5161
|
$vals=([])
|
|
5085
5162
|
for $n in $nodes { $vals.append($n.repr()) }
|
|
5086
|
-
$lib.print(
|
|
5163
|
+
$lib.print((',').join($vals))
|
|
5087
5164
|
}
|
|
5088
5165
|
'''
|
|
5089
5166
|
msgs = await core.stormlist(q)
|
|
@@ -5098,7 +5175,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
5098
5175
|
batch $lib.false --size 5 {
|
|
5099
5176
|
$vals=([])
|
|
5100
5177
|
for $n in $nodes { $vals.append($n.repr()) }
|
|
5101
|
-
$lib.print(
|
|
5178
|
+
$lib.print((',').join($vals))
|
|
5102
5179
|
}
|
|
5103
5180
|
'''
|
|
5104
5181
|
msgs = await core.stormlist(q)
|
|
@@ -5204,7 +5281,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
5204
5281
|
msgs = await core.stormlist(q)
|
|
5205
5282
|
self.stormHasNoErr(msgs)
|
|
5206
5283
|
|
|
5207
|
-
await core.nodes('''$token=foo $lib.print(({"Authorization"
|
|
5284
|
+
await core.nodes('''$token=foo $lib.print(({"Authorization":`Bearer {$token}`}))''')
|
|
5208
5285
|
|
|
5209
5286
|
q = '#rep.clearsky.dreamjob -># +syn:tag^=rep |uniq -syn:tag~=rep.clearsky'
|
|
5210
5287
|
msgs = await core.stormlist(q)
|
|
@@ -5218,7 +5295,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
5218
5295
|
msgs = await core.stormlist(q)
|
|
5219
5296
|
self.stormIsInWarn('Failed to decode iden: [ssl://svcrs:27492?certname=root=bar]', msgs)
|
|
5220
5297
|
|
|
5221
|
-
q = "$foo=one $bar=two $lib.print($
|
|
5298
|
+
q = "$foo=one $bar=two $lib.print(`{$foo}={$bar}`)"
|
|
5222
5299
|
msgs = await core.stormlist(q)
|
|
5223
5300
|
self.stormIsInPrint("one=two", msgs)
|
|
5224
5301
|
|
|
@@ -473,6 +473,19 @@ class StormHttpTest(s_test.SynTest):
|
|
|
473
473
|
data = resp.get('result')
|
|
474
474
|
self.eq(data.get('params'), {'foo': ['bar', 'baz'], 'key': ["('valu',)"]})
|
|
475
475
|
|
|
476
|
+
# headers are safe to serialize
|
|
477
|
+
q = '''
|
|
478
|
+
$headers = ({'Foo': 'Bar'})
|
|
479
|
+
$resp = $lib.inet.http.request(GET, $url, headers=$headers, ssl_verify=$lib.false)
|
|
480
|
+
return ( ($lib.json.save($resp.headers), $lib.json.save($resp.request_headers)) )
|
|
481
|
+
'''
|
|
482
|
+
resp = await core.callStorm(q, opts=opts)
|
|
483
|
+
(headers, req_headers) = resp
|
|
484
|
+
headers = s_json.loads(headers)
|
|
485
|
+
self.eq(headers.get('Content-Type'), 'application/json; charset=UTF-8')
|
|
486
|
+
req_headers = s_json.loads(req_headers)
|
|
487
|
+
self.eq(req_headers.get('Foo'), 'Bar')
|
|
488
|
+
|
|
476
489
|
async def test_storm_http_post(self):
|
|
477
490
|
|
|
478
491
|
async with self.getTestCore() as core:
|
|
@@ -481,7 +494,7 @@ class StormHttpTest(s_test.SynTest):
|
|
|
481
494
|
await root.setPasswd('root')
|
|
482
495
|
|
|
483
496
|
adduser = '''
|
|
484
|
-
$url =
|
|
497
|
+
$url = `https://root:root@127.0.0.1:{$port}/api/v1/auth/adduser`
|
|
485
498
|
$user = ({"name": $name, "passwd": $passwd})
|
|
486
499
|
$post = $lib.inet.http.post($url, json=$user, ssl_verify=$(0)).json().result.name
|
|
487
500
|
$lib.print($post)
|
|
@@ -493,8 +506,8 @@ class StormHttpTest(s_test.SynTest):
|
|
|
493
506
|
self.assertIn('foo', [u.name for u in core.auth.users()])
|
|
494
507
|
|
|
495
508
|
adduser = '''
|
|
496
|
-
$url =
|
|
497
|
-
$user = $lib.
|
|
509
|
+
$url = `https://root:root@127.0.0.1:{$port}/api/v1/auth/adduser`
|
|
510
|
+
$user = $lib.json.save( ({"name": $name, "passwd": $passwd}) )
|
|
498
511
|
$header = ({"Content-Type": "application/json"})
|
|
499
512
|
$post = $lib.inet.http.post($url, headers=$header, body=$user, ssl_verify=$(0)).json().result.name
|
|
500
513
|
[ test:str=$post ]
|
|
@@ -573,7 +586,7 @@ class StormHttpTest(s_test.SynTest):
|
|
|
573
586
|
root = await core.auth.getUserByName('root')
|
|
574
587
|
await root.setPasswd('root')
|
|
575
588
|
text = '''
|
|
576
|
-
$url =
|
|
589
|
+
$url = `https://root:root@127.0.0.1:{$port}/api/v1/storm`
|
|
577
590
|
$stormq = "($size, $sha2) = $lib.axon.put($lib.base64.decode('dmVydGV4')) [ test:str = $sha2 ] [ test:int = $size ]"
|
|
578
591
|
$json = ({"query": $stormq})
|
|
579
592
|
$bytez = $lib.inet.http.post($url, json=$json, ssl_verify=$(0))
|
|
@@ -589,7 +602,7 @@ class StormHttpTest(s_test.SynTest):
|
|
|
589
602
|
self.eq(nodes[0].ndef, ('test:int', 6))
|
|
590
603
|
|
|
591
604
|
text = '''
|
|
592
|
-
$url =
|
|
605
|
+
$url = `https://root:root@127.0.0.1:{$port}/api/v1/storm`
|
|
593
606
|
$json = ({"query": "test:str"})
|
|
594
607
|
$body = $json
|
|
595
608
|
$resp=$lib.inet.http.post($url, json=$json, body=$body, ssl_verify=$(0))
|
|
@@ -767,7 +780,7 @@ class StormHttpTest(s_test.SynTest):
|
|
|
767
780
|
mesg = await core.callStorm('''
|
|
768
781
|
$params = ( { "param1": "somevalu" } )
|
|
769
782
|
$hdr = ( { "key": $lib.false } )
|
|
770
|
-
$url =
|
|
783
|
+
$url = `https://127.0.0.1:{$port}/test/ws`
|
|
771
784
|
|
|
772
785
|
($ok, $sock) = $lib.inet.http.connect($url, headers=$hdr, params=$params, ssl_verify=$lib.false)
|
|
773
786
|
if (not $ok) { $lib.exit($sock) }
|
|
@@ -783,7 +796,7 @@ class StormHttpTest(s_test.SynTest):
|
|
|
783
796
|
|
|
784
797
|
mesg = await core.callStorm('''
|
|
785
798
|
$hdr = ( { "key": $lib.false } )
|
|
786
|
-
$url =
|
|
799
|
+
$url = `https://127.0.0.1:{$port}/test/ws`
|
|
787
800
|
|
|
788
801
|
($ok, $sock) = $lib.inet.http.connect($url, headers=$hdr, ssl_verify=$lib.false)
|
|
789
802
|
if (not $ok) { $lib.exit($sock) }
|
|
@@ -797,7 +810,7 @@ class StormHttpTest(s_test.SynTest):
|
|
|
797
810
|
self.none(mesg.get('params'))
|
|
798
811
|
|
|
799
812
|
query = '''
|
|
800
|
-
$url =
|
|
813
|
+
$url = `https://127.0.0.1:{$port}/test/ws`
|
|
801
814
|
|
|
802
815
|
($ok, $sock) = $lib.inet.http.connect($url, proxy=$proxy, ssl_verify=$lib.false)
|
|
803
816
|
if (not $ok) { $lib.exit($sock) }
|
|
@@ -120,12 +120,12 @@ class StormlibCacheTest(s_test.SynTest):
|
|
|
120
120
|
|
|
121
121
|
$rets.append($cache.get(foo))
|
|
122
122
|
$rets.append($sent)
|
|
123
|
-
$rets.append(
|
|
123
|
+
$rets.append((',').join($vals))
|
|
124
124
|
|
|
125
125
|
$val = one
|
|
126
126
|
$rets.append($cache.get(bar))
|
|
127
127
|
$rets.append($sent)
|
|
128
|
-
$rets.append(
|
|
128
|
+
$rets.append((',').join($vals))
|
|
129
129
|
|
|
130
130
|
return($rets)
|
|
131
131
|
''')
|
|
@@ -172,7 +172,7 @@ class StormScrapeTest(s_test.SynTest):
|
|
|
172
172
|
|
|
173
173
|
text = text + ' and then there was another 1.2.3.4 that happened at woot.com '
|
|
174
174
|
query = '''$tally = $lib.stats.tally() for ($form, $ndef) in $lib.scrape.ndefs($text)
|
|
175
|
-
{ $valu
|
|
175
|
+
{ $valu=`{$form}={$ndef}` $tally.inc($valu) }
|
|
176
176
|
fini { return ( $tally ) }
|
|
177
177
|
'''
|
|
178
178
|
varz = {'text': text}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
|
|
1
3
|
import synapse.exc as s_exc
|
|
2
4
|
import synapse.lib.parser as s_parser
|
|
3
5
|
|
|
@@ -142,3 +144,64 @@ class LibStormTest(s_test.SynTest):
|
|
|
142
144
|
}
|
|
143
145
|
''')
|
|
144
146
|
self.stormIsInPrint('mesg=hello', msgs)
|
|
147
|
+
|
|
148
|
+
async def test_lib_stormlib_storm_tasks(self):
|
|
149
|
+
|
|
150
|
+
with self.getStructuredAsyncLoggerStream('synapse') as stream:
|
|
151
|
+
|
|
152
|
+
async with self.getTestCore() as core:
|
|
153
|
+
|
|
154
|
+
async with core.getLocalProxy() as prox:
|
|
155
|
+
|
|
156
|
+
event = asyncio.Event()
|
|
157
|
+
|
|
158
|
+
q = 'for $mesg in $lib.storm.run("$lib.time.sleep(120)") { $lib.fire(storm, mesg=$mesg) }'
|
|
159
|
+
|
|
160
|
+
async def doit():
|
|
161
|
+
async for mesg in prox.storm(q):
|
|
162
|
+
if mesg[0] == 'storm:fire':
|
|
163
|
+
event.set()
|
|
164
|
+
|
|
165
|
+
task00 = core.schedCoro(doit())
|
|
166
|
+
await asyncio.wait_for(event.wait(), timeout=10)
|
|
167
|
+
|
|
168
|
+
viewiden = core.getView().iden
|
|
169
|
+
|
|
170
|
+
# One task for the main query
|
|
171
|
+
tasks = core.boss.ps()
|
|
172
|
+
self.len(1, tasks)
|
|
173
|
+
self.eq(tasks[0].name, 'storm')
|
|
174
|
+
self.eq(tasks[0].info, {'query': q, 'view': viewiden})
|
|
175
|
+
self.len(1, tasks[0].kids)
|
|
176
|
+
|
|
177
|
+
# The main query has one kid, the worker
|
|
178
|
+
kid = list(tasks[0].kids.values())[0]
|
|
179
|
+
self.nn(kid.iden)
|
|
180
|
+
self.nn(kid.user)
|
|
181
|
+
self.eq(kid.name, 'runstorm')
|
|
182
|
+
self.eq(kid.info, {})
|
|
183
|
+
|
|
184
|
+
# The worker has a kid which is the $lib.storm.run() worker
|
|
185
|
+
self.len(1, kid.kids)
|
|
186
|
+
gkid = list(kid.kids.values())[0]
|
|
187
|
+
self.eq(gkid.name, 'runstorm')
|
|
188
|
+
self.eq(gkid.info, {})
|
|
189
|
+
|
|
190
|
+
# Kill the main task
|
|
191
|
+
tasks = core.boss.ps()
|
|
192
|
+
self.len(1, tasks)
|
|
193
|
+
await tasks[0].kill()
|
|
194
|
+
|
|
195
|
+
# No tasks running
|
|
196
|
+
tasks = core.boss.ps()
|
|
197
|
+
self.len(0, tasks)
|
|
198
|
+
|
|
199
|
+
task00.cancel('oh bye')
|
|
200
|
+
|
|
201
|
+
# Verify we saw two storm runtime cancellations
|
|
202
|
+
msgs = stream.jsonlines()
|
|
203
|
+
self.gt(len(msgs), 0)
|
|
204
|
+
|
|
205
|
+
msgs = [(k.get('message'), k.get('text')) for k in msgs]
|
|
206
|
+
self.isin(('Storm runtime cancelled.', '$lib.time.sleep(120)'), msgs)
|
|
207
|
+
self.isin(('Storm runtime cancelled.', q), msgs)
|
|
@@ -156,7 +156,7 @@ class StormTypesTest(s_test.SynTest):
|
|
|
156
156
|
await core.callStorm(q, opts=opts)
|
|
157
157
|
|
|
158
158
|
# Push a handful of notifications and list a subset of them
|
|
159
|
-
q = '''$m
|
|
159
|
+
q = '''$m=`hello {$i}` return($lib.auth.users.byname(root).tell($m))'''
|
|
160
160
|
for i in range(5):
|
|
161
161
|
opts = {'user': visi.iden, 'vars': {'i': i}}
|
|
162
162
|
await core.callStorm(q, opts=opts)
|
|
@@ -1071,7 +1071,7 @@ class StormTypesTest(s_test.SynTest):
|
|
|
1071
1071
|
# exec vars do not populate upwards
|
|
1072
1072
|
q = '''
|
|
1073
1073
|
$foo = "that is one neato burrito"
|
|
1074
|
-
$baz = ${ $bar
|
|
1074
|
+
$baz = ${ $bar=`wompwomp{$lib.guid()}` $lib.print("in exec") }
|
|
1075
1075
|
$baz.exec()
|
|
1076
1076
|
$lib.print("post exec {bar}", bar=$bar)
|
|
1077
1077
|
[ test:str=$foo ]
|
|
@@ -1283,6 +1283,8 @@ class StormTypesTest(s_test.SynTest):
|
|
|
1283
1283
|
|
|
1284
1284
|
async def test_storm_lib_str(self):
|
|
1285
1285
|
async with self.getTestCore() as core:
|
|
1286
|
+
|
|
1287
|
+
# TODO $lib.str.concat and rmat are deprecated should be removed in 3.0.0
|
|
1286
1288
|
q = '$v=vertex $l=link $fqdn=$lib.str.concat($v, ".", $l)' \
|
|
1287
1289
|
' [ inet:email=$lib.str.format("visi@{domain}", domain=$fqdn) ]'
|
|
1288
1290
|
nodes = await core.nodes(q)
|
|
@@ -1310,7 +1312,7 @@ class StormTypesTest(s_test.SynTest):
|
|
|
1310
1312
|
sobj = s_stormtypes.Str('beepbeep')
|
|
1311
1313
|
self.len(8, sobj)
|
|
1312
1314
|
|
|
1313
|
-
nodes = await core.nodes(
|
|
1315
|
+
nodes = await core.nodes("$s = (foo, bar, baz) [ test:str=('.').join($s) ]")
|
|
1314
1316
|
self.eq('foo.bar.baz', nodes[0].ndef[1])
|
|
1315
1317
|
|
|
1316
1318
|
nodes = await core.nodes('$s = foo-bar-baz [ test:str=$s.replace("-", ".") ]')
|
|
@@ -1406,6 +1408,14 @@ class StormTypesTest(s_test.SynTest):
|
|
|
1406
1408
|
|
|
1407
1409
|
self.eq(((1, 2, 3)), await core.callStorm('return(("[1, 2, 3]").json())'))
|
|
1408
1410
|
|
|
1411
|
+
self.eq('hehe,haha', await core.callStorm("$sepr=',' $l=(hehe, haha) return( $sepr.join($l) )"))
|
|
1412
|
+
self.eq('hehehaha', await core.callStorm("$sepr='' $l=(hehe, haha) return( $sepr.join($l) )"))
|
|
1413
|
+
self.eq('a|++|b|++|c', await core.callStorm("$sepr='|++|' $l=(a, b, c) return( $sepr.join($l) )"))
|
|
1414
|
+
self.eq('hehe,haha', await core.callStorm("$l=(hehe, haha) return( (',').join($l) )"))
|
|
1415
|
+
self.eq('hehehaha', await core.callStorm("$l=(hehe, haha) return( ('').join($l) )"))
|
|
1416
|
+
self.eq('', await core.callStorm("$sepr=',' $l=() return( $sepr.join($l) )"))
|
|
1417
|
+
self.eq('', await core.callStorm("$sepr='' $l=() return( $sepr.join($l) )"))
|
|
1418
|
+
|
|
1409
1419
|
with self.raises(s_exc.BadJsonText):
|
|
1410
1420
|
await core.callStorm('return(("foo").json())')
|
|
1411
1421
|
|
|
@@ -1752,7 +1762,7 @@ class StormTypesTest(s_test.SynTest):
|
|
|
1752
1762
|
$ipv4 = $node.repr()
|
|
1753
1763
|
$loc = $node.repr(loc)
|
|
1754
1764
|
$latlong = $node.repr(latlong, defv="??")
|
|
1755
|
-
$valu = $
|
|
1765
|
+
$valu = `{$ipv4} in {$loc} at {$latlong}`
|
|
1756
1766
|
[ test:str=$valu ]
|
|
1757
1767
|
+test:str
|
|
1758
1768
|
'''
|
|
@@ -2454,7 +2464,7 @@ class StormTypesTest(s_test.SynTest):
|
|
|
2454
2464
|
self.stormIsInPrint('pop valu is 0', mesgs)
|
|
2455
2465
|
|
|
2456
2466
|
listq = '''for ($key, $valu) in $lib.globals.list() {
|
|
2457
|
-
$string = $
|
|
2467
|
+
$string = `{$key} is {$valu}`
|
|
2458
2468
|
$lib.print($string)
|
|
2459
2469
|
}
|
|
2460
2470
|
'''
|
|
@@ -2498,7 +2508,7 @@ class StormTypesTest(s_test.SynTest):
|
|
|
2498
2508
|
self.len(1, await core.nodes('test:str=hehe'))
|
|
2499
2509
|
|
|
2500
2510
|
listq = '''for ($key, $valu) in $lib.user.vars.list() {
|
|
2501
|
-
$string = $
|
|
2511
|
+
$string = `{$key} is {$valu}`
|
|
2502
2512
|
$lib.print($string)
|
|
2503
2513
|
}
|
|
2504
2514
|
'''
|
|
@@ -2571,7 +2581,7 @@ class StormTypesTest(s_test.SynTest):
|
|
|
2571
2581
|
# core.vars, they only get the values they can read.
|
|
2572
2582
|
corelistq = '''
|
|
2573
2583
|
for ($key, $valu) in $lib.globals.list() {
|
|
2574
|
-
$string = $
|
|
2584
|
+
$string = `{$key} is {$valu}`
|
|
2575
2585
|
$lib.print($string)
|
|
2576
2586
|
}
|
|
2577
2587
|
'''
|
|
@@ -2726,6 +2736,21 @@ class StormTypesTest(s_test.SynTest):
|
|
|
2726
2736
|
self.false(valu[0])
|
|
2727
2737
|
self.eq(valu[1]['err'], 'BadArg')
|
|
2728
2738
|
|
|
2739
|
+
query = '''$valu="2020-10-01 01:30:00"
|
|
2740
|
+
$parsed=$lib.time.parse($valu, "%Y-%m-%d %H:%M:%S")
|
|
2741
|
+
$lib.print($lib.time.toUTC($parsed, US/Eastern))
|
|
2742
|
+
'''
|
|
2743
|
+
mesgs = await core.stormlist(query)
|
|
2744
|
+
self.stormIsInPrint('1601530200000', mesgs)
|
|
2745
|
+
|
|
2746
|
+
query = '''$valu="2020-11-01 01:30:00"
|
|
2747
|
+
$parsed=$lib.time.parse($valu, "%Y-%m-%d %H:%M:%S")
|
|
2748
|
+
return($lib.time.toUTC($parsed, America/New_York))
|
|
2749
|
+
'''
|
|
2750
|
+
mesgs = await core.callStorm(query)
|
|
2751
|
+
self.false(mesgs[0])
|
|
2752
|
+
self.isin('Ambiguous time', mesgs[1]['errinfo']['mesg'])
|
|
2753
|
+
|
|
2729
2754
|
async def test_storm_lib_time_ticker(self):
|
|
2730
2755
|
|
|
2731
2756
|
async with self.getTestCore() as core:
|
|
@@ -4919,7 +4944,7 @@ class StormTypesTest(s_test.SynTest):
|
|
|
4919
4944
|
unixtime = datetime.datetime(year=2018, month=12, day=5, hour=7, minute=10,
|
|
4920
4945
|
tzinfo=tz.utc).timestamp()
|
|
4921
4946
|
|
|
4922
|
-
q = '{$lib.queue.get(foo).put(m3) $s
|
|
4947
|
+
q = '{$lib.queue.get(foo).put(m3) $s=`m3 {$auto.type} {$auto.iden}` $lib.log.info($s, ({"iden": $auto.iden})) }'
|
|
4923
4948
|
text = f'cron.add --minute 17 {q}'
|
|
4924
4949
|
async with getCronJob(text) as guid:
|
|
4925
4950
|
with self.getStructuredAsyncLoggerStream('synapse.storm.log', 'm3 cron') as stream:
|
|
@@ -6049,6 +6074,16 @@ class StormTypesTest(s_test.SynTest):
|
|
|
6049
6074
|
self.eq({'d', 'c'}, ret)
|
|
6050
6075
|
|
|
6051
6076
|
# str join
|
|
6077
|
+
ret = await core.callStorm('$x=(foo,bar,baz) $y=("-").join($x) return($y)')
|
|
6078
|
+
self.eq('foo-bar-baz', ret)
|
|
6079
|
+
|
|
6080
|
+
ret = await core.callStorm('$y=("-").join((foo, bar, baz)) return($y)')
|
|
6081
|
+
self.eq('foo-bar-baz', ret)
|
|
6082
|
+
|
|
6083
|
+
ret = await core.callStorm('$x=abcd $y=("-").join($x) return($y)')
|
|
6084
|
+
self.eq('a-b-c-d', ret)
|
|
6085
|
+
|
|
6086
|
+
# TODO $lib.str.join is deprecated and will be removed in 3.0.0
|
|
6052
6087
|
ret = await core.callStorm('$x=(foo,bar,baz) $y=$lib.str.join("-", $x) return($y)')
|
|
6053
6088
|
self.eq('foo-bar-baz', ret)
|
|
6054
6089
|
|
|
@@ -6069,7 +6104,7 @@ class StormTypesTest(s_test.SynTest):
|
|
|
6069
6104
|
|
|
6070
6105
|
opts = {'user': visi.iden, 'vars': {'port': port}}
|
|
6071
6106
|
wget = '''
|
|
6072
|
-
$url =
|
|
6107
|
+
$url = `https://visi:secret@127.0.0.1:{$port}/api/v1/healthcheck`
|
|
6073
6108
|
return($lib.axon.wget($url, ssl=$lib.false))
|
|
6074
6109
|
'''
|
|
6075
6110
|
with self.raises(s_exc.AuthDeny):
|
|
@@ -141,8 +141,7 @@ class TrigTest(s_t_utils.SynTest):
|
|
|
141
141
|
view = core.view
|
|
142
142
|
|
|
143
143
|
# node:add case
|
|
144
|
-
q = '
|
|
145
|
-
[ test:guid="*" +#nodeadd]'''
|
|
144
|
+
q = '$s=`f={$auto.opts.form} v={$auto.opts.valu}` $lib.log.info($s) [ test:guid="*" +#nodeadd]'
|
|
146
145
|
tdef = {'cond': 'node:add', 'form': 'test:str', 'storm': q}
|
|
147
146
|
await view.addTrigger(tdef)
|
|
148
147
|
with self.getAsyncLoggerStream('synapse.storm.log', 'f=') as stream:
|
|
@@ -206,7 +205,7 @@ class TrigTest(s_t_utils.SynTest):
|
|
|
206
205
|
self.len(0, await core.nodes('test:int=5'))
|
|
207
206
|
|
|
208
207
|
# Prop set
|
|
209
|
-
q = '''$s
|
|
208
|
+
q = '''$s=`pf={$auto.opts.propfull} pn={$auto.opts.propname}` $lib.log.info($s)
|
|
210
209
|
[ test:guid="*" +#propset ]'''
|
|
211
210
|
tdef = {'cond': 'prop:set',
|
|
212
211
|
'storm': q,
|
|
@@ -246,7 +245,7 @@ class TrigTest(s_t_utils.SynTest):
|
|
|
246
245
|
self.len(1, await core.nodes('test:int#withiden'))
|
|
247
246
|
|
|
248
247
|
# iden embedded in vars
|
|
249
|
-
q = '+test:str~=log $s
|
|
248
|
+
q = '+test:str~=log $s=`test {$auto.type} {$auto.iden}` $lib.log.info($s, ({"iden": $auto.iden}))'
|
|
250
249
|
tdef = {'cond': 'node:add', 'form': 'test:str', 'storm': q}
|
|
251
250
|
await view.addTrigger(tdef)
|
|
252
251
|
with self.getStructuredAsyncLoggerStream('synapse.storm.log', 'test trigger') as stream:
|