synapse 2.206.0__py311-none-any.whl → 2.208.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (37) hide show
  1. synapse/axon.py +4 -4
  2. synapse/cortex.py +12 -3
  3. synapse/lib/boss.py +5 -1
  4. synapse/lib/modelrev.py +27 -4
  5. synapse/lib/storm.lark +2 -2
  6. synapse/lib/storm.py +5 -5
  7. synapse/lib/stormhttp.py +6 -6
  8. synapse/lib/stormlib/auth.py +1 -1
  9. synapse/lib/stormlib/cortex.py +1 -1
  10. synapse/lib/stormlib/vault.py +1 -1
  11. synapse/lib/stormsvc.py +1 -1
  12. synapse/lib/stormtypes.py +64 -37
  13. synapse/lib/task.py +7 -1
  14. synapse/lib/time.py +8 -2
  15. synapse/lib/version.py +2 -2
  16. synapse/lib/view.py +6 -1
  17. synapse/tests/test_axon.py +10 -0
  18. synapse/tests/test_cortex.py +4 -4
  19. synapse/tests/test_datamodel.py +1 -1
  20. synapse/tests/test_lib_ast.py +5 -5
  21. synapse/tests/test_lib_boss.py +2 -2
  22. synapse/tests/test_lib_nexus.py +5 -11
  23. synapse/tests/test_lib_storm.py +83 -6
  24. synapse/tests/test_lib_stormhttp.py +21 -8
  25. synapse/tests/test_lib_stormlib_cache.py +2 -2
  26. synapse/tests/test_lib_stormlib_scrape.py +1 -1
  27. synapse/tests/test_lib_stormlib_storm.py +63 -0
  28. synapse/tests/test_lib_stormsvc.py +3 -0
  29. synapse/tests/test_lib_stormtypes.py +44 -9
  30. synapse/tests/test_lib_trigger.py +3 -4
  31. synapse/tests/test_lib_view.py +3 -3
  32. synapse/tests/test_tools_csvtool.py +1 -1
  33. {synapse-2.206.0.dist-info → synapse-2.208.0.dist-info}/METADATA +6 -6
  34. {synapse-2.206.0.dist-info → synapse-2.208.0.dist-info}/RECORD +37 -37
  35. {synapse-2.206.0.dist-info → synapse-2.208.0.dist-info}/WHEEL +0 -0
  36. {synapse-2.206.0.dist-info → synapse-2.208.0.dist-info}/licenses/LICENSE +0 -0
  37. {synapse-2.206.0.dist-info → synapse-2.208.0.dist-info}/top_level.txt +0 -0
@@ -19,6 +19,7 @@ import synapse.common as s_common
19
19
  import synapse.telepath as s_telepath
20
20
 
21
21
  import synapse.lib.coro as s_coro
22
+ import synapse.lib.json as s_json
22
23
  import synapse.lib.certdir as s_certdir
23
24
  import synapse.lib.httpapi as s_httpapi
24
25
  import synapse.lib.msgpack as s_msgpack
@@ -1005,6 +1006,15 @@ bar baz",vv
1005
1006
  self.eq(True, resp['ok'])
1006
1007
  self.eq(200, resp['code'])
1007
1008
 
1009
+ jsonq = f'''$resp = $lib.axon.wput($sha256, "https://127.0.0.1:{port}/api/v1/pushfile", ssl=(0))
1010
+ return ( $lib.json.save($resp) )
1011
+ '''
1012
+ resp = await core.callStorm(jsonq, opts=opts)
1013
+ self.isinstance(resp, str)
1014
+ resp = s_json.loads(resp)
1015
+ self.eq(True, resp['ok'])
1016
+ self.eq(200, resp['code'])
1017
+
1008
1018
  opts = {'vars': {'sha256': s_common.ehex(s_common.buid())}}
1009
1019
  resp = await core.callStorm(q, opts=opts)
1010
1020
  self.eq(False, resp['ok'])
@@ -1240,7 +1240,7 @@ class CortexTest(s_t_utils.SynTest):
1240
1240
  $ddef = $lib.dmon.add(${
1241
1241
  $lib.print(hi)
1242
1242
  $lib.warn(omg)
1243
- $s = $lib.str.format('Running {t} {i}', t=$auto.type, i=$auto.iden)
1243
+ $s = `Running {$auto.type} {$auto.iden}`
1244
1244
  $lib.log.info($s, ({"iden": $auto.iden}))
1245
1245
  $que = $lib.queue.get(foo)
1246
1246
  $que.put(done)
@@ -5224,7 +5224,7 @@ class CortexBasicTest(s_t_utils.SynTest):
5224
5224
  self.len(1, nodes)
5225
5225
  self.nn(nodes[0].getTag('bar'))
5226
5226
 
5227
- q = '[test:str=yop +#$lib.str.format("{first}.{last}", first=foo, last=bar)]'
5227
+ q = '$t="{first}.{last}" [test:str=yop +#$t.format(first=foo, last=bar)]'
5228
5228
  nodes = await core.nodes(q)
5229
5229
  self.len(1, nodes)
5230
5230
  self.nn(nodes[0].getTag('foo.bar'))
@@ -5695,7 +5695,7 @@ class CortexBasicTest(s_t_utils.SynTest):
5695
5695
  self.len(1, nodes)
5696
5696
 
5697
5697
  async def test_storm_order(self):
5698
- q = '''[test:str=foo :hehe=bar] $tvar=() $tvar.append(1) $tvar.append(:hehe) $lib.print($lib.str.join('', $tvar)) '''
5698
+ q = '''[test:str=foo :hehe=bar] $tvar=() $tvar.append(1) $tvar.append(:hehe) $lib.print(('').join($tvar)) '''
5699
5699
  async with self.getTestCore() as core:
5700
5700
  mesgs = await core.stormlist(q)
5701
5701
  self.stormIsInPrint('1bar', mesgs)
@@ -7731,7 +7731,7 @@ class CortexBasicTest(s_t_utils.SynTest):
7731
7731
  $ddef = $lib.dmon.add(${
7732
7732
  $lib.print(hi)
7733
7733
  $lib.warn(omg)
7734
- $s = $lib.str.format('Running {t} {i}', t=$auto.type, i=$auto.iden)
7734
+ $s = `Running {$auto.type} {$auto.iden}`
7735
7735
  $lib.log.info($s, ({"iden": $auto.iden}))
7736
7736
  })
7737
7737
  ''')
@@ -299,7 +299,7 @@ class DataModelTest(s_t_utils.SynTest):
299
299
  syn:type:subof=comp $opts=:opts
300
300
  -> syn:form:type $valu=$node.value()
301
301
  for ($name, $thing) in $opts.fields {
302
- $v=$lib.str.format('{v}:{t}', v=$valu, t=$name) syn:prop=$v
302
+ $v=`{$valu}:{$name}` syn:prop=$v
303
303
  }
304
304
  +syn:prop
305
305
  -:ro=1
@@ -74,9 +74,9 @@ foo_stormpkg = {
74
74
  }
75
75
 
76
76
  function outer(arg1, add) {
77
- $strbase = $lib.str.format("(Run: {c}) we got back ", c=$counter)
77
+ $strbase = `(Run: {$counter}) we got back `
78
78
  $reti = $inner($arg1, $add)
79
- $mesg = $lib.str.concat($strbase, $reti)
79
+ $mesg = `{$strbase}{$reti}`
80
80
  $counter = $( $counter + $add )
81
81
  $lib.print("foobar is {foobar}", foobar=$foobar)
82
82
  return ($mesg)
@@ -296,7 +296,7 @@ class AstTest(s_test.SynTest):
296
296
  $newvar=:hehe
297
297
  -.created
298
298
  $s.append("yar {x}", x=$newvar)
299
- $lib.print($lib.str.join('', $s))
299
+ $lib.print(('').join($s))
300
300
  '''
301
301
  mesgs = await core.stormlist(q)
302
302
  prints = [m[1]['mesg'] for m in mesgs if m[0] == 'print']
@@ -2029,7 +2029,7 @@ class AstTest(s_test.SynTest):
2029
2029
  q = '$val=$lib.base64.decode("dmlzaQ==") function x(parm1=$val) { return($parm1) }'
2030
2030
  self.len(0, await core.nodes(q))
2031
2031
 
2032
- self.eq('foo', await core.callStorm('return($lib.str.format("{func}", func=foo))'))
2032
+ self.eq('foo', await core.callStorm('$template="{func}" return($template.format(func=foo))'))
2033
2033
 
2034
2034
  msgs = await core.stormlist('$lib.null()')
2035
2035
  erfo = [m for m in msgs if m[0] == 'err'][0]
@@ -3580,7 +3580,7 @@ class AstTest(s_test.SynTest):
3580
3580
 
3581
3581
  q = '''
3582
3582
  init { $foo = bar }
3583
- init { $baz = $lib.str.format('foo={foo}', foo=$foo) }
3583
+ init { $baz = `foo={$foo}` }
3584
3584
  $lib.print($baz)
3585
3585
  '''
3586
3586
  msgs = await core.stormlist(q)
@@ -56,8 +56,8 @@ class BossTest(s_test.SynTest):
56
56
  iden = s_common.guid()
57
57
 
58
58
  async def double_promote():
59
- await boss.promote(f'double', root, taskiden=iden)
60
- await boss.promote(f'double', root, taskiden=iden + iden)
59
+ await boss.promote('double', root, taskiden=iden)
60
+ await boss.promote('double', root, taskiden=iden + iden)
61
61
 
62
62
  coro = boss.schedCoro(double_promote())
63
63
  self.true(await stream.wait(timeout=6))
@@ -385,19 +385,13 @@ class NexusTest(s_t_utils.SynTest):
385
385
  evnt = asyncio.Event()
386
386
 
387
387
  async with self.getTestCore() as core:
388
- orig = core._nexshands['view:delwithlayer'][0]
389
-
390
- async def holdlock(self, viewiden, layriden, nexsitem, newparent=None):
391
- evnt.set()
392
- await asyncio.sleep(1)
393
- await orig(self, viewiden, layriden, nexsitem, newparent=newparent)
394
-
395
- core._nexshands['view:delwithlayer'] = (holdlock, True)
396
-
397
388
  forkiden = await core.callStorm('return($lib.view.get().fork().iden)')
398
389
 
399
- core.schedCoro(core.delViewWithLayer(forkiden))
400
- await asyncio.wait_for(evnt.wait(), timeout=10)
390
+ # Remove the nexus handler for the fork's write layer to simulate a view delete/edit race
391
+ layriden = core.getView(forkiden).layers[0].iden
392
+ layr = core.nexsroot._nexskids.pop(layriden)
401
393
 
402
394
  with self.raises(s_exc.NoSuchIden):
403
395
  await core.nodes('[ it:dev:str=foo ]', opts={'view': forkiden})
396
+
397
+ core.nexsroot._nexskids[layriden] = layr
@@ -308,7 +308,7 @@ class StormTest(s_t_utils.SynTest):
308
308
  ''')
309
309
  self.eq('valu=12', retn)
310
310
 
311
- q = "$hehe=({'k': 'v'}) $fs=$lib.str.format('{v}56', v=$hehe) return((`{$hehe}56`, $fs))"
311
+ q = "$hehe=({'k': 'v'}) $fs=`{$hehe}56` return((`{$hehe}56`, $fs))"
312
312
  retn = await core.callStorm(q)
313
313
  self.eq("{'k': 'v'}56", retn[0])
314
314
  self.eq(retn[0], retn[1])
@@ -322,6 +322,14 @@ class StormTest(s_t_utils.SynTest):
322
322
 
323
323
  self.eq("foo 'bar'", await core.callStorm("$foo=bar return(`foo '{$foo}'`)"))
324
324
  self.eq(r"\'''''bar'''", await core.callStorm(r"$foo=bar return(`\\'\''''{$foo}'''`)"))
325
+ self.eq(r"\bar", await core.callStorm(r"$foo=bar return(`\\{$foo}`)"))
326
+ self.eq(r"\`bar", await core.callStorm(r"$foo=bar return(`\\\`{$foo}`)"))
327
+ self.eq(r"\{bar", await core.callStorm(r"$foo=bar return(`\\\{{$foo}`)"))
328
+ self.eq(r"foo\bar", await core.callStorm(r"$foo=foo $bar=bar return(`{$foo}\\{$bar}`)"))
329
+ self.eq(r"foo \bar", await core.callStorm(r"$foo=foo $bar=bar return(`{$foo} \\{$bar}`)"))
330
+
331
+ with self.raises(s_exc.BadSyntax):
332
+ await core.callStorm(r"$foo=bar return(`\\{{$foo}`)")
325
333
 
326
334
  async def test_lib_storm_emit(self):
327
335
  async with self.getTestCore() as core:
@@ -2179,7 +2187,7 @@ class StormTest(s_t_utils.SynTest):
2179
2187
  [ ou:org=(cov,) ]
2180
2188
 
2181
2189
  { for $i in $lib.range(1001) {
2182
- $prop = $lib.str.format('_test{i}', i=$i)
2190
+ $prop = `_test{$i}`
2183
2191
  [ :$prop = $i
2184
2192
  +#$prop:score = $i
2185
2193
  +($i)> { ou:org=(cov,) }
@@ -2826,6 +2834,75 @@ class StormTest(s_t_utils.SynTest):
2826
2834
  ]
2827
2835
  self.eq(exp, evnts)
2828
2836
 
2837
+ async def test_storm_pkg_onload_active(self):
2838
+ pkg = {
2839
+ 'name': 'testload',
2840
+ 'version': '0.3.0',
2841
+ 'modules': (
2842
+ {
2843
+ 'name': 'testload',
2844
+ 'storm': 'function x() { return((0)) }',
2845
+ },
2846
+ ),
2847
+ 'onload': '''
2848
+ $lib.print(testprint)
2849
+ $lib.warn(testwarn)
2850
+
2851
+ $queue = $lib.queue.gen(onload:test)
2852
+
2853
+ $vers = $lib.globals.get(testload:version, (0))
2854
+ $vers = ($vers + 1)
2855
+ $lib.globals.set(testload:version, $vers)
2856
+ $queue.put($vers)
2857
+ '''
2858
+ }
2859
+
2860
+ with self.getTestDir() as dirn:
2861
+ dirn00 = s_common.gendir(dirn, 'core00')
2862
+ dirn01 = s_common.gendir(dirn, 'core01')
2863
+
2864
+ async with self.getTestCore(dirn=dirn00) as core00:
2865
+
2866
+ waiter = core00.waiter(2, 'core:pkg:onload:start', 'core:pkg:onload:complete')
2867
+
2868
+ await core00.addStormPkg(pkg)
2869
+
2870
+ events = await waiter.wait(timeout=10)
2871
+ self.eq(events, [
2872
+ ('core:pkg:onload:start', {'pkg': 'testload'}),
2873
+ ('core:pkg:onload:complete', {'pkg': 'testload'}),
2874
+ ])
2875
+
2876
+ self.eq((0, 1), await core00.callStorm('return($lib.queue.gen(onload:test).get((0), cull=(false)))'))
2877
+
2878
+ s_tools_backup.backup(dirn00, dirn01)
2879
+
2880
+ async with self.getTestCore(dirn=dirn00) as core00:
2881
+
2882
+ self.eq((1, 2), await core00.callStorm('return($lib.queue.gen(onload:test).get((1), cull=(false)))'))
2883
+
2884
+ conf01 = {'mirror': core00.getLocalUrl()}
2885
+
2886
+ async with self.getTestCore(dirn=dirn01, conf=conf01) as core01:
2887
+
2888
+ await core01.sync()
2889
+
2890
+ waiter = core01.waiter(2, 'core:pkg:onload:start', 'core:pkg:onload:complete')
2891
+
2892
+ await core01.promote()
2893
+
2894
+ events = await waiter.wait(timeout=10)
2895
+ self.eq(events, [
2896
+ ('core:pkg:onload:start', {'pkg': 'testload'}),
2897
+ ('core:pkg:onload:complete', {'pkg': 'testload'}),
2898
+ ])
2899
+
2900
+ self.eq((2, 3), await core01.callStorm('return($lib.queue.gen(onload:test).get((2), cull=(false)))'))
2901
+
2902
+ await core01.waitfini()
2903
+
2904
+ await core00.waitfini()
2905
+
2829
2906
  async def test_storm_tree(self):
2830
2907
 
2831
2908
  async with self.getTestCore() as core:
@@ -5083,7 +5160,7 @@ class StormTest(s_t_utils.SynTest):
5083
5160
  batch $lib.true --size 5 ${
5084
5161
  $vals=([])
5085
5162
  for $n in $nodes { $vals.append($n.repr()) }
5086
- $lib.print($lib.str.join(',', $vals))
5163
+ $lib.print((',').join($vals))
5087
5164
  }
5088
5165
  '''
5089
5166
  msgs = await core.stormlist(q)
@@ -5098,7 +5175,7 @@ class StormTest(s_t_utils.SynTest):
5098
5175
  batch $lib.false --size 5 {
5099
5176
  $vals=([])
5100
5177
  for $n in $nodes { $vals.append($n.repr()) }
5101
- $lib.print($lib.str.join(',', $vals))
5178
+ $lib.print((',').join($vals))
5102
5179
  }
5103
5180
  '''
5104
5181
  msgs = await core.stormlist(q)
@@ -5204,7 +5281,7 @@ class StormTest(s_t_utils.SynTest):
5204
5281
  msgs = await core.stormlist(q)
5205
5282
  self.stormHasNoErr(msgs)
5206
5283
 
5207
- await core.nodes('''$token=foo $lib.print(({"Authorization":$lib.str.format("Bearer {token}", token=$token)}))''')
5284
+ await core.nodes('''$token=foo $lib.print(({"Authorization":`Bearer {$token}`}))''')
5208
5285
 
5209
5286
  q = '#rep.clearsky.dreamjob -># +syn:tag^=rep |uniq -syn:tag~=rep.clearsky'
5210
5287
  msgs = await core.stormlist(q)
@@ -5218,7 +5295,7 @@ class StormTest(s_t_utils.SynTest):
5218
5295
  msgs = await core.stormlist(q)
5219
5296
  self.stormIsInWarn('Failed to decode iden: [ssl://svcrs:27492?certname=root=bar]', msgs)
5220
5297
 
5221
- q = "$foo=one $bar=two $lib.print($lib.str.concat($foo, '=', $bar))"
5298
+ q = "$foo=one $bar=two $lib.print(`{$foo}={$bar}`)"
5222
5299
  msgs = await core.stormlist(q)
5223
5300
  self.stormIsInPrint("one=two", msgs)
5224
5301
 
@@ -473,6 +473,19 @@ class StormHttpTest(s_test.SynTest):
473
473
  data = resp.get('result')
474
474
  self.eq(data.get('params'), {'foo': ['bar', 'baz'], 'key': ["('valu',)"]})
475
475
 
476
+ # headers are safe to serialize
477
+ q = '''
478
+ $headers = ({'Foo': 'Bar'})
479
+ $resp = $lib.inet.http.request(GET, $url, headers=$headers, ssl_verify=$lib.false)
480
+ return ( ($lib.json.save($resp.headers), $lib.json.save($resp.request_headers)) )
481
+ '''
482
+ resp = await core.callStorm(q, opts=opts)
483
+ (headers, req_headers) = resp
484
+ headers = s_json.loads(headers)
485
+ self.eq(headers.get('Content-Type'), 'application/json; charset=UTF-8')
486
+ req_headers = s_json.loads(req_headers)
487
+ self.eq(req_headers.get('Foo'), 'Bar')
488
+
476
489
  async def test_storm_http_post(self):
477
490
 
478
491
  async with self.getTestCore() as core:
@@ -481,7 +494,7 @@ class StormHttpTest(s_test.SynTest):
481
494
  await root.setPasswd('root')
482
495
 
483
496
  adduser = '''
484
- $url = $lib.str.format("https://root:root@127.0.0.1:{port}/api/v1/auth/adduser", port=$port)
497
+ $url = `https://root:root@127.0.0.1:{$port}/api/v1/auth/adduser`
485
498
  $user = ({"name": $name, "passwd": $passwd})
486
499
  $post = $lib.inet.http.post($url, json=$user, ssl_verify=$(0)).json().result.name
487
500
  $lib.print($post)
@@ -493,8 +506,8 @@ class StormHttpTest(s_test.SynTest):
493
506
  self.assertIn('foo', [u.name for u in core.auth.users()])
494
507
 
495
508
  adduser = '''
496
- $url = $lib.str.format("https://root:root@127.0.0.1:{port}/api/v1/auth/adduser", port=$port)
497
- $user = $lib.str.format('{"name": "{name}", "passwd": "{passwd}"}', name=$name, passwd=$passwd)
509
+ $url = `https://root:root@127.0.0.1:{$port}/api/v1/auth/adduser`
510
+ $user = $lib.json.save( ({"name": $name, "passwd": $passwd}) )
498
511
  $header = ({"Content-Type": "application/json"})
499
512
  $post = $lib.inet.http.post($url, headers=$header, body=$user, ssl_verify=$(0)).json().result.name
500
513
  [ test:str=$post ]
@@ -573,7 +586,7 @@ class StormHttpTest(s_test.SynTest):
573
586
  root = await core.auth.getUserByName('root')
574
587
  await root.setPasswd('root')
575
588
  text = '''
576
- $url = $lib.str.format("https://root:root@127.0.0.1:{port}/api/v1/storm", port=$port)
589
+ $url = `https://root:root@127.0.0.1:{$port}/api/v1/storm`
577
590
  $stormq = "($size, $sha2) = $lib.axon.put($lib.base64.decode('dmVydGV4')) [ test:str = $sha2 ] [ test:int = $size ]"
578
591
  $json = ({"query": $stormq})
579
592
  $bytez = $lib.inet.http.post($url, json=$json, ssl_verify=$(0))
@@ -589,7 +602,7 @@ class StormHttpTest(s_test.SynTest):
589
602
  self.eq(nodes[0].ndef, ('test:int', 6))
590
603
 
591
604
  text = '''
592
- $url = $lib.str.format("https://root:root@127.0.0.1:{port}/api/v1/storm", port=$port)
605
+ $url = `https://root:root@127.0.0.1:{$port}/api/v1/storm`
593
606
  $json = ({"query": "test:str"})
594
607
  $body = $json
595
608
  $resp=$lib.inet.http.post($url, json=$json, body=$body, ssl_verify=$(0))
@@ -767,7 +780,7 @@ class StormHttpTest(s_test.SynTest):
767
780
  mesg = await core.callStorm('''
768
781
  $params = ( { "param1": "somevalu" } )
769
782
  $hdr = ( { "key": $lib.false } )
770
- $url = $lib.str.format('https://127.0.0.1:{port}/test/ws', port=$port)
783
+ $url = `https://127.0.0.1:{$port}/test/ws`
771
784
 
772
785
  ($ok, $sock) = $lib.inet.http.connect($url, headers=$hdr, params=$params, ssl_verify=$lib.false)
773
786
  if (not $ok) { $lib.exit($sock) }
@@ -783,7 +796,7 @@ class StormHttpTest(s_test.SynTest):
783
796
 
784
797
  mesg = await core.callStorm('''
785
798
  $hdr = ( { "key": $lib.false } )
786
- $url = $lib.str.format('https://127.0.0.1:{port}/test/ws', port=$port)
799
+ $url = `https://127.0.0.1:{$port}/test/ws`
787
800
 
788
801
  ($ok, $sock) = $lib.inet.http.connect($url, headers=$hdr, ssl_verify=$lib.false)
789
802
  if (not $ok) { $lib.exit($sock) }
@@ -797,7 +810,7 @@ class StormHttpTest(s_test.SynTest):
797
810
  self.none(mesg.get('params'))
798
811
 
799
812
  query = '''
800
- $url = $lib.str.format('https://127.0.0.1:{port}/test/ws', port=$port)
813
+ $url = `https://127.0.0.1:{$port}/test/ws`
801
814
 
802
815
  ($ok, $sock) = $lib.inet.http.connect($url, proxy=$proxy, ssl_verify=$lib.false)
803
816
  if (not $ok) { $lib.exit($sock) }
@@ -120,12 +120,12 @@ class StormlibCacheTest(s_test.SynTest):
120
120
 
121
121
  $rets.append($cache.get(foo))
122
122
  $rets.append($sent)
123
- $rets.append($lib.str.join(",", $vals))
123
+ $rets.append((',').join($vals))
124
124
 
125
125
  $val = one
126
126
  $rets.append($cache.get(bar))
127
127
  $rets.append($sent)
128
- $rets.append($lib.str.join(",", $vals))
128
+ $rets.append((',').join($vals))
129
129
 
130
130
  return($rets)
131
131
  ''')
@@ -172,7 +172,7 @@ class StormScrapeTest(s_test.SynTest):
172
172
 
173
173
  text = text + ' and then there was another 1.2.3.4 that happened at woot.com '
174
174
  query = '''$tally = $lib.stats.tally() for ($form, $ndef) in $lib.scrape.ndefs($text)
175
- { $valu=$lib.str.format('{f}={n}', f=$form, n=$ndef) $tally.inc($valu) }
175
+ { $valu=`{$form}={$ndef}` $tally.inc($valu) }
176
176
  fini { return ( $tally ) }
177
177
  '''
178
178
  varz = {'text': text}
@@ -1,3 +1,5 @@
1
+ import asyncio
2
+
1
3
  import synapse.exc as s_exc
2
4
  import synapse.lib.parser as s_parser
3
5
 
@@ -142,3 +144,64 @@ class LibStormTest(s_test.SynTest):
142
144
  }
143
145
  ''')
144
146
  self.stormIsInPrint('mesg=hello', msgs)
147
+
148
+ async def test_lib_stormlib_storm_tasks(self):
149
+
150
+ with self.getStructuredAsyncLoggerStream('synapse') as stream:
151
+
152
+ async with self.getTestCore() as core:
153
+
154
+ async with core.getLocalProxy() as prox:
155
+
156
+ event = asyncio.Event()
157
+
158
+ q = 'for $mesg in $lib.storm.run("$lib.time.sleep(120)") { $lib.fire(storm, mesg=$mesg) }'
159
+
160
+ async def doit():
161
+ async for mesg in prox.storm(q):
162
+ if mesg[0] == 'storm:fire':
163
+ event.set()
164
+
165
+ task00 = core.schedCoro(doit())
166
+ await asyncio.wait_for(event.wait(), timeout=10)
167
+
168
+ viewiden = core.getView().iden
169
+
170
+ # One task for the main query
171
+ tasks = core.boss.ps()
172
+ self.len(1, tasks)
173
+ self.eq(tasks[0].name, 'storm')
174
+ self.eq(tasks[0].info, {'query': q, 'view': viewiden})
175
+ self.len(1, tasks[0].kids)
176
+
177
+ # The main query has one kid, the worker
178
+ kid = list(tasks[0].kids.values())[0]
179
+ self.nn(kid.iden)
180
+ self.nn(kid.user)
181
+ self.eq(kid.name, 'runstorm')
182
+ self.eq(kid.info, {})
183
+
184
+ # The worker has a kid which is the $lib.storm.run() worker
185
+ self.len(1, kid.kids)
186
+ gkid = list(kid.kids.values())[0]
187
+ self.eq(gkid.name, 'runstorm')
188
+ self.eq(gkid.info, {})
189
+
190
+ # Kill the main task
191
+ tasks = core.boss.ps()
192
+ self.len(1, tasks)
193
+ await tasks[0].kill()
194
+
195
+ # No tasks running
196
+ tasks = core.boss.ps()
197
+ self.len(0, tasks)
198
+
199
+ task00.cancel('oh bye')
200
+
201
+ # Verify we saw two storm runtime cancellations
202
+ msgs = stream.jsonlines()
203
+ self.gt(len(msgs), 0)
204
+
205
+ msgs = [(k.get('message'), k.get('text')) for k in msgs]
206
+ self.isin(('Storm runtime cancelled.', '$lib.time.sleep(120)'), msgs)
207
+ self.isin(('Storm runtime cancelled.', q), msgs)
@@ -347,6 +347,9 @@ class StormvarServiceCell(s_cell.Cell):
347
347
  'bar': {
348
348
  'type': 'string',
349
349
  },
350
+ 'name': {
351
+ 'type': 'string',
352
+ },
350
353
  },
351
354
  'required': ['name', ],
352
355
  'additionalProperties': False,
@@ -156,7 +156,7 @@ class StormTypesTest(s_test.SynTest):
156
156
  await core.callStorm(q, opts=opts)
157
157
 
158
158
  # Push a handful of notifications and list a subset of them
159
- q = '''$m=$lib.str.format('hello {i}', i=$i) return($lib.auth.users.byname(root).tell($m))'''
159
+ q = '''$m=`hello {$i}` return($lib.auth.users.byname(root).tell($m))'''
160
160
  for i in range(5):
161
161
  opts = {'user': visi.iden, 'vars': {'i': i}}
162
162
  await core.callStorm(q, opts=opts)
@@ -1071,7 +1071,7 @@ class StormTypesTest(s_test.SynTest):
1071
1071
  # exec vars do not populate upwards
1072
1072
  q = '''
1073
1073
  $foo = "that is one neato burrito"
1074
- $baz = ${ $bar=$lib.str.concat(wompwomp, $lib.guid()) $lib.print("in exec") }
1074
+ $baz = ${ $bar=`wompwomp{$lib.guid()}` $lib.print("in exec") }
1075
1075
  $baz.exec()
1076
1076
  $lib.print("post exec {bar}", bar=$bar)
1077
1077
  [ test:str=$foo ]
@@ -1283,6 +1283,8 @@ class StormTypesTest(s_test.SynTest):
1283
1283
 
1284
1284
  async def test_storm_lib_str(self):
1285
1285
  async with self.getTestCore() as core:
1286
+
1287
+ # TODO $lib.str.concat and rmat are deprecated should be removed in 3.0.0
1286
1288
  q = '$v=vertex $l=link $fqdn=$lib.str.concat($v, ".", $l)' \
1287
1289
  ' [ inet:email=$lib.str.format("visi@{domain}", domain=$fqdn) ]'
1288
1290
  nodes = await core.nodes(q)
@@ -1310,7 +1312,7 @@ class StormTypesTest(s_test.SynTest):
1310
1312
  sobj = s_stormtypes.Str('beepbeep')
1311
1313
  self.len(8, sobj)
1312
1314
 
1313
- nodes = await core.nodes('$s = (foo, bar, baz) [ test:str=$lib.str.join(".", $s) ]')
1315
+ nodes = await core.nodes("$s = (foo, bar, baz) [ test:str=('.').join($s) ]")
1314
1316
  self.eq('foo.bar.baz', nodes[0].ndef[1])
1315
1317
 
1316
1318
  nodes = await core.nodes('$s = foo-bar-baz [ test:str=$s.replace("-", ".") ]')
@@ -1406,6 +1408,14 @@ class StormTypesTest(s_test.SynTest):
1406
1408
 
1407
1409
  self.eq(((1, 2, 3)), await core.callStorm('return(("[1, 2, 3]").json())'))
1408
1410
 
1411
+ self.eq('hehe,haha', await core.callStorm("$sepr=',' $l=(hehe, haha) return( $sepr.join($l) )"))
1412
+ self.eq('hehehaha', await core.callStorm("$sepr='' $l=(hehe, haha) return( $sepr.join($l) )"))
1413
+ self.eq('a|++|b|++|c', await core.callStorm("$sepr='|++|' $l=(a, b, c) return( $sepr.join($l) )"))
1414
+ self.eq('hehe,haha', await core.callStorm("$l=(hehe, haha) return( (',').join($l) )"))
1415
+ self.eq('hehehaha', await core.callStorm("$l=(hehe, haha) return( ('').join($l) )"))
1416
+ self.eq('', await core.callStorm("$sepr=',' $l=() return( $sepr.join($l) )"))
1417
+ self.eq('', await core.callStorm("$sepr='' $l=() return( $sepr.join($l) )"))
1418
+
1409
1419
  with self.raises(s_exc.BadJsonText):
1410
1420
  await core.callStorm('return(("foo").json())')
1411
1421
 
@@ -1752,7 +1762,7 @@ class StormTypesTest(s_test.SynTest):
1752
1762
  $ipv4 = $node.repr()
1753
1763
  $loc = $node.repr(loc)
1754
1764
  $latlong = $node.repr(latlong, defv="??")
1755
- $valu = $lib.str.format("{ipv4} in {loc} at {latlong}", ipv4=$ipv4, loc=$loc, latlong=$latlong)
1765
+ $valu = `{$ipv4} in {$loc} at {$latlong}`
1756
1766
  [ test:str=$valu ]
1757
1767
  +test:str
1758
1768
  '''
@@ -2454,7 +2464,7 @@ class StormTypesTest(s_test.SynTest):
2454
2464
  self.stormIsInPrint('pop valu is 0', mesgs)
2455
2465
 
2456
2466
  listq = '''for ($key, $valu) in $lib.globals.list() {
2457
- $string = $lib.str.format("{key} is {valu}", key=$key, valu=$valu)
2467
+ $string = `{$key} is {$valu}`
2458
2468
  $lib.print($string)
2459
2469
  }
2460
2470
  '''
@@ -2498,7 +2508,7 @@ class StormTypesTest(s_test.SynTest):
2498
2508
  self.len(1, await core.nodes('test:str=hehe'))
2499
2509
 
2500
2510
  listq = '''for ($key, $valu) in $lib.user.vars.list() {
2501
- $string = $lib.str.format("{key} is {valu}", key=$key, valu=$valu)
2511
+ $string = `{$key} is {$valu}`
2502
2512
  $lib.print($string)
2503
2513
  }
2504
2514
  '''
@@ -2571,7 +2581,7 @@ class StormTypesTest(s_test.SynTest):
2571
2581
  # core.vars, they only get the values they can read.
2572
2582
  corelistq = '''
2573
2583
  for ($key, $valu) in $lib.globals.list() {
2574
- $string = $lib.str.format("{key} is {valu}", key=$key, valu=$valu)
2584
+ $string = `{$key} is {$valu}`
2575
2585
  $lib.print($string)
2576
2586
  }
2577
2587
  '''
@@ -2726,6 +2736,21 @@ class StormTypesTest(s_test.SynTest):
2726
2736
  self.false(valu[0])
2727
2737
  self.eq(valu[1]['err'], 'BadArg')
2728
2738
 
2739
+ query = '''$valu="2020-10-01 01:30:00"
2740
+ $parsed=$lib.time.parse($valu, "%Y-%m-%d %H:%M:%S")
2741
+ $lib.print($lib.time.toUTC($parsed, US/Eastern))
2742
+ '''
2743
+ mesgs = await core.stormlist(query)
2744
+ self.stormIsInPrint('1601530200000', mesgs)
2745
+
2746
+ query = '''$valu="2020-11-01 01:30:00"
2747
+ $parsed=$lib.time.parse($valu, "%Y-%m-%d %H:%M:%S")
2748
+ return($lib.time.toUTC($parsed, America/New_York))
2749
+ '''
2750
+ mesgs = await core.callStorm(query)
2751
+ self.false(mesgs[0])
2752
+ self.isin('Ambiguous time', mesgs[1]['errinfo']['mesg'])
2753
+
2729
2754
  async def test_storm_lib_time_ticker(self):
2730
2755
 
2731
2756
  async with self.getTestCore() as core:
@@ -4919,7 +4944,7 @@ class StormTypesTest(s_test.SynTest):
4919
4944
  unixtime = datetime.datetime(year=2018, month=12, day=5, hour=7, minute=10,
4920
4945
  tzinfo=tz.utc).timestamp()
4921
4946
 
4922
- q = '{$lib.queue.get(foo).put(m3) $s=$lib.str.format("m3 {t} {i}", t=$auto.type, i=$auto.iden) $lib.log.info($s, ({"iden": $auto.iden})) }'
4947
+ q = '{$lib.queue.get(foo).put(m3) $s=`m3 {$auto.type} {$auto.iden}` $lib.log.info($s, ({"iden": $auto.iden})) }'
4923
4948
  text = f'cron.add --minute 17 {q}'
4924
4949
  async with getCronJob(text) as guid:
4925
4950
  with self.getStructuredAsyncLoggerStream('synapse.storm.log', 'm3 cron') as stream:
@@ -6049,6 +6074,16 @@ class StormTypesTest(s_test.SynTest):
6049
6074
  self.eq({'d', 'c'}, ret)
6050
6075
 
6051
6076
  # str join
6077
+ ret = await core.callStorm('$x=(foo,bar,baz) $y=("-").join($x) return($y)')
6078
+ self.eq('foo-bar-baz', ret)
6079
+
6080
+ ret = await core.callStorm('$y=("-").join((foo, bar, baz)) return($y)')
6081
+ self.eq('foo-bar-baz', ret)
6082
+
6083
+ ret = await core.callStorm('$x=abcd $y=("-").join($x) return($y)')
6084
+ self.eq('a-b-c-d', ret)
6085
+
6086
+ # TODO $lib.str.join is deprecated and will be removed in 3.0.0
6052
6087
  ret = await core.callStorm('$x=(foo,bar,baz) $y=$lib.str.join("-", $x) return($y)')
6053
6088
  self.eq('foo-bar-baz', ret)
6054
6089
 
@@ -6069,7 +6104,7 @@ class StormTypesTest(s_test.SynTest):
6069
6104
 
6070
6105
  opts = {'user': visi.iden, 'vars': {'port': port}}
6071
6106
  wget = '''
6072
- $url = $lib.str.format("https://visi:secret@127.0.0.1:{port}/api/v1/healthcheck", port=$port)
6107
+ $url = `https://visi:secret@127.0.0.1:{$port}/api/v1/healthcheck`
6073
6108
  return($lib.axon.wget($url, ssl=$lib.false))
6074
6109
  '''
6075
6110
  with self.raises(s_exc.AuthDeny):
@@ -141,8 +141,7 @@ class TrigTest(s_t_utils.SynTest):
141
141
  view = core.view
142
142
 
143
143
  # node:add case
144
- q = '''$s=$lib.str.format("f={f} v={v}", f=$auto.opts.form, v=$auto.opts.valu) $lib.log.info($s)
145
- [ test:guid="*" +#nodeadd]'''
144
+ q = '$s=`f={$auto.opts.form} v={$auto.opts.valu}` $lib.log.info($s) [ test:guid="*" +#nodeadd]'
146
145
  tdef = {'cond': 'node:add', 'form': 'test:str', 'storm': q}
147
146
  await view.addTrigger(tdef)
148
147
  with self.getAsyncLoggerStream('synapse.storm.log', 'f=') as stream:
@@ -206,7 +205,7 @@ class TrigTest(s_t_utils.SynTest):
206
205
  self.len(0, await core.nodes('test:int=5'))
207
206
 
208
207
  # Prop set
209
- q = '''$s=$lib.str.format("pf={f} pn={n}", f=$auto.opts.propfull, n=$auto.opts.propname) $lib.log.info($s)
208
+ q = '''$s=`pf={$auto.opts.propfull} pn={$auto.opts.propname}` $lib.log.info($s)
210
209
  [ test:guid="*" +#propset ]'''
211
210
  tdef = {'cond': 'prop:set',
212
211
  'storm': q,
@@ -246,7 +245,7 @@ class TrigTest(s_t_utils.SynTest):
246
245
  self.len(1, await core.nodes('test:int#withiden'))
247
246
 
248
247
  # iden embedded in vars
249
- q = '+test:str~=log $s=$lib.str.format("test {t} {i}", t=$auto.type, i=$auto.iden) $lib.log.info($s, ({"iden": $auto.iden}))'
248
+ q = '+test:str~=log $s=`test {$auto.type} {$auto.iden}` $lib.log.info($s, ({"iden": $auto.iden}))'
250
249
  tdef = {'cond': 'node:add', 'form': 'test:str', 'storm': q}
251
250
  await view.addTrigger(tdef)
252
251
  with self.getStructuredAsyncLoggerStream('synapse.storm.log', 'test trigger') as stream: