synapse 2.193.0__py311-none-any.whl → 2.195.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/cortex.py +9 -7
- synapse/datamodel.py +9 -6
- synapse/exc.py +1 -1
- synapse/lib/agenda.py +17 -4
- synapse/lib/ast.py +217 -86
- synapse/lib/auth.py +5 -2
- synapse/lib/link.py +33 -19
- synapse/lib/modelrev.py +6 -1
- synapse/lib/parser.py +4 -0
- synapse/lib/scrape.py +18 -1
- synapse/lib/snap.py +40 -11
- synapse/lib/storm.lark +16 -1
- synapse/lib/storm.py +6 -4
- synapse/lib/storm_format.py +1 -0
- synapse/lib/stormctrl.py +88 -6
- synapse/lib/stormlib/auth.py +15 -1
- synapse/lib/stormlib/cache.py +6 -2
- synapse/lib/stormlib/cell.py +11 -0
- synapse/lib/stormlib/infosec.py +2 -0
- synapse/lib/stormlib/scrape.py +1 -1
- synapse/lib/stormlib/stix.py +8 -8
- synapse/lib/stormtypes.py +13 -5
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +20 -3
- synapse/models/geopol.py +1 -0
- synapse/models/geospace.py +53 -10
- synapse/models/inet.py +3 -0
- synapse/models/infotech.py +12 -5
- synapse/models/material.py +67 -8
- synapse/models/orgs.py +11 -3
- synapse/models/person.py +28 -17
- synapse/models/risk.py +4 -1
- synapse/models/syn.py +3 -0
- synapse/models/telco.py +10 -3
- synapse/models/transport.py +382 -49
- synapse/tests/test_axon.py +6 -6
- synapse/tests/test_cortex.py +134 -12
- synapse/tests/test_exc.py +1 -0
- synapse/tests/test_lib_agenda.py +125 -1
- synapse/tests/test_lib_aha.py +13 -6
- synapse/tests/test_lib_ast.py +258 -9
- synapse/tests/test_lib_auth.py +6 -7
- synapse/tests/test_lib_cell.py +10 -0
- synapse/tests/test_lib_grammar.py +14 -0
- synapse/tests/test_lib_layer.py +1 -1
- synapse/tests/test_lib_link.py +6 -1
- synapse/tests/test_lib_lmdbslab.py +3 -3
- synapse/tests/test_lib_modelrev.py +7 -0
- synapse/tests/test_lib_scrape.py +8 -0
- synapse/tests/test_lib_storm.py +201 -25
- synapse/tests/test_lib_stormctrl.py +65 -0
- synapse/tests/test_lib_stormhttp.py +5 -5
- synapse/tests/test_lib_stormlib_auth.py +31 -5
- synapse/tests/test_lib_stormlib_cache.py +38 -6
- synapse/tests/test_lib_stormlib_cell.py +3 -0
- synapse/tests/test_lib_stormlib_modelext.py +3 -3
- synapse/tests/test_lib_stormlib_scrape.py +4 -4
- synapse/tests/test_lib_stormlib_spooled.py +1 -1
- synapse/tests/test_lib_stormlib_xml.py +5 -5
- synapse/tests/test_lib_stormtypes.py +54 -57
- synapse/tests/test_lib_view.py +1 -1
- synapse/tests/test_model_base.py +1 -2
- synapse/tests/test_model_geopol.py +4 -0
- synapse/tests/test_model_geospace.py +43 -4
- synapse/tests/test_model_inet.py +3 -0
- synapse/tests/test_model_infotech.py +31 -4
- synapse/tests/test_model_material.py +18 -0
- synapse/tests/test_model_orgs.py +25 -3
- synapse/tests/test_model_person.py +26 -1
- synapse/tests/test_model_risk.py +11 -0
- synapse/tests/test_model_syn.py +9 -3
- synapse/tests/test_model_transport.py +168 -0
- synapse/tests/test_telepath.py +24 -5
- synapse/tests/test_tools_healthcheck.py +4 -4
- synapse/tests/test_utils.py +17 -18
- synapse/tests/utils.py +0 -35
- synapse/tools/changelog.py +14 -5
- synapse/tools/storm.py +1 -1
- {synapse-2.193.0.dist-info → synapse-2.195.0.dist-info}/METADATA +5 -5
- {synapse-2.193.0.dist-info → synapse-2.195.0.dist-info}/RECORD +83 -82
- {synapse-2.193.0.dist-info → synapse-2.195.0.dist-info}/WHEEL +1 -1
- {synapse-2.193.0.dist-info → synapse-2.195.0.dist-info}/LICENSE +0 -0
- {synapse-2.193.0.dist-info → synapse-2.195.0.dist-info}/top_level.txt +0 -0
synapse/tests/test_lib_storm.py
CHANGED
|
@@ -53,10 +53,6 @@ class StormTest(s_t_utils.SynTest):
|
|
|
53
53
|
with self.raises(s_exc.BadTypeValu):
|
|
54
54
|
await core.nodes('[ ou:org=({"hq": "woot"}) ]')
|
|
55
55
|
|
|
56
|
-
msgs = await core.stormlist('[ ou:org=({"hq": "woot", "$try": true}) ]')
|
|
57
|
-
self.len(0, [m for m in msgs if m[0] == 'node'])
|
|
58
|
-
self.stormIsInWarn('Bad value for prop hq: valu is not a guid', msgs)
|
|
59
|
-
|
|
60
56
|
nodes05 = await core.nodes('[ ou:org=({"name": "vertex", "$props": {"motto": "for the people"}}) ]')
|
|
61
57
|
self.len(1, nodes05)
|
|
62
58
|
self.eq('vertex', nodes05[0].get('name'))
|
|
@@ -96,6 +92,57 @@ class StormTest(s_t_utils.SynTest):
|
|
|
96
92
|
self.len(1, nodes12)
|
|
97
93
|
self.ne(nodes11[0].ndef, nodes12[0].ndef)
|
|
98
94
|
|
|
95
|
+
# GUID ctor has a short-circuit where it tries to find an existing ndef before it does,
|
|
96
|
+
# some property deconfliction, and `<form>=({})` when pushed through guid generation gives
|
|
97
|
+
# back the same guid as `<form>=()`, which if we're not careful could lead to an
|
|
98
|
+
# inconsistent case where you fail to make a node because you don't provide any props,
|
|
99
|
+
# make a node with that matching ndef, and then run that invalid GUID ctor query again,
|
|
100
|
+
# and have it return back a node due to the short circuit. So test that we're consistent here.
|
|
101
|
+
with self.raises(s_exc.BadTypeValu):
|
|
102
|
+
await core.nodes('[ ou:org=({}) ]')
|
|
103
|
+
|
|
104
|
+
self.len(1, await core.nodes('[ ou:org=() ]'))
|
|
105
|
+
|
|
106
|
+
with self.raises(s_exc.BadTypeValu):
|
|
107
|
+
await core.nodes('[ ou:org=({}) ]')
|
|
108
|
+
|
|
109
|
+
msgs = await core.stormlist('[ ou:org=({"$props": {"desc": "lol"}})]')
|
|
110
|
+
self.len(0, [m for m in msgs if m[0] == 'node'])
|
|
111
|
+
self.stormIsInErr('No values provided for form ou:org', msgs)
|
|
112
|
+
|
|
113
|
+
msgs = await core.stormlist('[ou:org=({"name": "burrito corp", "$props": {"phone": "lolnope"}})]')
|
|
114
|
+
self.len(0, [m for m in msgs if m[0] == 'node'])
|
|
115
|
+
self.stormIsInErr('Bad value for prop ou:org:phone: requires a digit string', msgs)
|
|
116
|
+
|
|
117
|
+
with self.raises(s_exc.BadTypeValu):
|
|
118
|
+
await core.nodes('[ ou:org=({"$try": true}) ]')
|
|
119
|
+
|
|
120
|
+
# $try only affects $props
|
|
121
|
+
msgs = await core.stormlist('[ ou:org=({"founded": "lolnope", "$try": true}) ]')
|
|
122
|
+
self.len(0, [m for m in msgs if m[0] == 'node'])
|
|
123
|
+
self.stormIsInErr('Bad value for prop ou:org:founded: Unknown time format for lolnope', msgs)
|
|
124
|
+
|
|
125
|
+
msgs = await core.stormlist('[ou:org=({"name": "burrito corp", "$try": true, "$props": {"phone": "lolnope", "desc": "burritos man"}})]')
|
|
126
|
+
nodes = [m for m in msgs if m[0] == 'node']
|
|
127
|
+
self.len(1, nodes)
|
|
128
|
+
node = nodes[0][1]
|
|
129
|
+
props = node[1]['props']
|
|
130
|
+
self.none(props.get('phone'))
|
|
131
|
+
self.eq(props.get('name'), 'burrito corp')
|
|
132
|
+
self.eq(props.get('desc'), 'burritos man')
|
|
133
|
+
self.stormIsInWarn('Skipping bad value for prop ou:org:phone: requires a digit string', msgs)
|
|
134
|
+
|
|
135
|
+
await self.asyncraises(s_exc.BadTypeValu, core.addNode(core.auth.rootuser, 'ou:org', {'name': 'org name 77', 'phone': 'lolnope'}, props={'desc': 'an org desc'}))
|
|
136
|
+
|
|
137
|
+
await self.asyncraises(s_exc.BadTypeValu, core.addNode(core.auth.rootuser, 'ou:org', {'name': 'org name 77'}, props={'desc': 'an org desc', 'phone': 'lolnope'}))
|
|
138
|
+
|
|
139
|
+
node = await core.addNode(core.auth.rootuser, 'ou:org', {'$try': True, '$props': {'phone': 'invalid'}, 'name': 'org name 77'}, props={'desc': 'an org desc'})
|
|
140
|
+
self.nn(node)
|
|
141
|
+
props = node[1]['props']
|
|
142
|
+
self.none(props.get('phone'))
|
|
143
|
+
self.eq(props.get('name'), 'org name 77')
|
|
144
|
+
self.eq(props.get('desc'), 'an org desc')
|
|
145
|
+
|
|
99
146
|
async def test_lib_storm_jsonexpr(self):
|
|
100
147
|
async with self.getTestCore() as core:
|
|
101
148
|
|
|
@@ -254,7 +301,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
254
301
|
emit bar
|
|
255
302
|
}
|
|
256
303
|
function makelist() {
|
|
257
|
-
$retn =
|
|
304
|
+
$retn = ()
|
|
258
305
|
for $item in $generate() { $retn.append($item) }
|
|
259
306
|
return($retn)
|
|
260
307
|
}
|
|
@@ -267,7 +314,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
267
314
|
emit $node.repr()
|
|
268
315
|
}
|
|
269
316
|
function makelist() {
|
|
270
|
-
$retn =
|
|
317
|
+
$retn = ()
|
|
271
318
|
for $item in $generate() { $retn.append($item) }
|
|
272
319
|
return($retn)
|
|
273
320
|
}
|
|
@@ -339,9 +386,135 @@ class StormTest(s_t_utils.SynTest):
|
|
|
339
386
|
prnt = [m[1]['mesg'] for m in msgs if m[0] == 'print']
|
|
340
387
|
self.eq(prnt, ['inner 0', 'outer 0'])
|
|
341
388
|
|
|
342
|
-
|
|
389
|
+
# Emit outside an emitter function raises a runtime error with posinfo
|
|
390
|
+
with self.raises(s_exc.StormRuntimeError) as cm:
|
|
391
|
+
await core.nodes('emit foo')
|
|
392
|
+
self.nn(cm.exception.get('highlight'))
|
|
393
|
+
|
|
394
|
+
with self.raises(s_exc.StormRuntimeError) as cm:
|
|
395
|
+
await core.nodes('[test:str=emit] emit foo')
|
|
396
|
+
self.nn(cm.exception.get('highlight'))
|
|
397
|
+
|
|
398
|
+
# stop cannot cross function boundaries
|
|
399
|
+
q = '''
|
|
400
|
+
function inner(v) {
|
|
401
|
+
if ( $v = 2 ) {
|
|
402
|
+
stop
|
|
403
|
+
}
|
|
404
|
+
return ( $v )
|
|
405
|
+
}
|
|
406
|
+
function outer(n) {
|
|
407
|
+
for $i in $lib.range($n) {
|
|
408
|
+
emit $inner($i)
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
$N = (5)
|
|
412
|
+
for $valu in $outer($N) {
|
|
413
|
+
$lib.print(`{$valu}/{$N}`)
|
|
414
|
+
}
|
|
415
|
+
'''
|
|
416
|
+
msgs = await core.stormlist(q)
|
|
417
|
+
self.stormIsInPrint('1/5', msgs)
|
|
418
|
+
self.stormNotInPrint('2/5', msgs)
|
|
419
|
+
self.stormIsInErr('function inner - Generator control statement "stop" used outside of a generator '
|
|
420
|
+
'function.',
|
|
421
|
+
msgs)
|
|
422
|
+
|
|
423
|
+
# The function exception raised can be caught.
|
|
424
|
+
q = '''
|
|
425
|
+
function inner(v) {
|
|
426
|
+
if ( $v = 2 ) {
|
|
427
|
+
stop
|
|
428
|
+
}
|
|
429
|
+
return ( $v )
|
|
430
|
+
}
|
|
431
|
+
function outer(n) {
|
|
432
|
+
for $i in $lib.range($n) {
|
|
433
|
+
emit $inner($i)
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
$N = (5)
|
|
437
|
+
try {
|
|
438
|
+
for $valu in $outer($N) {
|
|
439
|
+
$lib.print(`{$valu}/{$N}`)
|
|
440
|
+
}
|
|
441
|
+
} catch StormRuntimeError as err {
|
|
442
|
+
$lib.print(`caught: {$err.mesg}`)
|
|
443
|
+
}
|
|
444
|
+
'''
|
|
445
|
+
msgs = await core.stormlist(q)
|
|
446
|
+
self.stormIsInPrint('1/5', msgs)
|
|
447
|
+
self.stormNotInPrint('2/5', msgs)
|
|
448
|
+
self.stormIsInPrint('caught: function inner - Generator control statement "stop" used outside of a'
|
|
449
|
+
' generator function.',
|
|
450
|
+
msgs)
|
|
451
|
+
|
|
452
|
+
# Outside a function, StopStorm is caught and converted into a StormRuntimeError for the message stream.
|
|
453
|
+
# Since this is tearing down the runtime, it cannot be caught.
|
|
454
|
+
q = '''
|
|
455
|
+
$N = (5)
|
|
456
|
+
for $j in $lib.range($N) {
|
|
457
|
+
if ($j = 2) {
|
|
458
|
+
stop
|
|
459
|
+
}
|
|
460
|
+
$lib.print(`{$j}/{$N}`)
|
|
461
|
+
}
|
|
462
|
+
'''
|
|
463
|
+
msgs = await core.stormlist(q)
|
|
464
|
+
self.stormIsInPrint('1/5', msgs)
|
|
465
|
+
self.stormNotInPrint('2/5', msgs)
|
|
466
|
+
self.stormIsInErr('Generator control statement "stop" used outside of a generator function.',
|
|
467
|
+
msgs)
|
|
468
|
+
errname = [m[1][0] for m in msgs if m[0] == 'err'][0]
|
|
469
|
+
self.eq(errname, 'StormRuntimeError')
|
|
343
470
|
|
|
344
|
-
|
|
471
|
+
q = '''
|
|
472
|
+
$N = (5)
|
|
473
|
+
try {
|
|
474
|
+
for $j in $lib.range($N) {
|
|
475
|
+
if ($j = 2) {
|
|
476
|
+
stop
|
|
477
|
+
}
|
|
478
|
+
$lib.print(`{$j}/{$N}`)
|
|
479
|
+
}
|
|
480
|
+
} catch StormRuntimeError as err {
|
|
481
|
+
$lib.print(`caught: {$err.mesg}`)
|
|
482
|
+
}
|
|
483
|
+
'''
|
|
484
|
+
msgs = await core.stormlist(q)
|
|
485
|
+
self.stormIsInPrint('1/5', msgs)
|
|
486
|
+
self.stormNotInPrint('2/5', msgs)
|
|
487
|
+
self.stormNotInPrint('caught:', msgs)
|
|
488
|
+
self.stormIsInErr('Generator control statement "stop" used outside of a generator function.',
|
|
489
|
+
msgs)
|
|
490
|
+
|
|
491
|
+
# Mixing a Loop control flow statement in an emitter to stop its processing
|
|
492
|
+
# will be converted into a catchable StormRuntimeError
|
|
493
|
+
q = '''
|
|
494
|
+
function inner(n) {
|
|
495
|
+
emit $n
|
|
496
|
+
$n = ( $n + 1 )
|
|
497
|
+
emit $n
|
|
498
|
+
$n = ( $n + 1 )
|
|
499
|
+
if ( $n >= 2 ) {
|
|
500
|
+
break
|
|
501
|
+
}
|
|
502
|
+
emit $n
|
|
503
|
+
}
|
|
504
|
+
$N = (0)
|
|
505
|
+
try {
|
|
506
|
+
for $valu in $inner($N) {
|
|
507
|
+
$lib.print(`got {$valu}`)
|
|
508
|
+
}
|
|
509
|
+
} catch StormRuntimeError as err {
|
|
510
|
+
$lib.print(`caught: {$err.mesg}`)
|
|
511
|
+
}
|
|
512
|
+
'''
|
|
513
|
+
msgs = await core.stormlist(q)
|
|
514
|
+
self.stormIsInPrint('got 1', msgs)
|
|
515
|
+
self.stormNotInPrint('got 2', msgs)
|
|
516
|
+
self.stormIsInPrint('caught: function inner - Loop control statement "break" used outside of a loop.',
|
|
517
|
+
msgs)
|
|
345
518
|
|
|
346
519
|
async def test_lib_storm_intersect(self):
|
|
347
520
|
async with self.getTestCore() as core:
|
|
@@ -580,7 +753,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
580
753
|
return((0))
|
|
581
754
|
}
|
|
582
755
|
|
|
583
|
-
$alerts =
|
|
756
|
+
$alerts = ()
|
|
584
757
|
{ $alerts.append($node.repr()) }
|
|
585
758
|
|
|
586
759
|
$bool = $stuff($alerts)
|
|
@@ -909,7 +1082,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
909
1082
|
|
|
910
1083
|
opts = {'view': view}
|
|
911
1084
|
self.len(0, await core.callStorm('''
|
|
912
|
-
$list =
|
|
1085
|
+
$list = ()
|
|
913
1086
|
$layr = $lib.view.get().layers.0
|
|
914
1087
|
for $item in $layr.getStorNodes() {
|
|
915
1088
|
$list.append($item)
|
|
@@ -922,7 +1095,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
922
1095
|
await core.callStorm('inet:ipv4=11.22.33.44 [ +(blahverb)> { inet:asn=99 } ]', opts=opts)
|
|
923
1096
|
|
|
924
1097
|
sodes = await core.callStorm('''
|
|
925
|
-
$list =
|
|
1098
|
+
$list = ()
|
|
926
1099
|
$layr = $lib.view.get().layers.0
|
|
927
1100
|
for $item in $layr.getStorNodes() {
|
|
928
1101
|
$list.append($item)
|
|
@@ -931,7 +1104,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
931
1104
|
self.len(2, sodes)
|
|
932
1105
|
|
|
933
1106
|
ipv4 = await core.callStorm('''
|
|
934
|
-
$list =
|
|
1107
|
+
$list = ()
|
|
935
1108
|
$layr = $lib.view.get().layers.0
|
|
936
1109
|
for ($buid, $sode) in $layr.getStorNodes() {
|
|
937
1110
|
yield $buid
|
|
@@ -1073,7 +1246,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
1073
1246
|
self.len(0, await core.nodes('diff', opts=opts))
|
|
1074
1247
|
|
|
1075
1248
|
self.len(0, await core.callStorm('''
|
|
1076
|
-
$list =
|
|
1249
|
+
$list = ()
|
|
1077
1250
|
for ($buid, $sode) in $lib.view.get().layers.0.getStorNodes() {
|
|
1078
1251
|
$list.append($buid)
|
|
1079
1252
|
}
|
|
@@ -1354,9 +1527,9 @@ class StormTest(s_t_utils.SynTest):
|
|
|
1354
1527
|
'''))
|
|
1355
1528
|
|
|
1356
1529
|
self.eq(('foo', 'bar', 'baz'), await core.callStorm('''
|
|
1357
|
-
return(
|
|
1358
|
-
/* hehe */ foo /* hehe */ , /* hehe */ bar /* hehe */ , /* hehe */ baz /* hehe */
|
|
1359
|
-
))
|
|
1530
|
+
return(([ // do foo thing
|
|
1531
|
+
/* hehe */ "foo" /* hehe */ , /* hehe */ "bar" /* hehe */ , /* hehe */ "baz" /* hehe */
|
|
1532
|
+
]))
|
|
1360
1533
|
'''))
|
|
1361
1534
|
|
|
1362
1535
|
# surrogate escapes are allowed
|
|
@@ -2569,6 +2742,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
2569
2742
|
class PkgHandler(s_httpapi.Handler):
|
|
2570
2743
|
|
|
2571
2744
|
async def get(self, name):
|
|
2745
|
+
assert self.request.headers.get('X-Synapse-Version') == s_version.verstring
|
|
2572
2746
|
|
|
2573
2747
|
if name == 'notok':
|
|
2574
2748
|
self.sendRestErr('FooBar', 'baz faz')
|
|
@@ -2578,6 +2752,8 @@ class StormTest(s_t_utils.SynTest):
|
|
|
2578
2752
|
|
|
2579
2753
|
class PkgHandlerRaw(s_httpapi.Handler):
|
|
2580
2754
|
async def get(self, name):
|
|
2755
|
+
assert self.request.headers.get('X-Synapse-Version') == s_version.verstring
|
|
2756
|
+
|
|
2581
2757
|
self.set_header('Content-Type', 'application/json')
|
|
2582
2758
|
return self.write(pkg)
|
|
2583
2759
|
|
|
@@ -3254,7 +3430,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
3254
3430
|
self.eq(nodes[4].ndef, ('inet:ipv4', 0x01020304))
|
|
3255
3431
|
|
|
3256
3432
|
# Queries can be a heavy list
|
|
3257
|
-
q = '$list =
|
|
3433
|
+
q = '$list = ([${ -> * }, ${ <- * }, ${ -> edge:refs:n2 :n1 -> * }]) inet:ipv4=1.2.3.4 | tee --join $list'
|
|
3258
3434
|
nodes = await core.nodes(q)
|
|
3259
3435
|
self.len(5, nodes)
|
|
3260
3436
|
self.eq(nodes[0].ndef, ('inet:asn', 0))
|
|
@@ -3264,22 +3440,22 @@ class StormTest(s_t_utils.SynTest):
|
|
|
3264
3440
|
self.eq(nodes[4].ndef, ('inet:ipv4', 0x01020304))
|
|
3265
3441
|
|
|
3266
3442
|
# A empty list of queries still works as an nop
|
|
3267
|
-
q = '$list =
|
|
3443
|
+
q = '$list = () | tee $list'
|
|
3268
3444
|
msgs = await core.stormlist(q)
|
|
3269
3445
|
self.len(2, msgs)
|
|
3270
3446
|
self.eq(('init', 'fini'), [m[0] for m in msgs])
|
|
3271
3447
|
|
|
3272
|
-
q = 'inet:ipv4=1.2.3.4 $list =
|
|
3448
|
+
q = 'inet:ipv4=1.2.3.4 $list = () | tee --join $list'
|
|
3273
3449
|
msgs = await core.stormlist(q)
|
|
3274
3450
|
self.len(3, msgs)
|
|
3275
3451
|
self.eq(('init', 'node', 'fini'), [m[0] for m in msgs])
|
|
3276
3452
|
|
|
3277
|
-
q = '$list =
|
|
3453
|
+
q = '$list = () | tee --parallel $list'
|
|
3278
3454
|
msgs = await core.stormlist(q)
|
|
3279
3455
|
self.len(2, msgs)
|
|
3280
3456
|
self.eq(('init', 'fini'), [m[0] for m in msgs])
|
|
3281
3457
|
|
|
3282
|
-
q = 'inet:ipv4=1.2.3.4 $list =
|
|
3458
|
+
q = 'inet:ipv4=1.2.3.4 $list = () | tee --parallel --join $list'
|
|
3283
3459
|
msgs = await core.stormlist(q)
|
|
3284
3460
|
self.len(3, msgs)
|
|
3285
3461
|
self.eq(('init', 'node', 'fini'), [m[0] for m in msgs])
|
|
@@ -3533,7 +3709,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
3533
3709
|
fork = await core.callStorm('return( $lib.view.get().fork().iden )')
|
|
3534
3710
|
|
|
3535
3711
|
q = '''
|
|
3536
|
-
$nodes =
|
|
3712
|
+
$nodes = ()
|
|
3537
3713
|
view.exec $view { inet:ipv4=1.2.3.4 $nodes.append($node) } |
|
|
3538
3714
|
for $n in $nodes {
|
|
3539
3715
|
yield $n
|
|
@@ -3543,7 +3719,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
3543
3719
|
self.stormIsInErr('Node is not from the current view.', msgs)
|
|
3544
3720
|
|
|
3545
3721
|
q = '''
|
|
3546
|
-
$nodes =
|
|
3722
|
+
$nodes = ()
|
|
3547
3723
|
view.exec $view { for $x in ${ inet:ipv4=1.2.3.4 } { $nodes.append($x) } } |
|
|
3548
3724
|
for $n in $nodes {
|
|
3549
3725
|
yield $n
|
|
@@ -3558,7 +3734,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
3558
3734
|
|
|
3559
3735
|
# Nodes lifted from another view and referred to by iden() works
|
|
3560
3736
|
q = '''
|
|
3561
|
-
$nodes =
|
|
3737
|
+
$nodes = ()
|
|
3562
3738
|
view.exec $view { inet:ipv4=1.2.3.4 $nodes.append($node) } |
|
|
3563
3739
|
for $n in $nodes {
|
|
3564
3740
|
yield $n.iden()
|
|
@@ -3568,7 +3744,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
3568
3744
|
self.len(1, nodes)
|
|
3569
3745
|
|
|
3570
3746
|
q = '''
|
|
3571
|
-
$nodes =
|
|
3747
|
+
$nodes = ()
|
|
3572
3748
|
view.exec $view { for $x in ${ inet:ipv4=1.2.3.4 } { $nodes.append($x) } } |
|
|
3573
3749
|
for $n in $nodes {
|
|
3574
3750
|
yield $n.iden()
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import pickle
|
|
2
|
+
|
|
3
|
+
import synapse.lib.stormctrl as s_stormctrl
|
|
4
|
+
|
|
5
|
+
import synapse.tests.utils as s_t_utils
|
|
6
|
+
|
|
7
|
+
class StormctrlTest(s_t_utils.SynTest):
|
|
8
|
+
def test_basic(self):
|
|
9
|
+
|
|
10
|
+
# Classes inherit as expected
|
|
11
|
+
self.isinstance(s_stormctrl.StormReturn(), s_stormctrl.StormCtrlFlow)
|
|
12
|
+
self.isinstance(s_stormctrl.StormExit(), s_stormctrl.StormCtrlFlow)
|
|
13
|
+
self.isinstance(s_stormctrl.StormBreak(), s_stormctrl.StormCtrlFlow)
|
|
14
|
+
self.isinstance(s_stormctrl.StormContinue(), s_stormctrl.StormCtrlFlow)
|
|
15
|
+
self.isinstance(s_stormctrl.StormStop(), s_stormctrl.StormCtrlFlow)
|
|
16
|
+
|
|
17
|
+
# Subtypes are noted as well
|
|
18
|
+
self.isinstance(s_stormctrl.StormBreak(), s_stormctrl.StormLoopCtrl)
|
|
19
|
+
self.isinstance(s_stormctrl.StormContinue(), s_stormctrl.StormLoopCtrl)
|
|
20
|
+
self.isinstance(s_stormctrl.StormStop(), s_stormctrl.StormGenrCtrl)
|
|
21
|
+
|
|
22
|
+
# control flow and exist constructs inherit from the SynErrMixin
|
|
23
|
+
# return does not to keep it thin. it is used often.
|
|
24
|
+
self.isinstance(s_stormctrl.StormExit(), s_stormctrl._SynErrMixin)
|
|
25
|
+
self.isinstance(s_stormctrl.StormBreak(), s_stormctrl._SynErrMixin)
|
|
26
|
+
self.isinstance(s_stormctrl.StormContinue(), s_stormctrl._SynErrMixin)
|
|
27
|
+
self.isinstance(s_stormctrl.StormStop(), s_stormctrl._SynErrMixin)
|
|
28
|
+
self.false(isinstance(s_stormctrl.StormReturn(), s_stormctrl._SynErrMixin))
|
|
29
|
+
|
|
30
|
+
# The base class cannot be used on its own.
|
|
31
|
+
with self.raises(NotImplementedError):
|
|
32
|
+
s_stormctrl.StormCtrlFlow()
|
|
33
|
+
|
|
34
|
+
# The _SynErrMixin classes have several methods that let us treat
|
|
35
|
+
# instance of them like SynErr exceptions.
|
|
36
|
+
e = s_stormctrl.StormExit(mesg='words', foo='bar')
|
|
37
|
+
self.eq(e.get('foo'), 'bar')
|
|
38
|
+
self.eq(e.items(), {'mesg': 'words', 'foo': 'bar'})
|
|
39
|
+
self.eq("StormExit: foo='bar' mesg='words'", str(e))
|
|
40
|
+
e.set('hehe', 1234)
|
|
41
|
+
e.set('foo', 'words')
|
|
42
|
+
self.eq("StormExit: foo='words' hehe=1234 mesg='words'", str(e))
|
|
43
|
+
|
|
44
|
+
e.setdefault('defv', 1)
|
|
45
|
+
self.eq("StormExit: defv=1 foo='words' hehe=1234 mesg='words'", str(e))
|
|
46
|
+
|
|
47
|
+
e.setdefault('defv', 2)
|
|
48
|
+
self.eq("StormExit: defv=1 foo='words' hehe=1234 mesg='words'", str(e))
|
|
49
|
+
|
|
50
|
+
e.update({'foo': 'newwords', 'bar': 'baz'})
|
|
51
|
+
self.eq("StormExit: bar='baz' defv=1 foo='newwords' hehe=1234 mesg='words'", str(e))
|
|
52
|
+
|
|
53
|
+
# But it does not have an errname property
|
|
54
|
+
self.false(hasattr(e, 'errname'))
|
|
55
|
+
|
|
56
|
+
# StormReturn is used to move objects around.
|
|
57
|
+
e = s_stormctrl.StormReturn('weee')
|
|
58
|
+
self.eq(e.item, 'weee')
|
|
59
|
+
|
|
60
|
+
async def test_pickled_stormctrlflow(self):
|
|
61
|
+
e = s_stormctrl.StormExit(mesg='words', foo='bar')
|
|
62
|
+
buf = pickle.dumps(e)
|
|
63
|
+
new_e = pickle.loads(buf)
|
|
64
|
+
self.eq(new_e.get('foo'), 'bar')
|
|
65
|
+
self.eq("StormExit: foo='bar' mesg='words'", str(new_e))
|
|
@@ -518,11 +518,11 @@ class StormHttpTest(s_test.SynTest):
|
|
|
518
518
|
self.eq(data.get('body'), 'MTIzNA==')
|
|
519
519
|
|
|
520
520
|
q = '''
|
|
521
|
-
$fields
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
)
|
|
521
|
+
$fields=([
|
|
522
|
+
{"name": "foo", "value": "bar"},
|
|
523
|
+
{"name": "foo", "value": "bar2"},
|
|
524
|
+
{"name": "baz", "value": "cool"}
|
|
525
|
+
])
|
|
526
526
|
$resp = $lib.inet.http.post($url, fields=$fields, ssl_verify=$lib.false)
|
|
527
527
|
return ( $resp.json() )
|
|
528
528
|
'''
|
|
@@ -320,6 +320,17 @@ class StormLibAuthTest(s_test.SynTest):
|
|
|
320
320
|
self.stormIsInPrint('Controls access to add a new view including forks.', msgs)
|
|
321
321
|
self.stormIsInPrint('default: false', msgs)
|
|
322
322
|
|
|
323
|
+
msgs = await core.stormlist('auth.perms.list --find macro.')
|
|
324
|
+
self.stormIsInPrint('storm.macro.add', msgs)
|
|
325
|
+
self.stormIsInPrint('storm.macro.admin', msgs)
|
|
326
|
+
self.stormIsInPrint('storm.macro.edit', msgs)
|
|
327
|
+
self.stormNotInPrint('node.add.<form>', msgs)
|
|
328
|
+
|
|
329
|
+
msgs = await core.stormlist('auth.perms.list --find url')
|
|
330
|
+
self.stormIsInPrint('storm.lib.telepath.open.<scheme>', msgs)
|
|
331
|
+
self.stormIsInPrint('Controls the ability to open a telepath URL with a specific URI scheme.', msgs)
|
|
332
|
+
self.stormNotInPrint('node.add.<form>', msgs)
|
|
333
|
+
|
|
323
334
|
async def test_stormlib_auth_default_allow(self):
|
|
324
335
|
async with self.getTestCore() as core:
|
|
325
336
|
|
|
@@ -427,7 +438,7 @@ class StormLibAuthTest(s_test.SynTest):
|
|
|
427
438
|
|
|
428
439
|
await core.callStorm('$lib.user.json.set(hi, hehe, prop=foo)')
|
|
429
440
|
items = await core.callStorm('''
|
|
430
|
-
$list =
|
|
441
|
+
$list = ()
|
|
431
442
|
for $item in $lib.user.json.iter() { $list.append($item) }
|
|
432
443
|
return($list)
|
|
433
444
|
''')
|
|
@@ -437,7 +448,7 @@ class StormLibAuthTest(s_test.SynTest):
|
|
|
437
448
|
))
|
|
438
449
|
|
|
439
450
|
items = await core.callStorm('''
|
|
440
|
-
$list =
|
|
451
|
+
$list = ()
|
|
441
452
|
for $item in $lib.user.json.iter(path=bye) { $list.append($item) }
|
|
442
453
|
return($list)
|
|
443
454
|
''')
|
|
@@ -731,7 +742,7 @@ class StormLibAuthTest(s_test.SynTest):
|
|
|
731
742
|
'''))
|
|
732
743
|
|
|
733
744
|
# user roles can be set in bulk
|
|
734
|
-
roles = await core.callStorm('''$roles
|
|
745
|
+
roles = await core.callStorm('''$roles=()
|
|
735
746
|
$role=$lib.auth.roles.byname(admins) $roles.append($role.iden)
|
|
736
747
|
$role=$lib.auth.roles.byname(all) $roles.append($role.iden)
|
|
737
748
|
$lib.auth.users.byname(visi).setRoles($roles)
|
|
@@ -791,7 +802,7 @@ class StormLibAuthTest(s_test.SynTest):
|
|
|
791
802
|
visi = await core.callStorm('''
|
|
792
803
|
$rule = $lib.auth.ruleFromText(hehe.haha)
|
|
793
804
|
$visi = $lib.auth.users.byname(visi)
|
|
794
|
-
$visi.setRules(
|
|
805
|
+
$visi.setRules(([$rule]))
|
|
795
806
|
return($visi)
|
|
796
807
|
''')
|
|
797
808
|
self.eq(((True, ('hehe', 'haha')),), visi['rules'])
|
|
@@ -834,7 +845,7 @@ class StormLibAuthTest(s_test.SynTest):
|
|
|
834
845
|
ninjas = await core.callStorm('''
|
|
835
846
|
$rule = $lib.auth.ruleFromText(hehe.haha)
|
|
836
847
|
$ninjas = $lib.auth.roles.byname(ninjas)
|
|
837
|
-
$ninjas.setRules(
|
|
848
|
+
$ninjas.setRules(([$rule]))
|
|
838
849
|
return($ninjas)
|
|
839
850
|
''')
|
|
840
851
|
self.eq(((True, ('hehe', 'haha')),), ninjas['rules'])
|
|
@@ -1003,6 +1014,21 @@ class StormLibAuthTest(s_test.SynTest):
|
|
|
1003
1014
|
with self.raises(s_exc.DupIden):
|
|
1004
1015
|
await core.callStorm('$lib.auth.roles.add(walkers, iden=$iden)', opts=opts)
|
|
1005
1016
|
|
|
1017
|
+
# The role & user.authgates local is a passthrough to the getRoleDef & getUserDef
|
|
1018
|
+
# results, which are a pack()'d structure. Modifying the results of that structure
|
|
1019
|
+
# does not persist.
|
|
1020
|
+
q = '$u = $lib.auth.users.byname(root) $u.authgates.newp = ({}) return ($u)'
|
|
1021
|
+
udef = await core.callStorm(q)
|
|
1022
|
+
self.notin('newp', udef.get('authgates'))
|
|
1023
|
+
q = '$u = $lib.auth.users.byname(root) return ( $lib.dict.has($u.authgates, newp) )'
|
|
1024
|
+
self.false(await core.callStorm(q))
|
|
1025
|
+
|
|
1026
|
+
q = '$r = $lib.auth.roles.byname(all) $r.authgates.newp = ({}) return ($r)'
|
|
1027
|
+
rdef = await core.callStorm(q)
|
|
1028
|
+
self.notin('newp', rdef.get('authgates'))
|
|
1029
|
+
q = '$r = $lib.auth.roles.byname(all) return ( $lib.dict.has($r.authgates, newp) )'
|
|
1030
|
+
self.false(await core.callStorm(q))
|
|
1031
|
+
|
|
1006
1032
|
async def test_stormlib_auth_gateadmin(self):
|
|
1007
1033
|
|
|
1008
1034
|
async with self.getTestCore() as core:
|
|
@@ -138,19 +138,51 @@ class StormlibCacheTest(s_test.SynTest):
|
|
|
138
138
|
self.none(await core.callStorm('return($lib.cache.fixed("if (0) { return(yup) }").get(foo))'))
|
|
139
139
|
|
|
140
140
|
## control flow exceptions don't propagate up
|
|
141
|
-
|
|
141
|
+
msgs = await core.stormlist('''
|
|
142
142
|
$cache = $lib.cache.fixed( ${ if ($cache_key < (2)) { return (`key={$cache_key}`) } else { break } } )
|
|
143
143
|
|
|
144
|
-
$
|
|
144
|
+
for $i in $lib.range(4) {
|
|
145
|
+
$lib.print(`{$cache.get($i)}`)
|
|
146
|
+
}
|
|
147
|
+
''')
|
|
148
|
+
self.stormIsInPrint('key=1', msgs)
|
|
149
|
+
self.stormNotInPrint('key=2', msgs)
|
|
150
|
+
self.stormIsInErr('Storm control flow "break" not allowed in cache callbacks.', msgs)
|
|
151
|
+
|
|
152
|
+
msgs = await core.stormlist('''
|
|
153
|
+
$cache = $lib.cache.fixed( ${ if ($cache_key < (2)) { return (`key={$cache_key}`) } else { continue } } )
|
|
145
154
|
|
|
146
155
|
for $i in $lib.range(4) {
|
|
147
|
-
$
|
|
156
|
+
$lib.print(`{$cache.get($i)}`)
|
|
148
157
|
}
|
|
158
|
+
''')
|
|
159
|
+
self.stormIsInPrint('key=1', msgs)
|
|
160
|
+
self.stormNotInPrint('key=2', msgs)
|
|
161
|
+
self.stormIsInErr('Storm control flow "continue" not allowed in cache callbacks.', msgs)
|
|
149
162
|
|
|
150
|
-
|
|
151
|
-
return($
|
|
163
|
+
msgs = await core.stormlist('''
|
|
164
|
+
$cache = $lib.cache.fixed( ${ if ($cache_key < (2)) { return (`key={$cache_key}`) } else { stop } } )
|
|
165
|
+
|
|
166
|
+
for $i in $lib.range(4) {
|
|
167
|
+
$lib.print(`{$cache.get($i)}`)
|
|
168
|
+
}
|
|
152
169
|
''')
|
|
153
|
-
self.
|
|
170
|
+
self.stormIsInPrint('key=1', msgs)
|
|
171
|
+
self.stormNotInPrint('key=2', msgs)
|
|
172
|
+
self.stormIsInErr('Storm control flow "stop" not allowed in cache callbacks.', msgs)
|
|
173
|
+
|
|
174
|
+
msgs = await core.stormlist('''
|
|
175
|
+
$cache = $lib.cache.fixed(
|
|
176
|
+
${ if ($cache_key < (2)) { return (`key={$cache_key}`) } else { $lib.exit(mesg=newp) } }
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
for $i in $lib.range(4) {
|
|
180
|
+
$lib.print(`{$cache.get($i)}`)
|
|
181
|
+
}
|
|
182
|
+
''')
|
|
183
|
+
self.stormIsInPrint('key=1', msgs)
|
|
184
|
+
self.stormNotInPrint('key=2', msgs)
|
|
185
|
+
self.stormIsInErr('Storm control flow "StormExit" not allowed in cache callbacks.', msgs)
|
|
154
186
|
|
|
155
187
|
## control flow scoped inside the callback
|
|
156
188
|
rets = await core.callStorm("""
|
|
@@ -14,6 +14,9 @@ class StormCellTest(s_test.SynTest):
|
|
|
14
14
|
|
|
15
15
|
async with self.getTestCore() as core:
|
|
16
16
|
|
|
17
|
+
ret = await core.callStorm('return ( $lib.cell.iden )')
|
|
18
|
+
self.eq(ret, core.getCellIden())
|
|
19
|
+
|
|
17
20
|
ret = await core.callStorm('return ( $lib.cell.getCellInfo() )')
|
|
18
21
|
self.eq(ret, await core.getCellInfo())
|
|
19
22
|
|
|
@@ -122,7 +122,7 @@ class StormtypesModelextTest(s_test.SynTest):
|
|
|
122
122
|
self.none(core.model.edge(('inet:user', '_copies', None)))
|
|
123
123
|
|
|
124
124
|
# Underscores can exist in extended names but only at specific locations
|
|
125
|
-
q = '''$l
|
|
125
|
+
q = '''$l =(['str', {}]) $d=({"doc": "Foo"})
|
|
126
126
|
$lib.model.ext.addFormProp('test:str', '_test:_myprop', $l, $d)
|
|
127
127
|
'''
|
|
128
128
|
self.none(await core.callStorm(q))
|
|
@@ -144,13 +144,13 @@ class StormtypesModelextTest(s_test.SynTest):
|
|
|
144
144
|
await core.callStorm(q)
|
|
145
145
|
|
|
146
146
|
with self.raises(s_exc.BadPropDef):
|
|
147
|
-
q = '''$l
|
|
147
|
+
q = '''$l =(['str', {}]) $d=({"doc": "Foo"})
|
|
148
148
|
$lib.model.ext.addFormProp('test:str', '_test:_my^prop', $l, $d)
|
|
149
149
|
'''
|
|
150
150
|
await core.callStorm(q)
|
|
151
151
|
|
|
152
152
|
with self.raises(s_exc.BadPropDef):
|
|
153
|
-
q = '''$l
|
|
153
|
+
q = '''$l =(['str', {}]) $d=({"doc": "Foo"})
|
|
154
154
|
$lib.model.ext.addFormProp('test:str', '_test::_myprop', $l, $d)
|
|
155
155
|
'''
|
|
156
156
|
await core.callStorm(q)
|
|
@@ -24,7 +24,7 @@ class StormScrapeTest(s_test.SynTest):
|
|
|
24
24
|
The helper does require a named match for valu this is extracted.
|
|
25
25
|
*/
|
|
26
26
|
function scrape(text) {
|
|
27
|
-
$ret =
|
|
27
|
+
$ret = ()
|
|
28
28
|
for ($valu, $info) in $lib.scrape.genMatches($text, $modRe) {
|
|
29
29
|
$ret.append(($modForm, $valu, $info))
|
|
30
30
|
}
|
|
@@ -49,7 +49,7 @@ class StormScrapeTest(s_test.SynTest):
|
|
|
49
49
|
Example of an storm module that scraps and matches on hzzp enfanged urls.
|
|
50
50
|
*/
|
|
51
51
|
function scrape(text) {
|
|
52
|
-
$ret =
|
|
52
|
+
$ret = ()
|
|
53
53
|
for ($valu, $info) in $lib.scrape.genMatches($text, $modRe, fangs=$modFangs) {
|
|
54
54
|
$ret.append(($modForm, $valu, $info))
|
|
55
55
|
}
|
|
@@ -105,7 +105,7 @@ class StormScrapeTest(s_test.SynTest):
|
|
|
105
105
|
self.stormIsInPrint('inet:url=https://giggles.com/mallory.html', msgs)
|
|
106
106
|
self.stormIsInPrint("'match': 'hzzps[:]\\\\giggles.com/mallory.html'", msgs)
|
|
107
107
|
|
|
108
|
-
cq = '''$ret
|
|
108
|
+
cq = '''$ret=()
|
|
109
109
|
for ($form, $valu) in $lib.scrape.ndefs($text) {
|
|
110
110
|
$ret.append(($form, $valu))
|
|
111
111
|
}
|
|
@@ -180,7 +180,7 @@ class StormScrapeTest(s_test.SynTest):
|
|
|
180
180
|
self.eq(result, {'inet:ipv4=16909060': 1, 'inet:fqdn=foo.bar': 1, 'inet:fqdn=woot.com': 1})
|
|
181
181
|
|
|
182
182
|
# $lib.scrape.context() - this is currently just wrapping s_scrape.contextscrape
|
|
183
|
-
query = '''$list =
|
|
183
|
+
query = '''$list = () for $info in $lib.scrape.context($text)
|
|
184
184
|
{ $list.append($info) }
|
|
185
185
|
fini { return ( $list ) }
|
|
186
186
|
'''
|