synapse 2.164.0__py311-none-any.whl → 2.166.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/axon.py +3 -3
- synapse/cmds/cortex.py +1 -6
- synapse/common.py +7 -1
- synapse/cortex.py +145 -192
- synapse/datamodel.py +36 -1
- synapse/lib/agenda.py +87 -97
- synapse/lib/aha.py +51 -0
- synapse/lib/ast.py +22 -23
- synapse/lib/base.py +0 -6
- synapse/lib/boss.py +3 -0
- synapse/lib/cell.py +70 -39
- synapse/lib/certdir.py +9 -0
- synapse/lib/hiveauth.py +65 -12
- synapse/lib/httpapi.py +1 -0
- synapse/lib/modelrev.py +121 -33
- synapse/lib/modules.py +1 -0
- synapse/lib/nexus.py +64 -26
- synapse/lib/parser.py +2 -0
- synapse/lib/schemas.py +14 -0
- synapse/lib/snap.py +50 -4
- synapse/lib/storm.lark +4 -3
- synapse/lib/storm.py +96 -22
- synapse/lib/storm_format.py +1 -0
- synapse/lib/stormlib/aha.py +7 -1
- synapse/lib/stormlib/auth.py +13 -5
- synapse/lib/stormlib/cache.py +202 -0
- synapse/lib/stormlib/cortex.py +147 -8
- synapse/lib/stormlib/gen.py +53 -6
- synapse/lib/stormlib/math.py +1 -1
- synapse/lib/stormlib/model.py +11 -1
- synapse/lib/stormlib/spooled.py +109 -0
- synapse/lib/stormlib/vault.py +1 -1
- synapse/lib/stormtypes.py +113 -17
- synapse/lib/trigger.py +36 -47
- synapse/lib/types.py +29 -2
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +80 -53
- synapse/models/economic.py +174 -5
- synapse/models/files.py +2 -0
- synapse/models/inet.py +77 -2
- synapse/models/infotech.py +12 -12
- synapse/models/orgs.py +72 -21
- synapse/models/person.py +40 -11
- synapse/models/risk.py +78 -24
- synapse/models/science.py +102 -0
- synapse/telepath.py +117 -35
- synapse/tests/test_cortex.py +84 -158
- synapse/tests/test_datamodel.py +22 -0
- synapse/tests/test_lib_agenda.py +52 -96
- synapse/tests/test_lib_aha.py +126 -4
- synapse/tests/test_lib_ast.py +412 -6
- synapse/tests/test_lib_cell.py +24 -8
- synapse/tests/test_lib_certdir.py +32 -0
- synapse/tests/test_lib_grammar.py +9 -1
- synapse/tests/test_lib_httpapi.py +0 -1
- synapse/tests/test_lib_jupyter.py +0 -1
- synapse/tests/test_lib_modelrev.py +41 -0
- synapse/tests/test_lib_nexus.py +38 -0
- synapse/tests/test_lib_storm.py +95 -5
- synapse/tests/test_lib_stormlib_cache.py +272 -0
- synapse/tests/test_lib_stormlib_cortex.py +71 -0
- synapse/tests/test_lib_stormlib_gen.py +37 -2
- synapse/tests/test_lib_stormlib_model.py +2 -0
- synapse/tests/test_lib_stormlib_spooled.py +190 -0
- synapse/tests/test_lib_stormlib_vault.py +12 -3
- synapse/tests/test_lib_stormsvc.py +0 -10
- synapse/tests/test_lib_stormtypes.py +60 -8
- synapse/tests/test_lib_trigger.py +20 -2
- synapse/tests/test_lib_types.py +17 -1
- synapse/tests/test_model_economic.py +114 -0
- synapse/tests/test_model_files.py +2 -0
- synapse/tests/test_model_inet.py +73 -1
- synapse/tests/test_model_infotech.py +2 -2
- synapse/tests/test_model_orgs.py +10 -1
- synapse/tests/test_model_risk.py +30 -2
- synapse/tests/test_model_science.py +59 -0
- synapse/tests/test_model_syn.py +0 -1
- synapse/tests/test_telepath.py +30 -7
- synapse/tests/test_tools_modrole.py +81 -0
- synapse/tests/test_tools_moduser.py +105 -0
- synapse/tools/modrole.py +59 -7
- synapse/tools/moduser.py +78 -10
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/METADATA +2 -2
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/RECORD +87 -83
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/WHEEL +1 -1
- synapse/lib/provenance.py +0 -111
- synapse/tests/test_lib_provenance.py +0 -37
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/LICENSE +0 -0
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/top_level.txt +0 -0
synapse/tests/test_lib_cell.py
CHANGED
|
@@ -25,6 +25,7 @@ import synapse.lib.base as s_base
|
|
|
25
25
|
import synapse.lib.cell as s_cell
|
|
26
26
|
import synapse.lib.coro as s_coro
|
|
27
27
|
import synapse.lib.link as s_link
|
|
28
|
+
import synapse.lib.nexus as s_nexus
|
|
28
29
|
import synapse.lib.certdir as s_certdir
|
|
29
30
|
import synapse.lib.msgpack as s_msgpack
|
|
30
31
|
import synapse.lib.version as s_version
|
|
@@ -2088,15 +2089,23 @@ class CellTest(s_t_utils.SynTest):
|
|
|
2088
2089
|
return _ntuple_diskusage(100, 96, 4)
|
|
2089
2090
|
|
|
2090
2091
|
revt = asyncio.Event()
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2092
|
+
addWriteHold = s_nexus.NexsRoot.addWriteHold
|
|
2093
|
+
delWriteHold = s_nexus.NexsRoot.delWriteHold
|
|
2094
|
+
async def wrapAddWriteHold(root, reason):
|
|
2095
|
+
retn = await addWriteHold(root, reason)
|
|
2094
2096
|
revt.set()
|
|
2097
|
+
return retn
|
|
2098
|
+
|
|
2099
|
+
async def wrapDelWriteHold(root, reason):
|
|
2100
|
+
retn = await delWriteHold(root, reason)
|
|
2101
|
+
revt.set()
|
|
2102
|
+
return retn
|
|
2095
2103
|
|
|
2096
2104
|
errmsg = 'Insufficient free space on disk.'
|
|
2097
2105
|
|
|
2098
2106
|
with mock.patch.object(s_cell.Cell, 'FREE_SPACE_CHECK_FREQ', 0.1), \
|
|
2099
|
-
mock.patch.object(
|
|
2107
|
+
mock.patch.object(s_nexus.NexsRoot, 'addWriteHold', wrapAddWriteHold), \
|
|
2108
|
+
mock.patch.object(s_nexus.NexsRoot, 'delWriteHold', wrapDelWriteHold):
|
|
2100
2109
|
|
|
2101
2110
|
async with self.getTestCore() as core:
|
|
2102
2111
|
|
|
@@ -2108,8 +2117,7 @@ class CellTest(s_t_utils.SynTest):
|
|
|
2108
2117
|
msgs = await core.stormlist('[inet:fqdn=newp.fail]')
|
|
2109
2118
|
self.stormIsInErr(errmsg, msgs)
|
|
2110
2119
|
|
|
2111
|
-
|
|
2112
|
-
|
|
2120
|
+
revt.clear()
|
|
2113
2121
|
self.true(await asyncio.wait_for(revt.wait(), 1))
|
|
2114
2122
|
|
|
2115
2123
|
self.len(1, await core.nodes('[inet:fqdn=foo.com]'))
|
|
@@ -2150,6 +2158,7 @@ class CellTest(s_t_utils.SynTest):
|
|
|
2150
2158
|
self.len(0, await core01.nodes('inet:ipv4=2.3.4.5'))
|
|
2151
2159
|
revt.clear()
|
|
2152
2160
|
|
|
2161
|
+
revt.clear()
|
|
2153
2162
|
self.true(await asyncio.wait_for(revt.wait(), 1))
|
|
2154
2163
|
await core01.sync()
|
|
2155
2164
|
|
|
@@ -2194,10 +2203,17 @@ class CellTest(s_t_utils.SynTest):
|
|
|
2194
2203
|
|
|
2195
2204
|
async with self.getTestCore() as core:
|
|
2196
2205
|
|
|
2197
|
-
core.nexsroot.
|
|
2206
|
+
await core.nexsroot.addWriteHold('LOLWRITE TESTING')
|
|
2198
2207
|
|
|
2199
2208
|
msgs = await core.stormlist('[inet:fqdn=newp.fail]')
|
|
2200
|
-
|
|
2209
|
+
|
|
2210
|
+
self.stormIsInErr('LOLWRITE TESTING', msgs)
|
|
2211
|
+
|
|
2212
|
+
await core.nexsroot.delWriteHold('LOLWRITE TESTING')
|
|
2213
|
+
|
|
2214
|
+
core.nexsroot.readonly = True
|
|
2215
|
+
with self.raises(s_exc.IsReadOnly):
|
|
2216
|
+
core.nexsroot.reqNotReadOnly()
|
|
2201
2217
|
|
|
2202
2218
|
async def test_cell_onboot_optimize(self):
|
|
2203
2219
|
|
|
@@ -340,6 +340,22 @@ class CertDirTest(s_t_utils.SynTest):
|
|
|
340
340
|
inter_cakey = cdir.getCaKey(inter_name)
|
|
341
341
|
self.basic_assertions(cdir, inter_cacert, inter_cakey, cacert=cacert)
|
|
342
342
|
|
|
343
|
+
# Per RFC5280 common-name has a length of 1 to 64 characters
|
|
344
|
+
# https://datatracker.ietf.org/doc/html/rfc5280#appendix-A.1
|
|
345
|
+
c64 = 'V' * 64
|
|
346
|
+
cakey64, cacert64 = cdir.genCaCert(c64)
|
|
347
|
+
self.basic_assertions(cdir, cacert64, cakey64)
|
|
348
|
+
|
|
349
|
+
c1 = 'V' * 1
|
|
350
|
+
cakey1, cacert1 = cdir.genCaCert(c1)
|
|
351
|
+
self.basic_assertions(cdir, cacert1, cakey1)
|
|
352
|
+
|
|
353
|
+
with self.raises(s_exc.CryptoErr) as cm:
|
|
354
|
+
cdir.genCaCert('V' * 65)
|
|
355
|
+
|
|
356
|
+
with self.raises(s_exc.CryptoErr) as cm:
|
|
357
|
+
cdir.genCaCert('')
|
|
358
|
+
|
|
343
359
|
def test_certdir_hosts(self):
|
|
344
360
|
with self.getCertDir() as cdir:
|
|
345
361
|
caname = 'syntest'
|
|
@@ -557,6 +573,14 @@ class CertDirTest(s_t_utils.SynTest):
|
|
|
557
573
|
key = cdir.getHostKey(hostname)
|
|
558
574
|
self.basic_assertions(cdir, cert, key, cacert=cacert)
|
|
559
575
|
|
|
576
|
+
# Per RFC5280 common-name has a length of 1 to 64 characters
|
|
577
|
+
# Do not generate CSRs which exceed that name range.
|
|
578
|
+
with self.raises(s_exc.CryptoErr) as cm:
|
|
579
|
+
cdir.genHostCsr('V' * 65)
|
|
580
|
+
|
|
581
|
+
with self.raises(s_exc.CryptoErr) as cm:
|
|
582
|
+
cdir.genHostCsr('')
|
|
583
|
+
|
|
560
584
|
def test_certdir_users_csr(self):
|
|
561
585
|
with self.getCertDir() as cdir:
|
|
562
586
|
caname = 'syntest'
|
|
@@ -580,6 +604,14 @@ class CertDirTest(s_t_utils.SynTest):
|
|
|
580
604
|
key = cdir.getUserKey(username)
|
|
581
605
|
self.basic_assertions(cdir, cert, key, cacert=cacert)
|
|
582
606
|
|
|
607
|
+
# Per RFC5280 common-name has a length of 1 to 64 characters
|
|
608
|
+
# Do not generate CSRs which exceed that name range.
|
|
609
|
+
with self.raises(s_exc.CryptoErr) as cm:
|
|
610
|
+
cdir.genUserCsr('V' * 65)
|
|
611
|
+
|
|
612
|
+
with self.raises(s_exc.CryptoErr) as cm:
|
|
613
|
+
cdir.genUserCsr('')
|
|
614
|
+
|
|
583
615
|
def test_certdir_importfile(self):
|
|
584
616
|
with self.getCertDir() as cdir:
|
|
585
617
|
with self.getTestDir() as testpath:
|
|
@@ -717,7 +717,11 @@ Queries = [
|
|
|
717
717
|
'inet:ipv4 <+(*)- media:news',
|
|
718
718
|
'media:news -(*)+> inet:fqdn',
|
|
719
719
|
'inet:ipv4 <+(*)- *',
|
|
720
|
-
'media:news -(*)+> *'
|
|
720
|
+
'media:news -(*)+> *',
|
|
721
|
+
'$foo=(null)',
|
|
722
|
+
'$foo=({"bar": null})',
|
|
723
|
+
'$p="names" ps:contact:name=foo [ :$p?+=bar ]',
|
|
724
|
+
'$p="names" ps:contact:name=foo [ :$p?-=bar ]',
|
|
721
725
|
]
|
|
722
726
|
|
|
723
727
|
# Generated with print_parse_list below
|
|
@@ -1340,6 +1344,10 @@ _ParseResults = [
|
|
|
1340
1344
|
'Query: [LiftProp: [Const: media:news], N1Walk: [Const: *, Const: inet:fqdn], isjoin=True]',
|
|
1341
1345
|
'Query: [LiftProp: [Const: inet:ipv4], N2Walk: [Const: *, Const: *], isjoin=True]',
|
|
1342
1346
|
'Query: [LiftProp: [Const: media:news], N1Walk: [Const: *, Const: *], isjoin=True]',
|
|
1347
|
+
'Query: [SetVarOper: [Const: foo, DollarExpr: [Const: None]]]',
|
|
1348
|
+
'Query: [SetVarOper: [Const: foo, DollarExpr: [ExprDict: [Const: bar, Const: None]]]]',
|
|
1349
|
+
'Query: [SetVarOper: [Const: p, Const: names], LiftPropBy: [Const: ps:contact:name, Const: =, Const: foo], EditPropSet: [RelProp: [VarValue: [Const: p]], Const: ?+=, Const: bar]]',
|
|
1350
|
+
'Query: [SetVarOper: [Const: p, Const: names], LiftPropBy: [Const: ps:contact:name, Const: =, Const: foo], EditPropSet: [RelProp: [VarValue: [Const: p]], Const: ?-=, Const: bar]]',
|
|
1343
1351
|
]
|
|
1344
1352
|
|
|
1345
1353
|
class GrammarTest(s_t_utils.SynTest):
|
|
@@ -840,7 +840,6 @@ class HttpApiTest(s_tests.SynTest):
|
|
|
840
840
|
spkg = {
|
|
841
841
|
'name': 'testy',
|
|
842
842
|
'version': (0, 0, 1),
|
|
843
|
-
'synapse_minversion': [2, 144, 0],
|
|
844
843
|
'synapse_version': '>=2.50.0,<3.0.0',
|
|
845
844
|
'modules': (
|
|
846
845
|
{'name': 'testy.ingest', 'storm': 'function punch(x, y) { return (($x + $y)) }'},
|
|
@@ -432,3 +432,44 @@ class ModelRevTest(s_tests.SynTest):
|
|
|
432
432
|
async def test_modelrev_0_2_23(self):
|
|
433
433
|
async with self.getRegrCore('model-0.2.23') as core:
|
|
434
434
|
self.len(1, await core.nodes('inet:ipv6="ff01::1" +:type=multicast +:scope=interface-local'))
|
|
435
|
+
|
|
436
|
+
async def test_modelrev_0_2_24(self):
|
|
437
|
+
async with self.getRegrCore('model-0.2.24') as core:
|
|
438
|
+
|
|
439
|
+
self.len(2, await core.nodes('transport:sea:telem:speed'))
|
|
440
|
+
|
|
441
|
+
self.len(1, await core.nodes('transport:air:telem:speed'))
|
|
442
|
+
self.len(1, await core.nodes('transport:air:telem:airspeed'))
|
|
443
|
+
self.len(1, await core.nodes('transport:air:telem:verticalspeed'))
|
|
444
|
+
|
|
445
|
+
self.len(2, await core.nodes('mat:item:_multispeed'))
|
|
446
|
+
nodes = await core.nodes('mat:item:_multispeed*[=5]')
|
|
447
|
+
self.len(1, nodes)
|
|
448
|
+
self.eq((5, 6), nodes[0].get('_multispeed'))
|
|
449
|
+
|
|
450
|
+
nodes = await core.nodes('transport:sea:telem:speed=4')
|
|
451
|
+
self.len(1, nodes)
|
|
452
|
+
self.eq(4, nodes[0].get('speed'))
|
|
453
|
+
|
|
454
|
+
nodes = await core.nodes('transport:air:telem')
|
|
455
|
+
node = nodes[0]
|
|
456
|
+
self.eq(1, node.get('speed'))
|
|
457
|
+
self.eq(2, node.get('airspeed'))
|
|
458
|
+
self.eq(3, node.get('verticalspeed'))
|
|
459
|
+
|
|
460
|
+
q = 'transport:sea:telem=(badvalu,) $node.data.load(_migrated:transport:sea:telem:speed)'
|
|
461
|
+
nodes = await core.nodes(q)
|
|
462
|
+
self.eq(-1.0, await nodes[0].getData('_migrated:transport:sea:telem:speed'))
|
|
463
|
+
|
|
464
|
+
nodes = await core.nodes('risk:mitigation=(foo,)')
|
|
465
|
+
self.len(1, nodes)
|
|
466
|
+
self.eq('foo bar', nodes[0].get('name'))
|
|
467
|
+
self.len(1, await core.nodes('risk:mitigation:name=" Foo Bar "'))
|
|
468
|
+
|
|
469
|
+
nodes = await core.nodes('it:mitre:attack:mitigation=M0100')
|
|
470
|
+
self.len(1, nodes)
|
|
471
|
+
self.eq('patchstuff', nodes[0].get('name'))
|
|
472
|
+
|
|
473
|
+
nodes = await core.nodes('it:mitre:attack:technique=T0100')
|
|
474
|
+
self.len(1, nodes)
|
|
475
|
+
self.eq('lockpicking', nodes[0].get('name'))
|
synapse/tests/test_lib_nexus.py
CHANGED
|
@@ -264,3 +264,41 @@ class NexusTest(s_t_utils.SynTest):
|
|
|
264
264
|
|
|
265
265
|
async with self.getTestCore(dirn=dirn) as core:
|
|
266
266
|
self.len(vcnt + viewadds, core.views)
|
|
267
|
+
|
|
268
|
+
async def test_mirror_version(self):
|
|
269
|
+
|
|
270
|
+
with self.getTestDir() as dirn:
|
|
271
|
+
|
|
272
|
+
s_common.yamlsave({'nexslog:en': True}, dirn, 'cell.yaml')
|
|
273
|
+
async with await s_cell.Cell.anit(dirn=dirn) as cell00:
|
|
274
|
+
|
|
275
|
+
getCellInfo = cell00.getCellInfo
|
|
276
|
+
async def getCrazyVersion():
|
|
277
|
+
info = await getCellInfo()
|
|
278
|
+
info['cell']['version'] = (9999, 0, 0)
|
|
279
|
+
return info
|
|
280
|
+
|
|
281
|
+
await cell00.runBackup(name='cell01')
|
|
282
|
+
|
|
283
|
+
path = s_common.genpath(dirn, 'backups', 'cell01')
|
|
284
|
+
|
|
285
|
+
conf = s_common.yamlload(path, 'cell.yaml')
|
|
286
|
+
conf['mirror'] = f'cell://{dirn}'
|
|
287
|
+
s_common.yamlsave(conf, path, 'cell.yaml')
|
|
288
|
+
|
|
289
|
+
evnt = asyncio.Event()
|
|
290
|
+
cell00.getCellInfo = getCrazyVersion
|
|
291
|
+
|
|
292
|
+
async with await s_cell.Cell.anit(dirn=path) as cell01:
|
|
293
|
+
addWriteHold = cell01.nexsroot.addWriteHold
|
|
294
|
+
def wrapAddWriteHold(reason):
|
|
295
|
+
retn = addWriteHold(reason)
|
|
296
|
+
evnt.set()
|
|
297
|
+
return retn
|
|
298
|
+
|
|
299
|
+
cell01.nexsroot.addWriteHold = wrapAddWriteHold
|
|
300
|
+
await asyncio.wait_for(evnt.wait(), timeout=3)
|
|
301
|
+
|
|
302
|
+
with self.raises(s_exc.IsReadOnly):
|
|
303
|
+
await cell01.sync()
|
|
304
|
+
self.isin(s_nexus.leaderversion, cell01.nexsroot.writeholds)
|
synapse/tests/test_lib_storm.py
CHANGED
|
@@ -1262,7 +1262,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
1262
1262
|
query = await core.getStormQuery('')
|
|
1263
1263
|
async with snap.getStormRuntime(query) as runt:
|
|
1264
1264
|
with self.raises(s_exc.AuthDeny):
|
|
1265
|
-
runt.reqAdmin(gateiden=
|
|
1265
|
+
runt.reqAdmin(gateiden=layr)
|
|
1266
1266
|
|
|
1267
1267
|
await core.stormlist('[ inet:fqdn=vertex.link ]')
|
|
1268
1268
|
fork = await core.callStorm('return($lib.view.get().fork().iden)')
|
|
@@ -1772,13 +1772,92 @@ class StormTest(s_t_utils.SynTest):
|
|
|
1772
1772
|
self.eq('796d67b92a6ffe9b88fa19d115b46ab6712d673a06ae602d41de84b1464782f2', node[1]['embeds']['asn']['*'])
|
|
1773
1773
|
|
|
1774
1774
|
opts = {'embeds': {'ou:org': {'hq::email': ('user',)}}}
|
|
1775
|
-
msgs = await core.stormlist('[ ou:org=* :hq=* ] { -> ps:contact [ :email=visi@vertex.link ] }', opts=opts)
|
|
1775
|
+
msgs = await core.stormlist('[ ou:org=* :country=* :hq=* ] { -> ps:contact [ :email=visi@vertex.link ] }', opts=opts)
|
|
1776
1776
|
nodes = [m[1] for m in msgs if m[0] == 'node']
|
|
1777
|
-
|
|
1778
1777
|
node = nodes[0]
|
|
1778
|
+
|
|
1779
1779
|
self.eq('visi', node[1]['embeds']['hq::email']['user'])
|
|
1780
1780
|
self.eq('2346d7bed4b0fae05e00a413bbf8716c9e08857eb71a1ecf303b8972823f2899', node[1]['embeds']['hq::email']['*'])
|
|
1781
1781
|
|
|
1782
|
+
fork = await core.callStorm('return($lib.view.get().fork().iden)')
|
|
1783
|
+
|
|
1784
|
+
opts['vars'] = {
|
|
1785
|
+
'md5': '12345a5758eea935f817dd1490a322a5',
|
|
1786
|
+
'sha1': '40b8e76cff472e593bd0ba148c09fec66ae72362'
|
|
1787
|
+
}
|
|
1788
|
+
opts['view'] = fork
|
|
1789
|
+
opts['show:storage'] = True
|
|
1790
|
+
opts['embeds']['ou:org']['lol::nope'] = ('notreal',)
|
|
1791
|
+
opts['embeds']['ou:org']['country::flag'] = ('md5', 'sha1')
|
|
1792
|
+
opts['embeds']['ou:org']['country::tld'] = ('domain',)
|
|
1793
|
+
|
|
1794
|
+
await core.stormlist('pol:country [ :flag={[ file:bytes=* :md5=fa818a259cbed7ce8bc2a22d35a464fc ]} ]')
|
|
1795
|
+
|
|
1796
|
+
msgs = await core.stormlist('''
|
|
1797
|
+
ou:org {
|
|
1798
|
+
-> pol:country
|
|
1799
|
+
[ :tld=co.uk ]
|
|
1800
|
+
{
|
|
1801
|
+
:flag -> file:bytes [ :md5=$md5 :sha1=$sha1 ]
|
|
1802
|
+
}
|
|
1803
|
+
}
|
|
1804
|
+
''', opts=opts)
|
|
1805
|
+
nodes = [m[1] for m in msgs if m[0] == 'node']
|
|
1806
|
+
node = nodes[0]
|
|
1807
|
+
|
|
1808
|
+
storage = node[1]['storage']
|
|
1809
|
+
self.len(2, storage)
|
|
1810
|
+
top = storage[0].get('embeds')
|
|
1811
|
+
bot = storage[1].get('embeds')
|
|
1812
|
+
self.nn(top)
|
|
1813
|
+
self.nn(bot)
|
|
1814
|
+
|
|
1815
|
+
self.nn(top.get('country::flag::md5'))
|
|
1816
|
+
self.eq(top['country::flag::md5'][0], '12345a5758eea935f817dd1490a322a5')
|
|
1817
|
+
|
|
1818
|
+
self.nn(top.get('country::flag::sha1'))
|
|
1819
|
+
self.eq(top['country::flag::sha1'][0], '40b8e76cff472e593bd0ba148c09fec66ae72362')
|
|
1820
|
+
|
|
1821
|
+
self.nn(top.get('country::tld::domain'))
|
|
1822
|
+
self.eq(top['country::tld::domain'][0], 'uk')
|
|
1823
|
+
|
|
1824
|
+
self.nn(bot.get('hq::email::user'))
|
|
1825
|
+
self.eq(bot['hq::email::user'][0], 'visi')
|
|
1826
|
+
|
|
1827
|
+
self.nn(bot.get('country::flag::md5'))
|
|
1828
|
+
self.eq(bot['country::flag::md5'][0], 'fa818a259cbed7ce8bc2a22d35a464fc')
|
|
1829
|
+
|
|
1830
|
+
empty = await core.callStorm('return($lib.view.get().fork().iden)', opts=opts)
|
|
1831
|
+
opts['view'] = empty
|
|
1832
|
+
|
|
1833
|
+
msgs = await core.stormlist('ou:org', opts=opts)
|
|
1834
|
+
nodes = [m[1] for m in msgs if m[0] == 'node']
|
|
1835
|
+
node = nodes[0]
|
|
1836
|
+
storage = node[1]['storage']
|
|
1837
|
+
self.len(3, storage)
|
|
1838
|
+
top = storage[0].get('embeds')
|
|
1839
|
+
mid = storage[1].get('embeds')
|
|
1840
|
+
bot = storage[2].get('embeds')
|
|
1841
|
+
self.none(top)
|
|
1842
|
+
|
|
1843
|
+
self.nn(mid)
|
|
1844
|
+
self.nn(bot)
|
|
1845
|
+
|
|
1846
|
+
self.nn(mid.get('country::flag::md5'))
|
|
1847
|
+
self.eq(mid['country::flag::md5'][0], '12345a5758eea935f817dd1490a322a5')
|
|
1848
|
+
|
|
1849
|
+
self.nn(mid.get('country::flag::sha1'))
|
|
1850
|
+
self.eq(mid['country::flag::sha1'][0], '40b8e76cff472e593bd0ba148c09fec66ae72362')
|
|
1851
|
+
|
|
1852
|
+
self.nn(mid.get('country::tld::domain'))
|
|
1853
|
+
self.eq(mid['country::tld::domain'][0], 'uk')
|
|
1854
|
+
|
|
1855
|
+
self.nn(bot.get('hq::email::user'))
|
|
1856
|
+
self.eq(bot['hq::email::user'][0], 'visi')
|
|
1857
|
+
|
|
1858
|
+
self.nn(bot.get('country::flag::md5'))
|
|
1859
|
+
self.eq(bot['country::flag::md5'][0], 'fa818a259cbed7ce8bc2a22d35a464fc')
|
|
1860
|
+
|
|
1782
1861
|
async def test_storm_wget(self):
|
|
1783
1862
|
|
|
1784
1863
|
async def _getRespFromSha(core, mesgs):
|
|
@@ -2151,7 +2230,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
2151
2230
|
|
|
2152
2231
|
# Max recursion fail
|
|
2153
2232
|
q = '[ inet:fqdn=www.vertex.link ] | tree { inet:fqdn=www.vertex.link }'
|
|
2154
|
-
await self.asyncraises(s_exc.
|
|
2233
|
+
await self.asyncraises(s_exc.RecursionLimitHit, core.nodes(q))
|
|
2155
2234
|
|
|
2156
2235
|
# Runtsafety test
|
|
2157
2236
|
q = '[ inet:fqdn=www.vertex.link ] $q=:domain | tree $q'
|
|
@@ -2359,6 +2438,18 @@ class StormTest(s_t_utils.SynTest):
|
|
|
2359
2438
|
with self.raises(s_exc.BadOperArg):
|
|
2360
2439
|
await core.nodes('movetag this is')
|
|
2361
2440
|
|
|
2441
|
+
async with self.getTestCore() as core:
|
|
2442
|
+
await core.nodes('[ syn:tag=hehe :isnow=haha ]')
|
|
2443
|
+
nodes = await core.nodes('[ ou:org=* +#hehe.qwer ]')
|
|
2444
|
+
self.len(1, nodes)
|
|
2445
|
+
self.nn(nodes[0].getTag('haha.qwer'))
|
|
2446
|
+
self.none(nodes[0].getTag('hehe.qwer'))
|
|
2447
|
+
self.len(1, await core.nodes('syn:tag=haha.qwer'))
|
|
2448
|
+
|
|
2449
|
+
# this should hit the already existing redirected tag now...
|
|
2450
|
+
nodes = await core.nodes('[ ou:org=* +#hehe.qwer ]')
|
|
2451
|
+
self.len(1, nodes)
|
|
2452
|
+
|
|
2362
2453
|
# Sad path
|
|
2363
2454
|
async with self.getTestCore() as core:
|
|
2364
2455
|
# Test moving a tag to itself
|
|
@@ -3331,7 +3422,6 @@ class StormTest(s_t_utils.SynTest):
|
|
|
3331
3422
|
otherpkg = {
|
|
3332
3423
|
'name': 'foosball',
|
|
3333
3424
|
'version': '0.0.1',
|
|
3334
|
-
'synapse_minversion': [2, 144, 0],
|
|
3335
3425
|
'synapse_version': '>=2.8.0,<3.0.0',
|
|
3336
3426
|
'commands': ({
|
|
3337
3427
|
'name': 'testcmd',
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
import synapse.exc as s_exc
|
|
2
|
+
|
|
3
|
+
import synapse.tests.utils as s_test
|
|
4
|
+
|
|
5
|
+
class StormlibCacheTest(s_test.SynTest):
|
|
6
|
+
|
|
7
|
+
async def test_storm_lib_cache_fixed(self):
|
|
8
|
+
|
|
9
|
+
async with self.getTestCore() as core:
|
|
10
|
+
|
|
11
|
+
# basics
|
|
12
|
+
|
|
13
|
+
rets = await core.callStorm('''
|
|
14
|
+
$rets = ([])
|
|
15
|
+
$cache = $lib.cache.fixed("return(`{$cache_key}-ret`)")
|
|
16
|
+
|
|
17
|
+
$cache.clear()
|
|
18
|
+
|
|
19
|
+
$rets.append($lib.len($cache))
|
|
20
|
+
|
|
21
|
+
$rets.append($cache.get(key))
|
|
22
|
+
$rets.append($lib.len($cache))
|
|
23
|
+
|
|
24
|
+
$cache.put(key, key-put)
|
|
25
|
+
$rets.append($cache.get(key))
|
|
26
|
+
|
|
27
|
+
$cache.clear()
|
|
28
|
+
$rets.append($cache.get(key))
|
|
29
|
+
|
|
30
|
+
$cache.put(key, key-put)
|
|
31
|
+
$rets.append($cache.get(key))
|
|
32
|
+
$rets.append($cache.pop(key))
|
|
33
|
+
$rets.append($cache.get(key))
|
|
34
|
+
|
|
35
|
+
$rets.append($cache.pop(newp))
|
|
36
|
+
|
|
37
|
+
$rets.append($cache.query)
|
|
38
|
+
|
|
39
|
+
return($rets)
|
|
40
|
+
''')
|
|
41
|
+
self.eq([
|
|
42
|
+
0,
|
|
43
|
+
'key-ret', 1,
|
|
44
|
+
'key-put',
|
|
45
|
+
'key-ret',
|
|
46
|
+
'key-put', 'key-put', 'key-ret',
|
|
47
|
+
None,
|
|
48
|
+
'return(`{$cache_key}-ret`)'
|
|
49
|
+
], rets)
|
|
50
|
+
|
|
51
|
+
# exhaust size
|
|
52
|
+
|
|
53
|
+
rets = await core.callStorm('''
|
|
54
|
+
$rets = ([])
|
|
55
|
+
$cache = $lib.cache.fixed("return(`{$cache_key}-ret`)", size=2)
|
|
56
|
+
|
|
57
|
+
$cache.put(one, one-put)
|
|
58
|
+
$cache.put(two, two-put)
|
|
59
|
+
$rets.append($cache.get(one))
|
|
60
|
+
$rets.append($cache.get(two))
|
|
61
|
+
|
|
62
|
+
$rets.append($cache.get(three))
|
|
63
|
+
$rets.append($cache.get(one))
|
|
64
|
+
|
|
65
|
+
return($rets)
|
|
66
|
+
''')
|
|
67
|
+
self.eq(['one-put', 'two-put', 'three-ret', 'one-ret'], rets)
|
|
68
|
+
|
|
69
|
+
# also accept a storm query object
|
|
70
|
+
|
|
71
|
+
ret = await core.callStorm('''
|
|
72
|
+
$cache = $lib.cache.fixed(${ $suf=ret return(`{$cache_key}-{$suf}`)})
|
|
73
|
+
return($cache.get(foo))
|
|
74
|
+
''')
|
|
75
|
+
self.eq('foo-ret', ret)
|
|
76
|
+
|
|
77
|
+
# callback runtime scoping
|
|
78
|
+
|
|
79
|
+
## a function still has the outer scope as its root
|
|
80
|
+
rets = await core.callStorm('''
|
|
81
|
+
$val = zero
|
|
82
|
+
$sent = $lib.null
|
|
83
|
+
|
|
84
|
+
function cb(key) {
|
|
85
|
+
$sent = $val
|
|
86
|
+
return(`{$key}-{$val}`)
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
$cache = $lib.cache.fixed("return($cb($cache_key))")
|
|
90
|
+
|
|
91
|
+
$rets = ([])
|
|
92
|
+
|
|
93
|
+
$rets.append($cache.get(foo))
|
|
94
|
+
$rets.append($sent)
|
|
95
|
+
|
|
96
|
+
$val = one
|
|
97
|
+
$rets.append($cache.get(bar))
|
|
98
|
+
$rets.append($sent)
|
|
99
|
+
|
|
100
|
+
return($rets)
|
|
101
|
+
''')
|
|
102
|
+
self.eq([
|
|
103
|
+
'foo-zero', 'zero',
|
|
104
|
+
'bar-one', 'one'
|
|
105
|
+
], rets)
|
|
106
|
+
|
|
107
|
+
## runtime also can modify refs to the outer scope
|
|
108
|
+
rets = await core.callStorm('''
|
|
109
|
+
$val = zero
|
|
110
|
+
$vals = ([])
|
|
111
|
+
$sent = $lib.null
|
|
112
|
+
|
|
113
|
+
$cache = $lib.cache.fixed(${
|
|
114
|
+
$sent = $val
|
|
115
|
+
$vals.append($cache_key)
|
|
116
|
+
return(`{$cache_key}-{$val}`)
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
$rets = ([])
|
|
120
|
+
|
|
121
|
+
$rets.append($cache.get(foo))
|
|
122
|
+
$rets.append($sent)
|
|
123
|
+
$rets.append($lib.str.join(",", $vals))
|
|
124
|
+
|
|
125
|
+
$val = one
|
|
126
|
+
$rets.append($cache.get(bar))
|
|
127
|
+
$rets.append($sent)
|
|
128
|
+
$rets.append($lib.str.join(",", $vals))
|
|
129
|
+
|
|
130
|
+
return($rets)
|
|
131
|
+
''')
|
|
132
|
+
self.eq([
|
|
133
|
+
'foo-zero', None, 'foo',
|
|
134
|
+
'bar-one', None, 'foo,bar',
|
|
135
|
+
], rets)
|
|
136
|
+
|
|
137
|
+
## default to null w/o a return
|
|
138
|
+
self.none(await core.callStorm('return($lib.cache.fixed("if (0) { return(yup) }").get(foo))'))
|
|
139
|
+
|
|
140
|
+
## control flow exceptions don't propagate up
|
|
141
|
+
rets = await core.callStorm('''
|
|
142
|
+
$cache = $lib.cache.fixed( ${ if ($cache_key < (2)) { return (`key={$cache_key}`) } else { break } } )
|
|
143
|
+
|
|
144
|
+
$rets = ([])
|
|
145
|
+
|
|
146
|
+
for $i in $lib.range(4) {
|
|
147
|
+
$rets.append($cache.get($i))
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
$rets.append(`i={$i}`)
|
|
151
|
+
return($rets)
|
|
152
|
+
''')
|
|
153
|
+
self.eq(['key=0', 'key=1', None, None, 'i=3'], rets)
|
|
154
|
+
|
|
155
|
+
## control flow scoped inside the callback
|
|
156
|
+
rets = await core.callStorm("""
|
|
157
|
+
$cache = $lib.cache.fixed('''
|
|
158
|
+
$vals = ([])
|
|
159
|
+
for $i in $lib.range(3) {
|
|
160
|
+
$vals.append(`key={$cache_key} i={$i}`)
|
|
161
|
+
break
|
|
162
|
+
}
|
|
163
|
+
return($vals)
|
|
164
|
+
''')
|
|
165
|
+
|
|
166
|
+
$rets = ([])
|
|
167
|
+
|
|
168
|
+
for $k in (foo, bar, baz) {
|
|
169
|
+
$rets.append($cache.get($k))
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return($rets)
|
|
173
|
+
""")
|
|
174
|
+
self.eq([('key=foo i=0',), ('key=bar i=0',), ('key=baz i=0',),], rets)
|
|
175
|
+
|
|
176
|
+
## coverage for the cb runtime emiting nodes
|
|
177
|
+
rets = await core.callStorm('''
|
|
178
|
+
$rets = ([])
|
|
179
|
+
$cache = $lib.cache.fixed(${ if (0) { return(yup) } [ inet:ipv4=$cache_key ] })
|
|
180
|
+
|
|
181
|
+
for $i in (0, 1) {
|
|
182
|
+
$rets.append($cache.get($i))
|
|
183
|
+
}
|
|
184
|
+
return($rets)
|
|
185
|
+
''')
|
|
186
|
+
self.eq([None, None], rets)
|
|
187
|
+
|
|
188
|
+
# stormrepr
|
|
189
|
+
|
|
190
|
+
msgs = await core.stormlist('''
|
|
191
|
+
$lib.print($lib.cache.fixed(${return(cool)}))
|
|
192
|
+
$lib.print($lib.cache.fixed($longq))
|
|
193
|
+
''', opts={'vars': {'longq': f'return({"a" * 150})'}})
|
|
194
|
+
self.stormIsInPrint('cache:fixed: size=10000 query="return(cool)"', msgs)
|
|
195
|
+
self.stormIsInPrint('aaaaa...', msgs)
|
|
196
|
+
|
|
197
|
+
# sad
|
|
198
|
+
|
|
199
|
+
## bad storm query
|
|
200
|
+
|
|
201
|
+
with self.raises(s_exc.BadArg) as ectx:
|
|
202
|
+
await core.nodes('$lib.cache.fixed("function x -> newp")')
|
|
203
|
+
self.isin('Invalid callback query', ectx.exception.errinfo.get('mesg'))
|
|
204
|
+
|
|
205
|
+
## no return
|
|
206
|
+
|
|
207
|
+
with self.raises(s_exc.BadArg) as ectx:
|
|
208
|
+
await core.nodes('$lib.cache.fixed("$x=1")')
|
|
209
|
+
self.eq('Callback query must return a value', ectx.exception.errinfo.get('mesg'))
|
|
210
|
+
|
|
211
|
+
## bad size
|
|
212
|
+
|
|
213
|
+
with self.raises(s_exc.BadArg) as ectx:
|
|
214
|
+
await core.nodes('$lib.cache.fixed("return()", size=(-1))')
|
|
215
|
+
self.eq('Cache size must be between 1-10000', ectx.exception.errinfo.get('mesg'))
|
|
216
|
+
|
|
217
|
+
with self.raises(s_exc.BadArg) as ectx:
|
|
218
|
+
await core.nodes('$lib.cache.fixed("return()", size=(1000000))')
|
|
219
|
+
self.eq('Cache size must be between 1-10000', ectx.exception.errinfo.get('mesg'))
|
|
220
|
+
|
|
221
|
+
## callback raises an exception
|
|
222
|
+
|
|
223
|
+
rets = await core.callStorm('''
|
|
224
|
+
function cb(key) {
|
|
225
|
+
if ($key = "sad") {
|
|
226
|
+
$lib.raise(Bad, Time)
|
|
227
|
+
}
|
|
228
|
+
return(`{$key}-happy`)
|
|
229
|
+
}
|
|
230
|
+
$cache = $lib.cache.fixed("return($cb($cache_key))")
|
|
231
|
+
|
|
232
|
+
$rets = ([])
|
|
233
|
+
|
|
234
|
+
try {
|
|
235
|
+
$rets.append($cache.get(sad))
|
|
236
|
+
} catch Bad as e {
|
|
237
|
+
$rets.append(badtime)
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
$rets.append($cache.get(foo))
|
|
241
|
+
|
|
242
|
+
return($rets)
|
|
243
|
+
''')
|
|
244
|
+
self.eq(['badtime', 'foo-happy'], rets)
|
|
245
|
+
|
|
246
|
+
with self.raises(s_exc.BadCast) as ectx:
|
|
247
|
+
await core.nodes('$lib.cache.fixed(${ return(($cache_key * 3)) }).get(foo)')
|
|
248
|
+
self.eq('Failed to make an integer from \'foo\'.', ectx.exception.errinfo.get('mesg'))
|
|
249
|
+
|
|
250
|
+
## mutable key
|
|
251
|
+
|
|
252
|
+
with self.raises(s_exc.BadArg) as ectx:
|
|
253
|
+
await core.nodes('$lib.cache.fixed("return()").get((foo,))')
|
|
254
|
+
self.eq('Mutable values are not allowed as cache keys', ectx.exception.errinfo.get('mesg'))
|
|
255
|
+
|
|
256
|
+
with self.raises(s_exc.BadArg) as ectx:
|
|
257
|
+
await core.nodes('$lib.cache.fixed("return()").put((foo,), bar)')
|
|
258
|
+
self.eq('Mutable values are not allowed as cache keys', ectx.exception.errinfo.get('mesg'))
|
|
259
|
+
|
|
260
|
+
with self.raises(s_exc.BadArg) as ectx:
|
|
261
|
+
await core.nodes('$lib.cache.fixed("return()").pop((foo,))')
|
|
262
|
+
self.eq('Mutable values are not allowed as cache keys', ectx.exception.errinfo.get('mesg'))
|
|
263
|
+
|
|
264
|
+
## non-primable key
|
|
265
|
+
|
|
266
|
+
with self.raises(s_exc.NoSuchType) as ectx:
|
|
267
|
+
await core.nodes('$cache = $lib.cache.fixed("return()") $cache.get($cache)')
|
|
268
|
+
self.eq('Unable to convert object to Storm primitive.', ectx.exception.errinfo.get('mesg'))
|
|
269
|
+
|
|
270
|
+
## missing use of $cache_key - no error
|
|
271
|
+
|
|
272
|
+
self.eq('newp', await core.callStorm('return($lib.cache.fixed("return(newp)").get(foo))'))
|