synapse 2.164.0__py311-none-any.whl → 2.166.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (89) hide show
  1. synapse/axon.py +3 -3
  2. synapse/cmds/cortex.py +1 -6
  3. synapse/common.py +7 -1
  4. synapse/cortex.py +145 -192
  5. synapse/datamodel.py +36 -1
  6. synapse/lib/agenda.py +87 -97
  7. synapse/lib/aha.py +51 -0
  8. synapse/lib/ast.py +22 -23
  9. synapse/lib/base.py +0 -6
  10. synapse/lib/boss.py +3 -0
  11. synapse/lib/cell.py +70 -39
  12. synapse/lib/certdir.py +9 -0
  13. synapse/lib/hiveauth.py +65 -12
  14. synapse/lib/httpapi.py +1 -0
  15. synapse/lib/modelrev.py +121 -33
  16. synapse/lib/modules.py +1 -0
  17. synapse/lib/nexus.py +64 -26
  18. synapse/lib/parser.py +2 -0
  19. synapse/lib/schemas.py +14 -0
  20. synapse/lib/snap.py +50 -4
  21. synapse/lib/storm.lark +4 -3
  22. synapse/lib/storm.py +96 -22
  23. synapse/lib/storm_format.py +1 -0
  24. synapse/lib/stormlib/aha.py +7 -1
  25. synapse/lib/stormlib/auth.py +13 -5
  26. synapse/lib/stormlib/cache.py +202 -0
  27. synapse/lib/stormlib/cortex.py +147 -8
  28. synapse/lib/stormlib/gen.py +53 -6
  29. synapse/lib/stormlib/math.py +1 -1
  30. synapse/lib/stormlib/model.py +11 -1
  31. synapse/lib/stormlib/spooled.py +109 -0
  32. synapse/lib/stormlib/vault.py +1 -1
  33. synapse/lib/stormtypes.py +113 -17
  34. synapse/lib/trigger.py +36 -47
  35. synapse/lib/types.py +29 -2
  36. synapse/lib/version.py +2 -2
  37. synapse/lib/view.py +80 -53
  38. synapse/models/economic.py +174 -5
  39. synapse/models/files.py +2 -0
  40. synapse/models/inet.py +77 -2
  41. synapse/models/infotech.py +12 -12
  42. synapse/models/orgs.py +72 -21
  43. synapse/models/person.py +40 -11
  44. synapse/models/risk.py +78 -24
  45. synapse/models/science.py +102 -0
  46. synapse/telepath.py +117 -35
  47. synapse/tests/test_cortex.py +84 -158
  48. synapse/tests/test_datamodel.py +22 -0
  49. synapse/tests/test_lib_agenda.py +52 -96
  50. synapse/tests/test_lib_aha.py +126 -4
  51. synapse/tests/test_lib_ast.py +412 -6
  52. synapse/tests/test_lib_cell.py +24 -8
  53. synapse/tests/test_lib_certdir.py +32 -0
  54. synapse/tests/test_lib_grammar.py +9 -1
  55. synapse/tests/test_lib_httpapi.py +0 -1
  56. synapse/tests/test_lib_jupyter.py +0 -1
  57. synapse/tests/test_lib_modelrev.py +41 -0
  58. synapse/tests/test_lib_nexus.py +38 -0
  59. synapse/tests/test_lib_storm.py +95 -5
  60. synapse/tests/test_lib_stormlib_cache.py +272 -0
  61. synapse/tests/test_lib_stormlib_cortex.py +71 -0
  62. synapse/tests/test_lib_stormlib_gen.py +37 -2
  63. synapse/tests/test_lib_stormlib_model.py +2 -0
  64. synapse/tests/test_lib_stormlib_spooled.py +190 -0
  65. synapse/tests/test_lib_stormlib_vault.py +12 -3
  66. synapse/tests/test_lib_stormsvc.py +0 -10
  67. synapse/tests/test_lib_stormtypes.py +60 -8
  68. synapse/tests/test_lib_trigger.py +20 -2
  69. synapse/tests/test_lib_types.py +17 -1
  70. synapse/tests/test_model_economic.py +114 -0
  71. synapse/tests/test_model_files.py +2 -0
  72. synapse/tests/test_model_inet.py +73 -1
  73. synapse/tests/test_model_infotech.py +2 -2
  74. synapse/tests/test_model_orgs.py +10 -1
  75. synapse/tests/test_model_risk.py +30 -2
  76. synapse/tests/test_model_science.py +59 -0
  77. synapse/tests/test_model_syn.py +0 -1
  78. synapse/tests/test_telepath.py +30 -7
  79. synapse/tests/test_tools_modrole.py +81 -0
  80. synapse/tests/test_tools_moduser.py +105 -0
  81. synapse/tools/modrole.py +59 -7
  82. synapse/tools/moduser.py +78 -10
  83. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/METADATA +2 -2
  84. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/RECORD +87 -83
  85. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/WHEEL +1 -1
  86. synapse/lib/provenance.py +0 -111
  87. synapse/tests/test_lib_provenance.py +0 -37
  88. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/LICENSE +0 -0
  89. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/top_level.txt +0 -0
@@ -25,6 +25,7 @@ import synapse.lib.base as s_base
25
25
  import synapse.lib.cell as s_cell
26
26
  import synapse.lib.coro as s_coro
27
27
  import synapse.lib.link as s_link
28
+ import synapse.lib.nexus as s_nexus
28
29
  import synapse.lib.certdir as s_certdir
29
30
  import synapse.lib.msgpack as s_msgpack
30
31
  import synapse.lib.version as s_version
@@ -2088,15 +2089,23 @@ class CellTest(s_t_utils.SynTest):
2088
2089
  return _ntuple_diskusage(100, 96, 4)
2089
2090
 
2090
2091
  revt = asyncio.Event()
2091
- orig = s_cortex.Cortex._setReadOnly
2092
- async def wrapReadOnly(self, valu, reason=None):
2093
- await orig(self, valu, reason=reason)
2092
+ addWriteHold = s_nexus.NexsRoot.addWriteHold
2093
+ delWriteHold = s_nexus.NexsRoot.delWriteHold
2094
+ async def wrapAddWriteHold(root, reason):
2095
+ retn = await addWriteHold(root, reason)
2094
2096
  revt.set()
2097
+ return retn
2098
+
2099
+ async def wrapDelWriteHold(root, reason):
2100
+ retn = await delWriteHold(root, reason)
2101
+ revt.set()
2102
+ return retn
2095
2103
 
2096
2104
  errmsg = 'Insufficient free space on disk.'
2097
2105
 
2098
2106
  with mock.patch.object(s_cell.Cell, 'FREE_SPACE_CHECK_FREQ', 0.1), \
2099
- mock.patch.object(s_cortex.Cortex, '_setReadOnly', wrapReadOnly):
2107
+ mock.patch.object(s_nexus.NexsRoot, 'addWriteHold', wrapAddWriteHold), \
2108
+ mock.patch.object(s_nexus.NexsRoot, 'delWriteHold', wrapDelWriteHold):
2100
2109
 
2101
2110
  async with self.getTestCore() as core:
2102
2111
 
@@ -2108,8 +2117,7 @@ class CellTest(s_t_utils.SynTest):
2108
2117
  msgs = await core.stormlist('[inet:fqdn=newp.fail]')
2109
2118
  self.stormIsInErr(errmsg, msgs)
2110
2119
 
2111
- revt.clear()
2112
-
2120
+ revt.clear()
2113
2121
  self.true(await asyncio.wait_for(revt.wait(), 1))
2114
2122
 
2115
2123
  self.len(1, await core.nodes('[inet:fqdn=foo.com]'))
@@ -2150,6 +2158,7 @@ class CellTest(s_t_utils.SynTest):
2150
2158
  self.len(0, await core01.nodes('inet:ipv4=2.3.4.5'))
2151
2159
  revt.clear()
2152
2160
 
2161
+ revt.clear()
2153
2162
  self.true(await asyncio.wait_for(revt.wait(), 1))
2154
2163
  await core01.sync()
2155
2164
 
@@ -2194,10 +2203,17 @@ class CellTest(s_t_utils.SynTest):
2194
2203
 
2195
2204
  async with self.getTestCore() as core:
2196
2205
 
2197
- core.nexsroot.setReadOnly(True)
2206
+ await core.nexsroot.addWriteHold('LOLWRITE TESTING')
2198
2207
 
2199
2208
  msgs = await core.stormlist('[inet:fqdn=newp.fail]')
2200
- self.stormIsInErr('Unable to issue Nexus events when readonly is set.', msgs)
2209
+
2210
+ self.stormIsInErr('LOLWRITE TESTING', msgs)
2211
+
2212
+ await core.nexsroot.delWriteHold('LOLWRITE TESTING')
2213
+
2214
+ core.nexsroot.readonly = True
2215
+ with self.raises(s_exc.IsReadOnly):
2216
+ core.nexsroot.reqNotReadOnly()
2201
2217
 
2202
2218
  async def test_cell_onboot_optimize(self):
2203
2219
 
@@ -340,6 +340,22 @@ class CertDirTest(s_t_utils.SynTest):
340
340
  inter_cakey = cdir.getCaKey(inter_name)
341
341
  self.basic_assertions(cdir, inter_cacert, inter_cakey, cacert=cacert)
342
342
 
343
+ # Per RFC5280 common-name has a length of 1 to 64 characters
344
+ # https://datatracker.ietf.org/doc/html/rfc5280#appendix-A.1
345
+ c64 = 'V' * 64
346
+ cakey64, cacert64 = cdir.genCaCert(c64)
347
+ self.basic_assertions(cdir, cacert64, cakey64)
348
+
349
+ c1 = 'V' * 1
350
+ cakey1, cacert1 = cdir.genCaCert(c1)
351
+ self.basic_assertions(cdir, cacert1, cakey1)
352
+
353
+ with self.raises(s_exc.CryptoErr) as cm:
354
+ cdir.genCaCert('V' * 65)
355
+
356
+ with self.raises(s_exc.CryptoErr) as cm:
357
+ cdir.genCaCert('')
358
+
343
359
  def test_certdir_hosts(self):
344
360
  with self.getCertDir() as cdir:
345
361
  caname = 'syntest'
@@ -557,6 +573,14 @@ class CertDirTest(s_t_utils.SynTest):
557
573
  key = cdir.getHostKey(hostname)
558
574
  self.basic_assertions(cdir, cert, key, cacert=cacert)
559
575
 
576
+ # Per RFC5280 common-name has a length of 1 to 64 characters
577
+ # Do not generate CSRs which exceed that name range.
578
+ with self.raises(s_exc.CryptoErr) as cm:
579
+ cdir.genHostCsr('V' * 65)
580
+
581
+ with self.raises(s_exc.CryptoErr) as cm:
582
+ cdir.genHostCsr('')
583
+
560
584
  def test_certdir_users_csr(self):
561
585
  with self.getCertDir() as cdir:
562
586
  caname = 'syntest'
@@ -580,6 +604,14 @@ class CertDirTest(s_t_utils.SynTest):
580
604
  key = cdir.getUserKey(username)
581
605
  self.basic_assertions(cdir, cert, key, cacert=cacert)
582
606
 
607
+ # Per RFC5280 common-name has a length of 1 to 64 characters
608
+ # Do not generate CSRs which exceed that name range.
609
+ with self.raises(s_exc.CryptoErr) as cm:
610
+ cdir.genUserCsr('V' * 65)
611
+
612
+ with self.raises(s_exc.CryptoErr) as cm:
613
+ cdir.genUserCsr('')
614
+
583
615
  def test_certdir_importfile(self):
584
616
  with self.getCertDir() as cdir:
585
617
  with self.getTestDir() as testpath:
@@ -717,7 +717,11 @@ Queries = [
717
717
  'inet:ipv4 <+(*)- media:news',
718
718
  'media:news -(*)+> inet:fqdn',
719
719
  'inet:ipv4 <+(*)- *',
720
- 'media:news -(*)+> *'
720
+ 'media:news -(*)+> *',
721
+ '$foo=(null)',
722
+ '$foo=({"bar": null})',
723
+ '$p="names" ps:contact:name=foo [ :$p?+=bar ]',
724
+ '$p="names" ps:contact:name=foo [ :$p?-=bar ]',
721
725
  ]
722
726
 
723
727
  # Generated with print_parse_list below
@@ -1340,6 +1344,10 @@ _ParseResults = [
1340
1344
  'Query: [LiftProp: [Const: media:news], N1Walk: [Const: *, Const: inet:fqdn], isjoin=True]',
1341
1345
  'Query: [LiftProp: [Const: inet:ipv4], N2Walk: [Const: *, Const: *], isjoin=True]',
1342
1346
  'Query: [LiftProp: [Const: media:news], N1Walk: [Const: *, Const: *], isjoin=True]',
1347
+ 'Query: [SetVarOper: [Const: foo, DollarExpr: [Const: None]]]',
1348
+ 'Query: [SetVarOper: [Const: foo, DollarExpr: [ExprDict: [Const: bar, Const: None]]]]',
1349
+ 'Query: [SetVarOper: [Const: p, Const: names], LiftPropBy: [Const: ps:contact:name, Const: =, Const: foo], EditPropSet: [RelProp: [VarValue: [Const: p]], Const: ?+=, Const: bar]]',
1350
+ 'Query: [SetVarOper: [Const: p, Const: names], LiftPropBy: [Const: ps:contact:name, Const: =, Const: foo], EditPropSet: [RelProp: [VarValue: [Const: p]], Const: ?-=, Const: bar]]',
1343
1351
  ]
1344
1352
 
1345
1353
  class GrammarTest(s_t_utils.SynTest):
@@ -840,7 +840,6 @@ class HttpApiTest(s_tests.SynTest):
840
840
  spkg = {
841
841
  'name': 'testy',
842
842
  'version': (0, 0, 1),
843
- 'synapse_minversion': [2, 144, 0],
844
843
  'synapse_version': '>=2.50.0,<3.0.0',
845
844
  'modules': (
846
845
  {'name': 'testy.ingest', 'storm': 'function punch(x, y) { return (($x + $y)) }'},
@@ -16,7 +16,6 @@ class TstsvcApi(s_cell.CellApi, s_stormsvc.StormSvc):
16
16
  {
17
17
  'name': 'testsvc',
18
18
  'version': (0, 0, 1),
19
- 'synapse_minversion': [2, 144, 0],
20
19
  'synapse_version': '>=2.8.0,<3.0.0',
21
20
  'commands': (
22
21
  {
@@ -432,3 +432,44 @@ class ModelRevTest(s_tests.SynTest):
432
432
  async def test_modelrev_0_2_23(self):
433
433
  async with self.getRegrCore('model-0.2.23') as core:
434
434
  self.len(1, await core.nodes('inet:ipv6="ff01::1" +:type=multicast +:scope=interface-local'))
435
+
436
+ async def test_modelrev_0_2_24(self):
437
+ async with self.getRegrCore('model-0.2.24') as core:
438
+
439
+ self.len(2, await core.nodes('transport:sea:telem:speed'))
440
+
441
+ self.len(1, await core.nodes('transport:air:telem:speed'))
442
+ self.len(1, await core.nodes('transport:air:telem:airspeed'))
443
+ self.len(1, await core.nodes('transport:air:telem:verticalspeed'))
444
+
445
+ self.len(2, await core.nodes('mat:item:_multispeed'))
446
+ nodes = await core.nodes('mat:item:_multispeed*[=5]')
447
+ self.len(1, nodes)
448
+ self.eq((5, 6), nodes[0].get('_multispeed'))
449
+
450
+ nodes = await core.nodes('transport:sea:telem:speed=4')
451
+ self.len(1, nodes)
452
+ self.eq(4, nodes[0].get('speed'))
453
+
454
+ nodes = await core.nodes('transport:air:telem')
455
+ node = nodes[0]
456
+ self.eq(1, node.get('speed'))
457
+ self.eq(2, node.get('airspeed'))
458
+ self.eq(3, node.get('verticalspeed'))
459
+
460
+ q = 'transport:sea:telem=(badvalu,) $node.data.load(_migrated:transport:sea:telem:speed)'
461
+ nodes = await core.nodes(q)
462
+ self.eq(-1.0, await nodes[0].getData('_migrated:transport:sea:telem:speed'))
463
+
464
+ nodes = await core.nodes('risk:mitigation=(foo,)')
465
+ self.len(1, nodes)
466
+ self.eq('foo bar', nodes[0].get('name'))
467
+ self.len(1, await core.nodes('risk:mitigation:name=" Foo Bar "'))
468
+
469
+ nodes = await core.nodes('it:mitre:attack:mitigation=M0100')
470
+ self.len(1, nodes)
471
+ self.eq('patchstuff', nodes[0].get('name'))
472
+
473
+ nodes = await core.nodes('it:mitre:attack:technique=T0100')
474
+ self.len(1, nodes)
475
+ self.eq('lockpicking', nodes[0].get('name'))
@@ -264,3 +264,41 @@ class NexusTest(s_t_utils.SynTest):
264
264
 
265
265
  async with self.getTestCore(dirn=dirn) as core:
266
266
  self.len(vcnt + viewadds, core.views)
267
+
268
+ async def test_mirror_version(self):
269
+
270
+ with self.getTestDir() as dirn:
271
+
272
+ s_common.yamlsave({'nexslog:en': True}, dirn, 'cell.yaml')
273
+ async with await s_cell.Cell.anit(dirn=dirn) as cell00:
274
+
275
+ getCellInfo = cell00.getCellInfo
276
+ async def getCrazyVersion():
277
+ info = await getCellInfo()
278
+ info['cell']['version'] = (9999, 0, 0)
279
+ return info
280
+
281
+ await cell00.runBackup(name='cell01')
282
+
283
+ path = s_common.genpath(dirn, 'backups', 'cell01')
284
+
285
+ conf = s_common.yamlload(path, 'cell.yaml')
286
+ conf['mirror'] = f'cell://{dirn}'
287
+ s_common.yamlsave(conf, path, 'cell.yaml')
288
+
289
+ evnt = asyncio.Event()
290
+ cell00.getCellInfo = getCrazyVersion
291
+
292
+ async with await s_cell.Cell.anit(dirn=path) as cell01:
293
+ addWriteHold = cell01.nexsroot.addWriteHold
294
+ def wrapAddWriteHold(reason):
295
+ retn = addWriteHold(reason)
296
+ evnt.set()
297
+ return retn
298
+
299
+ cell01.nexsroot.addWriteHold = wrapAddWriteHold
300
+ await asyncio.wait_for(evnt.wait(), timeout=3)
301
+
302
+ with self.raises(s_exc.IsReadOnly):
303
+ await cell01.sync()
304
+ self.isin(s_nexus.leaderversion, cell01.nexsroot.writeholds)
@@ -1262,7 +1262,7 @@ class StormTest(s_t_utils.SynTest):
1262
1262
  query = await core.getStormQuery('')
1263
1263
  async with snap.getStormRuntime(query) as runt:
1264
1264
  with self.raises(s_exc.AuthDeny):
1265
- runt.reqAdmin(gateiden='cortex')
1265
+ runt.reqAdmin(gateiden=layr)
1266
1266
 
1267
1267
  await core.stormlist('[ inet:fqdn=vertex.link ]')
1268
1268
  fork = await core.callStorm('return($lib.view.get().fork().iden)')
@@ -1772,13 +1772,92 @@ class StormTest(s_t_utils.SynTest):
1772
1772
  self.eq('796d67b92a6ffe9b88fa19d115b46ab6712d673a06ae602d41de84b1464782f2', node[1]['embeds']['asn']['*'])
1773
1773
 
1774
1774
  opts = {'embeds': {'ou:org': {'hq::email': ('user',)}}}
1775
- msgs = await core.stormlist('[ ou:org=* :hq=* ] { -> ps:contact [ :email=visi@vertex.link ] }', opts=opts)
1775
+ msgs = await core.stormlist('[ ou:org=* :country=* :hq=* ] { -> ps:contact [ :email=visi@vertex.link ] }', opts=opts)
1776
1776
  nodes = [m[1] for m in msgs if m[0] == 'node']
1777
-
1778
1777
  node = nodes[0]
1778
+
1779
1779
  self.eq('visi', node[1]['embeds']['hq::email']['user'])
1780
1780
  self.eq('2346d7bed4b0fae05e00a413bbf8716c9e08857eb71a1ecf303b8972823f2899', node[1]['embeds']['hq::email']['*'])
1781
1781
 
1782
+ fork = await core.callStorm('return($lib.view.get().fork().iden)')
1783
+
1784
+ opts['vars'] = {
1785
+ 'md5': '12345a5758eea935f817dd1490a322a5',
1786
+ 'sha1': '40b8e76cff472e593bd0ba148c09fec66ae72362'
1787
+ }
1788
+ opts['view'] = fork
1789
+ opts['show:storage'] = True
1790
+ opts['embeds']['ou:org']['lol::nope'] = ('notreal',)
1791
+ opts['embeds']['ou:org']['country::flag'] = ('md5', 'sha1')
1792
+ opts['embeds']['ou:org']['country::tld'] = ('domain',)
1793
+
1794
+ await core.stormlist('pol:country [ :flag={[ file:bytes=* :md5=fa818a259cbed7ce8bc2a22d35a464fc ]} ]')
1795
+
1796
+ msgs = await core.stormlist('''
1797
+ ou:org {
1798
+ -> pol:country
1799
+ [ :tld=co.uk ]
1800
+ {
1801
+ :flag -> file:bytes [ :md5=$md5 :sha1=$sha1 ]
1802
+ }
1803
+ }
1804
+ ''', opts=opts)
1805
+ nodes = [m[1] for m in msgs if m[0] == 'node']
1806
+ node = nodes[0]
1807
+
1808
+ storage = node[1]['storage']
1809
+ self.len(2, storage)
1810
+ top = storage[0].get('embeds')
1811
+ bot = storage[1].get('embeds')
1812
+ self.nn(top)
1813
+ self.nn(bot)
1814
+
1815
+ self.nn(top.get('country::flag::md5'))
1816
+ self.eq(top['country::flag::md5'][0], '12345a5758eea935f817dd1490a322a5')
1817
+
1818
+ self.nn(top.get('country::flag::sha1'))
1819
+ self.eq(top['country::flag::sha1'][0], '40b8e76cff472e593bd0ba148c09fec66ae72362')
1820
+
1821
+ self.nn(top.get('country::tld::domain'))
1822
+ self.eq(top['country::tld::domain'][0], 'uk')
1823
+
1824
+ self.nn(bot.get('hq::email::user'))
1825
+ self.eq(bot['hq::email::user'][0], 'visi')
1826
+
1827
+ self.nn(bot.get('country::flag::md5'))
1828
+ self.eq(bot['country::flag::md5'][0], 'fa818a259cbed7ce8bc2a22d35a464fc')
1829
+
1830
+ empty = await core.callStorm('return($lib.view.get().fork().iden)', opts=opts)
1831
+ opts['view'] = empty
1832
+
1833
+ msgs = await core.stormlist('ou:org', opts=opts)
1834
+ nodes = [m[1] for m in msgs if m[0] == 'node']
1835
+ node = nodes[0]
1836
+ storage = node[1]['storage']
1837
+ self.len(3, storage)
1838
+ top = storage[0].get('embeds')
1839
+ mid = storage[1].get('embeds')
1840
+ bot = storage[2].get('embeds')
1841
+ self.none(top)
1842
+
1843
+ self.nn(mid)
1844
+ self.nn(bot)
1845
+
1846
+ self.nn(mid.get('country::flag::md5'))
1847
+ self.eq(mid['country::flag::md5'][0], '12345a5758eea935f817dd1490a322a5')
1848
+
1849
+ self.nn(mid.get('country::flag::sha1'))
1850
+ self.eq(mid['country::flag::sha1'][0], '40b8e76cff472e593bd0ba148c09fec66ae72362')
1851
+
1852
+ self.nn(mid.get('country::tld::domain'))
1853
+ self.eq(mid['country::tld::domain'][0], 'uk')
1854
+
1855
+ self.nn(bot.get('hq::email::user'))
1856
+ self.eq(bot['hq::email::user'][0], 'visi')
1857
+
1858
+ self.nn(bot.get('country::flag::md5'))
1859
+ self.eq(bot['country::flag::md5'][0], 'fa818a259cbed7ce8bc2a22d35a464fc')
1860
+
1782
1861
  async def test_storm_wget(self):
1783
1862
 
1784
1863
  async def _getRespFromSha(core, mesgs):
@@ -2151,7 +2230,7 @@ class StormTest(s_t_utils.SynTest):
2151
2230
 
2152
2231
  # Max recursion fail
2153
2232
  q = '[ inet:fqdn=www.vertex.link ] | tree { inet:fqdn=www.vertex.link }'
2154
- await self.asyncraises(s_exc.StormRuntimeError, core.nodes(q))
2233
+ await self.asyncraises(s_exc.RecursionLimitHit, core.nodes(q))
2155
2234
 
2156
2235
  # Runtsafety test
2157
2236
  q = '[ inet:fqdn=www.vertex.link ] $q=:domain | tree $q'
@@ -2359,6 +2438,18 @@ class StormTest(s_t_utils.SynTest):
2359
2438
  with self.raises(s_exc.BadOperArg):
2360
2439
  await core.nodes('movetag this is')
2361
2440
 
2441
+ async with self.getTestCore() as core:
2442
+ await core.nodes('[ syn:tag=hehe :isnow=haha ]')
2443
+ nodes = await core.nodes('[ ou:org=* +#hehe.qwer ]')
2444
+ self.len(1, nodes)
2445
+ self.nn(nodes[0].getTag('haha.qwer'))
2446
+ self.none(nodes[0].getTag('hehe.qwer'))
2447
+ self.len(1, await core.nodes('syn:tag=haha.qwer'))
2448
+
2449
+ # this should hit the already existing redirected tag now...
2450
+ nodes = await core.nodes('[ ou:org=* +#hehe.qwer ]')
2451
+ self.len(1, nodes)
2452
+
2362
2453
  # Sad path
2363
2454
  async with self.getTestCore() as core:
2364
2455
  # Test moving a tag to itself
@@ -3331,7 +3422,6 @@ class StormTest(s_t_utils.SynTest):
3331
3422
  otherpkg = {
3332
3423
  'name': 'foosball',
3333
3424
  'version': '0.0.1',
3334
- 'synapse_minversion': [2, 144, 0],
3335
3425
  'synapse_version': '>=2.8.0,<3.0.0',
3336
3426
  'commands': ({
3337
3427
  'name': 'testcmd',
@@ -0,0 +1,272 @@
1
+ import synapse.exc as s_exc
2
+
3
+ import synapse.tests.utils as s_test
4
+
5
+ class StormlibCacheTest(s_test.SynTest):
6
+
7
+ async def test_storm_lib_cache_fixed(self):
8
+
9
+ async with self.getTestCore() as core:
10
+
11
+ # basics
12
+
13
+ rets = await core.callStorm('''
14
+ $rets = ([])
15
+ $cache = $lib.cache.fixed("return(`{$cache_key}-ret`)")
16
+
17
+ $cache.clear()
18
+
19
+ $rets.append($lib.len($cache))
20
+
21
+ $rets.append($cache.get(key))
22
+ $rets.append($lib.len($cache))
23
+
24
+ $cache.put(key, key-put)
25
+ $rets.append($cache.get(key))
26
+
27
+ $cache.clear()
28
+ $rets.append($cache.get(key))
29
+
30
+ $cache.put(key, key-put)
31
+ $rets.append($cache.get(key))
32
+ $rets.append($cache.pop(key))
33
+ $rets.append($cache.get(key))
34
+
35
+ $rets.append($cache.pop(newp))
36
+
37
+ $rets.append($cache.query)
38
+
39
+ return($rets)
40
+ ''')
41
+ self.eq([
42
+ 0,
43
+ 'key-ret', 1,
44
+ 'key-put',
45
+ 'key-ret',
46
+ 'key-put', 'key-put', 'key-ret',
47
+ None,
48
+ 'return(`{$cache_key}-ret`)'
49
+ ], rets)
50
+
51
+ # exhaust size
52
+
53
+ rets = await core.callStorm('''
54
+ $rets = ([])
55
+ $cache = $lib.cache.fixed("return(`{$cache_key}-ret`)", size=2)
56
+
57
+ $cache.put(one, one-put)
58
+ $cache.put(two, two-put)
59
+ $rets.append($cache.get(one))
60
+ $rets.append($cache.get(two))
61
+
62
+ $rets.append($cache.get(three))
63
+ $rets.append($cache.get(one))
64
+
65
+ return($rets)
66
+ ''')
67
+ self.eq(['one-put', 'two-put', 'three-ret', 'one-ret'], rets)
68
+
69
+ # also accept a storm query object
70
+
71
+ ret = await core.callStorm('''
72
+ $cache = $lib.cache.fixed(${ $suf=ret return(`{$cache_key}-{$suf}`)})
73
+ return($cache.get(foo))
74
+ ''')
75
+ self.eq('foo-ret', ret)
76
+
77
+ # callback runtime scoping
78
+
79
+ ## a function still has the outer scope as its root
80
+ rets = await core.callStorm('''
81
+ $val = zero
82
+ $sent = $lib.null
83
+
84
+ function cb(key) {
85
+ $sent = $val
86
+ return(`{$key}-{$val}`)
87
+ }
88
+
89
+ $cache = $lib.cache.fixed("return($cb($cache_key))")
90
+
91
+ $rets = ([])
92
+
93
+ $rets.append($cache.get(foo))
94
+ $rets.append($sent)
95
+
96
+ $val = one
97
+ $rets.append($cache.get(bar))
98
+ $rets.append($sent)
99
+
100
+ return($rets)
101
+ ''')
102
+ self.eq([
103
+ 'foo-zero', 'zero',
104
+ 'bar-one', 'one'
105
+ ], rets)
106
+
107
+ ## runtime also can modify refs to the outer scope
108
+ rets = await core.callStorm('''
109
+ $val = zero
110
+ $vals = ([])
111
+ $sent = $lib.null
112
+
113
+ $cache = $lib.cache.fixed(${
114
+ $sent = $val
115
+ $vals.append($cache_key)
116
+ return(`{$cache_key}-{$val}`)
117
+ })
118
+
119
+ $rets = ([])
120
+
121
+ $rets.append($cache.get(foo))
122
+ $rets.append($sent)
123
+ $rets.append($lib.str.join(",", $vals))
124
+
125
+ $val = one
126
+ $rets.append($cache.get(bar))
127
+ $rets.append($sent)
128
+ $rets.append($lib.str.join(",", $vals))
129
+
130
+ return($rets)
131
+ ''')
132
+ self.eq([
133
+ 'foo-zero', None, 'foo',
134
+ 'bar-one', None, 'foo,bar',
135
+ ], rets)
136
+
137
+ ## default to null w/o a return
138
+ self.none(await core.callStorm('return($lib.cache.fixed("if (0) { return(yup) }").get(foo))'))
139
+
140
+ ## control flow exceptions don't propagate up
141
+ rets = await core.callStorm('''
142
+ $cache = $lib.cache.fixed( ${ if ($cache_key < (2)) { return (`key={$cache_key}`) } else { break } } )
143
+
144
+ $rets = ([])
145
+
146
+ for $i in $lib.range(4) {
147
+ $rets.append($cache.get($i))
148
+ }
149
+
150
+ $rets.append(`i={$i}`)
151
+ return($rets)
152
+ ''')
153
+ self.eq(['key=0', 'key=1', None, None, 'i=3'], rets)
154
+
155
+ ## control flow scoped inside the callback
156
+ rets = await core.callStorm("""
157
+ $cache = $lib.cache.fixed('''
158
+ $vals = ([])
159
+ for $i in $lib.range(3) {
160
+ $vals.append(`key={$cache_key} i={$i}`)
161
+ break
162
+ }
163
+ return($vals)
164
+ ''')
165
+
166
+ $rets = ([])
167
+
168
+ for $k in (foo, bar, baz) {
169
+ $rets.append($cache.get($k))
170
+ }
171
+
172
+ return($rets)
173
+ """)
174
+ self.eq([('key=foo i=0',), ('key=bar i=0',), ('key=baz i=0',),], rets)
175
+
176
+ ## coverage for the cb runtime emiting nodes
177
+ rets = await core.callStorm('''
178
+ $rets = ([])
179
+ $cache = $lib.cache.fixed(${ if (0) { return(yup) } [ inet:ipv4=$cache_key ] })
180
+
181
+ for $i in (0, 1) {
182
+ $rets.append($cache.get($i))
183
+ }
184
+ return($rets)
185
+ ''')
186
+ self.eq([None, None], rets)
187
+
188
+ # stormrepr
189
+
190
+ msgs = await core.stormlist('''
191
+ $lib.print($lib.cache.fixed(${return(cool)}))
192
+ $lib.print($lib.cache.fixed($longq))
193
+ ''', opts={'vars': {'longq': f'return({"a" * 150})'}})
194
+ self.stormIsInPrint('cache:fixed: size=10000 query="return(cool)"', msgs)
195
+ self.stormIsInPrint('aaaaa...', msgs)
196
+
197
+ # sad
198
+
199
+ ## bad storm query
200
+
201
+ with self.raises(s_exc.BadArg) as ectx:
202
+ await core.nodes('$lib.cache.fixed("function x -> newp")')
203
+ self.isin('Invalid callback query', ectx.exception.errinfo.get('mesg'))
204
+
205
+ ## no return
206
+
207
+ with self.raises(s_exc.BadArg) as ectx:
208
+ await core.nodes('$lib.cache.fixed("$x=1")')
209
+ self.eq('Callback query must return a value', ectx.exception.errinfo.get('mesg'))
210
+
211
+ ## bad size
212
+
213
+ with self.raises(s_exc.BadArg) as ectx:
214
+ await core.nodes('$lib.cache.fixed("return()", size=(-1))')
215
+ self.eq('Cache size must be between 1-10000', ectx.exception.errinfo.get('mesg'))
216
+
217
+ with self.raises(s_exc.BadArg) as ectx:
218
+ await core.nodes('$lib.cache.fixed("return()", size=(1000000))')
219
+ self.eq('Cache size must be between 1-10000', ectx.exception.errinfo.get('mesg'))
220
+
221
+ ## callback raises an exception
222
+
223
+ rets = await core.callStorm('''
224
+ function cb(key) {
225
+ if ($key = "sad") {
226
+ $lib.raise(Bad, Time)
227
+ }
228
+ return(`{$key}-happy`)
229
+ }
230
+ $cache = $lib.cache.fixed("return($cb($cache_key))")
231
+
232
+ $rets = ([])
233
+
234
+ try {
235
+ $rets.append($cache.get(sad))
236
+ } catch Bad as e {
237
+ $rets.append(badtime)
238
+ }
239
+
240
+ $rets.append($cache.get(foo))
241
+
242
+ return($rets)
243
+ ''')
244
+ self.eq(['badtime', 'foo-happy'], rets)
245
+
246
+ with self.raises(s_exc.BadCast) as ectx:
247
+ await core.nodes('$lib.cache.fixed(${ return(($cache_key * 3)) }).get(foo)')
248
+ self.eq('Failed to make an integer from \'foo\'.', ectx.exception.errinfo.get('mesg'))
249
+
250
+ ## mutable key
251
+
252
+ with self.raises(s_exc.BadArg) as ectx:
253
+ await core.nodes('$lib.cache.fixed("return()").get((foo,))')
254
+ self.eq('Mutable values are not allowed as cache keys', ectx.exception.errinfo.get('mesg'))
255
+
256
+ with self.raises(s_exc.BadArg) as ectx:
257
+ await core.nodes('$lib.cache.fixed("return()").put((foo,), bar)')
258
+ self.eq('Mutable values are not allowed as cache keys', ectx.exception.errinfo.get('mesg'))
259
+
260
+ with self.raises(s_exc.BadArg) as ectx:
261
+ await core.nodes('$lib.cache.fixed("return()").pop((foo,))')
262
+ self.eq('Mutable values are not allowed as cache keys', ectx.exception.errinfo.get('mesg'))
263
+
264
+ ## non-primable key
265
+
266
+ with self.raises(s_exc.NoSuchType) as ectx:
267
+ await core.nodes('$cache = $lib.cache.fixed("return()") $cache.get($cache)')
268
+ self.eq('Unable to convert object to Storm primitive.', ectx.exception.errinfo.get('mesg'))
269
+
270
+ ## missing use of $cache_key - no error
271
+
272
+ self.eq('newp', await core.callStorm('return($lib.cache.fixed("return(newp)").get(foo))'))