synapse 2.155.0__py311-none-any.whl → 2.156.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (64) hide show
  1. synapse/cmds/cortex.py +2 -14
  2. synapse/common.py +1 -28
  3. synapse/cortex.py +10 -510
  4. synapse/lib/ast.py +60 -1
  5. synapse/lib/cell.py +33 -8
  6. synapse/lib/certdir.py +11 -0
  7. synapse/lib/cmdr.py +0 -5
  8. synapse/lib/gis.py +2 -2
  9. synapse/lib/httpapi.py +1 -43
  10. synapse/lib/layer.py +64 -201
  11. synapse/lib/lmdbslab.py +11 -0
  12. synapse/lib/node.py +1 -3
  13. synapse/lib/parser.py +10 -0
  14. synapse/lib/snap.py +121 -21
  15. synapse/lib/storm.lark +23 -6
  16. synapse/lib/storm.py +15 -338
  17. synapse/lib/storm_format.py +5 -0
  18. synapse/lib/stormlib/gen.py +1 -2
  19. synapse/lib/stormlib/gis.py +41 -0
  20. synapse/lib/stormlib/stats.py +21 -2
  21. synapse/lib/stormlib/storm.py +16 -1
  22. synapse/lib/stormtypes.py +225 -12
  23. synapse/lib/version.py +2 -2
  24. synapse/lib/view.py +96 -21
  25. synapse/models/inet.py +60 -30
  26. synapse/models/infotech.py +56 -1
  27. synapse/models/orgs.py +3 -0
  28. synapse/models/risk.py +15 -0
  29. synapse/models/syn.py +0 -38
  30. synapse/tests/test_cmds_cortex.py +1 -1
  31. synapse/tests/test_cortex.py +32 -336
  32. synapse/tests/test_lib_agenda.py +19 -54
  33. synapse/tests/test_lib_aha.py +97 -0
  34. synapse/tests/test_lib_ast.py +402 -0
  35. synapse/tests/test_lib_grammar.py +30 -10
  36. synapse/tests/test_lib_httpapi.py +0 -46
  37. synapse/tests/test_lib_layer.py +19 -234
  38. synapse/tests/test_lib_lmdbslab.py +22 -0
  39. synapse/tests/test_lib_snap.py +9 -0
  40. synapse/tests/test_lib_storm.py +16 -309
  41. synapse/tests/test_lib_stormlib_gis.py +21 -0
  42. synapse/tests/test_lib_stormlib_stats.py +107 -20
  43. synapse/tests/test_lib_stormlib_storm.py +25 -0
  44. synapse/tests/test_lib_stormtypes.py +231 -8
  45. synapse/tests/test_lib_view.py +6 -13
  46. synapse/tests/test_model_base.py +1 -1
  47. synapse/tests/test_model_inet.py +15 -0
  48. synapse/tests/test_model_infotech.py +60 -0
  49. synapse/tests/test_model_orgs.py +10 -0
  50. synapse/tests/test_model_person.py +0 -3
  51. synapse/tests/test_model_risk.py +20 -0
  52. synapse/tests/test_model_syn.py +20 -34
  53. synapse/tests/test_tools_csvtool.py +2 -1
  54. synapse/tests/test_tools_feed.py +4 -30
  55. synapse/tools/csvtool.py +2 -1
  56. {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/METADATA +3 -3
  57. {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/RECORD +60 -62
  58. {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/WHEEL +1 -1
  59. synapse/cmds/cron.py +0 -726
  60. synapse/cmds/trigger.py +0 -319
  61. synapse/tests/test_cmds_cron.py +0 -453
  62. synapse/tests/test_cmds_trigger.py +0 -176
  63. {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/LICENSE +0 -0
  64. {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/top_level.txt +0 -0
@@ -408,6 +408,10 @@ class ItModule(s_module.CoreModule):
408
408
  'doc': 'A Mitre ATT&CK Software ID.',
409
409
  'ex': 'S0154',
410
410
  }),
411
+ ('it:mitre:attack:campaign', ('str', {'regex': r'^C[0-9]{4}$'}), {
412
+ 'doc': 'A Mitre ATT&CK Campaign ID.',
413
+ 'ex': 'C0028',
414
+ }),
411
415
  ('it:mitre:attack:flow', ('guid', {}), {
412
416
  'doc': 'A Mitre ATT&CK Flow diagram.',
413
417
  }),
@@ -490,7 +494,8 @@ class ItModule(s_module.CoreModule):
490
494
  ('it:adid', ('str', {'lower': True, 'strip': True}), {
491
495
  'doc': 'An advertising identification string.'}),
492
496
 
493
- ('it:os:windows:sid', ('str', {'regex': r'^S-1-[0-59]-\d{2}-\d{8,10}-\d{8,10}-\d{8,10}-[1-9]\d{3}$'}), {
497
+ # https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-dtyp/c92a27b1-c772-4fa7-a432-15df5f1b66a1
498
+ ('it:os:windows:sid', ('str', {'regex': r'^S-1-(?:\d{1,10}|0x[0-9a-fA-F]{12})(?:-(?:\d+|0x[0-9a-fA-F]{2,}))*$'}), {
494
499
  'doc': 'A Microsoft Windows Security Identifier.',
495
500
  'ex': 'S-1-5-21-1220945662-1202665555-839525555-5555',
496
501
  }),
@@ -1255,6 +1260,50 @@ class ItModule(s_module.CoreModule):
1255
1260
  'doc': 'An array of ATT&CK technique IDs addressed by the mitigation.',
1256
1261
  }),
1257
1262
  )),
1263
+ ('it:mitre:attack:campaign', {}, (
1264
+ ('name', ('ou:campname', {}), {
1265
+ 'doc': 'The primary name for the ATT&CK campaign.',
1266
+ }),
1267
+ ('names', ('array', {'type': 'ou:campname', 'uniq': True, 'sorted': True}), {
1268
+ 'doc': 'An array of alternate names for the ATT&CK campaign.',
1269
+ }),
1270
+ ('desc', ('str', {'strip': True}), {
1271
+ 'doc': 'A description of the ATT&CK campaign.',
1272
+ 'disp': {'hint': 'text'},
1273
+ }),
1274
+ ('url', ('inet:url', {}), {
1275
+ 'doc': 'The URL that documents the ATT&CK campaign.',
1276
+ }),
1277
+ ('groups', ('array', {'type': 'it:mitre:attack:group',
1278
+ 'uniq': True, 'sorted': True, 'split': ','}), {
1279
+ 'doc': 'An array of ATT&CK group IDs attributed to the campaign.',
1280
+ }),
1281
+ ('software', ('array', {'type': 'it:mitre:attack:software',
1282
+ 'uniq': True, 'sorted': True, 'split': ','}), {
1283
+ 'doc': 'An array of ATT&CK software IDs used in the campaign.',
1284
+ }),
1285
+ ('techniques', ('array', {'type': 'it:mitre:attack:technique',
1286
+ 'uniq': True, 'sorted': True, 'split': ','}), {
1287
+ 'doc': 'An array of ATT&CK technique IDs used in the campaign.',
1288
+ }),
1289
+ ('matrices', ('array', {'type': 'it:mitre:attack:matrix',
1290
+ 'uniq': True, 'sorted': True, 'split': ','}), {
1291
+ 'doc': 'The ATT&CK matrices which define the campaign.',
1292
+ }),
1293
+ ('references', ('array', {'type': 'inet:url', 'uniq': True}), {
1294
+ 'doc': 'An array of URLs that document the ATT&CK campaign.',
1295
+ }),
1296
+ ('period', ('ival', {}), {
1297
+ 'doc': 'The time interval when the campaign was active.'}),
1298
+ ('created', ('time', {}), {
1299
+ 'doc': 'The time that the campaign was created by Mitre.'}),
1300
+ ('updated', ('time', {}), {
1301
+ 'doc': 'The time that the campaign was last updated by Mitre.'}),
1302
+ ('tag', ('syn:tag', {}), {
1303
+ 'doc': 'The synapse tag used to annotate nodes included in this ATT&CK campaign.',
1304
+ 'ex': 'cno.mitre.c0028',
1305
+ }),
1306
+ )),
1258
1307
  ('it:mitre:attack:flow', {}, (
1259
1308
  ('name', ('str', {}), {
1260
1309
  'doc': 'The name of the attack-flow diagram.'}),
@@ -1789,6 +1838,12 @@ class ItModule(s_module.CoreModule):
1789
1838
  ('target:url', ('inet:url', {}), {
1790
1839
  'doc': 'The URL that was scanned to produce the result.'}),
1791
1840
 
1841
+ ('target:ipv4', ('inet:ipv4', {}), {
1842
+ 'doc': 'The IPv4 address that was scanned to produce the result.'}),
1843
+
1844
+ ('target:ipv6', ('inet:ipv6', {}), {
1845
+ 'doc': 'The IPv6 address that was scanned to produce the result.'}),
1846
+
1792
1847
  ('multi:scan', ('it:av:scan:result', {}), {
1793
1848
  'doc': 'Set if this result was part of running multiple scanners.'}),
1794
1849
 
synapse/models/orgs.py CHANGED
@@ -503,6 +503,9 @@ class OuModule(s_module.CoreModule):
503
503
 
504
504
  ('tag', ('syn:tag', {}), {
505
505
  'doc': 'The tag used to annotate nodes that are associated with the campaign.'}),
506
+
507
+ ('mitre:attack:campaign', ('it:mitre:attack:campaign', {}), {
508
+ 'doc': 'A mapping to a Mitre ATT&CK campaign if applicable.'}),
506
509
  )),
507
510
  ('ou:conflict', {}, (
508
511
  ('name', ('str', {'onespace': True}), {
synapse/models/risk.py CHANGED
@@ -115,6 +115,8 @@ class RiskModule(s_module.CoreModule):
115
115
  ('risk:extortion:type:taxonomy', ('taxonomy', {}), {
116
116
  'interfaces': ('taxonomy',),
117
117
  'doc': 'A taxonomy of extortion event types.'}),
118
+ ('risk:technique:masquerade', ('guid', {}), {
119
+ 'doc': 'Represents the assessment that a node is designed to resemble another in order to mislead.'}),
118
120
  ),
119
121
  'edges': (
120
122
  # some explicit examples...
@@ -337,6 +339,9 @@ class RiskModule(s_module.CoreModule):
337
339
  ('timeline:exploited', ('time', {"ismin": True}), {
338
340
  'doc': 'The earliest known time when the vulnerability was exploited in the wild.'}),
339
341
 
342
+ ('id', ('str', {'strip': True}), {
343
+ 'doc': 'An identifier for the vulnerability.'}),
344
+
340
345
  ('cve', ('it:sec:cve', {}), {
341
346
  'doc': 'The CVE ID of the vulnerability.'}),
342
347
 
@@ -942,6 +947,16 @@ class RiskModule(s_module.CoreModule):
942
947
  'doc': 'The currency in which payment was demanded.'}),
943
948
 
944
949
  )),
950
+ ('risk:technique:masquerade', {}, (
951
+ ('node', ('ndef', {}), {
952
+ 'doc': 'The node masquerading as another.'}),
953
+ ('period', ('ival', {}), {
954
+ 'doc': 'The time period when the masquerading was active.'}),
955
+ ('target', ('ndef', {}), {
956
+ 'doc': 'The being masqueraded as.'}),
957
+ ('technique', ('ou:technique', {}), {
958
+ 'doc': 'The specific technique which describes the type of masquerading.'}),
959
+ )),
945
960
  ),
946
961
  }
947
962
  name = 'risk'
synapse/models/syn.py CHANGED
@@ -127,9 +127,6 @@ class SynModule(s_module.CoreModule):
127
127
  ('syn:cmd', ('str', {'strip': True}), {
128
128
  'doc': 'A Synapse storm command.'
129
129
  }),
130
- ('syn:splice', ('guid', {'strip': True}), {
131
- 'doc': 'A splice from a layer.'
132
- }),
133
130
  ('syn:nodedata', ('comp', {'fields': (('key', 'str'), ('form', 'syn:form'))}), {
134
131
  'doc': 'A nodedata key and the form it may be present on.',
135
132
  }),
@@ -282,40 +279,5 @@ class SynModule(s_module.CoreModule):
282
279
  ('nodedata', ('array', {'type': 'syn:nodedata'}), {
283
280
  'doc': 'The list of nodedata that may be added by the command.', 'uniq': True, 'sorted': True, 'ro': True}),
284
281
  )),
285
- ('syn:splice', {'runt': True}, (
286
- ('type', ('str', {'strip': True}), {
287
- 'doc': 'Type of splice.', 'ro': True
288
- }),
289
- ('iden', ('str', {}), {
290
- 'doc': 'The iden of the node involved in the splice.', 'ro': True,
291
- }),
292
- ('form', ('syn:form', {'strip': True}), {
293
- 'doc': 'The form involved in the splice.', 'ro': True
294
- }),
295
- ('prop', ('syn:prop', {'strip': True}), {
296
- 'doc': 'Property modified in the splice.', 'ro': True
297
- }),
298
- ('tag', ('syn:tag', {'strip': True}), {
299
- 'doc': 'Tag modified in the splice.', 'ro': True
300
- }),
301
- ('valu', ('data', {}), {
302
- 'doc': 'The value being set in the splice.', 'ro': True
303
- }),
304
- ('oldv', ('data', {}), {
305
- 'doc': 'The value before the splice.', 'ro': True
306
- }),
307
- ('user', ('guid', {}), {
308
- 'doc': 'The user who caused the splice.', 'ro': True,
309
- }),
310
- ('prov', ('guid', {}), {
311
- 'doc': 'The provenance stack of the splice.', 'ro': True,
312
- }),
313
- ('time', ('time', {}), {
314
- 'doc': 'The time the splice occurred.', 'ro': True,
315
- }),
316
- ('splice', ('data', {}), {
317
- 'doc': 'The splice.', 'ro': True
318
- }),
319
- )),
320
282
  ),
321
283
  }),)
@@ -245,7 +245,7 @@ class CmdCoreTest(s_t_utils.SynTest):
245
245
  cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
246
246
  await cmdr.runCmdLine('log --on --format jsonl')
247
247
  fp = cmdr.locs.get('log:fp')
248
- await cmdr.runCmdLine('storm --editformat splices [test:str=hi :tick=2018 +#haha.hehe]')
248
+ await cmdr.runCmdLine('storm [test:str=hi :tick=2018 +#haha.hehe]')
249
249
 
250
250
  await cmdr.runCmdLine('storm --editformat nodeedits [test:str=hi2 :tick=2018 +#haha.hehe]')
251
251
  await cmdr.runCmdLine('storm [test:comp=(42, bar)]')
@@ -937,14 +937,6 @@ class CortexTest(s_t_utils.SynTest):
937
937
  return($list.size())
938
938
  '''))
939
939
 
940
- # check that edge node edits dont bork up legacy splice generation
941
- nodeedits = [(ipv4.buid, 'inet:ipv4', (
942
- (s_layer.EDIT_EDGE_ADD, (), ()),
943
- (s_layer.EDIT_EDGE_DEL, (), ()),
944
- ))]
945
-
946
- self.eq((), await alist(core.view.layers[0].makeSplices(0, nodeedits, {})))
947
-
948
940
  # Run multiple nodes through edge creation/deletion ( test coverage for perm caching )
949
941
  await core.nodes('inet:ipv4 [ <(test)+ { meta:source:name=test }]')
950
942
  self.len(2, await core.nodes('meta:source:name=test -(test)> *'))
@@ -1799,45 +1791,41 @@ class CortexTest(s_t_utils.SynTest):
1799
1791
 
1800
1792
  async with self.getTestCore(dirn=dirn) as core:
1801
1793
 
1802
- async with core.getLocalProxy() as prox:
1803
-
1804
- await prox.setStormCmd(cdef0)
1794
+ await core.setStormCmd(cdef0)
1805
1795
 
1806
- nodes = await core.nodes('[ inet:asn=10 ] | testcmd0 zoinks')
1807
- self.true(nodes[0].tags.get('zoinks'))
1796
+ nodes = await core.nodes('[ inet:asn=10 ] | testcmd0 zoinks')
1797
+ self.true(nodes[0].tags.get('zoinks'))
1808
1798
 
1809
- nodes = await core.nodes('[ inet:asn=11 ] | testcmd0 zoinks --domore')
1799
+ nodes = await core.nodes('[ inet:asn=11 ] | testcmd0 zoinks --domore')
1810
1800
 
1811
- self.true(nodes[0].tags.get('haha'))
1812
- self.true(nodes[0].tags.get('zoinks'))
1801
+ self.true(nodes[0].tags.get('haha'))
1802
+ self.true(nodes[0].tags.get('zoinks'))
1813
1803
 
1814
- # test that cmdopts/cmdconf/locals dont leak
1815
- with self.raises(s_exc.NoSuchVar):
1816
- q = '[ inet:asn=11 ] | testcmd0 zoinks --domore | if ($cmdopts) {[ +#hascmdopts ]}'
1817
- nodes = await core.nodes(q)
1804
+ # test that cmdopts/cmdconf/locals dont leak
1805
+ with self.raises(s_exc.NoSuchVar):
1806
+ q = '[ inet:asn=11 ] | testcmd0 zoinks --domore | if ($cmdopts) {[ +#hascmdopts ]}'
1807
+ nodes = await core.nodes(q)
1818
1808
 
1819
- with self.raises(s_exc.NoSuchVar):
1820
- q = '[ inet:asn=11 ] | testcmd0 zoinks --domore | if ($cmdconf) {[ +#hascmdconf ]}'
1821
- nodes = await core.nodes(q)
1809
+ with self.raises(s_exc.NoSuchVar):
1810
+ q = '[ inet:asn=11 ] | testcmd0 zoinks --domore | if ($cmdconf) {[ +#hascmdconf ]}'
1811
+ nodes = await core.nodes(q)
1822
1812
 
1823
- with self.raises(s_exc.NoSuchVar):
1824
- q = '[ inet:asn=11 ] | testcmd0 zoinks --domore | if ($foo) {[ +#hasfoo ]}'
1825
- nodes = await core.nodes(q)
1813
+ with self.raises(s_exc.NoSuchVar):
1814
+ q = '[ inet:asn=11 ] | testcmd0 zoinks --domore | if ($foo) {[ +#hasfoo ]}'
1815
+ nodes = await core.nodes(q)
1826
1816
 
1827
1817
  # make sure it's still loaded...
1828
1818
  async with self.getTestCore(dirn=dirn) as core:
1829
1819
 
1830
- async with core.getLocalProxy() as prox:
1831
-
1832
- await core.nodes('[ inet:asn=30 ] | testcmd0 zoinks')
1820
+ await core.nodes('[ inet:asn=30 ] | testcmd0 zoinks')
1833
1821
 
1834
- await prox.delStormCmd('testcmd0')
1822
+ await core.delStormCmd('testcmd0')
1835
1823
 
1836
- with self.raises(s_exc.NoSuchCmd):
1837
- await prox.delStormCmd('newpcmd')
1824
+ with self.raises(s_exc.NoSuchCmd):
1825
+ await core.delStormCmd('newpcmd')
1838
1826
 
1839
- with self.raises(s_exc.NoSuchName):
1840
- await core.nodes('[ inet:asn=31 ] | testcmd0 zoinks')
1827
+ with self.raises(s_exc.NoSuchName):
1828
+ await core.nodes('[ inet:asn=31 ] | testcmd0 zoinks')
1841
1829
 
1842
1830
  async def test_base_types2(self):
1843
1831
 
@@ -3130,6 +3118,7 @@ class CortexBasicTest(s_t_utils.SynTest):
3130
3118
  self.eq(corever, s_version.version)
3131
3119
  self.eq(corever, cellver)
3132
3120
 
3121
+ # NOTE: addNode / addNodes are deprecated in 3.0.0
3133
3122
  nodes = ((('inet:user', 'visi'), {}),)
3134
3123
 
3135
3124
  nodes = await alist(proxy.addNodes(nodes))
@@ -3137,24 +3126,6 @@ class CortexBasicTest(s_t_utils.SynTest):
3137
3126
 
3138
3127
  node = await proxy.addNode('test:str', 'foo')
3139
3128
 
3140
- pack = await proxy.addNodeTag(node[1].get('iden'), '#foo.bar')
3141
- self.eq(pack[1]['tags'].get('foo.bar'), (None, None))
3142
-
3143
- pack = await proxy.setNodeProp(node[1].get('iden'), 'tick', '2015')
3144
- self.eq(pack[1]['props'].get('tick'), 1420070400000)
3145
-
3146
- self.eq(1, await proxy.count('test:str#foo.bar'))
3147
- self.eq(1, await proxy.count('test:str:tick=2015'))
3148
-
3149
- pack = await proxy.delNodeProp(node[1].get('iden'), 'tick')
3150
- self.none(pack[1]['props'].get('tick'))
3151
-
3152
- iden = s_common.ehex(s_common.buid('newp'))
3153
- await self.asyncraises(s_exc.NoSuchIden, proxy.delNodeProp(iden, 'tick'))
3154
-
3155
- await proxy.delNodeTag(node[1].get('iden'), '#foo.bar')
3156
- self.eq(0, await proxy.count('test:str#foo.bar'))
3157
-
3158
3129
  opts = {'ndefs': [('inet:user', 'visi')]}
3159
3130
 
3160
3131
  msgs = await proxy.storm('', opts=opts).list()
@@ -3174,9 +3145,7 @@ class CortexBasicTest(s_t_utils.SynTest):
3174
3145
  ret = await proxy.getFeedFuncs()
3175
3146
  resp = {rec.get('name'): rec for rec in ret}
3176
3147
  self.isin('com.test.record', resp)
3177
- self.isin('syn.splice', resp)
3178
3148
  self.isin('syn.nodes', resp)
3179
- self.isin('syn.nodeedits', resp)
3180
3149
  rec = resp.get('syn.nodes')
3181
3150
  self.eq(rec.get('name'), 'syn.nodes')
3182
3151
  self.eq(rec.get('desc'), 'Add nodes to the Cortex via the packed node format.')
@@ -4452,46 +4421,6 @@ class CortexBasicTest(s_t_utils.SynTest):
4452
4421
 
4453
4422
  self.len(0, await core.nodes('test:cycle0=foo | delnode --force', opts=opts))
4454
4423
 
4455
- async def test_cortex_cell_splices(self):
4456
-
4457
- async with self.getTestCore() as core:
4458
-
4459
- async with core.getLocalProxy() as prox:
4460
- # TestModule creates one node and 3 splices
4461
- await self.agenlen(3, prox.splices((0, 0, 0), 1000))
4462
-
4463
- await alist(prox.eval('[ test:str=foo ]'))
4464
-
4465
- splicelist = await alist(prox.splices((0, 0, 0), 1000))
4466
- splicecount = len(splicelist)
4467
- self.ge(splicecount, 3)
4468
-
4469
- # should get the same splices in reverse order
4470
- splicelist.reverse()
4471
- self.eq(await alist(prox.splicesBack(splicelist[0][0], 1000)), splicelist)
4472
- self.eq(await alist(prox.splicesBack(splicelist[0][0], 3)), splicelist[:3])
4473
-
4474
- self.eq(await alist(prox.spliceHistory()), [s[1] for s in splicelist])
4475
-
4476
- visi = await prox.addUser('visi')
4477
- await prox.setUserPasswd(visi['iden'], 'secret')
4478
-
4479
- await prox.addUserRule(visi['iden'], (True, ('node', 'add')))
4480
- await prox.addUserRule(visi['iden'], (True, ('prop', 'set')))
4481
-
4482
- async with core.getLocalProxy(user='visi') as asvisi:
4483
-
4484
- # normal user can't user splicesBack
4485
- await self.agenraises(s_exc.AuthDeny, asvisi.splicesBack((1000, 0, 0), 1000))
4486
-
4487
- # make sure a normal user only gets their own splices
4488
- await alist(asvisi.eval('[ test:str=bar ]'))
4489
- await self.agenlen(2, asvisi.spliceHistory())
4490
-
4491
- # should get all splices now as an admin
4492
- await prox.setUserAdmin(visi['iden'], True)
4493
- await self.agenlen(splicecount + 2, asvisi.spliceHistory())
4494
-
4495
4424
  async def test_node_repr(self):
4496
4425
 
4497
4426
  async with self.getTestCore() as core:
@@ -4590,12 +4519,13 @@ class CortexBasicTest(s_t_utils.SynTest):
4590
4519
  self.len(2, await core.nodes('test:str'))
4591
4520
 
4592
4521
  layr = core.getLayer()
4593
- await self.agenlen(0, layr.splices())
4594
- await self.agenlen(0, layr.splicesBack())
4595
- await self.agenlen(0, layr.syncNodeEdits(0))
4596
- self.eq(0, await layr.getEditIndx())
4522
+ await self.agenlen(0, layr.syncNodeEdits(0, wait=False))
4523
+ await self.agenlen(0, layr.syncNodeEdits2(0, wait=False))
4524
+ # We can still generate synthetic edits though
4525
+ ndedits = await alist(layr.iterLayerNodeEdits())
4526
+ self.gt(len(ndedits), 0)
4597
4527
 
4598
- self.nn(await core.stat())
4528
+ self.eq(0, await layr.getEditIndx())
4599
4529
 
4600
4530
  async def test_cortex_layer_settings(self):
4601
4531
  '''
@@ -4789,181 +4719,6 @@ class CortexBasicTest(s_t_utils.SynTest):
4789
4719
  q = '[test:deprform=dform :deprprop=(1, 2)]'
4790
4720
  await core1.nodes(q, opts={'view': view2_iden})
4791
4721
 
4792
- async def test_feed_syn_splice(self):
4793
-
4794
- async with self.getTestCoreAndProxy() as (core, prox):
4795
-
4796
- mesg = ('node:add', {'ndef': ('test:str', 'foo')})
4797
- await core.addFeedData('syn.splice', [mesg])
4798
-
4799
- async with await core.snap() as snap:
4800
- node = await snap.getNodeByNdef(('test:str', 'foo'))
4801
- self.nn(node)
4802
-
4803
- # test coreapi addFeedData
4804
- mesg = ('node:add', {'ndef': ('test:str', 'foobar')})
4805
- await prox.addFeedData('syn.splice', [mesg])
4806
-
4807
- async with await core.snap() as snap:
4808
- node = await snap.getNodeByNdef(('test:str', 'foobar'))
4809
- self.nn(node)
4810
-
4811
- mesg = ('prop:set', {'ndef': ('test:str', 'foo'), 'prop': 'tick', 'valu': 200})
4812
- await core.addFeedData('syn.splice', [mesg])
4813
-
4814
- async with await core.snap() as snap:
4815
- node = await snap.getNodeByNdef(('test:str', 'foo'))
4816
- self.eq(200, node.get('tick'))
4817
-
4818
- mesg = ('prop:del', {'ndef': ('test:str', 'foo'), 'prop': 'tick'})
4819
- await core.addFeedData('syn.splice', [mesg])
4820
-
4821
- async with await core.snap() as snap:
4822
- node = await snap.getNodeByNdef(('test:str', 'foo'))
4823
- self.none(node.get('tick'))
4824
-
4825
- mesg = ('tag:add', {'ndef': ('test:str', 'foo'), 'tag': 'bar', 'valu': (200, 300)})
4826
- await core.addFeedData('syn.splice', [mesg])
4827
-
4828
- async with await core.snap() as snap:
4829
- node = await snap.getNodeByNdef(('test:str', 'foo'))
4830
- self.eq((200, 300), node.getTag('bar'))
4831
-
4832
- mesg = ('tag:del', {'ndef': ('test:str', 'foo'), 'tag': 'bar'})
4833
- await core.addFeedData('syn.splice', [mesg])
4834
-
4835
- async with await core.snap() as snap:
4836
- node = await snap.getNodeByNdef(('test:str', 'foo'))
4837
- self.none(node.getTag('bar'))
4838
-
4839
- await core.addTagProp('score', ('int', {}), {})
4840
- splice = ('tag:prop:set', {'ndef': ('test:str', 'foo'), 'tag': 'lol', 'prop': 'score', 'valu': 100,
4841
- 'curv': None})
4842
- await core.addFeedData('syn.splice', [splice])
4843
-
4844
- self.len(1, await core.nodes('#lol:score=100'))
4845
-
4846
- splice = ('tag:prop:del', {'ndef': ('test:str', 'foo'), 'tag': 'lol', 'prop': 'score', 'valu': 100})
4847
- await core.addFeedData('syn.splice', [splice])
4848
-
4849
- self.len(0, await core.nodes('#lol:score=100'))
4850
-
4851
- mesg = ('node:del', {'ndef': ('test:str', 'foo')})
4852
- await core.addFeedData('syn.splice', [mesg])
4853
-
4854
- async with await core.snap() as snap:
4855
- node = await snap.getNodeByNdef(('test:str', 'foo'))
4856
- self.none(node)
4857
-
4858
- # test feeding to a different view
4859
- vdef2 = await core.view.fork()
4860
- view2_iden = vdef2.get('iden')
4861
- view2 = core.getView(view2_iden)
4862
-
4863
- mesg = ('node:add', {'ndef': ('test:str', 'bar')})
4864
- await core.addFeedData('syn.splice', [mesg], viewiden=view2_iden)
4865
-
4866
- async with await core.snap(view=view2) as snap:
4867
- node = await snap.getNodeByNdef(('test:str', 'bar'))
4868
- self.nn(node)
4869
-
4870
- async with await core.snap() as snap:
4871
- node = await snap.getNodeByNdef(('test:str', 'bar'))
4872
- self.none(node)
4873
-
4874
- # test coreapi addFeedData to a different view
4875
- mesg = ('node:add', {'ndef': ('test:str', 'baz')})
4876
- await prox.addFeedData('syn.splice', [mesg], viewiden=view2_iden)
4877
-
4878
- async with await core.snap(view=view2) as snap:
4879
- node = await snap.getNodeByNdef(('test:str', 'baz'))
4880
- self.nn(node)
4881
-
4882
- async with await core.snap() as snap:
4883
- node = await snap.getNodeByNdef(('test:str', 'baz'))
4884
- self.none(node)
4885
-
4886
- # sad paths
4887
- await self.asyncraises(s_exc.NoSuchView, core.addFeedData('syn.splice', [mesg], viewiden='badiden'))
4888
- await self.asyncraises(s_exc.NoSuchView, prox.addFeedData('syn.splice', [mesg], viewiden='badiden'))
4889
-
4890
- async def test_feed_syn_nodeedits(self):
4891
-
4892
- async with self.getTestCoreAndProxy() as (core0, prox0):
4893
-
4894
- nodelist0 = []
4895
- nodelist0.extend(await core0.nodes('[ test:str=foo ]'))
4896
- nodelist0.extend(await core0.nodes('[ inet:ipv4=1.2.3.4 .seen=(2012,2014) +#foo.bar=(2012, 2014) ]'))
4897
- nodelist0.extend(await core0.nodes('[ test:int=42 ]'))
4898
- await core0.nodes('test:int=42 | delnode')
4899
-
4900
- with self.raises(s_exc.NoSuchLayer):
4901
- async for _, nodeedits in prox0.syncLayerNodeEdits(0, layriden='asdf', wait=False):
4902
- pass
4903
-
4904
- with self.raises(s_exc.NoSuchLayer):
4905
- async for _, nodeedits in core0.syncLayerNodeEdits('asdf', 0, wait=False):
4906
- pass
4907
-
4908
- editlist = []
4909
- async for _, nodeedits in prox0.syncLayerNodeEdits(0, wait=False):
4910
- editlist.append(nodeedits)
4911
-
4912
- deledit = editlist.pop(len(editlist) - 1)
4913
-
4914
- async with self.getTestCoreAndProxy() as (core1, prox1):
4915
-
4916
- await prox1.addFeedData('syn.nodeedits', editlist)
4917
-
4918
- nodelist1 = []
4919
- nodelist1.extend(await core1.nodes('test:str'))
4920
- nodelist1.extend(await core1.nodes('inet:ipv4'))
4921
- nodelist1.extend(await core1.nodes('test:int'))
4922
-
4923
- nodelist0 = [node.pack() for node in nodelist0]
4924
- nodelist1 = [node.pack() for node in nodelist1]
4925
- self.eq(nodelist0, nodelist1)
4926
-
4927
- await core1.nodes('trigger.add node:del --form test:int --query {[test:int=7]}')
4928
-
4929
- self.len(1, await core1.nodes('test:int=42'))
4930
-
4931
- await prox1.addFeedData('syn.nodeedits', [deledit])
4932
-
4933
- self.len(0, await core1.nodes('test:int=42'))
4934
- self.len(1, await core1.nodes('test:int=7'))
4935
-
4936
- # Try a nodeedits we might get from cmdr
4937
- cmdrnodeedits = s_common.jsonsafe_nodeedits(editlist[1])
4938
- await core0.nodes('test:str=foo | delnode')
4939
-
4940
- await prox1.addFeedData('syn.nodeedits', [cmdrnodeedits])
4941
- self.len(1, await core1.nodes('test:str'))
4942
-
4943
- async def test_stat(self):
4944
-
4945
- async with self.getTestCoreAndProxy() as (realcore, core):
4946
- coreiden = realcore.iden
4947
- ostat = await core.stat()
4948
- self.eq(ostat.get('iden'), coreiden)
4949
- self.isin('layer', ostat)
4950
- self.len(1, await realcore.nodes('[test:str=123 :tick=2018]'))
4951
- nstat = await core.stat()
4952
-
4953
- counts = nstat.get('formcounts')
4954
- self.eq(counts.get('test:str'), 1)
4955
-
4956
- async def test_stat_lock(self):
4957
- self.thisHostMust(hasmemlocking=True)
4958
- conf = {'layers:lockmemory': True}
4959
- async with self.getTestCoreAndProxy(conf=conf) as (realcore, core):
4960
- slab = realcore.view.layers[0].layrslab
4961
- self.true(await asyncio.wait_for(slab.lockdoneevent.wait(), 8))
4962
-
4963
- nstat = await core.stat()
4964
- layr = nstat.get('layer')
4965
- self.gt(layr.get('lock_goal'), 0)
4966
-
4967
4722
  async def test_storm_sub_query(self):
4968
4723
 
4969
4724
  async with self.getTestCore() as core:
@@ -5124,6 +4879,9 @@ class CortexBasicTest(s_t_utils.SynTest):
5124
4879
  self.nn(node.getTag('known'))
5125
4880
  self.none(node.getTag('unknown'))
5126
4881
 
4882
+ q = '$valu={[test:str=foo]} switch $valu { foo: {test:str=foo return($node.value()) } }'
4883
+ self.eq('foo', await core.callStorm(q))
4884
+
5127
4885
  async def test_storm_tagvar(self):
5128
4886
 
5129
4887
  async with self.getTestCore() as core:
@@ -6663,35 +6421,6 @@ class CortexBasicTest(s_t_utils.SynTest):
6663
6421
  self.eq(view, core.getView(core.iden))
6664
6422
  self.none(core.getView('xxx'))
6665
6423
 
6666
- async def test_cortex_cronjob_perms(self):
6667
- async with self.getTestCore() as realcore:
6668
- async with realcore.getLocalProxy() as core:
6669
- fred = await core.addUser('fred')
6670
- await core.setUserPasswd(fred['iden'], 'secret')
6671
- cdef = {'storm': '[test:str=foo]', 'reqs': {'dayofmonth': 1},
6672
- 'incunit': None, 'incvals': None}
6673
- adef = await core.addCronJob(cdef)
6674
- iden = adef.get('iden')
6675
-
6676
- async with realcore.getLocalProxy(user='fred') as core:
6677
- # Rando user can't make cron jobs
6678
- cdef = {'storm': '[test:int=1]', 'reqs': {'month': 1},
6679
- 'incunit': None, 'incvals': None}
6680
- await self.asyncraises(s_exc.AuthDeny, core.addCronJob(cdef))
6681
-
6682
- # Rando user can't mod cron jobs
6683
- await self.asyncraises(s_exc.AuthDeny, core.updateCronJob(iden, '[test:str=bar]'))
6684
-
6685
- # Rando user doesn't see any cron jobs
6686
- self.len(0, await core.listCronJobs())
6687
-
6688
- # Rando user can't delete cron jobs
6689
- await self.asyncraises(s_exc.AuthDeny, core.delCronJob(iden))
6690
-
6691
- # Rando user can't enable/disable cron jobs
6692
- await self.asyncraises(s_exc.AuthDeny, core.enableCronJob(iden))
6693
- await self.asyncraises(s_exc.AuthDeny, core.disableCronJob(iden))
6694
-
6695
6424
  async def test_cortex_cron_deluser(self):
6696
6425
 
6697
6426
  async with self.getTestCore() as core:
@@ -6767,39 +6496,6 @@ class CortexBasicTest(s_t_utils.SynTest):
6767
6496
  await self.asyncraises(s_exc.AuthDeny, prox.enableMigrationMode())
6768
6497
  await self.asyncraises(s_exc.AuthDeny, prox.disableMigrationMode())
6769
6498
 
6770
- async def test_cortex_watch(self):
6771
-
6772
- async with self.getTestCore() as core:
6773
-
6774
- async with core.getLocalProxy() as prox:
6775
-
6776
- async def nodes():
6777
- await asyncio.sleep(0.1) # due to telepath proxy causing task switch
6778
- await core.nodes('[ test:int=10 +#foo.bar +#baz.faz ]')
6779
- await core.nodes('test:int=10 [ -#foo.bar -#baz.faz ]')
6780
-
6781
- task = core.schedCoro(nodes())
6782
-
6783
- data = []
6784
- async for mesg in prox.watch({'tags': ['foo.bar', 'baz.*']}):
6785
- data.append(mesg)
6786
- if len(data) == 4:
6787
- break
6788
-
6789
- await asyncio.wait_for(task, timeout=1)
6790
-
6791
- self.eq(data[0][0], 'tag:add')
6792
- self.eq(data[0][1]['tag'], 'foo.bar')
6793
-
6794
- self.eq(data[1][0], 'tag:add')
6795
- self.eq(data[1][1]['tag'], 'baz.faz')
6796
-
6797
- self.eq(data[2][0], 'tag:del')
6798
- self.eq(data[2][1]['tag'], 'foo.bar')
6799
-
6800
- self.eq(data[3][0], 'tag:del')
6801
- self.eq(data[3][1]['tag'], 'baz.faz')
6802
-
6803
6499
  async def test_cortex_behold(self):
6804
6500
  async with self.getTestCore() as core:
6805
6501
  async with core.getLocalProxy() as prox: