synapse 2.155.0__py311-none-any.whl → 2.156.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/cmds/cortex.py +2 -14
- synapse/common.py +1 -28
- synapse/cortex.py +10 -510
- synapse/lib/ast.py +60 -1
- synapse/lib/cell.py +33 -8
- synapse/lib/certdir.py +11 -0
- synapse/lib/cmdr.py +0 -5
- synapse/lib/gis.py +2 -2
- synapse/lib/httpapi.py +1 -43
- synapse/lib/layer.py +64 -201
- synapse/lib/lmdbslab.py +11 -0
- synapse/lib/node.py +1 -3
- synapse/lib/parser.py +10 -0
- synapse/lib/snap.py +121 -21
- synapse/lib/storm.lark +23 -6
- synapse/lib/storm.py +15 -338
- synapse/lib/storm_format.py +5 -0
- synapse/lib/stormlib/gen.py +1 -2
- synapse/lib/stormlib/gis.py +41 -0
- synapse/lib/stormlib/stats.py +21 -2
- synapse/lib/stormlib/storm.py +16 -1
- synapse/lib/stormtypes.py +225 -12
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +96 -21
- synapse/models/inet.py +60 -30
- synapse/models/infotech.py +56 -1
- synapse/models/orgs.py +3 -0
- synapse/models/risk.py +15 -0
- synapse/models/syn.py +0 -38
- synapse/tests/test_cmds_cortex.py +1 -1
- synapse/tests/test_cortex.py +32 -336
- synapse/tests/test_lib_agenda.py +19 -54
- synapse/tests/test_lib_aha.py +97 -0
- synapse/tests/test_lib_ast.py +402 -0
- synapse/tests/test_lib_grammar.py +30 -10
- synapse/tests/test_lib_httpapi.py +0 -46
- synapse/tests/test_lib_layer.py +19 -234
- synapse/tests/test_lib_lmdbslab.py +22 -0
- synapse/tests/test_lib_snap.py +9 -0
- synapse/tests/test_lib_storm.py +16 -309
- synapse/tests/test_lib_stormlib_gis.py +21 -0
- synapse/tests/test_lib_stormlib_stats.py +107 -20
- synapse/tests/test_lib_stormlib_storm.py +25 -0
- synapse/tests/test_lib_stormtypes.py +231 -8
- synapse/tests/test_lib_view.py +6 -13
- synapse/tests/test_model_base.py +1 -1
- synapse/tests/test_model_inet.py +15 -0
- synapse/tests/test_model_infotech.py +60 -0
- synapse/tests/test_model_orgs.py +10 -0
- synapse/tests/test_model_person.py +0 -3
- synapse/tests/test_model_risk.py +20 -0
- synapse/tests/test_model_syn.py +20 -34
- synapse/tests/test_tools_csvtool.py +2 -1
- synapse/tests/test_tools_feed.py +4 -30
- synapse/tools/csvtool.py +2 -1
- {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/METADATA +3 -3
- {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/RECORD +60 -62
- {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/WHEEL +1 -1
- synapse/cmds/cron.py +0 -726
- synapse/cmds/trigger.py +0 -319
- synapse/tests/test_cmds_cron.py +0 -453
- synapse/tests/test_cmds_trigger.py +0 -176
- {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/LICENSE +0 -0
- {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/top_level.txt +0 -0
synapse/models/infotech.py
CHANGED
|
@@ -408,6 +408,10 @@ class ItModule(s_module.CoreModule):
|
|
|
408
408
|
'doc': 'A Mitre ATT&CK Software ID.',
|
|
409
409
|
'ex': 'S0154',
|
|
410
410
|
}),
|
|
411
|
+
('it:mitre:attack:campaign', ('str', {'regex': r'^C[0-9]{4}$'}), {
|
|
412
|
+
'doc': 'A Mitre ATT&CK Campaign ID.',
|
|
413
|
+
'ex': 'C0028',
|
|
414
|
+
}),
|
|
411
415
|
('it:mitre:attack:flow', ('guid', {}), {
|
|
412
416
|
'doc': 'A Mitre ATT&CK Flow diagram.',
|
|
413
417
|
}),
|
|
@@ -490,7 +494,8 @@ class ItModule(s_module.CoreModule):
|
|
|
490
494
|
('it:adid', ('str', {'lower': True, 'strip': True}), {
|
|
491
495
|
'doc': 'An advertising identification string.'}),
|
|
492
496
|
|
|
493
|
-
|
|
497
|
+
# https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-dtyp/c92a27b1-c772-4fa7-a432-15df5f1b66a1
|
|
498
|
+
('it:os:windows:sid', ('str', {'regex': r'^S-1-(?:\d{1,10}|0x[0-9a-fA-F]{12})(?:-(?:\d+|0x[0-9a-fA-F]{2,}))*$'}), {
|
|
494
499
|
'doc': 'A Microsoft Windows Security Identifier.',
|
|
495
500
|
'ex': 'S-1-5-21-1220945662-1202665555-839525555-5555',
|
|
496
501
|
}),
|
|
@@ -1255,6 +1260,50 @@ class ItModule(s_module.CoreModule):
|
|
|
1255
1260
|
'doc': 'An array of ATT&CK technique IDs addressed by the mitigation.',
|
|
1256
1261
|
}),
|
|
1257
1262
|
)),
|
|
1263
|
+
('it:mitre:attack:campaign', {}, (
|
|
1264
|
+
('name', ('ou:campname', {}), {
|
|
1265
|
+
'doc': 'The primary name for the ATT&CK campaign.',
|
|
1266
|
+
}),
|
|
1267
|
+
('names', ('array', {'type': 'ou:campname', 'uniq': True, 'sorted': True}), {
|
|
1268
|
+
'doc': 'An array of alternate names for the ATT&CK campaign.',
|
|
1269
|
+
}),
|
|
1270
|
+
('desc', ('str', {'strip': True}), {
|
|
1271
|
+
'doc': 'A description of the ATT&CK campaign.',
|
|
1272
|
+
'disp': {'hint': 'text'},
|
|
1273
|
+
}),
|
|
1274
|
+
('url', ('inet:url', {}), {
|
|
1275
|
+
'doc': 'The URL that documents the ATT&CK campaign.',
|
|
1276
|
+
}),
|
|
1277
|
+
('groups', ('array', {'type': 'it:mitre:attack:group',
|
|
1278
|
+
'uniq': True, 'sorted': True, 'split': ','}), {
|
|
1279
|
+
'doc': 'An array of ATT&CK group IDs attributed to the campaign.',
|
|
1280
|
+
}),
|
|
1281
|
+
('software', ('array', {'type': 'it:mitre:attack:software',
|
|
1282
|
+
'uniq': True, 'sorted': True, 'split': ','}), {
|
|
1283
|
+
'doc': 'An array of ATT&CK software IDs used in the campaign.',
|
|
1284
|
+
}),
|
|
1285
|
+
('techniques', ('array', {'type': 'it:mitre:attack:technique',
|
|
1286
|
+
'uniq': True, 'sorted': True, 'split': ','}), {
|
|
1287
|
+
'doc': 'An array of ATT&CK technique IDs used in the campaign.',
|
|
1288
|
+
}),
|
|
1289
|
+
('matrices', ('array', {'type': 'it:mitre:attack:matrix',
|
|
1290
|
+
'uniq': True, 'sorted': True, 'split': ','}), {
|
|
1291
|
+
'doc': 'The ATT&CK matrices which define the campaign.',
|
|
1292
|
+
}),
|
|
1293
|
+
('references', ('array', {'type': 'inet:url', 'uniq': True}), {
|
|
1294
|
+
'doc': 'An array of URLs that document the ATT&CK campaign.',
|
|
1295
|
+
}),
|
|
1296
|
+
('period', ('ival', {}), {
|
|
1297
|
+
'doc': 'The time interval when the campaign was active.'}),
|
|
1298
|
+
('created', ('time', {}), {
|
|
1299
|
+
'doc': 'The time that the campaign was created by Mitre.'}),
|
|
1300
|
+
('updated', ('time', {}), {
|
|
1301
|
+
'doc': 'The time that the campaign was last updated by Mitre.'}),
|
|
1302
|
+
('tag', ('syn:tag', {}), {
|
|
1303
|
+
'doc': 'The synapse tag used to annotate nodes included in this ATT&CK campaign.',
|
|
1304
|
+
'ex': 'cno.mitre.c0028',
|
|
1305
|
+
}),
|
|
1306
|
+
)),
|
|
1258
1307
|
('it:mitre:attack:flow', {}, (
|
|
1259
1308
|
('name', ('str', {}), {
|
|
1260
1309
|
'doc': 'The name of the attack-flow diagram.'}),
|
|
@@ -1789,6 +1838,12 @@ class ItModule(s_module.CoreModule):
|
|
|
1789
1838
|
('target:url', ('inet:url', {}), {
|
|
1790
1839
|
'doc': 'The URL that was scanned to produce the result.'}),
|
|
1791
1840
|
|
|
1841
|
+
('target:ipv4', ('inet:ipv4', {}), {
|
|
1842
|
+
'doc': 'The IPv4 address that was scanned to produce the result.'}),
|
|
1843
|
+
|
|
1844
|
+
('target:ipv6', ('inet:ipv6', {}), {
|
|
1845
|
+
'doc': 'The IPv6 address that was scanned to produce the result.'}),
|
|
1846
|
+
|
|
1792
1847
|
('multi:scan', ('it:av:scan:result', {}), {
|
|
1793
1848
|
'doc': 'Set if this result was part of running multiple scanners.'}),
|
|
1794
1849
|
|
synapse/models/orgs.py
CHANGED
|
@@ -503,6 +503,9 @@ class OuModule(s_module.CoreModule):
|
|
|
503
503
|
|
|
504
504
|
('tag', ('syn:tag', {}), {
|
|
505
505
|
'doc': 'The tag used to annotate nodes that are associated with the campaign.'}),
|
|
506
|
+
|
|
507
|
+
('mitre:attack:campaign', ('it:mitre:attack:campaign', {}), {
|
|
508
|
+
'doc': 'A mapping to a Mitre ATT&CK campaign if applicable.'}),
|
|
506
509
|
)),
|
|
507
510
|
('ou:conflict', {}, (
|
|
508
511
|
('name', ('str', {'onespace': True}), {
|
synapse/models/risk.py
CHANGED
|
@@ -115,6 +115,8 @@ class RiskModule(s_module.CoreModule):
|
|
|
115
115
|
('risk:extortion:type:taxonomy', ('taxonomy', {}), {
|
|
116
116
|
'interfaces': ('taxonomy',),
|
|
117
117
|
'doc': 'A taxonomy of extortion event types.'}),
|
|
118
|
+
('risk:technique:masquerade', ('guid', {}), {
|
|
119
|
+
'doc': 'Represents the assessment that a node is designed to resemble another in order to mislead.'}),
|
|
118
120
|
),
|
|
119
121
|
'edges': (
|
|
120
122
|
# some explicit examples...
|
|
@@ -337,6 +339,9 @@ class RiskModule(s_module.CoreModule):
|
|
|
337
339
|
('timeline:exploited', ('time', {"ismin": True}), {
|
|
338
340
|
'doc': 'The earliest known time when the vulnerability was exploited in the wild.'}),
|
|
339
341
|
|
|
342
|
+
('id', ('str', {'strip': True}), {
|
|
343
|
+
'doc': 'An identifier for the vulnerability.'}),
|
|
344
|
+
|
|
340
345
|
('cve', ('it:sec:cve', {}), {
|
|
341
346
|
'doc': 'The CVE ID of the vulnerability.'}),
|
|
342
347
|
|
|
@@ -942,6 +947,16 @@ class RiskModule(s_module.CoreModule):
|
|
|
942
947
|
'doc': 'The currency in which payment was demanded.'}),
|
|
943
948
|
|
|
944
949
|
)),
|
|
950
|
+
('risk:technique:masquerade', {}, (
|
|
951
|
+
('node', ('ndef', {}), {
|
|
952
|
+
'doc': 'The node masquerading as another.'}),
|
|
953
|
+
('period', ('ival', {}), {
|
|
954
|
+
'doc': 'The time period when the masquerading was active.'}),
|
|
955
|
+
('target', ('ndef', {}), {
|
|
956
|
+
'doc': 'The being masqueraded as.'}),
|
|
957
|
+
('technique', ('ou:technique', {}), {
|
|
958
|
+
'doc': 'The specific technique which describes the type of masquerading.'}),
|
|
959
|
+
)),
|
|
945
960
|
),
|
|
946
961
|
}
|
|
947
962
|
name = 'risk'
|
synapse/models/syn.py
CHANGED
|
@@ -127,9 +127,6 @@ class SynModule(s_module.CoreModule):
|
|
|
127
127
|
('syn:cmd', ('str', {'strip': True}), {
|
|
128
128
|
'doc': 'A Synapse storm command.'
|
|
129
129
|
}),
|
|
130
|
-
('syn:splice', ('guid', {'strip': True}), {
|
|
131
|
-
'doc': 'A splice from a layer.'
|
|
132
|
-
}),
|
|
133
130
|
('syn:nodedata', ('comp', {'fields': (('key', 'str'), ('form', 'syn:form'))}), {
|
|
134
131
|
'doc': 'A nodedata key and the form it may be present on.',
|
|
135
132
|
}),
|
|
@@ -282,40 +279,5 @@ class SynModule(s_module.CoreModule):
|
|
|
282
279
|
('nodedata', ('array', {'type': 'syn:nodedata'}), {
|
|
283
280
|
'doc': 'The list of nodedata that may be added by the command.', 'uniq': True, 'sorted': True, 'ro': True}),
|
|
284
281
|
)),
|
|
285
|
-
('syn:splice', {'runt': True}, (
|
|
286
|
-
('type', ('str', {'strip': True}), {
|
|
287
|
-
'doc': 'Type of splice.', 'ro': True
|
|
288
|
-
}),
|
|
289
|
-
('iden', ('str', {}), {
|
|
290
|
-
'doc': 'The iden of the node involved in the splice.', 'ro': True,
|
|
291
|
-
}),
|
|
292
|
-
('form', ('syn:form', {'strip': True}), {
|
|
293
|
-
'doc': 'The form involved in the splice.', 'ro': True
|
|
294
|
-
}),
|
|
295
|
-
('prop', ('syn:prop', {'strip': True}), {
|
|
296
|
-
'doc': 'Property modified in the splice.', 'ro': True
|
|
297
|
-
}),
|
|
298
|
-
('tag', ('syn:tag', {'strip': True}), {
|
|
299
|
-
'doc': 'Tag modified in the splice.', 'ro': True
|
|
300
|
-
}),
|
|
301
|
-
('valu', ('data', {}), {
|
|
302
|
-
'doc': 'The value being set in the splice.', 'ro': True
|
|
303
|
-
}),
|
|
304
|
-
('oldv', ('data', {}), {
|
|
305
|
-
'doc': 'The value before the splice.', 'ro': True
|
|
306
|
-
}),
|
|
307
|
-
('user', ('guid', {}), {
|
|
308
|
-
'doc': 'The user who caused the splice.', 'ro': True,
|
|
309
|
-
}),
|
|
310
|
-
('prov', ('guid', {}), {
|
|
311
|
-
'doc': 'The provenance stack of the splice.', 'ro': True,
|
|
312
|
-
}),
|
|
313
|
-
('time', ('time', {}), {
|
|
314
|
-
'doc': 'The time the splice occurred.', 'ro': True,
|
|
315
|
-
}),
|
|
316
|
-
('splice', ('data', {}), {
|
|
317
|
-
'doc': 'The splice.', 'ro': True
|
|
318
|
-
}),
|
|
319
|
-
)),
|
|
320
282
|
),
|
|
321
283
|
}),)
|
|
@@ -245,7 +245,7 @@ class CmdCoreTest(s_t_utils.SynTest):
|
|
|
245
245
|
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
|
|
246
246
|
await cmdr.runCmdLine('log --on --format jsonl')
|
|
247
247
|
fp = cmdr.locs.get('log:fp')
|
|
248
|
-
await cmdr.runCmdLine('storm
|
|
248
|
+
await cmdr.runCmdLine('storm [test:str=hi :tick=2018 +#haha.hehe]')
|
|
249
249
|
|
|
250
250
|
await cmdr.runCmdLine('storm --editformat nodeedits [test:str=hi2 :tick=2018 +#haha.hehe]')
|
|
251
251
|
await cmdr.runCmdLine('storm [test:comp=(42, bar)]')
|
synapse/tests/test_cortex.py
CHANGED
|
@@ -937,14 +937,6 @@ class CortexTest(s_t_utils.SynTest):
|
|
|
937
937
|
return($list.size())
|
|
938
938
|
'''))
|
|
939
939
|
|
|
940
|
-
# check that edge node edits dont bork up legacy splice generation
|
|
941
|
-
nodeedits = [(ipv4.buid, 'inet:ipv4', (
|
|
942
|
-
(s_layer.EDIT_EDGE_ADD, (), ()),
|
|
943
|
-
(s_layer.EDIT_EDGE_DEL, (), ()),
|
|
944
|
-
))]
|
|
945
|
-
|
|
946
|
-
self.eq((), await alist(core.view.layers[0].makeSplices(0, nodeedits, {})))
|
|
947
|
-
|
|
948
940
|
# Run multiple nodes through edge creation/deletion ( test coverage for perm caching )
|
|
949
941
|
await core.nodes('inet:ipv4 [ <(test)+ { meta:source:name=test }]')
|
|
950
942
|
self.len(2, await core.nodes('meta:source:name=test -(test)> *'))
|
|
@@ -1799,45 +1791,41 @@ class CortexTest(s_t_utils.SynTest):
|
|
|
1799
1791
|
|
|
1800
1792
|
async with self.getTestCore(dirn=dirn) as core:
|
|
1801
1793
|
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
await prox.setStormCmd(cdef0)
|
|
1794
|
+
await core.setStormCmd(cdef0)
|
|
1805
1795
|
|
|
1806
|
-
|
|
1807
|
-
|
|
1796
|
+
nodes = await core.nodes('[ inet:asn=10 ] | testcmd0 zoinks')
|
|
1797
|
+
self.true(nodes[0].tags.get('zoinks'))
|
|
1808
1798
|
|
|
1809
|
-
|
|
1799
|
+
nodes = await core.nodes('[ inet:asn=11 ] | testcmd0 zoinks --domore')
|
|
1810
1800
|
|
|
1811
|
-
|
|
1812
|
-
|
|
1801
|
+
self.true(nodes[0].tags.get('haha'))
|
|
1802
|
+
self.true(nodes[0].tags.get('zoinks'))
|
|
1813
1803
|
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1804
|
+
# test that cmdopts/cmdconf/locals dont leak
|
|
1805
|
+
with self.raises(s_exc.NoSuchVar):
|
|
1806
|
+
q = '[ inet:asn=11 ] | testcmd0 zoinks --domore | if ($cmdopts) {[ +#hascmdopts ]}'
|
|
1807
|
+
nodes = await core.nodes(q)
|
|
1818
1808
|
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1809
|
+
with self.raises(s_exc.NoSuchVar):
|
|
1810
|
+
q = '[ inet:asn=11 ] | testcmd0 zoinks --domore | if ($cmdconf) {[ +#hascmdconf ]}'
|
|
1811
|
+
nodes = await core.nodes(q)
|
|
1822
1812
|
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1813
|
+
with self.raises(s_exc.NoSuchVar):
|
|
1814
|
+
q = '[ inet:asn=11 ] | testcmd0 zoinks --domore | if ($foo) {[ +#hasfoo ]}'
|
|
1815
|
+
nodes = await core.nodes(q)
|
|
1826
1816
|
|
|
1827
1817
|
# make sure it's still loaded...
|
|
1828
1818
|
async with self.getTestCore(dirn=dirn) as core:
|
|
1829
1819
|
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
await core.nodes('[ inet:asn=30 ] | testcmd0 zoinks')
|
|
1820
|
+
await core.nodes('[ inet:asn=30 ] | testcmd0 zoinks')
|
|
1833
1821
|
|
|
1834
|
-
|
|
1822
|
+
await core.delStormCmd('testcmd0')
|
|
1835
1823
|
|
|
1836
|
-
|
|
1837
|
-
|
|
1824
|
+
with self.raises(s_exc.NoSuchCmd):
|
|
1825
|
+
await core.delStormCmd('newpcmd')
|
|
1838
1826
|
|
|
1839
|
-
|
|
1840
|
-
|
|
1827
|
+
with self.raises(s_exc.NoSuchName):
|
|
1828
|
+
await core.nodes('[ inet:asn=31 ] | testcmd0 zoinks')
|
|
1841
1829
|
|
|
1842
1830
|
async def test_base_types2(self):
|
|
1843
1831
|
|
|
@@ -3130,6 +3118,7 @@ class CortexBasicTest(s_t_utils.SynTest):
|
|
|
3130
3118
|
self.eq(corever, s_version.version)
|
|
3131
3119
|
self.eq(corever, cellver)
|
|
3132
3120
|
|
|
3121
|
+
# NOTE: addNode / addNodes are deprecated in 3.0.0
|
|
3133
3122
|
nodes = ((('inet:user', 'visi'), {}),)
|
|
3134
3123
|
|
|
3135
3124
|
nodes = await alist(proxy.addNodes(nodes))
|
|
@@ -3137,24 +3126,6 @@ class CortexBasicTest(s_t_utils.SynTest):
|
|
|
3137
3126
|
|
|
3138
3127
|
node = await proxy.addNode('test:str', 'foo')
|
|
3139
3128
|
|
|
3140
|
-
pack = await proxy.addNodeTag(node[1].get('iden'), '#foo.bar')
|
|
3141
|
-
self.eq(pack[1]['tags'].get('foo.bar'), (None, None))
|
|
3142
|
-
|
|
3143
|
-
pack = await proxy.setNodeProp(node[1].get('iden'), 'tick', '2015')
|
|
3144
|
-
self.eq(pack[1]['props'].get('tick'), 1420070400000)
|
|
3145
|
-
|
|
3146
|
-
self.eq(1, await proxy.count('test:str#foo.bar'))
|
|
3147
|
-
self.eq(1, await proxy.count('test:str:tick=2015'))
|
|
3148
|
-
|
|
3149
|
-
pack = await proxy.delNodeProp(node[1].get('iden'), 'tick')
|
|
3150
|
-
self.none(pack[1]['props'].get('tick'))
|
|
3151
|
-
|
|
3152
|
-
iden = s_common.ehex(s_common.buid('newp'))
|
|
3153
|
-
await self.asyncraises(s_exc.NoSuchIden, proxy.delNodeProp(iden, 'tick'))
|
|
3154
|
-
|
|
3155
|
-
await proxy.delNodeTag(node[1].get('iden'), '#foo.bar')
|
|
3156
|
-
self.eq(0, await proxy.count('test:str#foo.bar'))
|
|
3157
|
-
|
|
3158
3129
|
opts = {'ndefs': [('inet:user', 'visi')]}
|
|
3159
3130
|
|
|
3160
3131
|
msgs = await proxy.storm('', opts=opts).list()
|
|
@@ -3174,9 +3145,7 @@ class CortexBasicTest(s_t_utils.SynTest):
|
|
|
3174
3145
|
ret = await proxy.getFeedFuncs()
|
|
3175
3146
|
resp = {rec.get('name'): rec for rec in ret}
|
|
3176
3147
|
self.isin('com.test.record', resp)
|
|
3177
|
-
self.isin('syn.splice', resp)
|
|
3178
3148
|
self.isin('syn.nodes', resp)
|
|
3179
|
-
self.isin('syn.nodeedits', resp)
|
|
3180
3149
|
rec = resp.get('syn.nodes')
|
|
3181
3150
|
self.eq(rec.get('name'), 'syn.nodes')
|
|
3182
3151
|
self.eq(rec.get('desc'), 'Add nodes to the Cortex via the packed node format.')
|
|
@@ -4452,46 +4421,6 @@ class CortexBasicTest(s_t_utils.SynTest):
|
|
|
4452
4421
|
|
|
4453
4422
|
self.len(0, await core.nodes('test:cycle0=foo | delnode --force', opts=opts))
|
|
4454
4423
|
|
|
4455
|
-
async def test_cortex_cell_splices(self):
|
|
4456
|
-
|
|
4457
|
-
async with self.getTestCore() as core:
|
|
4458
|
-
|
|
4459
|
-
async with core.getLocalProxy() as prox:
|
|
4460
|
-
# TestModule creates one node and 3 splices
|
|
4461
|
-
await self.agenlen(3, prox.splices((0, 0, 0), 1000))
|
|
4462
|
-
|
|
4463
|
-
await alist(prox.eval('[ test:str=foo ]'))
|
|
4464
|
-
|
|
4465
|
-
splicelist = await alist(prox.splices((0, 0, 0), 1000))
|
|
4466
|
-
splicecount = len(splicelist)
|
|
4467
|
-
self.ge(splicecount, 3)
|
|
4468
|
-
|
|
4469
|
-
# should get the same splices in reverse order
|
|
4470
|
-
splicelist.reverse()
|
|
4471
|
-
self.eq(await alist(prox.splicesBack(splicelist[0][0], 1000)), splicelist)
|
|
4472
|
-
self.eq(await alist(prox.splicesBack(splicelist[0][0], 3)), splicelist[:3])
|
|
4473
|
-
|
|
4474
|
-
self.eq(await alist(prox.spliceHistory()), [s[1] for s in splicelist])
|
|
4475
|
-
|
|
4476
|
-
visi = await prox.addUser('visi')
|
|
4477
|
-
await prox.setUserPasswd(visi['iden'], 'secret')
|
|
4478
|
-
|
|
4479
|
-
await prox.addUserRule(visi['iden'], (True, ('node', 'add')))
|
|
4480
|
-
await prox.addUserRule(visi['iden'], (True, ('prop', 'set')))
|
|
4481
|
-
|
|
4482
|
-
async with core.getLocalProxy(user='visi') as asvisi:
|
|
4483
|
-
|
|
4484
|
-
# normal user can't user splicesBack
|
|
4485
|
-
await self.agenraises(s_exc.AuthDeny, asvisi.splicesBack((1000, 0, 0), 1000))
|
|
4486
|
-
|
|
4487
|
-
# make sure a normal user only gets their own splices
|
|
4488
|
-
await alist(asvisi.eval('[ test:str=bar ]'))
|
|
4489
|
-
await self.agenlen(2, asvisi.spliceHistory())
|
|
4490
|
-
|
|
4491
|
-
# should get all splices now as an admin
|
|
4492
|
-
await prox.setUserAdmin(visi['iden'], True)
|
|
4493
|
-
await self.agenlen(splicecount + 2, asvisi.spliceHistory())
|
|
4494
|
-
|
|
4495
4424
|
async def test_node_repr(self):
|
|
4496
4425
|
|
|
4497
4426
|
async with self.getTestCore() as core:
|
|
@@ -4590,12 +4519,13 @@ class CortexBasicTest(s_t_utils.SynTest):
|
|
|
4590
4519
|
self.len(2, await core.nodes('test:str'))
|
|
4591
4520
|
|
|
4592
4521
|
layr = core.getLayer()
|
|
4593
|
-
await self.agenlen(0, layr.
|
|
4594
|
-
await self.agenlen(0, layr.
|
|
4595
|
-
|
|
4596
|
-
|
|
4522
|
+
await self.agenlen(0, layr.syncNodeEdits(0, wait=False))
|
|
4523
|
+
await self.agenlen(0, layr.syncNodeEdits2(0, wait=False))
|
|
4524
|
+
# We can still generate synthetic edits though
|
|
4525
|
+
ndedits = await alist(layr.iterLayerNodeEdits())
|
|
4526
|
+
self.gt(len(ndedits), 0)
|
|
4597
4527
|
|
|
4598
|
-
self.
|
|
4528
|
+
self.eq(0, await layr.getEditIndx())
|
|
4599
4529
|
|
|
4600
4530
|
async def test_cortex_layer_settings(self):
|
|
4601
4531
|
'''
|
|
@@ -4789,181 +4719,6 @@ class CortexBasicTest(s_t_utils.SynTest):
|
|
|
4789
4719
|
q = '[test:deprform=dform :deprprop=(1, 2)]'
|
|
4790
4720
|
await core1.nodes(q, opts={'view': view2_iden})
|
|
4791
4721
|
|
|
4792
|
-
async def test_feed_syn_splice(self):
|
|
4793
|
-
|
|
4794
|
-
async with self.getTestCoreAndProxy() as (core, prox):
|
|
4795
|
-
|
|
4796
|
-
mesg = ('node:add', {'ndef': ('test:str', 'foo')})
|
|
4797
|
-
await core.addFeedData('syn.splice', [mesg])
|
|
4798
|
-
|
|
4799
|
-
async with await core.snap() as snap:
|
|
4800
|
-
node = await snap.getNodeByNdef(('test:str', 'foo'))
|
|
4801
|
-
self.nn(node)
|
|
4802
|
-
|
|
4803
|
-
# test coreapi addFeedData
|
|
4804
|
-
mesg = ('node:add', {'ndef': ('test:str', 'foobar')})
|
|
4805
|
-
await prox.addFeedData('syn.splice', [mesg])
|
|
4806
|
-
|
|
4807
|
-
async with await core.snap() as snap:
|
|
4808
|
-
node = await snap.getNodeByNdef(('test:str', 'foobar'))
|
|
4809
|
-
self.nn(node)
|
|
4810
|
-
|
|
4811
|
-
mesg = ('prop:set', {'ndef': ('test:str', 'foo'), 'prop': 'tick', 'valu': 200})
|
|
4812
|
-
await core.addFeedData('syn.splice', [mesg])
|
|
4813
|
-
|
|
4814
|
-
async with await core.snap() as snap:
|
|
4815
|
-
node = await snap.getNodeByNdef(('test:str', 'foo'))
|
|
4816
|
-
self.eq(200, node.get('tick'))
|
|
4817
|
-
|
|
4818
|
-
mesg = ('prop:del', {'ndef': ('test:str', 'foo'), 'prop': 'tick'})
|
|
4819
|
-
await core.addFeedData('syn.splice', [mesg])
|
|
4820
|
-
|
|
4821
|
-
async with await core.snap() as snap:
|
|
4822
|
-
node = await snap.getNodeByNdef(('test:str', 'foo'))
|
|
4823
|
-
self.none(node.get('tick'))
|
|
4824
|
-
|
|
4825
|
-
mesg = ('tag:add', {'ndef': ('test:str', 'foo'), 'tag': 'bar', 'valu': (200, 300)})
|
|
4826
|
-
await core.addFeedData('syn.splice', [mesg])
|
|
4827
|
-
|
|
4828
|
-
async with await core.snap() as snap:
|
|
4829
|
-
node = await snap.getNodeByNdef(('test:str', 'foo'))
|
|
4830
|
-
self.eq((200, 300), node.getTag('bar'))
|
|
4831
|
-
|
|
4832
|
-
mesg = ('tag:del', {'ndef': ('test:str', 'foo'), 'tag': 'bar'})
|
|
4833
|
-
await core.addFeedData('syn.splice', [mesg])
|
|
4834
|
-
|
|
4835
|
-
async with await core.snap() as snap:
|
|
4836
|
-
node = await snap.getNodeByNdef(('test:str', 'foo'))
|
|
4837
|
-
self.none(node.getTag('bar'))
|
|
4838
|
-
|
|
4839
|
-
await core.addTagProp('score', ('int', {}), {})
|
|
4840
|
-
splice = ('tag:prop:set', {'ndef': ('test:str', 'foo'), 'tag': 'lol', 'prop': 'score', 'valu': 100,
|
|
4841
|
-
'curv': None})
|
|
4842
|
-
await core.addFeedData('syn.splice', [splice])
|
|
4843
|
-
|
|
4844
|
-
self.len(1, await core.nodes('#lol:score=100'))
|
|
4845
|
-
|
|
4846
|
-
splice = ('tag:prop:del', {'ndef': ('test:str', 'foo'), 'tag': 'lol', 'prop': 'score', 'valu': 100})
|
|
4847
|
-
await core.addFeedData('syn.splice', [splice])
|
|
4848
|
-
|
|
4849
|
-
self.len(0, await core.nodes('#lol:score=100'))
|
|
4850
|
-
|
|
4851
|
-
mesg = ('node:del', {'ndef': ('test:str', 'foo')})
|
|
4852
|
-
await core.addFeedData('syn.splice', [mesg])
|
|
4853
|
-
|
|
4854
|
-
async with await core.snap() as snap:
|
|
4855
|
-
node = await snap.getNodeByNdef(('test:str', 'foo'))
|
|
4856
|
-
self.none(node)
|
|
4857
|
-
|
|
4858
|
-
# test feeding to a different view
|
|
4859
|
-
vdef2 = await core.view.fork()
|
|
4860
|
-
view2_iden = vdef2.get('iden')
|
|
4861
|
-
view2 = core.getView(view2_iden)
|
|
4862
|
-
|
|
4863
|
-
mesg = ('node:add', {'ndef': ('test:str', 'bar')})
|
|
4864
|
-
await core.addFeedData('syn.splice', [mesg], viewiden=view2_iden)
|
|
4865
|
-
|
|
4866
|
-
async with await core.snap(view=view2) as snap:
|
|
4867
|
-
node = await snap.getNodeByNdef(('test:str', 'bar'))
|
|
4868
|
-
self.nn(node)
|
|
4869
|
-
|
|
4870
|
-
async with await core.snap() as snap:
|
|
4871
|
-
node = await snap.getNodeByNdef(('test:str', 'bar'))
|
|
4872
|
-
self.none(node)
|
|
4873
|
-
|
|
4874
|
-
# test coreapi addFeedData to a different view
|
|
4875
|
-
mesg = ('node:add', {'ndef': ('test:str', 'baz')})
|
|
4876
|
-
await prox.addFeedData('syn.splice', [mesg], viewiden=view2_iden)
|
|
4877
|
-
|
|
4878
|
-
async with await core.snap(view=view2) as snap:
|
|
4879
|
-
node = await snap.getNodeByNdef(('test:str', 'baz'))
|
|
4880
|
-
self.nn(node)
|
|
4881
|
-
|
|
4882
|
-
async with await core.snap() as snap:
|
|
4883
|
-
node = await snap.getNodeByNdef(('test:str', 'baz'))
|
|
4884
|
-
self.none(node)
|
|
4885
|
-
|
|
4886
|
-
# sad paths
|
|
4887
|
-
await self.asyncraises(s_exc.NoSuchView, core.addFeedData('syn.splice', [mesg], viewiden='badiden'))
|
|
4888
|
-
await self.asyncraises(s_exc.NoSuchView, prox.addFeedData('syn.splice', [mesg], viewiden='badiden'))
|
|
4889
|
-
|
|
4890
|
-
async def test_feed_syn_nodeedits(self):
|
|
4891
|
-
|
|
4892
|
-
async with self.getTestCoreAndProxy() as (core0, prox0):
|
|
4893
|
-
|
|
4894
|
-
nodelist0 = []
|
|
4895
|
-
nodelist0.extend(await core0.nodes('[ test:str=foo ]'))
|
|
4896
|
-
nodelist0.extend(await core0.nodes('[ inet:ipv4=1.2.3.4 .seen=(2012,2014) +#foo.bar=(2012, 2014) ]'))
|
|
4897
|
-
nodelist0.extend(await core0.nodes('[ test:int=42 ]'))
|
|
4898
|
-
await core0.nodes('test:int=42 | delnode')
|
|
4899
|
-
|
|
4900
|
-
with self.raises(s_exc.NoSuchLayer):
|
|
4901
|
-
async for _, nodeedits in prox0.syncLayerNodeEdits(0, layriden='asdf', wait=False):
|
|
4902
|
-
pass
|
|
4903
|
-
|
|
4904
|
-
with self.raises(s_exc.NoSuchLayer):
|
|
4905
|
-
async for _, nodeedits in core0.syncLayerNodeEdits('asdf', 0, wait=False):
|
|
4906
|
-
pass
|
|
4907
|
-
|
|
4908
|
-
editlist = []
|
|
4909
|
-
async for _, nodeedits in prox0.syncLayerNodeEdits(0, wait=False):
|
|
4910
|
-
editlist.append(nodeedits)
|
|
4911
|
-
|
|
4912
|
-
deledit = editlist.pop(len(editlist) - 1)
|
|
4913
|
-
|
|
4914
|
-
async with self.getTestCoreAndProxy() as (core1, prox1):
|
|
4915
|
-
|
|
4916
|
-
await prox1.addFeedData('syn.nodeedits', editlist)
|
|
4917
|
-
|
|
4918
|
-
nodelist1 = []
|
|
4919
|
-
nodelist1.extend(await core1.nodes('test:str'))
|
|
4920
|
-
nodelist1.extend(await core1.nodes('inet:ipv4'))
|
|
4921
|
-
nodelist1.extend(await core1.nodes('test:int'))
|
|
4922
|
-
|
|
4923
|
-
nodelist0 = [node.pack() for node in nodelist0]
|
|
4924
|
-
nodelist1 = [node.pack() for node in nodelist1]
|
|
4925
|
-
self.eq(nodelist0, nodelist1)
|
|
4926
|
-
|
|
4927
|
-
await core1.nodes('trigger.add node:del --form test:int --query {[test:int=7]}')
|
|
4928
|
-
|
|
4929
|
-
self.len(1, await core1.nodes('test:int=42'))
|
|
4930
|
-
|
|
4931
|
-
await prox1.addFeedData('syn.nodeedits', [deledit])
|
|
4932
|
-
|
|
4933
|
-
self.len(0, await core1.nodes('test:int=42'))
|
|
4934
|
-
self.len(1, await core1.nodes('test:int=7'))
|
|
4935
|
-
|
|
4936
|
-
# Try a nodeedits we might get from cmdr
|
|
4937
|
-
cmdrnodeedits = s_common.jsonsafe_nodeedits(editlist[1])
|
|
4938
|
-
await core0.nodes('test:str=foo | delnode')
|
|
4939
|
-
|
|
4940
|
-
await prox1.addFeedData('syn.nodeedits', [cmdrnodeedits])
|
|
4941
|
-
self.len(1, await core1.nodes('test:str'))
|
|
4942
|
-
|
|
4943
|
-
async def test_stat(self):
|
|
4944
|
-
|
|
4945
|
-
async with self.getTestCoreAndProxy() as (realcore, core):
|
|
4946
|
-
coreiden = realcore.iden
|
|
4947
|
-
ostat = await core.stat()
|
|
4948
|
-
self.eq(ostat.get('iden'), coreiden)
|
|
4949
|
-
self.isin('layer', ostat)
|
|
4950
|
-
self.len(1, await realcore.nodes('[test:str=123 :tick=2018]'))
|
|
4951
|
-
nstat = await core.stat()
|
|
4952
|
-
|
|
4953
|
-
counts = nstat.get('formcounts')
|
|
4954
|
-
self.eq(counts.get('test:str'), 1)
|
|
4955
|
-
|
|
4956
|
-
async def test_stat_lock(self):
|
|
4957
|
-
self.thisHostMust(hasmemlocking=True)
|
|
4958
|
-
conf = {'layers:lockmemory': True}
|
|
4959
|
-
async with self.getTestCoreAndProxy(conf=conf) as (realcore, core):
|
|
4960
|
-
slab = realcore.view.layers[0].layrslab
|
|
4961
|
-
self.true(await asyncio.wait_for(slab.lockdoneevent.wait(), 8))
|
|
4962
|
-
|
|
4963
|
-
nstat = await core.stat()
|
|
4964
|
-
layr = nstat.get('layer')
|
|
4965
|
-
self.gt(layr.get('lock_goal'), 0)
|
|
4966
|
-
|
|
4967
4722
|
async def test_storm_sub_query(self):
|
|
4968
4723
|
|
|
4969
4724
|
async with self.getTestCore() as core:
|
|
@@ -5124,6 +4879,9 @@ class CortexBasicTest(s_t_utils.SynTest):
|
|
|
5124
4879
|
self.nn(node.getTag('known'))
|
|
5125
4880
|
self.none(node.getTag('unknown'))
|
|
5126
4881
|
|
|
4882
|
+
q = '$valu={[test:str=foo]} switch $valu { foo: {test:str=foo return($node.value()) } }'
|
|
4883
|
+
self.eq('foo', await core.callStorm(q))
|
|
4884
|
+
|
|
5127
4885
|
async def test_storm_tagvar(self):
|
|
5128
4886
|
|
|
5129
4887
|
async with self.getTestCore() as core:
|
|
@@ -6663,35 +6421,6 @@ class CortexBasicTest(s_t_utils.SynTest):
|
|
|
6663
6421
|
self.eq(view, core.getView(core.iden))
|
|
6664
6422
|
self.none(core.getView('xxx'))
|
|
6665
6423
|
|
|
6666
|
-
async def test_cortex_cronjob_perms(self):
|
|
6667
|
-
async with self.getTestCore() as realcore:
|
|
6668
|
-
async with realcore.getLocalProxy() as core:
|
|
6669
|
-
fred = await core.addUser('fred')
|
|
6670
|
-
await core.setUserPasswd(fred['iden'], 'secret')
|
|
6671
|
-
cdef = {'storm': '[test:str=foo]', 'reqs': {'dayofmonth': 1},
|
|
6672
|
-
'incunit': None, 'incvals': None}
|
|
6673
|
-
adef = await core.addCronJob(cdef)
|
|
6674
|
-
iden = adef.get('iden')
|
|
6675
|
-
|
|
6676
|
-
async with realcore.getLocalProxy(user='fred') as core:
|
|
6677
|
-
# Rando user can't make cron jobs
|
|
6678
|
-
cdef = {'storm': '[test:int=1]', 'reqs': {'month': 1},
|
|
6679
|
-
'incunit': None, 'incvals': None}
|
|
6680
|
-
await self.asyncraises(s_exc.AuthDeny, core.addCronJob(cdef))
|
|
6681
|
-
|
|
6682
|
-
# Rando user can't mod cron jobs
|
|
6683
|
-
await self.asyncraises(s_exc.AuthDeny, core.updateCronJob(iden, '[test:str=bar]'))
|
|
6684
|
-
|
|
6685
|
-
# Rando user doesn't see any cron jobs
|
|
6686
|
-
self.len(0, await core.listCronJobs())
|
|
6687
|
-
|
|
6688
|
-
# Rando user can't delete cron jobs
|
|
6689
|
-
await self.asyncraises(s_exc.AuthDeny, core.delCronJob(iden))
|
|
6690
|
-
|
|
6691
|
-
# Rando user can't enable/disable cron jobs
|
|
6692
|
-
await self.asyncraises(s_exc.AuthDeny, core.enableCronJob(iden))
|
|
6693
|
-
await self.asyncraises(s_exc.AuthDeny, core.disableCronJob(iden))
|
|
6694
|
-
|
|
6695
6424
|
async def test_cortex_cron_deluser(self):
|
|
6696
6425
|
|
|
6697
6426
|
async with self.getTestCore() as core:
|
|
@@ -6767,39 +6496,6 @@ class CortexBasicTest(s_t_utils.SynTest):
|
|
|
6767
6496
|
await self.asyncraises(s_exc.AuthDeny, prox.enableMigrationMode())
|
|
6768
6497
|
await self.asyncraises(s_exc.AuthDeny, prox.disableMigrationMode())
|
|
6769
6498
|
|
|
6770
|
-
async def test_cortex_watch(self):
|
|
6771
|
-
|
|
6772
|
-
async with self.getTestCore() as core:
|
|
6773
|
-
|
|
6774
|
-
async with core.getLocalProxy() as prox:
|
|
6775
|
-
|
|
6776
|
-
async def nodes():
|
|
6777
|
-
await asyncio.sleep(0.1) # due to telepath proxy causing task switch
|
|
6778
|
-
await core.nodes('[ test:int=10 +#foo.bar +#baz.faz ]')
|
|
6779
|
-
await core.nodes('test:int=10 [ -#foo.bar -#baz.faz ]')
|
|
6780
|
-
|
|
6781
|
-
task = core.schedCoro(nodes())
|
|
6782
|
-
|
|
6783
|
-
data = []
|
|
6784
|
-
async for mesg in prox.watch({'tags': ['foo.bar', 'baz.*']}):
|
|
6785
|
-
data.append(mesg)
|
|
6786
|
-
if len(data) == 4:
|
|
6787
|
-
break
|
|
6788
|
-
|
|
6789
|
-
await asyncio.wait_for(task, timeout=1)
|
|
6790
|
-
|
|
6791
|
-
self.eq(data[0][0], 'tag:add')
|
|
6792
|
-
self.eq(data[0][1]['tag'], 'foo.bar')
|
|
6793
|
-
|
|
6794
|
-
self.eq(data[1][0], 'tag:add')
|
|
6795
|
-
self.eq(data[1][1]['tag'], 'baz.faz')
|
|
6796
|
-
|
|
6797
|
-
self.eq(data[2][0], 'tag:del')
|
|
6798
|
-
self.eq(data[2][1]['tag'], 'foo.bar')
|
|
6799
|
-
|
|
6800
|
-
self.eq(data[3][0], 'tag:del')
|
|
6801
|
-
self.eq(data[3][1]['tag'], 'baz.faz')
|
|
6802
|
-
|
|
6803
6499
|
async def test_cortex_behold(self):
|
|
6804
6500
|
async with self.getTestCore() as core:
|
|
6805
6501
|
async with core.getLocalProxy() as prox:
|