synapse 2.193.0__py311-none-any.whl → 2.195.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/cortex.py +9 -7
- synapse/datamodel.py +9 -6
- synapse/exc.py +1 -1
- synapse/lib/agenda.py +17 -4
- synapse/lib/ast.py +217 -86
- synapse/lib/auth.py +5 -2
- synapse/lib/link.py +33 -19
- synapse/lib/modelrev.py +6 -1
- synapse/lib/parser.py +4 -0
- synapse/lib/scrape.py +18 -1
- synapse/lib/snap.py +40 -11
- synapse/lib/storm.lark +16 -1
- synapse/lib/storm.py +6 -4
- synapse/lib/storm_format.py +1 -0
- synapse/lib/stormctrl.py +88 -6
- synapse/lib/stormlib/auth.py +15 -1
- synapse/lib/stormlib/cache.py +6 -2
- synapse/lib/stormlib/cell.py +11 -0
- synapse/lib/stormlib/infosec.py +2 -0
- synapse/lib/stormlib/scrape.py +1 -1
- synapse/lib/stormlib/stix.py +8 -8
- synapse/lib/stormtypes.py +13 -5
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +20 -3
- synapse/models/geopol.py +1 -0
- synapse/models/geospace.py +53 -10
- synapse/models/inet.py +3 -0
- synapse/models/infotech.py +12 -5
- synapse/models/material.py +67 -8
- synapse/models/orgs.py +11 -3
- synapse/models/person.py +28 -17
- synapse/models/risk.py +4 -1
- synapse/models/syn.py +3 -0
- synapse/models/telco.py +10 -3
- synapse/models/transport.py +382 -49
- synapse/tests/test_axon.py +6 -6
- synapse/tests/test_cortex.py +134 -12
- synapse/tests/test_exc.py +1 -0
- synapse/tests/test_lib_agenda.py +125 -1
- synapse/tests/test_lib_aha.py +13 -6
- synapse/tests/test_lib_ast.py +258 -9
- synapse/tests/test_lib_auth.py +6 -7
- synapse/tests/test_lib_cell.py +10 -0
- synapse/tests/test_lib_grammar.py +14 -0
- synapse/tests/test_lib_layer.py +1 -1
- synapse/tests/test_lib_link.py +6 -1
- synapse/tests/test_lib_lmdbslab.py +3 -3
- synapse/tests/test_lib_modelrev.py +7 -0
- synapse/tests/test_lib_scrape.py +8 -0
- synapse/tests/test_lib_storm.py +201 -25
- synapse/tests/test_lib_stormctrl.py +65 -0
- synapse/tests/test_lib_stormhttp.py +5 -5
- synapse/tests/test_lib_stormlib_auth.py +31 -5
- synapse/tests/test_lib_stormlib_cache.py +38 -6
- synapse/tests/test_lib_stormlib_cell.py +3 -0
- synapse/tests/test_lib_stormlib_modelext.py +3 -3
- synapse/tests/test_lib_stormlib_scrape.py +4 -4
- synapse/tests/test_lib_stormlib_spooled.py +1 -1
- synapse/tests/test_lib_stormlib_xml.py +5 -5
- synapse/tests/test_lib_stormtypes.py +54 -57
- synapse/tests/test_lib_view.py +1 -1
- synapse/tests/test_model_base.py +1 -2
- synapse/tests/test_model_geopol.py +4 -0
- synapse/tests/test_model_geospace.py +43 -4
- synapse/tests/test_model_inet.py +3 -0
- synapse/tests/test_model_infotech.py +31 -4
- synapse/tests/test_model_material.py +18 -0
- synapse/tests/test_model_orgs.py +25 -3
- synapse/tests/test_model_person.py +26 -1
- synapse/tests/test_model_risk.py +11 -0
- synapse/tests/test_model_syn.py +9 -3
- synapse/tests/test_model_transport.py +168 -0
- synapse/tests/test_telepath.py +24 -5
- synapse/tests/test_tools_healthcheck.py +4 -4
- synapse/tests/test_utils.py +17 -18
- synapse/tests/utils.py +0 -35
- synapse/tools/changelog.py +14 -5
- synapse/tools/storm.py +1 -1
- {synapse-2.193.0.dist-info → synapse-2.195.0.dist-info}/METADATA +5 -5
- {synapse-2.193.0.dist-info → synapse-2.195.0.dist-info}/RECORD +83 -82
- {synapse-2.193.0.dist-info → synapse-2.195.0.dist-info}/WHEEL +1 -1
- {synapse-2.193.0.dist-info → synapse-2.195.0.dist-info}/LICENSE +0 -0
- {synapse-2.193.0.dist-info → synapse-2.195.0.dist-info}/top_level.txt +0 -0
synapse/lib/link.py
CHANGED
|
@@ -15,7 +15,8 @@ import synapse.lib.base as s_base
|
|
|
15
15
|
import synapse.lib.const as s_const
|
|
16
16
|
import synapse.lib.msgpack as s_msgpack
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
READSIZE = 16 * s_const.mebibyte
|
|
19
|
+
MAXWRITE = 64 * s_const.mebibyte
|
|
19
20
|
|
|
20
21
|
async def connect(host, port, ssl=None, hostname=None, linkinfo=None):
|
|
21
22
|
'''
|
|
@@ -114,8 +115,8 @@ class Link(s_base.Base):
|
|
|
114
115
|
self.sock = self.writer.get_extra_info('socket')
|
|
115
116
|
self.peercert = self.writer.get_extra_info('peercert')
|
|
116
117
|
|
|
118
|
+
self._txlock = asyncio.Lock()
|
|
117
119
|
self._forceclose = forceclose
|
|
118
|
-
self._drain_lock = asyncio.Lock()
|
|
119
120
|
|
|
120
121
|
if info is None:
|
|
121
122
|
info = {}
|
|
@@ -234,11 +235,18 @@ class Link(s_base.Base):
|
|
|
234
235
|
return dict(self._addrinfo)
|
|
235
236
|
|
|
236
237
|
async def send(self, byts):
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
238
|
+
|
|
239
|
+
offs = 0
|
|
240
|
+
size = len(byts)
|
|
241
|
+
|
|
242
|
+
async with self._txlock:
|
|
243
|
+
|
|
244
|
+
while offs < size:
|
|
245
|
+
|
|
246
|
+
self.writer.write(byts[offs:offs + MAXWRITE])
|
|
247
|
+
offs += MAXWRITE
|
|
248
|
+
|
|
249
|
+
await self.writer.drain()
|
|
242
250
|
|
|
243
251
|
async def tx(self, mesg):
|
|
244
252
|
'''
|
|
@@ -247,23 +255,29 @@ class Link(s_base.Base):
|
|
|
247
255
|
if self.isfini:
|
|
248
256
|
raise s_exc.IsFini()
|
|
249
257
|
|
|
258
|
+
offs = 0
|
|
250
259
|
byts = s_msgpack.en(mesg)
|
|
251
|
-
|
|
260
|
+
size = len(byts)
|
|
252
261
|
|
|
253
|
-
|
|
254
|
-
# Avoid Python bug. See https://github.com/python/cpython/issues/74116
|
|
255
|
-
# TODO Remove drain lock in 3.10+
|
|
256
|
-
async with self._drain_lock:
|
|
257
|
-
await self.writer.drain()
|
|
262
|
+
async with self._txlock:
|
|
258
263
|
|
|
259
|
-
|
|
264
|
+
try:
|
|
265
|
+
|
|
266
|
+
while offs < size:
|
|
260
267
|
|
|
261
|
-
|
|
268
|
+
self.writer.write(byts[offs:offs + MAXWRITE])
|
|
269
|
+
offs += MAXWRITE
|
|
262
270
|
|
|
263
|
-
|
|
264
|
-
logger.debug('link.tx connection trouble %s', einfo)
|
|
271
|
+
await self.writer.drain()
|
|
265
272
|
|
|
266
|
-
|
|
273
|
+
except (asyncio.CancelledError, Exception) as e:
|
|
274
|
+
|
|
275
|
+
await self.fini()
|
|
276
|
+
|
|
277
|
+
einfo = s_common.retnexc(e)
|
|
278
|
+
logger.debug('link.tx connection trouble %s', einfo)
|
|
279
|
+
|
|
280
|
+
raise
|
|
267
281
|
|
|
268
282
|
def txfini(self):
|
|
269
283
|
self.sock.shutdown(1)
|
|
@@ -294,7 +308,7 @@ class Link(s_base.Base):
|
|
|
294
308
|
|
|
295
309
|
try:
|
|
296
310
|
|
|
297
|
-
byts = await self.reader.read(
|
|
311
|
+
byts = await self.reader.read(READSIZE)
|
|
298
312
|
if not byts:
|
|
299
313
|
await self.fini()
|
|
300
314
|
return None
|
synapse/lib/modelrev.py
CHANGED
|
@@ -13,7 +13,7 @@ import synapse.models.infotech as s_infotech
|
|
|
13
13
|
|
|
14
14
|
logger = logging.getLogger(__name__)
|
|
15
15
|
|
|
16
|
-
maxvers = (0, 2,
|
|
16
|
+
maxvers = (0, 2, 32)
|
|
17
17
|
|
|
18
18
|
class ModelRev:
|
|
19
19
|
|
|
@@ -50,6 +50,7 @@ class ModelRev:
|
|
|
50
50
|
((0, 2, 29), self.revModel_0_2_29),
|
|
51
51
|
((0, 2, 30), self.revModel_0_2_30),
|
|
52
52
|
((0, 2, 31), self.revModel_0_2_31),
|
|
53
|
+
((0, 2, 32), self.revModel_0_2_32),
|
|
53
54
|
)
|
|
54
55
|
|
|
55
56
|
async def _uniqSortArray(self, todoprops, layers):
|
|
@@ -815,6 +816,10 @@ class ModelRev:
|
|
|
815
816
|
await migr.revModel_0_2_31()
|
|
816
817
|
await self._normFormSubs(layers, 'it:sec:cpe')
|
|
817
818
|
|
|
819
|
+
async def revModel_0_2_32(self, layers):
|
|
820
|
+
await self._normPropValu(layers, 'transport:air:craft:model')
|
|
821
|
+
await self._normPropValu(layers, 'transport:sea:vessel:model')
|
|
822
|
+
|
|
818
823
|
async def runStorm(self, text, opts=None):
|
|
819
824
|
'''
|
|
820
825
|
Run storm code in a schedcoro and log the output messages.
|
synapse/lib/parser.py
CHANGED
|
@@ -95,6 +95,7 @@ terminalEnglishMap = {
|
|
|
95
95
|
'TRYSETPLUS': '?+=',
|
|
96
96
|
'TRYSETMINUS': '?-=',
|
|
97
97
|
'UNIVNAME': 'universal property',
|
|
98
|
+
'UNSET': 'unset',
|
|
98
99
|
'EXPRUNIVNAME': 'universal property',
|
|
99
100
|
'VARTOKN': 'variable',
|
|
100
101
|
'EXPRVARTOKN': 'variable',
|
|
@@ -642,6 +643,8 @@ ruleClassMap = {
|
|
|
642
643
|
'andexpr': s_ast.AndCond,
|
|
643
644
|
'baresubquery': s_ast.SubQuery,
|
|
644
645
|
'catchblock': s_ast.CatchBlock,
|
|
646
|
+
'condsetoper': s_ast.CondSetOper,
|
|
647
|
+
'condtrysetoper': lambda astinfo, kids: s_ast.CondSetOper(astinfo, kids, errok=True),
|
|
645
648
|
'condsubq': s_ast.SubqCond,
|
|
646
649
|
'dollarexpr': s_ast.DollarExpr,
|
|
647
650
|
'edgeaddn1': s_ast.EditEdgeAdd,
|
|
@@ -657,6 +660,7 @@ ruleClassMap = {
|
|
|
657
660
|
'formname': s_ast.FormName,
|
|
658
661
|
'editpropdel': lambda astinfo, kids: s_ast.EditPropDel(astinfo, kids[1:]),
|
|
659
662
|
'editpropset': s_ast.EditPropSet,
|
|
663
|
+
'editcondpropset': s_ast.EditCondPropSet,
|
|
660
664
|
'edittagadd': s_ast.EditTagAdd,
|
|
661
665
|
'edittagdel': lambda astinfo, kids: s_ast.EditTagDel(astinfo, kids[1:]),
|
|
662
666
|
'edittagpropset': s_ast.EditTagPropSet,
|
synapse/lib/scrape.py
CHANGED
|
@@ -128,6 +128,8 @@ _cpe23_regex = r'''(?P<valu>cpe:2\.3:[aho\*-]
|
|
|
128
128
|
(?::(([?]+|\*)?([a-z0-9-._]|\\[\\?*!"#$%&\'()+,/:;<=>@\[\]^`{|}~])+([?]+|\*)?|[*-])){4})
|
|
129
129
|
'''
|
|
130
130
|
|
|
131
|
+
path_parts_limit = 1024
|
|
132
|
+
|
|
131
133
|
linux_path_regex = r'''
|
|
132
134
|
(?<![\w\d]+)
|
|
133
135
|
(?P<valu>
|
|
@@ -142,17 +144,23 @@ linux_path_rootdirs = (
|
|
|
142
144
|
'sys', 'tmp', 'usr', 'var'
|
|
143
145
|
)
|
|
144
146
|
|
|
147
|
+
# https://docs.kernel.org/filesystems/path-lookup.html#the-symlink-stack
|
|
148
|
+
linux_path_limit = 4096
|
|
149
|
+
|
|
145
150
|
def linux_path_check(match: regex.Match):
|
|
146
151
|
mnfo = match.groupdict()
|
|
147
152
|
valu = mnfo.get('valu')
|
|
148
153
|
|
|
154
|
+
if len(valu) > linux_path_limit:
|
|
155
|
+
return None, {}
|
|
156
|
+
|
|
149
157
|
path = pathlib.PurePosixPath(valu)
|
|
150
158
|
parts = path.parts
|
|
151
159
|
|
|
152
160
|
if parts[0] != '/':
|
|
153
161
|
return None, {}
|
|
154
162
|
|
|
155
|
-
if len(parts) < 2:
|
|
163
|
+
if len(parts) < 2 or len(parts) > path_parts_limit:
|
|
156
164
|
return None, {}
|
|
157
165
|
|
|
158
166
|
if parts[1] not in linux_path_rootdirs:
|
|
@@ -177,17 +185,26 @@ windows_path_reserved = (
|
|
|
177
185
|
|
|
178
186
|
windows_drive_paths = [f'{letter}:\\' for letter in string.ascii_lowercase]
|
|
179
187
|
|
|
188
|
+
# https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation
|
|
189
|
+
windows_path_limit = 32_767
|
|
190
|
+
|
|
180
191
|
# https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions
|
|
181
192
|
def windows_path_check(match: regex.Match):
|
|
182
193
|
mnfo = match.groupdict()
|
|
183
194
|
valu = mnfo.get('valu')
|
|
184
195
|
|
|
196
|
+
if len(valu) > windows_path_limit:
|
|
197
|
+
return None, {}
|
|
198
|
+
|
|
185
199
|
path = pathlib.PureWindowsPath(valu)
|
|
186
200
|
parts = path.parts
|
|
187
201
|
|
|
188
202
|
if parts[0].lower() not in windows_drive_paths:
|
|
189
203
|
return None, {}
|
|
190
204
|
|
|
205
|
+
if len(parts) > path_parts_limit:
|
|
206
|
+
return None, {}
|
|
207
|
+
|
|
191
208
|
for part in parts:
|
|
192
209
|
if part in windows_path_reserved:
|
|
193
210
|
return None, {}
|
synapse/lib/snap.py
CHANGED
|
@@ -362,7 +362,7 @@ class ProtoNode:
|
|
|
362
362
|
try:
|
|
363
363
|
valu, norminfo = prop.type.norm(valu)
|
|
364
364
|
except s_exc.BadTypeValu as e:
|
|
365
|
-
oldm = e.
|
|
365
|
+
oldm = e.get('mesg')
|
|
366
366
|
e.update({'prop': prop.name,
|
|
367
367
|
'form': prop.form.name,
|
|
368
368
|
'mesg': f'Bad prop value {prop.full}={valu!r} : {oldm}'})
|
|
@@ -1404,25 +1404,54 @@ class Snap(s_base.Base):
|
|
|
1404
1404
|
|
|
1405
1405
|
trycast = vals.pop('$try', False)
|
|
1406
1406
|
addprops = vals.pop('$props', None)
|
|
1407
|
-
if addprops is not None:
|
|
1408
|
-
props.update(addprops)
|
|
1409
1407
|
|
|
1410
|
-
|
|
1411
|
-
|
|
1408
|
+
if not vals:
|
|
1409
|
+
mesg = f'No values provided for form {form.full}'
|
|
1410
|
+
raise s_exc.BadTypeValu(mesg=mesg)
|
|
1411
|
+
|
|
1412
|
+
for name, valu in list(props.items()):
|
|
1413
|
+
try:
|
|
1412
1414
|
props[name] = form.reqProp(name).type.norm(valu)
|
|
1415
|
+
except s_exc.BadTypeValu as e:
|
|
1416
|
+
mesg = e.get('mesg')
|
|
1417
|
+
e.update({
|
|
1418
|
+
'prop': name,
|
|
1419
|
+
'form': form.name,
|
|
1420
|
+
'mesg': f'Bad value for prop {form.name}:{name}: {mesg}',
|
|
1421
|
+
})
|
|
1422
|
+
raise e
|
|
1413
1423
|
|
|
1414
|
-
|
|
1424
|
+
if addprops is not None:
|
|
1425
|
+
for name, valu in addprops.items():
|
|
1426
|
+
try:
|
|
1427
|
+
props[name] = form.reqProp(name).type.norm(valu)
|
|
1428
|
+
except s_exc.BadTypeValu as e:
|
|
1429
|
+
mesg = e.get("mesg")
|
|
1430
|
+
if not trycast:
|
|
1431
|
+
e.update({
|
|
1432
|
+
'prop': name,
|
|
1433
|
+
'form': form.name,
|
|
1434
|
+
'mesg': f'Bad value for prop {form.name}:{name}: {mesg}'
|
|
1435
|
+
})
|
|
1436
|
+
raise e
|
|
1437
|
+
await self.warn(f'Skipping bad value for prop {form.name}:{name}: {mesg}')
|
|
1438
|
+
|
|
1439
|
+
for name, valu in vals.items():
|
|
1415
1440
|
|
|
1441
|
+
try:
|
|
1416
1442
|
prop = form.reqProp(name)
|
|
1417
1443
|
norm, norminfo = prop.type.norm(valu)
|
|
1418
1444
|
|
|
1419
1445
|
norms[name] = (prop, norm, norminfo)
|
|
1420
1446
|
proplist.append((name, norm))
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1447
|
+
except s_exc.BadTypeValu as e:
|
|
1448
|
+
mesg = e.get('mesg')
|
|
1449
|
+
e.update({
|
|
1450
|
+
'prop': name,
|
|
1451
|
+
'form': form.name,
|
|
1452
|
+
'mesg': f'Bad value for prop {form.name}:{name}: {mesg}',
|
|
1453
|
+
})
|
|
1454
|
+
raise e
|
|
1426
1455
|
|
|
1427
1456
|
proplist.sort()
|
|
1428
1457
|
|
synapse/lib/storm.lark
CHANGED
|
@@ -39,7 +39,7 @@ _editblock: "[" _editoper* "]"
|
|
|
39
39
|
|
|
40
40
|
// A single edit operation
|
|
41
41
|
_editoper: editnodeadd
|
|
42
|
-
| editpropset | editunivset | edittagpropset | edittagadd
|
|
42
|
+
| editpropset | editunivset | edittagpropset | edittagadd | editcondpropset
|
|
43
43
|
| editpropdel | editunivdel | edittagpropdel | edittagdel
|
|
44
44
|
| editparens | edgeaddn1 | edgedeln1 | edgeaddn2 | edgedeln2
|
|
45
45
|
|
|
@@ -49,11 +49,13 @@ edittagadd: "+" [SETTAGOPER] tagname [(EQSPACE | EQNOSPACE) _valu]
|
|
|
49
49
|
editunivdel: EXPRMINUS univprop
|
|
50
50
|
edittagdel: EXPRMINUS tagname
|
|
51
51
|
editpropset: relprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu
|
|
52
|
+
editcondpropset: relprop condsetoper _valu
|
|
52
53
|
editpropdel: EXPRMINUS relprop
|
|
53
54
|
editunivset: univprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu
|
|
54
55
|
editnodeadd: formname (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu
|
|
55
56
|
edittagpropset: "+" tagprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu
|
|
56
57
|
edittagpropdel: EXPRMINUS tagprop
|
|
58
|
+
|
|
57
59
|
EQSPACE: /((?<=\s)=|=(?=\s))/
|
|
58
60
|
MODSET.4: "+=" | "-="
|
|
59
61
|
TRYSETPLUS.1: "?+="
|
|
@@ -61,6 +63,19 @@ TRYSETMINUS.1: "?-="
|
|
|
61
63
|
TRYSET.1: "?="
|
|
62
64
|
SETTAGOPER: "?"
|
|
63
65
|
|
|
66
|
+
condsetoper: ("*" UNSET | _DEREF "$" _condvarvaluatom) "="
|
|
67
|
+
| ("*" UNSET | _DEREF "$" _condvarvaluatom) "?=" -> condtrysetoper
|
|
68
|
+
UNSET: "unset"
|
|
69
|
+
_condvarvaluatom: condvarvalue | condvarderef | condfunccall
|
|
70
|
+
condvarvalue: VARTOKN -> varvalue
|
|
71
|
+
|
|
72
|
+
!condvarderef: _condvarvaluatom "." (VARTOKN | "$" VARTOKN | _condderefexpr) -> varderef
|
|
73
|
+
_condderefexpr: "$"? conddollarexpr
|
|
74
|
+
conddollarexpr: "(" expror ")" -> dollarexpr
|
|
75
|
+
|
|
76
|
+
condfunccall: _condvarvaluatom _condcallargs -> funccall
|
|
77
|
+
_condcallargs: _LPARNOSPACE [(_valu | VARTOKN | (VARTOKN | NONQUOTEWORD) (EQSPACE | EQNOSPACE) _valu) ("," (_valu | VARTOKN | (VARTOKN | NONQUOTEWORD) (EQSPACE | EQNOSPACE) _valu))*] ","? ")"
|
|
78
|
+
|
|
64
79
|
// The set of non-edit non-commands in storm
|
|
65
80
|
|
|
66
81
|
_oper: stormfunc | initblock | emptyblock | finiblock | trycatch | subquery | _formpivot | formjoin
|
synapse/lib/storm.py
CHANGED
|
@@ -984,7 +984,9 @@ stormcmds = (
|
|
|
984
984
|
$ssl = $lib.true
|
|
985
985
|
if $cmdopts.ssl_noverify { $ssl = $lib.false }
|
|
986
986
|
|
|
987
|
-
$
|
|
987
|
+
$headers = ({'X-Synapse-Version': $lib.str.join('.', $lib.version.synapse())})
|
|
988
|
+
|
|
989
|
+
$resp = $lib.inet.http.get($cmdopts.url, ssl_verify=$ssl, headers=$headers)
|
|
988
990
|
|
|
989
991
|
if ($resp.code != 200) {
|
|
990
992
|
$lib.warn("pkg.load got HTTP code: {code} for URL: {url}", code=$resp.code, url=$cmdopts.url)
|
|
@@ -1603,7 +1605,7 @@ stormcmds = (
|
|
|
1603
1605
|
function fetchnodes(url, ssl) {
|
|
1604
1606
|
$resp = $lib.inet.http.get($url, ssl_verify=$ssl)
|
|
1605
1607
|
if ($resp.code = 200) {
|
|
1606
|
-
$nodes =
|
|
1608
|
+
$nodes = ()
|
|
1607
1609
|
for $valu in $resp.msgpack() {
|
|
1608
1610
|
$nodes.append($valu)
|
|
1609
1611
|
}
|
|
@@ -3552,7 +3554,7 @@ class HelpCmd(Cmd):
|
|
|
3552
3554
|
await runt.printf(line)
|
|
3553
3555
|
|
|
3554
3556
|
else: # pragma: no cover
|
|
3555
|
-
raise s_exc.StormRuntimeError(
|
|
3557
|
+
raise s_exc.StormRuntimeError(mesg=f'Unknown bound method {func}')
|
|
3556
3558
|
|
|
3557
3559
|
async def _handleStormLibMethod(self, func, runt: Runtime, verbose: bool =False):
|
|
3558
3560
|
# Storm library methods must be derived from a library definition.
|
|
@@ -3583,7 +3585,7 @@ class HelpCmd(Cmd):
|
|
|
3583
3585
|
await runt.printf(line)
|
|
3584
3586
|
|
|
3585
3587
|
else: # pragma: no cover
|
|
3586
|
-
raise s_exc.StormRuntimeError(
|
|
3588
|
+
raise s_exc.StormRuntimeError(mesg=f'Unknown runtime lib method {func} {cls} {fname}')
|
|
3587
3589
|
|
|
3588
3590
|
class DiffCmd(Cmd):
|
|
3589
3591
|
'''
|
synapse/lib/storm_format.py
CHANGED
synapse/lib/stormctrl.py
CHANGED
|
@@ -1,9 +1,91 @@
|
|
|
1
1
|
class StormCtrlFlow(Exception):
|
|
2
|
+
'''
|
|
3
|
+
Base class all StormCtrlFlow exceptions derive from.
|
|
4
|
+
'''
|
|
5
|
+
def __init__(self):
|
|
6
|
+
raise NotImplementedError
|
|
7
|
+
|
|
8
|
+
class _SynErrMixin(Exception):
|
|
9
|
+
'''
|
|
10
|
+
An exception mixin to give some control flow classes functionality like SynErr.
|
|
11
|
+
'''
|
|
12
|
+
def __init__(self, *args, **info):
|
|
13
|
+
self.errinfo = info
|
|
14
|
+
Exception.__init__(self, self._getExcMsg())
|
|
15
|
+
|
|
16
|
+
def _getExcMsg(self):
|
|
17
|
+
props = sorted(self.errinfo.items())
|
|
18
|
+
displ = ' '.join(['%s=%r' % (p, v) for (p, v) in props])
|
|
19
|
+
return '%s: %s' % (self.__class__.__name__, displ)
|
|
20
|
+
|
|
21
|
+
def _setExcMesg(self):
|
|
22
|
+
'''Should be called when self.errinfo is modified.'''
|
|
23
|
+
self.args = (self._getExcMsg(),)
|
|
24
|
+
|
|
25
|
+
def __setstate__(self, state):
|
|
26
|
+
'''Pickle support.'''
|
|
27
|
+
super(StormCtrlFlow, self).__setstate__(state)
|
|
28
|
+
self._setExcMesg()
|
|
29
|
+
|
|
30
|
+
def items(self):
|
|
31
|
+
return {k: v for k, v in self.errinfo.items()}
|
|
32
|
+
|
|
33
|
+
def get(self, name, defv=None):
|
|
34
|
+
'''
|
|
35
|
+
Return a value from the errinfo dict.
|
|
36
|
+
|
|
37
|
+
Example:
|
|
38
|
+
|
|
39
|
+
try:
|
|
40
|
+
foothing()
|
|
41
|
+
except SynErr as e:
|
|
42
|
+
blah = e.get('blah')
|
|
43
|
+
|
|
44
|
+
'''
|
|
45
|
+
return self.errinfo.get(name, defv)
|
|
46
|
+
|
|
47
|
+
def set(self, name, valu):
|
|
48
|
+
'''
|
|
49
|
+
Set a value in the errinfo dict.
|
|
50
|
+
'''
|
|
51
|
+
self.errinfo[name] = valu
|
|
52
|
+
self._setExcMesg()
|
|
53
|
+
|
|
54
|
+
def setdefault(self, name, valu):
|
|
55
|
+
'''
|
|
56
|
+
Set a value in errinfo dict if it is not already set.
|
|
57
|
+
'''
|
|
58
|
+
if name in self.errinfo:
|
|
59
|
+
return
|
|
60
|
+
self.errinfo[name] = valu
|
|
61
|
+
self._setExcMesg()
|
|
62
|
+
|
|
63
|
+
def update(self, items: dict):
|
|
64
|
+
'''Update multiple items in the errinfo dict at once.'''
|
|
65
|
+
self.errinfo.update(items)
|
|
66
|
+
self._setExcMesg()
|
|
67
|
+
|
|
68
|
+
class StormLoopCtrl(_SynErrMixin):
|
|
69
|
+
# Control flow statements for WHILE and FOR loop control
|
|
70
|
+
statement = ''
|
|
71
|
+
|
|
72
|
+
class StormGenrCtrl(_SynErrMixin):
|
|
73
|
+
# Control flow statements for GENERATOR control
|
|
74
|
+
statement = ''
|
|
75
|
+
|
|
76
|
+
class StormStop(StormGenrCtrl, StormCtrlFlow):
|
|
77
|
+
statement = 'stop'
|
|
78
|
+
|
|
79
|
+
class StormBreak(StormLoopCtrl, StormCtrlFlow):
|
|
80
|
+
statement = 'break'
|
|
81
|
+
|
|
82
|
+
class StormContinue(StormLoopCtrl, StormCtrlFlow):
|
|
83
|
+
statement = 'continue'
|
|
84
|
+
|
|
85
|
+
class StormExit(_SynErrMixin, StormCtrlFlow): pass
|
|
86
|
+
|
|
87
|
+
# StormReturn is kept thin since it is commonly used and just
|
|
88
|
+
# needs to be the container for moving an item up a frame.
|
|
89
|
+
class StormReturn(StormCtrlFlow):
|
|
2
90
|
def __init__(self, item=None):
|
|
3
91
|
self.item = item
|
|
4
|
-
|
|
5
|
-
class StormExit(StormCtrlFlow): pass
|
|
6
|
-
class StormStop(StormCtrlFlow): pass
|
|
7
|
-
class StormBreak(StormCtrlFlow): pass
|
|
8
|
-
class StormReturn(StormCtrlFlow): pass
|
|
9
|
-
class StormContinue(StormCtrlFlow): pass
|
synapse/lib/stormlib/auth.py
CHANGED
|
@@ -583,12 +583,26 @@ stormcmds = (
|
|
|
583
583
|
{
|
|
584
584
|
'name': 'auth.perms.list',
|
|
585
585
|
'descr': 'Display a list of the current permissions defined within the Cortex.',
|
|
586
|
-
'cmdargs': (
|
|
586
|
+
'cmdargs': (
|
|
587
|
+
('--find', {'type': 'str', 'help': 'A search string for permissions.'}),
|
|
588
|
+
),
|
|
587
589
|
'storm': '''
|
|
588
590
|
|
|
589
591
|
for $pdef in $lib.auth.getPermDefs() {
|
|
590
592
|
$perm = $lib.str.join(".", $pdef.perm)
|
|
591
593
|
|
|
594
|
+
if $cmdopts.find {
|
|
595
|
+
$find = $cmdopts.find.lower()
|
|
596
|
+
$match = (
|
|
597
|
+
$perm.lower().find($find) != (null) or
|
|
598
|
+
$pdef.desc.lower().find($find) != (null) or
|
|
599
|
+
$pdef.gate.lower().find($find) != (null) or
|
|
600
|
+
($pdef.ex and $pdef.ex.lower().find($find) != (null))
|
|
601
|
+
)
|
|
602
|
+
|
|
603
|
+
if (not $match) { continue }
|
|
604
|
+
}
|
|
605
|
+
|
|
592
606
|
$lib.print($perm)
|
|
593
607
|
$lib.print(` {$pdef.desc}`)
|
|
594
608
|
$lib.print(` gate: {$pdef.gate}`)
|
synapse/lib/stormlib/cache.py
CHANGED
|
@@ -172,8 +172,12 @@ class FixedCache(s_stormtypes.StormType):
|
|
|
172
172
|
await asyncio.sleep(0)
|
|
173
173
|
except s_stormctrl.StormReturn as e:
|
|
174
174
|
return await s_stormtypes.toprim(e.item)
|
|
175
|
-
except s_stormctrl.StormCtrlFlow:
|
|
176
|
-
|
|
175
|
+
except s_stormctrl.StormCtrlFlow as e:
|
|
176
|
+
name = e.__class__.__name__
|
|
177
|
+
if hasattr(e, 'statement'):
|
|
178
|
+
name = e.statement
|
|
179
|
+
exc = s_exc.StormRuntimeError(mesg=f'Storm control flow "{name}" not allowed in cache callbacks.')
|
|
180
|
+
raise exc from None
|
|
177
181
|
|
|
178
182
|
async def _reqKey(self, key):
|
|
179
183
|
if s_stormtypes.ismutable(key):
|
synapse/lib/stormlib/cell.py
CHANGED
|
@@ -120,6 +120,9 @@ class CellLib(s_stormtypes.Lib):
|
|
|
120
120
|
A Storm Library for interacting with the Cortex.
|
|
121
121
|
'''
|
|
122
122
|
_storm_locals = (
|
|
123
|
+
{'name': 'iden', 'desc': 'The Cortex service identifier.',
|
|
124
|
+
'type': {'type': 'gtor', '_gtorfunc': '_getCellIden',
|
|
125
|
+
'returns': {'type': 'str', 'desc': 'The Cortex service identifier.'}}},
|
|
123
126
|
{'name': 'getCellInfo', 'desc': 'Return metadata specific for the Cortex.',
|
|
124
127
|
'type': {'type': 'function', '_funcname': '_getCellInfo', 'args': (),
|
|
125
128
|
'returns': {'type': 'dict', 'desc': 'A dictionary containing metadata.', }}},
|
|
@@ -174,6 +177,10 @@ class CellLib(s_stormtypes.Lib):
|
|
|
174
177
|
)
|
|
175
178
|
_storm_lib_path = ('cell',)
|
|
176
179
|
|
|
180
|
+
def __init__(self, runt, name=()):
|
|
181
|
+
s_stormtypes.Lib.__init__(self, runt, name=name)
|
|
182
|
+
self.gtors['iden'] = self._getCellIden
|
|
183
|
+
|
|
177
184
|
def getObjLocals(self):
|
|
178
185
|
return {
|
|
179
186
|
'getCellInfo': self._getCellInfo,
|
|
@@ -187,6 +194,10 @@ class CellLib(s_stormtypes.Lib):
|
|
|
187
194
|
'uptime': self._uptime,
|
|
188
195
|
}
|
|
189
196
|
|
|
197
|
+
@s_stormtypes.stormfunc(readonly=True)
|
|
198
|
+
async def _getCellIden(self):
|
|
199
|
+
return self.runt.snap.core.getCellIden()
|
|
200
|
+
|
|
190
201
|
async def _hotFixesApply(self):
|
|
191
202
|
if not self.runt.isAdmin():
|
|
192
203
|
mesg = '$lib.cell.stormFixesApply() requires admin privs.'
|
synapse/lib/stormlib/infosec.py
CHANGED
|
@@ -515,6 +515,7 @@ class CvssLib(s_stormtypes.Lib):
|
|
|
515
515
|
'''
|
|
516
516
|
_storm_locals = (
|
|
517
517
|
{'name': 'calculate', 'desc': 'Calculate the CVSS score values for an input risk:vuln node.',
|
|
518
|
+
'deprecated': {'eolvers': 'v3.0.0'},
|
|
518
519
|
'type': {'type': 'function', '_funcname': 'calculate',
|
|
519
520
|
'args': (
|
|
520
521
|
{'name': 'node', 'type': 'node',
|
|
@@ -527,6 +528,7 @@ class CvssLib(s_stormtypes.Lib):
|
|
|
527
528
|
'returns': {'type': 'dict', 'desc': 'A dictionary containing the computed score and subscores.', }
|
|
528
529
|
}},
|
|
529
530
|
{'name': 'calculateFromProps', 'desc': 'Calculate the CVSS score values from a props dict.',
|
|
531
|
+
'deprecated': {'eolvers': 'v3.0.0'},
|
|
530
532
|
'type': {'type': 'function', '_funcname': 'calculateFromProps',
|
|
531
533
|
'args': (
|
|
532
534
|
{'name': 'props', 'type': 'dict',
|
synapse/lib/stormlib/scrape.py
CHANGED
synapse/lib/stormlib/stix.py
CHANGED
|
@@ -74,7 +74,7 @@ _DefaultConfig = {
|
|
|
74
74
|
'created': 'return($lib.stix.export.timestamp(.created))',
|
|
75
75
|
'modified': 'return($lib.stix.export.timestamp(.created))',
|
|
76
76
|
'sectors': '''
|
|
77
|
-
init { $list =
|
|
77
|
+
init { $list = () }
|
|
78
78
|
-> ou:industry +:name $list.append(:name)
|
|
79
79
|
fini { if $list { return($list) } }
|
|
80
80
|
''',
|
|
@@ -88,7 +88,7 @@ _DefaultConfig = {
|
|
|
88
88
|
'first_seen': '+.seen $seen=.seen return($lib.stix.export.timestamp($seen.0))',
|
|
89
89
|
'last_seen': '+.seen $seen=.seen return($lib.stix.export.timestamp($seen.1))',
|
|
90
90
|
'goals': '''
|
|
91
|
-
init { $goals =
|
|
91
|
+
init { $goals = () }
|
|
92
92
|
-> ou:campaign:org -> ou:goal | uniq | +:name $goals.append(:name)
|
|
93
93
|
fini { if $goals { return($goals) } }
|
|
94
94
|
''',
|
|
@@ -183,7 +183,7 @@ _DefaultConfig = {
|
|
|
183
183
|
'props': {
|
|
184
184
|
'value': 'return($node.repr())',
|
|
185
185
|
'resolves_to_refs': '''
|
|
186
|
-
init { $refs =
|
|
186
|
+
init { $refs = () }
|
|
187
187
|
{ -> inet:dns:a -> inet:ipv4 $refs.append($bundle.add($node)) }
|
|
188
188
|
{ -> inet:dns:aaaa -> inet:ipv6 $refs.append($bundle.add($node)) }
|
|
189
189
|
{ -> inet:dns:cname:fqdn :cname -> inet:fqdn $refs.append($bundle.add($node)) }
|
|
@@ -257,7 +257,7 @@ _DefaultConfig = {
|
|
|
257
257
|
''',
|
|
258
258
|
'mime_type': '+:mime return(:mime)',
|
|
259
259
|
'contains_refs': '''
|
|
260
|
-
init { $refs =
|
|
260
|
+
init { $refs = () }
|
|
261
261
|
-(refs)> *
|
|
262
262
|
$stixid = $bundle.add($node)
|
|
263
263
|
if $stixid { $refs.append($stixid) }
|
|
@@ -279,7 +279,7 @@ _DefaultConfig = {
|
|
|
279
279
|
'is_multipart': 'return($lib.false)',
|
|
280
280
|
'from_ref': ':from -> inet:email return($bundle.add($node))',
|
|
281
281
|
'to_refs': '''
|
|
282
|
-
init { $refs =
|
|
282
|
+
init { $refs = () }
|
|
283
283
|
{ :to -> inet:email $refs.append($bundle.add($node)) }
|
|
284
284
|
fini { if $refs { return($refs) } }
|
|
285
285
|
''',
|
|
@@ -311,7 +311,7 @@ _DefaultConfig = {
|
|
|
311
311
|
'created': 'return($lib.stix.export.timestamp(.created))',
|
|
312
312
|
'modified': 'return($lib.stix.export.timestamp(.created))',
|
|
313
313
|
'sample_refs': '''
|
|
314
|
-
init { $refs =
|
|
314
|
+
init { $refs = () }
|
|
315
315
|
-> file:bytes $refs.append($bundle.add($node))
|
|
316
316
|
fini { if $refs { return($refs) } }
|
|
317
317
|
''',
|
|
@@ -393,7 +393,7 @@ _DefaultConfig = {
|
|
|
393
393
|
'description': 'if (:desc) { return (:desc) }',
|
|
394
394
|
'created': 'return($lib.stix.export.timestamp(.created))',
|
|
395
395
|
'modified': 'return($lib.stix.export.timestamp(.created))',
|
|
396
|
-
'external_references': 'if :cve { $cve=:cve $cve=$cve.upper()
|
|
396
|
+
'external_references': 'if :cve { $cve=:cve $cve=$cve.upper() return(([{"source_name": "cve", "external_id": $cve}])) }'
|
|
397
397
|
},
|
|
398
398
|
'rels': (
|
|
399
399
|
|
|
@@ -439,7 +439,7 @@ _DefaultConfig = {
|
|
|
439
439
|
'modified': 'return($lib.stix.export.timestamp(.created))',
|
|
440
440
|
'published': 'return($lib.stix.export.timestamp(:published))',
|
|
441
441
|
'object_refs': '''
|
|
442
|
-
init { $refs =
|
|
442
|
+
init { $refs = () }
|
|
443
443
|
-(refs)> *
|
|
444
444
|
$stixid = $bundle.add($node)
|
|
445
445
|
if $stixid { $refs.append($stixid) }
|