synapse 2.165.0__py311-none-any.whl → 2.167.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/axon.py +4 -10
- synapse/cmds/cortex.py +1 -6
- synapse/common.py +6 -0
- synapse/cortex.py +104 -57
- synapse/datamodel.py +32 -0
- synapse/exc.py +1 -0
- synapse/lib/agenda.py +81 -51
- synapse/lib/aha.py +2 -0
- synapse/lib/ast.py +21 -23
- synapse/lib/base.py +11 -10
- synapse/lib/cell.py +24 -24
- synapse/lib/hive.py +11 -0
- synapse/lib/httpapi.py +1 -0
- synapse/lib/nexus.py +3 -2
- synapse/lib/node.py +4 -2
- synapse/lib/schemas.py +3 -1
- synapse/lib/snap.py +50 -0
- synapse/lib/storm.py +19 -17
- synapse/lib/stormlib/aha.py +370 -17
- synapse/lib/stormlib/auth.py +11 -4
- synapse/lib/stormlib/cache.py +202 -0
- synapse/lib/stormlib/cortex.py +69 -7
- synapse/lib/stormlib/macro.py +11 -18
- synapse/lib/stormlib/spooled.py +109 -0
- synapse/lib/stormlib/stix.py +1 -1
- synapse/lib/stormtypes.py +61 -17
- synapse/lib/trigger.py +10 -12
- synapse/lib/types.py +3 -1
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +16 -3
- synapse/models/base.py +8 -0
- synapse/models/files.py +3 -0
- synapse/models/inet.py +74 -2
- synapse/models/orgs.py +52 -8
- synapse/models/person.py +30 -11
- synapse/models/risk.py +44 -3
- synapse/telepath.py +115 -32
- synapse/tests/files/stormpkg/dotstorm/dotstorm.yaml +3 -0
- synapse/tests/test_cortex.py +79 -8
- synapse/tests/test_datamodel.py +22 -0
- synapse/tests/test_lib_agenda.py +8 -1
- synapse/tests/test_lib_aha.py +19 -6
- synapse/tests/test_lib_cell.py +6 -2
- synapse/tests/test_lib_grammar.py +62 -64
- synapse/tests/test_lib_httpapi.py +1 -1
- synapse/tests/test_lib_rstorm.py +4 -4
- synapse/tests/test_lib_storm.py +98 -7
- synapse/tests/test_lib_stormlib_aha.py +196 -0
- synapse/tests/test_lib_stormlib_cache.py +272 -0
- synapse/tests/test_lib_stormlib_compression.py +12 -12
- synapse/tests/test_lib_stormlib_cortex.py +71 -0
- synapse/tests/test_lib_stormlib_macro.py +94 -0
- synapse/tests/test_lib_stormlib_spooled.py +190 -0
- synapse/tests/test_lib_stormtypes.py +71 -37
- synapse/tests/test_lib_view.py +50 -3
- synapse/tests/test_model_files.py +3 -0
- synapse/tests/test_model_inet.py +67 -0
- synapse/tests/test_model_risk.py +6 -0
- synapse/tests/test_telepath.py +30 -7
- synapse/tests/test_tools_genpkg.py +26 -0
- synapse/tests/test_tools_hiveload.py +1 -0
- synapse/tests/test_tools_hivesave.py +1 -0
- synapse/tests/test_tools_modrole.py +81 -0
- synapse/tests/test_tools_moduser.py +105 -0
- synapse/tests/utils.py +22 -3
- synapse/tools/autodoc.py +1 -1
- synapse/tools/hive/load.py +3 -0
- synapse/tools/hive/save.py +3 -0
- synapse/tools/modrole.py +59 -7
- synapse/tools/moduser.py +78 -10
- {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/METADATA +3 -3
- {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/RECORD +75 -72
- synapse/lib/provenance.py +0 -111
- synapse/tests/test_lib_provenance.py +0 -37
- {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/LICENSE +0 -0
- {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/WHEEL +0 -0
- {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
|
|
3
|
+
import synapse.exc as s_exc
|
|
4
|
+
|
|
5
|
+
import synapse.lib.ast as s_ast
|
|
6
|
+
import synapse.lib.cache as s_cache
|
|
7
|
+
import synapse.lib.stormctrl as s_stormctrl
|
|
8
|
+
import synapse.lib.stormtypes as s_stormtypes
|
|
9
|
+
|
|
10
|
+
CACHE_SIZE_MAX = 10_000
|
|
11
|
+
CACHE_SIZE_DEFAULT = 10_000
|
|
12
|
+
|
|
13
|
+
@s_stormtypes.registry.registerLib
|
|
14
|
+
class LibCache(s_stormtypes.Lib):
|
|
15
|
+
'''
|
|
16
|
+
A Storm Library for interacting with Cache Objects.
|
|
17
|
+
'''
|
|
18
|
+
_storm_locals = (
|
|
19
|
+
{'name': 'fixed', 'desc': '''
|
|
20
|
+
Get a new Fixed Cache object.
|
|
21
|
+
|
|
22
|
+
On a cache-miss when calling .get(), the callback Storm query is executed in a sub-runtime
|
|
23
|
+
in the current execution context. A special variable, $cache_key, will be set
|
|
24
|
+
to the key argument provided to .get().
|
|
25
|
+
|
|
26
|
+
The callback Storm query must contain a return statement, and if it does not return a value
|
|
27
|
+
when executed with the input, $lib.null will be set as the value.
|
|
28
|
+
|
|
29
|
+
The fixed cache uses FIFO to evict items once the maximum size is reached.
|
|
30
|
+
|
|
31
|
+
Examples:
|
|
32
|
+
|
|
33
|
+
// Use a callback query with a function that modifies the outer runtime,
|
|
34
|
+
// since it will run in the scope where it was defined.
|
|
35
|
+
$test = foo
|
|
36
|
+
|
|
37
|
+
function callback(key) {
|
|
38
|
+
$test = $key // this will modify $test in the outer runtime
|
|
39
|
+
return(`{$key}-val`)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
$cache = $lib.cache.fixed(${ return($callback($cache_key)) })
|
|
43
|
+
$value = $cache.get(bar)
|
|
44
|
+
$lib.print($test) // this will equal "bar"
|
|
45
|
+
|
|
46
|
+
// Use a callback query that will not modify the outer runtime,
|
|
47
|
+
// except for variables accessible as references.
|
|
48
|
+
$test = foo
|
|
49
|
+
$tests = ([])
|
|
50
|
+
|
|
51
|
+
$cache = $lib.cache.fixed(${
|
|
52
|
+
$test = $cache_key // this will *not* modify $test in the outer runtime
|
|
53
|
+
$tests.append($cache_key) // this will modify $tests in the outer runtime
|
|
54
|
+
return(`{$cache_key}-val`)
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
$value = $cache.get(bar)
|
|
58
|
+
$lib.print($test) // this will equal "foo"
|
|
59
|
+
$lib.print($tests) // this will equal (foo,)
|
|
60
|
+
''',
|
|
61
|
+
'type': {'type': 'function', '_funcname': '_methFixedCache',
|
|
62
|
+
'args': (
|
|
63
|
+
{'name': 'callback', 'type': ['str', 'storm:query'],
|
|
64
|
+
'desc': 'A Storm query that will return a value for $cache_key on a cache miss.', },
|
|
65
|
+
{'name': 'size', 'type': 'int', 'default': CACHE_SIZE_DEFAULT,
|
|
66
|
+
'desc': 'The maximum size of the cache.', },
|
|
67
|
+
),
|
|
68
|
+
'returns': {'type': 'cache:fixed', 'desc': 'A new ``cache:fixed`` object.'}}},
|
|
69
|
+
)
|
|
70
|
+
_storm_lib_path = ('cache',)
|
|
71
|
+
|
|
72
|
+
def getObjLocals(self):
|
|
73
|
+
return {
|
|
74
|
+
'fixed': self._methFixedCache,
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
@s_stormtypes.stormfunc(readonly=True)
|
|
78
|
+
async def _methFixedCache(self, callback, size=CACHE_SIZE_DEFAULT):
|
|
79
|
+
size = await s_stormtypes.toint(size)
|
|
80
|
+
callback = await s_stormtypes.tostr(callback)
|
|
81
|
+
|
|
82
|
+
if size < 1 or size > CACHE_SIZE_MAX:
|
|
83
|
+
raise s_exc.BadArg(mesg=f'Cache size must be between 1-{CACHE_SIZE_MAX}')
|
|
84
|
+
|
|
85
|
+
try:
|
|
86
|
+
query = await self.runt.getStormQuery(callback)
|
|
87
|
+
except s_exc.BadSyntax as e:
|
|
88
|
+
raise s_exc.BadArg(mesg=f'Invalid callback query: {e.errinfo.get("mesg")}')
|
|
89
|
+
|
|
90
|
+
if not query.hasAstClass(s_ast.Return):
|
|
91
|
+
raise s_exc.BadArg(mesg='Callback query must return a value')
|
|
92
|
+
|
|
93
|
+
return FixedCache(self.runt, query, size=size)
|
|
94
|
+
|
|
95
|
+
@s_stormtypes.registry.registerType
|
|
96
|
+
class FixedCache(s_stormtypes.StormType):
|
|
97
|
+
'''
|
|
98
|
+
A StormLib API instance of a Storm Fixed Cache.
|
|
99
|
+
'''
|
|
100
|
+
_storm_locals = (
|
|
101
|
+
{'name': 'query', 'desc': 'Get the callback Storm query as string.',
|
|
102
|
+
'type': {'type': 'gtor', '_gtorfunc': '_gtorQuery',
|
|
103
|
+
'returns': {'type': 'str', 'desc': 'The callback Storm query text.', }}},
|
|
104
|
+
{'name': 'get', 'desc': 'Get an item from the cache by key.',
|
|
105
|
+
'type': {'type': 'function', '_funcname': '_methGet',
|
|
106
|
+
'args': (
|
|
107
|
+
{'name': 'key', 'type': 'any', 'desc': 'The key to lookup.'},
|
|
108
|
+
),
|
|
109
|
+
'returns': {'type': 'any',
|
|
110
|
+
'desc': 'The value from the cache, or the callback query if it does not exist', }}},
|
|
111
|
+
{'name': 'pop', 'desc': 'Pop an item from the cache.',
|
|
112
|
+
'type': {'type': 'function', '_funcname': '_methPop',
|
|
113
|
+
'args': (
|
|
114
|
+
{'name': 'key', 'type': 'any', 'desc': 'The key to pop.'},
|
|
115
|
+
),
|
|
116
|
+
'returns': {'type': 'any',
|
|
117
|
+
'desc': 'The value from the cache, or $lib.null if it does not exist', }}},
|
|
118
|
+
{'name': 'put', 'desc': 'Put an item into the cache.',
|
|
119
|
+
'type': {'type': 'function', '_funcname': '_methPut',
|
|
120
|
+
'args': (
|
|
121
|
+
{'name': 'key', 'type': 'any', 'desc': 'The key put in the cache.'},
|
|
122
|
+
{'name': 'value', 'type': 'any', 'desc': 'The value to assign to the key.'},
|
|
123
|
+
),
|
|
124
|
+
'returns': {'type': 'null', }}},
|
|
125
|
+
{'name': 'clear', 'desc': 'Clear all items from the cache.',
|
|
126
|
+
'type': {'type': 'function', '_funcname': '_methClear',
|
|
127
|
+
'returns': {'type': 'null', }}},
|
|
128
|
+
)
|
|
129
|
+
_storm_typename = 'cache:fixed'
|
|
130
|
+
_ismutable = False
|
|
131
|
+
|
|
132
|
+
def __init__(self, runt, query, size=CACHE_SIZE_DEFAULT):
|
|
133
|
+
s_stormtypes.StormType.__init__(self)
|
|
134
|
+
self.runt = runt
|
|
135
|
+
self.size = size
|
|
136
|
+
self.query = query
|
|
137
|
+
self.locls.update(self.getObjLocals())
|
|
138
|
+
self.gtors.update({
|
|
139
|
+
'query': self._gtorQuery,
|
|
140
|
+
})
|
|
141
|
+
|
|
142
|
+
self.cache = s_cache.FixedCache(self._runCallback, size=size)
|
|
143
|
+
|
|
144
|
+
def __len__(self):
|
|
145
|
+
return len(self.cache)
|
|
146
|
+
|
|
147
|
+
async def stormrepr(self):
|
|
148
|
+
if len(qtext := self.query.text) > 100:
|
|
149
|
+
qtext = qtext[:100] + '...'
|
|
150
|
+
return f'{self._storm_typename}: size={self.size} query="{qtext}"'
|
|
151
|
+
|
|
152
|
+
def getObjLocals(self):
|
|
153
|
+
return {
|
|
154
|
+
'pop': self._methPop,
|
|
155
|
+
'put': self._methPut,
|
|
156
|
+
'get': self._methGet,
|
|
157
|
+
'clear': self._methClear,
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
async def _gtorQuery(self):
|
|
161
|
+
return self.query.text
|
|
162
|
+
|
|
163
|
+
async def _runCallback(self, key):
|
|
164
|
+
|
|
165
|
+
varz = self.runt.getScopeVars()
|
|
166
|
+
varz['cache_key'] = key
|
|
167
|
+
|
|
168
|
+
opts = {'vars': varz}
|
|
169
|
+
async with self.runt.getCmdRuntime(self.query, opts=opts) as runt:
|
|
170
|
+
try:
|
|
171
|
+
async for _ in runt.execute():
|
|
172
|
+
await asyncio.sleep(0)
|
|
173
|
+
except s_stormctrl.StormReturn as e:
|
|
174
|
+
return await s_stormtypes.toprim(e.item)
|
|
175
|
+
except s_stormctrl.StormCtrlFlow:
|
|
176
|
+
pass
|
|
177
|
+
|
|
178
|
+
async def _reqKey(self, key):
|
|
179
|
+
if s_stormtypes.ismutable(key):
|
|
180
|
+
mesg = 'Mutable values are not allowed as cache keys'
|
|
181
|
+
raise s_exc.BadArg(mesg=mesg, name=await s_stormtypes.torepr(key))
|
|
182
|
+
return await s_stormtypes.toprim(key)
|
|
183
|
+
|
|
184
|
+
@s_stormtypes.stormfunc(readonly=True)
|
|
185
|
+
async def _methPop(self, key):
|
|
186
|
+
key = await self._reqKey(key)
|
|
187
|
+
return self.cache.pop(key)
|
|
188
|
+
|
|
189
|
+
@s_stormtypes.stormfunc(readonly=True)
|
|
190
|
+
async def _methPut(self, key, value):
|
|
191
|
+
key = await self._reqKey(key)
|
|
192
|
+
val = await s_stormtypes.toprim(value)
|
|
193
|
+
self.cache.put(key, val)
|
|
194
|
+
|
|
195
|
+
@s_stormtypes.stormfunc(readonly=True)
|
|
196
|
+
async def _methGet(self, key):
|
|
197
|
+
key = await self._reqKey(key)
|
|
198
|
+
return await self.cache.aget(key)
|
|
199
|
+
|
|
200
|
+
@s_stormtypes.stormfunc(readonly=True)
|
|
201
|
+
async def _methClear(self):
|
|
202
|
+
self.cache.clear()
|
synapse/lib/stormlib/cortex.py
CHANGED
|
@@ -3,6 +3,7 @@ import json
|
|
|
3
3
|
import logging
|
|
4
4
|
|
|
5
5
|
import synapse.exc as s_exc
|
|
6
|
+
import synapse.telepath as s_telepath
|
|
6
7
|
|
|
7
8
|
import synapse.lib.storm as s_storm
|
|
8
9
|
import synapse.lib.stormtypes as s_stormtypes
|
|
@@ -86,6 +87,7 @@ stormcmds = [
|
|
|
86
87
|
$lib.print(`!View: No view found ({$err.info.iden})`)
|
|
87
88
|
}
|
|
88
89
|
$lib.print(`Readonly: {$api.readonly}`)
|
|
90
|
+
$lib.print(`Pool enabled: {$api.pool}`)
|
|
89
91
|
$lib.print(`Authenticated: {$api.authenticated}`)
|
|
90
92
|
$lib.print(`Name: {$api.name}`)
|
|
91
93
|
$lib.print(`Description: {$api.desc}`)
|
|
@@ -224,6 +226,9 @@ class HttpApi(s_stormtypes.StormType):
|
|
|
224
226
|
''',
|
|
225
227
|
'type': {'type': ['stor', 'gtor'], '_storfunc': '_storPath', '_gtorfunc': '_gtorPath',
|
|
226
228
|
'returns': {'type': 'str'}}},
|
|
229
|
+
{'name': 'pool', 'desc': 'Boolean value indicating if the handler responses may be executed as part of a Storm pool.',
|
|
230
|
+
'type': {'type': ['stor', 'gtor'], '_storfunc': '_storPool', '_gtorfunc': '_gtorPool',
|
|
231
|
+
'returns': {'type': 'boolean'}}},
|
|
227
232
|
{'name': 'vars', 'desc': 'The Storm runtime variables specific for the API instance.',
|
|
228
233
|
'type': {'type': ['stor', 'ctor'], '_storfunc': '_storVars', '_ctorfunc': '_ctorVars',
|
|
229
234
|
'returns': {'type': 'http:api:vars'}}},
|
|
@@ -242,6 +247,8 @@ class HttpApi(s_stormtypes.StormType):
|
|
|
242
247
|
{'name': 'authenticated', 'desc': 'Boolean value indicating if the Extended HTTP API requires an authenticated user or session.',
|
|
243
248
|
'type': {'type': ['stor', 'gtor'], '_storfunc': '_storAuthenticated', '_gtorfunc': '_gtorAuthenticated',
|
|
244
249
|
'returns': {'type': 'boolean'}}},
|
|
250
|
+
{'name': 'methods', 'desc': 'The dictionary containing the Storm code used to implement the HTTP methods.',
|
|
251
|
+
'type': {'type': ['ctor'], '_ctorfunc': '_ctorMethods', 'returns': {'type': 'http:api:methods'}}}
|
|
245
252
|
)
|
|
246
253
|
|
|
247
254
|
def __init__(self, runt, info):
|
|
@@ -257,6 +264,7 @@ class HttpApi(s_stormtypes.StormType):
|
|
|
257
264
|
'name': self._storName,
|
|
258
265
|
'desc': self._storDesc,
|
|
259
266
|
'path': self._storPath,
|
|
267
|
+
'pool': self._storPool,
|
|
260
268
|
'vars': self._storVars,
|
|
261
269
|
'view': self._storView,
|
|
262
270
|
'runas': self._storRunas,
|
|
@@ -270,6 +278,7 @@ class HttpApi(s_stormtypes.StormType):
|
|
|
270
278
|
'name': self._gtorName,
|
|
271
279
|
'desc': self._gtorDesc,
|
|
272
280
|
'path': self._gtorPath,
|
|
281
|
+
'pool': self._gtorPool,
|
|
273
282
|
'view': self._gtorView,
|
|
274
283
|
'runas': self._gtorRunas,
|
|
275
284
|
'owner': self._gtorOwner,
|
|
@@ -292,6 +301,13 @@ class HttpApi(s_stormtypes.StormType):
|
|
|
292
301
|
'created': self.info.get('created'),
|
|
293
302
|
})
|
|
294
303
|
|
|
304
|
+
async def stormrepr(self):
|
|
305
|
+
name = await self._gtorName()
|
|
306
|
+
if not name:
|
|
307
|
+
name = '<no name>'
|
|
308
|
+
path = await self._gtorPath()
|
|
309
|
+
return f'{self._storm_typename}: {name} ({self.iden}), path={path}'
|
|
310
|
+
|
|
295
311
|
def getObjLocals(self):
|
|
296
312
|
return {
|
|
297
313
|
'pack': self._methPack,
|
|
@@ -299,7 +315,10 @@ class HttpApi(s_stormtypes.StormType):
|
|
|
299
315
|
|
|
300
316
|
@s_stormtypes.stormfunc(readonly=True)
|
|
301
317
|
async def _methPack(self):
|
|
302
|
-
|
|
318
|
+
# TODO: Remove this when we've migrated the HTTPAPI data to set this value.
|
|
319
|
+
ret = copy.deepcopy(self.info)
|
|
320
|
+
ret.setdefault('pool', False)
|
|
321
|
+
return ret
|
|
303
322
|
|
|
304
323
|
@s_stormtypes.stormfunc(readonly=True)
|
|
305
324
|
def _ctorMethods(self, path=None):
|
|
@@ -316,6 +335,17 @@ class HttpApi(s_stormtypes.StormType):
|
|
|
316
335
|
async def _gtorPath(self):
|
|
317
336
|
return self.info.get('path')
|
|
318
337
|
|
|
338
|
+
async def _storPool(self, pool):
|
|
339
|
+
s_stormtypes.confirm(('storm', 'lib', 'cortex', 'httpapi', 'set'))
|
|
340
|
+
pool = await s_stormtypes.tobool(pool)
|
|
341
|
+
adef = await self.runt.snap.core.modHttpExtApi(self.iden, 'pool', pool)
|
|
342
|
+
self.info['pool'] = pool
|
|
343
|
+
self.info['updated'] = adef.get('updated')
|
|
344
|
+
|
|
345
|
+
@s_stormtypes.stormfunc(readonly=True)
|
|
346
|
+
async def _gtorPool(self):
|
|
347
|
+
return self.info.get('pool')
|
|
348
|
+
|
|
319
349
|
async def _storName(self, name):
|
|
320
350
|
s_stormtypes.confirm(('storm', 'lib', 'cortex', 'httpapi', 'set'))
|
|
321
351
|
name = await s_stormtypes.tostr(name)
|
|
@@ -1055,7 +1085,7 @@ class CortexHttpApi(s_stormtypes.Lib):
|
|
|
1055
1085
|
{'name': 'readonly', 'type': 'boolean',
|
|
1056
1086
|
'desc': 'Run the Extended HTTP Storm methods in readonly mode.', 'default': False},
|
|
1057
1087
|
),
|
|
1058
|
-
'returns': {'type': 'http:api', 'desc': 'A new http:api object.'}}},
|
|
1088
|
+
'returns': {'type': 'http:api', 'desc': 'A new ``http:api`` object.'}}},
|
|
1059
1089
|
{'name': 'del', 'desc': 'Delete an Extended HTTP API endpoint.',
|
|
1060
1090
|
'type': {'type': 'function', '_funcname': 'delHttpApi',
|
|
1061
1091
|
'args': (
|
|
@@ -1063,16 +1093,29 @@ class CortexHttpApi(s_stormtypes.Lib):
|
|
|
1063
1093
|
'desc': 'The iden of the API to delete.'},
|
|
1064
1094
|
),
|
|
1065
1095
|
'returns': {'type': 'null'}}},
|
|
1066
|
-
{'name': 'get', 'desc': 'Get an Extended
|
|
1096
|
+
{'name': 'get', 'desc': 'Get an Extended ``http:api`` object.',
|
|
1067
1097
|
'type': {'type': 'function', '_funcname': 'getHttpApi',
|
|
1068
1098
|
'args': (
|
|
1069
1099
|
{'name': 'iden', 'type': 'string',
|
|
1070
|
-
'desc': 'The iden of the API to
|
|
1100
|
+
'desc': 'The iden of the API to retrieve.'},
|
|
1101
|
+
),
|
|
1102
|
+
'returns': {'type': 'http:api', 'desc': 'The ``http:api`` object.'}}},
|
|
1103
|
+
{'name': 'getByPath', 'desc': '''
|
|
1104
|
+
Get an Extended ``http:api`` object by path.
|
|
1105
|
+
|
|
1106
|
+
Notes:
|
|
1107
|
+
The path argument is evaluated as a regular expression input, and will be
|
|
1108
|
+
used to get the first HTTP API handler whose path value has a match.
|
|
1109
|
+
''',
|
|
1110
|
+
'type': {'type': 'function', '_funcname': 'getHttpApiByPath',
|
|
1111
|
+
'args': (
|
|
1112
|
+
{'name': 'path', 'type': 'string',
|
|
1113
|
+
'desc': 'Path to use to retrieve an object.'},
|
|
1071
1114
|
),
|
|
1072
|
-
'returns': {'type': 'http:api', 'desc': 'The http:api object.'}}},
|
|
1073
|
-
{'name': 'list', 'desc': 'Get all the
|
|
1115
|
+
'returns': {'type': ['http:api', 'null'], 'desc': 'The ``http:api`` object or ``$lib.null`` if there is no match.'}}},
|
|
1116
|
+
{'name': 'list', 'desc': 'Get all the Extended HTTP APIs on the Cortex',
|
|
1074
1117
|
'type': {'type': 'function', '_funcname': 'listHttpApis', 'args': (),
|
|
1075
|
-
|
|
1118
|
+
'returns': {'type': 'list', 'desc': 'A list of ``http:api`` objects'}}},
|
|
1076
1119
|
{'name': 'index', 'desc': 'Set the index for a given Extended HTTP API.',
|
|
1077
1120
|
'type': {'type': 'function', '_funcname': 'setHttpApiIndx',
|
|
1078
1121
|
'args': (
|
|
@@ -1111,6 +1154,7 @@ class CortexHttpApi(s_stormtypes.Lib):
|
|
|
1111
1154
|
'list': self.listHttpApis,
|
|
1112
1155
|
'index': self.setHttpApiIndx,
|
|
1113
1156
|
'response': self.makeHttpResponse,
|
|
1157
|
+
'getByPath': self.getHttpApiByPath,
|
|
1114
1158
|
}
|
|
1115
1159
|
|
|
1116
1160
|
@s_stormtypes.stormfunc(readonly=True)
|
|
@@ -1125,6 +1169,15 @@ class CortexHttpApi(s_stormtypes.Lib):
|
|
|
1125
1169
|
adef = await self.runt.snap.core.getHttpExtApi(iden)
|
|
1126
1170
|
return HttpApi(self.runt, adef)
|
|
1127
1171
|
|
|
1172
|
+
@s_stormtypes.stormfunc(readonly=True)
|
|
1173
|
+
async def getHttpApiByPath(self, path):
|
|
1174
|
+
s_stormtypes.confirm(('storm', 'lib', 'cortex', 'httpapi', 'get'))
|
|
1175
|
+
path = await s_stormtypes.tostr(path)
|
|
1176
|
+
adef, _ = await self.runt.snap.core.getHttpExtApiByPath(path)
|
|
1177
|
+
if adef is None:
|
|
1178
|
+
return None
|
|
1179
|
+
return HttpApi(self.runt, adef)
|
|
1180
|
+
|
|
1128
1181
|
@s_stormtypes.stormfunc(readonly=True)
|
|
1129
1182
|
async def listHttpApis(self):
|
|
1130
1183
|
s_stormtypes.confirm(('storm', 'lib', 'cortex', 'httpapi', 'get'))
|
|
@@ -1195,6 +1248,11 @@ class StormPoolSetCmd(s_storm.Cmd):
|
|
|
1195
1248
|
async for node, path in genr: # pragma: no cover
|
|
1196
1249
|
yield node, path
|
|
1197
1250
|
|
|
1251
|
+
try:
|
|
1252
|
+
s_telepath.chopurl(self.opts.url)
|
|
1253
|
+
except s_exc.BadUrl as e:
|
|
1254
|
+
raise s_exc.BadArg(mesg=f'Unable to set Storm pool URL from url={self.opts.url} : {e.get("mesg")}') from None
|
|
1255
|
+
|
|
1198
1256
|
opts = {
|
|
1199
1257
|
'timeout:sync': self.opts.sync_timeout,
|
|
1200
1258
|
'timeout:connection': self.opts.connection_timeout,
|
|
@@ -1206,6 +1264,10 @@ class StormPoolSetCmd(s_storm.Cmd):
|
|
|
1206
1264
|
class StormPoolDelCmd(s_storm.Cmd):
|
|
1207
1265
|
'''
|
|
1208
1266
|
Remove a Storm query offload mirror pool configuration.
|
|
1267
|
+
|
|
1268
|
+
Notes:
|
|
1269
|
+
This will result in tearing down any Storm queries currently being serviced by the Storm pool.
|
|
1270
|
+
This may result in this command raising an exception if it was offloaded to a pool member. That would be an expected behavior.
|
|
1209
1271
|
'''
|
|
1210
1272
|
name = 'cortex.storm.pool.del'
|
|
1211
1273
|
|
synapse/lib/stormlib/macro.py
CHANGED
|
@@ -34,7 +34,7 @@ stormcmds = [
|
|
|
34
34
|
'name': 'macro.del',
|
|
35
35
|
'descr': macro_del_descr,
|
|
36
36
|
'cmdargs': (
|
|
37
|
-
('name', {'help': 'The name of the macro to delete.'}),
|
|
37
|
+
('name', {'type': 'str', 'help': 'The name of the macro to delete.'}),
|
|
38
38
|
),
|
|
39
39
|
'storm': '''
|
|
40
40
|
$lib.macro.del($cmdopts.name)
|
|
@@ -45,7 +45,7 @@ stormcmds = [
|
|
|
45
45
|
'name': 'macro.set',
|
|
46
46
|
'descr': macro_set_descr,
|
|
47
47
|
'cmdargs': (
|
|
48
|
-
('name', {'help': 'The name of the macro to set.'}),
|
|
48
|
+
('name', {'type': 'str', 'help': 'The name of the macro to set.'}),
|
|
49
49
|
('storm', {'help': 'The storm command string or embedded query to set.'}),
|
|
50
50
|
),
|
|
51
51
|
'storm': '''
|
|
@@ -57,7 +57,7 @@ stormcmds = [
|
|
|
57
57
|
'name': 'macro.get',
|
|
58
58
|
'descr': macro_get_descr,
|
|
59
59
|
'cmdargs': (
|
|
60
|
-
('name', {'help': 'The name of the macro to display.'}),
|
|
60
|
+
('name', {'type': 'str', 'help': 'The name of the macro to display.'}),
|
|
61
61
|
),
|
|
62
62
|
'storm': '''
|
|
63
63
|
$mdef = $lib.macro.get($cmdopts.name)
|
|
@@ -74,8 +74,14 @@ stormcmds = [
|
|
|
74
74
|
'storm': '''
|
|
75
75
|
$count = $(0)
|
|
76
76
|
for ($name, $mdef) in $lib.macro.list() {
|
|
77
|
-
$user = $lib.auth.users.get($mdef.
|
|
78
|
-
$
|
|
77
|
+
$user = $lib.auth.users.get($mdef.creator)
|
|
78
|
+
$username = $lib.null
|
|
79
|
+
if (not $user) {
|
|
80
|
+
$username = `User not found ({$mdef.creator})`
|
|
81
|
+
} else {
|
|
82
|
+
$username = $user.name
|
|
83
|
+
}
|
|
84
|
+
$lib.print('{name} (owner: {user})', name=$name.ljust(20), user=$username)
|
|
79
85
|
$count = $($count + 1)
|
|
80
86
|
}
|
|
81
87
|
$lib.print('{count} macros found', count=$count)
|
|
@@ -185,27 +191,17 @@ class LibMacro(s_stormtypes.Lib):
|
|
|
185
191
|
async def _funcMacroGet(self, name):
|
|
186
192
|
name = await s_stormtypes.tostr(name)
|
|
187
193
|
|
|
188
|
-
if len(name) > 491:
|
|
189
|
-
raise s_exc.BadArg(mesg='Macro names may only be up to 491 chars.')
|
|
190
|
-
|
|
191
194
|
return self.runt.snap.core.getStormMacro(name, user=self.runt.user)
|
|
192
195
|
|
|
193
196
|
async def _funcMacroDel(self, name):
|
|
194
|
-
|
|
195
197
|
name = await s_stormtypes.tostr(name)
|
|
196
198
|
|
|
197
|
-
if len(name) > 491:
|
|
198
|
-
raise s_exc.BadArg(mesg='Macro names may only be up to 491 chars.')
|
|
199
|
-
|
|
200
199
|
return await self.runt.snap.core.delStormMacro(name, user=self.runt.user)
|
|
201
200
|
|
|
202
201
|
async def _funcMacroSet(self, name, storm):
|
|
203
202
|
name = await s_stormtypes.tostr(name)
|
|
204
203
|
storm = await s_stormtypes.tostr(storm)
|
|
205
204
|
|
|
206
|
-
if len(name) > 491:
|
|
207
|
-
raise s_exc.BadArg(mesg='Macro names may only be up to 491 chars.')
|
|
208
|
-
|
|
209
205
|
await self.runt.getStormQuery(storm)
|
|
210
206
|
|
|
211
207
|
if self.runt.snap.core.getStormMacro(name) is None:
|
|
@@ -220,9 +216,6 @@ class LibMacro(s_stormtypes.Lib):
|
|
|
220
216
|
name = await s_stormtypes.tostr(name)
|
|
221
217
|
info = await s_stormtypes.toprim(info)
|
|
222
218
|
|
|
223
|
-
if len(name) > 491:
|
|
224
|
-
raise s_exc.BadArg(mesg='Macro names may only be up to 491 chars.')
|
|
225
|
-
|
|
226
219
|
if not isinstance(info, dict):
|
|
227
220
|
raise s_exc.BadArg(mesg='Macro info must be a dictionary object.')
|
|
228
221
|
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import synapse.exc as s_exc
|
|
2
|
+
import synapse.lib.msgpack as s_msgpack
|
|
3
|
+
import synapse.lib.spooled as s_spooled
|
|
4
|
+
import synapse.lib.stormtypes as s_stormtypes
|
|
5
|
+
|
|
6
|
+
@s_stormtypes.registry.registerLib
|
|
7
|
+
class LibSpooled(s_stormtypes.Lib):
|
|
8
|
+
'''
|
|
9
|
+
A Storm Library for interacting with Spooled Objects.
|
|
10
|
+
'''
|
|
11
|
+
_storm_locals = (
|
|
12
|
+
{'name': 'set', 'desc': '''
|
|
13
|
+
Get a Spooled Storm Set object.
|
|
14
|
+
|
|
15
|
+
A Spooled Storm Set object is memory-safe to grow to extraordinarily large sizes,
|
|
16
|
+
as it will fallback to file backed storage, with two restrictions. First
|
|
17
|
+
is that all items in the set can be serialized to a file if the set grows too large,
|
|
18
|
+
so all items added must be a serializable Storm primitive. Second is that when an
|
|
19
|
+
item is added to the Set, because it could be immediately written disk,
|
|
20
|
+
do not hold any references to it outside of the Set itself, as the two objects could
|
|
21
|
+
differ.
|
|
22
|
+
''',
|
|
23
|
+
'type': {'type': 'function', '_funcname': '_methSet',
|
|
24
|
+
'args': (
|
|
25
|
+
{'name': '*vals', 'type': 'any', 'desc': 'Initial values to place in the set.', },
|
|
26
|
+
),
|
|
27
|
+
'returns': {'type': 'set', 'desc': 'The new set.'}}},
|
|
28
|
+
)
|
|
29
|
+
_storm_lib_path = ('spooled',)
|
|
30
|
+
|
|
31
|
+
def getObjLocals(self):
|
|
32
|
+
return {
|
|
33
|
+
'set': self._methSet,
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
@s_stormtypes.stormfunc(readonly=True)
|
|
37
|
+
async def _methSet(self, *vals):
|
|
38
|
+
core = self.runt.snap.core
|
|
39
|
+
spool = await s_spooled.Set.anit(dirn=core.dirn, cell=core, size=1000)
|
|
40
|
+
|
|
41
|
+
valu = list(vals)
|
|
42
|
+
for item in valu:
|
|
43
|
+
if s_stormtypes.ismutable(item):
|
|
44
|
+
mesg = f'{await s_stormtypes.torepr(item)} is mutable and cannot be used in a set.'
|
|
45
|
+
raise s_exc.StormRuntimeError(mesg=mesg)
|
|
46
|
+
|
|
47
|
+
if not s_msgpack.isok(item):
|
|
48
|
+
mesg = f'{await s_stormtypes.torepr(item)} is not safe to be used in a SpooledSet.'
|
|
49
|
+
raise s_exc.StormRuntimeError(mesg=mesg)
|
|
50
|
+
|
|
51
|
+
await spool.add(item)
|
|
52
|
+
|
|
53
|
+
return SpooledSet(spool)
|
|
54
|
+
|
|
55
|
+
@s_stormtypes.registry.registerType
|
|
56
|
+
class SpooledSet(s_stormtypes.Set):
|
|
57
|
+
'''
|
|
58
|
+
A StormLib API instance of a Storm Set object that can fallback to lmdb.
|
|
59
|
+
'''
|
|
60
|
+
_storm_typename = 'spooled:set'
|
|
61
|
+
_ismutable = True
|
|
62
|
+
|
|
63
|
+
def __init__(self, valu, path=None):
|
|
64
|
+
|
|
65
|
+
s_stormtypes.Prim.__init__(self, valu, path=path)
|
|
66
|
+
self.locls.update(self.getObjLocals())
|
|
67
|
+
|
|
68
|
+
async def iter(self):
|
|
69
|
+
async for item in self.valu:
|
|
70
|
+
yield item
|
|
71
|
+
|
|
72
|
+
@s_stormtypes.stormfunc(readonly=True)
|
|
73
|
+
async def _methSetAdd(self, *items):
|
|
74
|
+
for i in items:
|
|
75
|
+
if s_stormtypes.ismutable(i):
|
|
76
|
+
mesg = f'{await s_stormtypes.torepr(i)} is mutable and cannot be used in a set.'
|
|
77
|
+
raise s_exc.StormRuntimeError(mesg=mesg)
|
|
78
|
+
|
|
79
|
+
if not s_msgpack.isok(i):
|
|
80
|
+
mesg = f'{await s_stormtypes.torepr(i)} is not safe to be used in a SpooledSet.'
|
|
81
|
+
raise s_exc.StormRuntimeError(mesg=mesg)
|
|
82
|
+
|
|
83
|
+
await self.valu.add(i)
|
|
84
|
+
|
|
85
|
+
@s_stormtypes.stormfunc(readonly=True)
|
|
86
|
+
async def _methSetAdds(self, *items):
|
|
87
|
+
for item in items:
|
|
88
|
+
async for i in s_stormtypes.toiter(item):
|
|
89
|
+
if s_stormtypes.ismutable(i):
|
|
90
|
+
mesg = f'{await s_stormtypes.torepr(i)} is mutable and cannot be used in a set.'
|
|
91
|
+
raise s_exc.StormRuntimeError(mesg=mesg)
|
|
92
|
+
|
|
93
|
+
if not s_msgpack.isok(i):
|
|
94
|
+
mesg = f'{await s_stormtypes.torepr(i)} is not safe to be used in a SpooledSet.'
|
|
95
|
+
raise s_exc.StormRuntimeError(mesg=mesg)
|
|
96
|
+
|
|
97
|
+
await self.valu.add(i)
|
|
98
|
+
|
|
99
|
+
@s_stormtypes.stormfunc(readonly=True)
|
|
100
|
+
async def _methSetList(self):
|
|
101
|
+
return [x async for x in self.valu]
|
|
102
|
+
|
|
103
|
+
async def stormrepr(self):
|
|
104
|
+
reprs = [await s_stormtypes.torepr(k) async for k in self.valu]
|
|
105
|
+
rval = ', '.join(reprs)
|
|
106
|
+
return f'{{{rval}}}'
|
|
107
|
+
|
|
108
|
+
async def value(self):
|
|
109
|
+
return set([x async for x in self.valu])
|
synapse/lib/stormlib/stix.py
CHANGED
|
@@ -585,7 +585,7 @@ def validateStix(bundle, version='2.1'):
|
|
|
585
585
|
'result': {},
|
|
586
586
|
}
|
|
587
587
|
bundle = json.loads(json.dumps(bundle))
|
|
588
|
-
opts = stix2validator.ValidationOptions(strict=True, version=version
|
|
588
|
+
opts = stix2validator.ValidationOptions(strict=True, version=version)
|
|
589
589
|
try:
|
|
590
590
|
results = stix2validator.validate_parsed_json(bundle, options=opts)
|
|
591
591
|
except stix2validator.ValidationError as e:
|