synapse 2.165.0__py311-none-any.whl → 2.167.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (77) hide show
  1. synapse/axon.py +4 -10
  2. synapse/cmds/cortex.py +1 -6
  3. synapse/common.py +6 -0
  4. synapse/cortex.py +104 -57
  5. synapse/datamodel.py +32 -0
  6. synapse/exc.py +1 -0
  7. synapse/lib/agenda.py +81 -51
  8. synapse/lib/aha.py +2 -0
  9. synapse/lib/ast.py +21 -23
  10. synapse/lib/base.py +11 -10
  11. synapse/lib/cell.py +24 -24
  12. synapse/lib/hive.py +11 -0
  13. synapse/lib/httpapi.py +1 -0
  14. synapse/lib/nexus.py +3 -2
  15. synapse/lib/node.py +4 -2
  16. synapse/lib/schemas.py +3 -1
  17. synapse/lib/snap.py +50 -0
  18. synapse/lib/storm.py +19 -17
  19. synapse/lib/stormlib/aha.py +370 -17
  20. synapse/lib/stormlib/auth.py +11 -4
  21. synapse/lib/stormlib/cache.py +202 -0
  22. synapse/lib/stormlib/cortex.py +69 -7
  23. synapse/lib/stormlib/macro.py +11 -18
  24. synapse/lib/stormlib/spooled.py +109 -0
  25. synapse/lib/stormlib/stix.py +1 -1
  26. synapse/lib/stormtypes.py +61 -17
  27. synapse/lib/trigger.py +10 -12
  28. synapse/lib/types.py +3 -1
  29. synapse/lib/version.py +2 -2
  30. synapse/lib/view.py +16 -3
  31. synapse/models/base.py +8 -0
  32. synapse/models/files.py +3 -0
  33. synapse/models/inet.py +74 -2
  34. synapse/models/orgs.py +52 -8
  35. synapse/models/person.py +30 -11
  36. synapse/models/risk.py +44 -3
  37. synapse/telepath.py +115 -32
  38. synapse/tests/files/stormpkg/dotstorm/dotstorm.yaml +3 -0
  39. synapse/tests/test_cortex.py +79 -8
  40. synapse/tests/test_datamodel.py +22 -0
  41. synapse/tests/test_lib_agenda.py +8 -1
  42. synapse/tests/test_lib_aha.py +19 -6
  43. synapse/tests/test_lib_cell.py +6 -2
  44. synapse/tests/test_lib_grammar.py +62 -64
  45. synapse/tests/test_lib_httpapi.py +1 -1
  46. synapse/tests/test_lib_rstorm.py +4 -4
  47. synapse/tests/test_lib_storm.py +98 -7
  48. synapse/tests/test_lib_stormlib_aha.py +196 -0
  49. synapse/tests/test_lib_stormlib_cache.py +272 -0
  50. synapse/tests/test_lib_stormlib_compression.py +12 -12
  51. synapse/tests/test_lib_stormlib_cortex.py +71 -0
  52. synapse/tests/test_lib_stormlib_macro.py +94 -0
  53. synapse/tests/test_lib_stormlib_spooled.py +190 -0
  54. synapse/tests/test_lib_stormtypes.py +71 -37
  55. synapse/tests/test_lib_view.py +50 -3
  56. synapse/tests/test_model_files.py +3 -0
  57. synapse/tests/test_model_inet.py +67 -0
  58. synapse/tests/test_model_risk.py +6 -0
  59. synapse/tests/test_telepath.py +30 -7
  60. synapse/tests/test_tools_genpkg.py +26 -0
  61. synapse/tests/test_tools_hiveload.py +1 -0
  62. synapse/tests/test_tools_hivesave.py +1 -0
  63. synapse/tests/test_tools_modrole.py +81 -0
  64. synapse/tests/test_tools_moduser.py +105 -0
  65. synapse/tests/utils.py +22 -3
  66. synapse/tools/autodoc.py +1 -1
  67. synapse/tools/hive/load.py +3 -0
  68. synapse/tools/hive/save.py +3 -0
  69. synapse/tools/modrole.py +59 -7
  70. synapse/tools/moduser.py +78 -10
  71. {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/METADATA +3 -3
  72. {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/RECORD +75 -72
  73. synapse/lib/provenance.py +0 -111
  74. synapse/tests/test_lib_provenance.py +0 -37
  75. {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/LICENSE +0 -0
  76. {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/WHEEL +0 -0
  77. {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,202 @@
1
+ import asyncio
2
+
3
+ import synapse.exc as s_exc
4
+
5
+ import synapse.lib.ast as s_ast
6
+ import synapse.lib.cache as s_cache
7
+ import synapse.lib.stormctrl as s_stormctrl
8
+ import synapse.lib.stormtypes as s_stormtypes
9
+
10
+ CACHE_SIZE_MAX = 10_000
11
+ CACHE_SIZE_DEFAULT = 10_000
12
+
13
+ @s_stormtypes.registry.registerLib
14
+ class LibCache(s_stormtypes.Lib):
15
+ '''
16
+ A Storm Library for interacting with Cache Objects.
17
+ '''
18
+ _storm_locals = (
19
+ {'name': 'fixed', 'desc': '''
20
+ Get a new Fixed Cache object.
21
+
22
+ On a cache-miss when calling .get(), the callback Storm query is executed in a sub-runtime
23
+ in the current execution context. A special variable, $cache_key, will be set
24
+ to the key argument provided to .get().
25
+
26
+ The callback Storm query must contain a return statement, and if it does not return a value
27
+ when executed with the input, $lib.null will be set as the value.
28
+
29
+ The fixed cache uses FIFO to evict items once the maximum size is reached.
30
+
31
+ Examples:
32
+
33
+ // Use a callback query with a function that modifies the outer runtime,
34
+ // since it will run in the scope where it was defined.
35
+ $test = foo
36
+
37
+ function callback(key) {
38
+ $test = $key // this will modify $test in the outer runtime
39
+ return(`{$key}-val`)
40
+ }
41
+
42
+ $cache = $lib.cache.fixed(${ return($callback($cache_key)) })
43
+ $value = $cache.get(bar)
44
+ $lib.print($test) // this will equal "bar"
45
+
46
+ // Use a callback query that will not modify the outer runtime,
47
+ // except for variables accessible as references.
48
+ $test = foo
49
+ $tests = ([])
50
+
51
+ $cache = $lib.cache.fixed(${
52
+ $test = $cache_key // this will *not* modify $test in the outer runtime
53
+ $tests.append($cache_key) // this will modify $tests in the outer runtime
54
+ return(`{$cache_key}-val`)
55
+ })
56
+
57
+ $value = $cache.get(bar)
58
+ $lib.print($test) // this will equal "foo"
59
+ $lib.print($tests) // this will equal (foo,)
60
+ ''',
61
+ 'type': {'type': 'function', '_funcname': '_methFixedCache',
62
+ 'args': (
63
+ {'name': 'callback', 'type': ['str', 'storm:query'],
64
+ 'desc': 'A Storm query that will return a value for $cache_key on a cache miss.', },
65
+ {'name': 'size', 'type': 'int', 'default': CACHE_SIZE_DEFAULT,
66
+ 'desc': 'The maximum size of the cache.', },
67
+ ),
68
+ 'returns': {'type': 'cache:fixed', 'desc': 'A new ``cache:fixed`` object.'}}},
69
+ )
70
+ _storm_lib_path = ('cache',)
71
+
72
+ def getObjLocals(self):
73
+ return {
74
+ 'fixed': self._methFixedCache,
75
+ }
76
+
77
+ @s_stormtypes.stormfunc(readonly=True)
78
+ async def _methFixedCache(self, callback, size=CACHE_SIZE_DEFAULT):
79
+ size = await s_stormtypes.toint(size)
80
+ callback = await s_stormtypes.tostr(callback)
81
+
82
+ if size < 1 or size > CACHE_SIZE_MAX:
83
+ raise s_exc.BadArg(mesg=f'Cache size must be between 1-{CACHE_SIZE_MAX}')
84
+
85
+ try:
86
+ query = await self.runt.getStormQuery(callback)
87
+ except s_exc.BadSyntax as e:
88
+ raise s_exc.BadArg(mesg=f'Invalid callback query: {e.errinfo.get("mesg")}')
89
+
90
+ if not query.hasAstClass(s_ast.Return):
91
+ raise s_exc.BadArg(mesg='Callback query must return a value')
92
+
93
+ return FixedCache(self.runt, query, size=size)
94
+
95
+ @s_stormtypes.registry.registerType
96
+ class FixedCache(s_stormtypes.StormType):
97
+ '''
98
+ A StormLib API instance of a Storm Fixed Cache.
99
+ '''
100
+ _storm_locals = (
101
+ {'name': 'query', 'desc': 'Get the callback Storm query as string.',
102
+ 'type': {'type': 'gtor', '_gtorfunc': '_gtorQuery',
103
+ 'returns': {'type': 'str', 'desc': 'The callback Storm query text.', }}},
104
+ {'name': 'get', 'desc': 'Get an item from the cache by key.',
105
+ 'type': {'type': 'function', '_funcname': '_methGet',
106
+ 'args': (
107
+ {'name': 'key', 'type': 'any', 'desc': 'The key to lookup.'},
108
+ ),
109
+ 'returns': {'type': 'any',
110
+ 'desc': 'The value from the cache, or the callback query if it does not exist', }}},
111
+ {'name': 'pop', 'desc': 'Pop an item from the cache.',
112
+ 'type': {'type': 'function', '_funcname': '_methPop',
113
+ 'args': (
114
+ {'name': 'key', 'type': 'any', 'desc': 'The key to pop.'},
115
+ ),
116
+ 'returns': {'type': 'any',
117
+ 'desc': 'The value from the cache, or $lib.null if it does not exist', }}},
118
+ {'name': 'put', 'desc': 'Put an item into the cache.',
119
+ 'type': {'type': 'function', '_funcname': '_methPut',
120
+ 'args': (
121
+ {'name': 'key', 'type': 'any', 'desc': 'The key put in the cache.'},
122
+ {'name': 'value', 'type': 'any', 'desc': 'The value to assign to the key.'},
123
+ ),
124
+ 'returns': {'type': 'null', }}},
125
+ {'name': 'clear', 'desc': 'Clear all items from the cache.',
126
+ 'type': {'type': 'function', '_funcname': '_methClear',
127
+ 'returns': {'type': 'null', }}},
128
+ )
129
+ _storm_typename = 'cache:fixed'
130
+ _ismutable = False
131
+
132
+ def __init__(self, runt, query, size=CACHE_SIZE_DEFAULT):
133
+ s_stormtypes.StormType.__init__(self)
134
+ self.runt = runt
135
+ self.size = size
136
+ self.query = query
137
+ self.locls.update(self.getObjLocals())
138
+ self.gtors.update({
139
+ 'query': self._gtorQuery,
140
+ })
141
+
142
+ self.cache = s_cache.FixedCache(self._runCallback, size=size)
143
+
144
+ def __len__(self):
145
+ return len(self.cache)
146
+
147
+ async def stormrepr(self):
148
+ if len(qtext := self.query.text) > 100:
149
+ qtext = qtext[:100] + '...'
150
+ return f'{self._storm_typename}: size={self.size} query="{qtext}"'
151
+
152
+ def getObjLocals(self):
153
+ return {
154
+ 'pop': self._methPop,
155
+ 'put': self._methPut,
156
+ 'get': self._methGet,
157
+ 'clear': self._methClear,
158
+ }
159
+
160
+ async def _gtorQuery(self):
161
+ return self.query.text
162
+
163
+ async def _runCallback(self, key):
164
+
165
+ varz = self.runt.getScopeVars()
166
+ varz['cache_key'] = key
167
+
168
+ opts = {'vars': varz}
169
+ async with self.runt.getCmdRuntime(self.query, opts=opts) as runt:
170
+ try:
171
+ async for _ in runt.execute():
172
+ await asyncio.sleep(0)
173
+ except s_stormctrl.StormReturn as e:
174
+ return await s_stormtypes.toprim(e.item)
175
+ except s_stormctrl.StormCtrlFlow:
176
+ pass
177
+
178
+ async def _reqKey(self, key):
179
+ if s_stormtypes.ismutable(key):
180
+ mesg = 'Mutable values are not allowed as cache keys'
181
+ raise s_exc.BadArg(mesg=mesg, name=await s_stormtypes.torepr(key))
182
+ return await s_stormtypes.toprim(key)
183
+
184
+ @s_stormtypes.stormfunc(readonly=True)
185
+ async def _methPop(self, key):
186
+ key = await self._reqKey(key)
187
+ return self.cache.pop(key)
188
+
189
+ @s_stormtypes.stormfunc(readonly=True)
190
+ async def _methPut(self, key, value):
191
+ key = await self._reqKey(key)
192
+ val = await s_stormtypes.toprim(value)
193
+ self.cache.put(key, val)
194
+
195
+ @s_stormtypes.stormfunc(readonly=True)
196
+ async def _methGet(self, key):
197
+ key = await self._reqKey(key)
198
+ return await self.cache.aget(key)
199
+
200
+ @s_stormtypes.stormfunc(readonly=True)
201
+ async def _methClear(self):
202
+ self.cache.clear()
@@ -3,6 +3,7 @@ import json
3
3
  import logging
4
4
 
5
5
  import synapse.exc as s_exc
6
+ import synapse.telepath as s_telepath
6
7
 
7
8
  import synapse.lib.storm as s_storm
8
9
  import synapse.lib.stormtypes as s_stormtypes
@@ -86,6 +87,7 @@ stormcmds = [
86
87
  $lib.print(`!View: No view found ({$err.info.iden})`)
87
88
  }
88
89
  $lib.print(`Readonly: {$api.readonly}`)
90
+ $lib.print(`Pool enabled: {$api.pool}`)
89
91
  $lib.print(`Authenticated: {$api.authenticated}`)
90
92
  $lib.print(`Name: {$api.name}`)
91
93
  $lib.print(`Description: {$api.desc}`)
@@ -224,6 +226,9 @@ class HttpApi(s_stormtypes.StormType):
224
226
  ''',
225
227
  'type': {'type': ['stor', 'gtor'], '_storfunc': '_storPath', '_gtorfunc': '_gtorPath',
226
228
  'returns': {'type': 'str'}}},
229
+ {'name': 'pool', 'desc': 'Boolean value indicating if the handler responses may be executed as part of a Storm pool.',
230
+ 'type': {'type': ['stor', 'gtor'], '_storfunc': '_storPool', '_gtorfunc': '_gtorPool',
231
+ 'returns': {'type': 'boolean'}}},
227
232
  {'name': 'vars', 'desc': 'The Storm runtime variables specific for the API instance.',
228
233
  'type': {'type': ['stor', 'ctor'], '_storfunc': '_storVars', '_ctorfunc': '_ctorVars',
229
234
  'returns': {'type': 'http:api:vars'}}},
@@ -242,6 +247,8 @@ class HttpApi(s_stormtypes.StormType):
242
247
  {'name': 'authenticated', 'desc': 'Boolean value indicating if the Extended HTTP API requires an authenticated user or session.',
243
248
  'type': {'type': ['stor', 'gtor'], '_storfunc': '_storAuthenticated', '_gtorfunc': '_gtorAuthenticated',
244
249
  'returns': {'type': 'boolean'}}},
250
+ {'name': 'methods', 'desc': 'The dictionary containing the Storm code used to implement the HTTP methods.',
251
+ 'type': {'type': ['ctor'], '_ctorfunc': '_ctorMethods', 'returns': {'type': 'http:api:methods'}}}
245
252
  )
246
253
 
247
254
  def __init__(self, runt, info):
@@ -257,6 +264,7 @@ class HttpApi(s_stormtypes.StormType):
257
264
  'name': self._storName,
258
265
  'desc': self._storDesc,
259
266
  'path': self._storPath,
267
+ 'pool': self._storPool,
260
268
  'vars': self._storVars,
261
269
  'view': self._storView,
262
270
  'runas': self._storRunas,
@@ -270,6 +278,7 @@ class HttpApi(s_stormtypes.StormType):
270
278
  'name': self._gtorName,
271
279
  'desc': self._gtorDesc,
272
280
  'path': self._gtorPath,
281
+ 'pool': self._gtorPool,
273
282
  'view': self._gtorView,
274
283
  'runas': self._gtorRunas,
275
284
  'owner': self._gtorOwner,
@@ -292,6 +301,13 @@ class HttpApi(s_stormtypes.StormType):
292
301
  'created': self.info.get('created'),
293
302
  })
294
303
 
304
+ async def stormrepr(self):
305
+ name = await self._gtorName()
306
+ if not name:
307
+ name = '<no name>'
308
+ path = await self._gtorPath()
309
+ return f'{self._storm_typename}: {name} ({self.iden}), path={path}'
310
+
295
311
  def getObjLocals(self):
296
312
  return {
297
313
  'pack': self._methPack,
@@ -299,7 +315,10 @@ class HttpApi(s_stormtypes.StormType):
299
315
 
300
316
  @s_stormtypes.stormfunc(readonly=True)
301
317
  async def _methPack(self):
302
- return copy.deepcopy(self.info)
318
+ # TODO: Remove this when we've migrated the HTTPAPI data to set this value.
319
+ ret = copy.deepcopy(self.info)
320
+ ret.setdefault('pool', False)
321
+ return ret
303
322
 
304
323
  @s_stormtypes.stormfunc(readonly=True)
305
324
  def _ctorMethods(self, path=None):
@@ -316,6 +335,17 @@ class HttpApi(s_stormtypes.StormType):
316
335
  async def _gtorPath(self):
317
336
  return self.info.get('path')
318
337
 
338
+ async def _storPool(self, pool):
339
+ s_stormtypes.confirm(('storm', 'lib', 'cortex', 'httpapi', 'set'))
340
+ pool = await s_stormtypes.tobool(pool)
341
+ adef = await self.runt.snap.core.modHttpExtApi(self.iden, 'pool', pool)
342
+ self.info['pool'] = pool
343
+ self.info['updated'] = adef.get('updated')
344
+
345
+ @s_stormtypes.stormfunc(readonly=True)
346
+ async def _gtorPool(self):
347
+ return self.info.get('pool')
348
+
319
349
  async def _storName(self, name):
320
350
  s_stormtypes.confirm(('storm', 'lib', 'cortex', 'httpapi', 'set'))
321
351
  name = await s_stormtypes.tostr(name)
@@ -1055,7 +1085,7 @@ class CortexHttpApi(s_stormtypes.Lib):
1055
1085
  {'name': 'readonly', 'type': 'boolean',
1056
1086
  'desc': 'Run the Extended HTTP Storm methods in readonly mode.', 'default': False},
1057
1087
  ),
1058
- 'returns': {'type': 'http:api', 'desc': 'A new http:api object.'}}},
1088
+ 'returns': {'type': 'http:api', 'desc': 'A new ``http:api`` object.'}}},
1059
1089
  {'name': 'del', 'desc': 'Delete an Extended HTTP API endpoint.',
1060
1090
  'type': {'type': 'function', '_funcname': 'delHttpApi',
1061
1091
  'args': (
@@ -1063,16 +1093,29 @@ class CortexHttpApi(s_stormtypes.Lib):
1063
1093
  'desc': 'The iden of the API to delete.'},
1064
1094
  ),
1065
1095
  'returns': {'type': 'null'}}},
1066
- {'name': 'get', 'desc': 'Get an Extended HTTP API object.',
1096
+ {'name': 'get', 'desc': 'Get an Extended ``http:api`` object.',
1067
1097
  'type': {'type': 'function', '_funcname': 'getHttpApi',
1068
1098
  'args': (
1069
1099
  {'name': 'iden', 'type': 'string',
1070
- 'desc': 'The iden of the API to retreive.'},
1100
+ 'desc': 'The iden of the API to retrieve.'},
1101
+ ),
1102
+ 'returns': {'type': 'http:api', 'desc': 'The ``http:api`` object.'}}},
1103
+ {'name': 'getByPath', 'desc': '''
1104
+ Get an Extended ``http:api`` object by path.
1105
+
1106
+ Notes:
1107
+ The path argument is evaluated as a regular expression input, and will be
1108
+ used to get the first HTTP API handler whose path value has a match.
1109
+ ''',
1110
+ 'type': {'type': 'function', '_funcname': 'getHttpApiByPath',
1111
+ 'args': (
1112
+ {'name': 'path', 'type': 'string',
1113
+ 'desc': 'Path to use to retrieve an object.'},
1071
1114
  ),
1072
- 'returns': {'type': 'http:api', 'desc': 'The http:api object.'}}},
1073
- {'name': 'list', 'desc': 'Get all the Extneded HTTP APIs on the Cortex',
1115
+ 'returns': {'type': ['http:api', 'null'], 'desc': 'The ``http:api`` object or ``$lib.null`` if there is no match.'}}},
1116
+ {'name': 'list', 'desc': 'Get all the Extended HTTP APIs on the Cortex',
1074
1117
  'type': {'type': 'function', '_funcname': 'listHttpApis', 'args': (),
1075
- 'returns': {'type': 'list', 'desc': 'A list of http:api objects'}}},
1118
+ 'returns': {'type': 'list', 'desc': 'A list of ``http:api`` objects'}}},
1076
1119
  {'name': 'index', 'desc': 'Set the index for a given Extended HTTP API.',
1077
1120
  'type': {'type': 'function', '_funcname': 'setHttpApiIndx',
1078
1121
  'args': (
@@ -1111,6 +1154,7 @@ class CortexHttpApi(s_stormtypes.Lib):
1111
1154
  'list': self.listHttpApis,
1112
1155
  'index': self.setHttpApiIndx,
1113
1156
  'response': self.makeHttpResponse,
1157
+ 'getByPath': self.getHttpApiByPath,
1114
1158
  }
1115
1159
 
1116
1160
  @s_stormtypes.stormfunc(readonly=True)
@@ -1125,6 +1169,15 @@ class CortexHttpApi(s_stormtypes.Lib):
1125
1169
  adef = await self.runt.snap.core.getHttpExtApi(iden)
1126
1170
  return HttpApi(self.runt, adef)
1127
1171
 
1172
+ @s_stormtypes.stormfunc(readonly=True)
1173
+ async def getHttpApiByPath(self, path):
1174
+ s_stormtypes.confirm(('storm', 'lib', 'cortex', 'httpapi', 'get'))
1175
+ path = await s_stormtypes.tostr(path)
1176
+ adef, _ = await self.runt.snap.core.getHttpExtApiByPath(path)
1177
+ if adef is None:
1178
+ return None
1179
+ return HttpApi(self.runt, adef)
1180
+
1128
1181
  @s_stormtypes.stormfunc(readonly=True)
1129
1182
  async def listHttpApis(self):
1130
1183
  s_stormtypes.confirm(('storm', 'lib', 'cortex', 'httpapi', 'get'))
@@ -1195,6 +1248,11 @@ class StormPoolSetCmd(s_storm.Cmd):
1195
1248
  async for node, path in genr: # pragma: no cover
1196
1249
  yield node, path
1197
1250
 
1251
+ try:
1252
+ s_telepath.chopurl(self.opts.url)
1253
+ except s_exc.BadUrl as e:
1254
+ raise s_exc.BadArg(mesg=f'Unable to set Storm pool URL from url={self.opts.url} : {e.get("mesg")}') from None
1255
+
1198
1256
  opts = {
1199
1257
  'timeout:sync': self.opts.sync_timeout,
1200
1258
  'timeout:connection': self.opts.connection_timeout,
@@ -1206,6 +1264,10 @@ class StormPoolSetCmd(s_storm.Cmd):
1206
1264
  class StormPoolDelCmd(s_storm.Cmd):
1207
1265
  '''
1208
1266
  Remove a Storm query offload mirror pool configuration.
1267
+
1268
+ Notes:
1269
+ This will result in tearing down any Storm queries currently being serviced by the Storm pool.
1270
+ This may result in this command raising an exception if it was offloaded to a pool member. That would be an expected behavior.
1209
1271
  '''
1210
1272
  name = 'cortex.storm.pool.del'
1211
1273
 
@@ -34,7 +34,7 @@ stormcmds = [
34
34
  'name': 'macro.del',
35
35
  'descr': macro_del_descr,
36
36
  'cmdargs': (
37
- ('name', {'help': 'The name of the macro to delete.'}),
37
+ ('name', {'type': 'str', 'help': 'The name of the macro to delete.'}),
38
38
  ),
39
39
  'storm': '''
40
40
  $lib.macro.del($cmdopts.name)
@@ -45,7 +45,7 @@ stormcmds = [
45
45
  'name': 'macro.set',
46
46
  'descr': macro_set_descr,
47
47
  'cmdargs': (
48
- ('name', {'help': 'The name of the macro to set.'}),
48
+ ('name', {'type': 'str', 'help': 'The name of the macro to set.'}),
49
49
  ('storm', {'help': 'The storm command string or embedded query to set.'}),
50
50
  ),
51
51
  'storm': '''
@@ -57,7 +57,7 @@ stormcmds = [
57
57
  'name': 'macro.get',
58
58
  'descr': macro_get_descr,
59
59
  'cmdargs': (
60
- ('name', {'help': 'The name of the macro to display.'}),
60
+ ('name', {'type': 'str', 'help': 'The name of the macro to display.'}),
61
61
  ),
62
62
  'storm': '''
63
63
  $mdef = $lib.macro.get($cmdopts.name)
@@ -74,8 +74,14 @@ stormcmds = [
74
74
  'storm': '''
75
75
  $count = $(0)
76
76
  for ($name, $mdef) in $lib.macro.list() {
77
- $user = $lib.auth.users.get($mdef.user)
78
- $lib.print('{name} (owner: {user})', name=$name.ljust(20), user=$user.name)
77
+ $user = $lib.auth.users.get($mdef.creator)
78
+ $username = $lib.null
79
+ if (not $user) {
80
+ $username = `User not found ({$mdef.creator})`
81
+ } else {
82
+ $username = $user.name
83
+ }
84
+ $lib.print('{name} (owner: {user})', name=$name.ljust(20), user=$username)
79
85
  $count = $($count + 1)
80
86
  }
81
87
  $lib.print('{count} macros found', count=$count)
@@ -185,27 +191,17 @@ class LibMacro(s_stormtypes.Lib):
185
191
  async def _funcMacroGet(self, name):
186
192
  name = await s_stormtypes.tostr(name)
187
193
 
188
- if len(name) > 491:
189
- raise s_exc.BadArg(mesg='Macro names may only be up to 491 chars.')
190
-
191
194
  return self.runt.snap.core.getStormMacro(name, user=self.runt.user)
192
195
 
193
196
  async def _funcMacroDel(self, name):
194
-
195
197
  name = await s_stormtypes.tostr(name)
196
198
 
197
- if len(name) > 491:
198
- raise s_exc.BadArg(mesg='Macro names may only be up to 491 chars.')
199
-
200
199
  return await self.runt.snap.core.delStormMacro(name, user=self.runt.user)
201
200
 
202
201
  async def _funcMacroSet(self, name, storm):
203
202
  name = await s_stormtypes.tostr(name)
204
203
  storm = await s_stormtypes.tostr(storm)
205
204
 
206
- if len(name) > 491:
207
- raise s_exc.BadArg(mesg='Macro names may only be up to 491 chars.')
208
-
209
205
  await self.runt.getStormQuery(storm)
210
206
 
211
207
  if self.runt.snap.core.getStormMacro(name) is None:
@@ -220,9 +216,6 @@ class LibMacro(s_stormtypes.Lib):
220
216
  name = await s_stormtypes.tostr(name)
221
217
  info = await s_stormtypes.toprim(info)
222
218
 
223
- if len(name) > 491:
224
- raise s_exc.BadArg(mesg='Macro names may only be up to 491 chars.')
225
-
226
219
  if not isinstance(info, dict):
227
220
  raise s_exc.BadArg(mesg='Macro info must be a dictionary object.')
228
221
 
@@ -0,0 +1,109 @@
1
+ import synapse.exc as s_exc
2
+ import synapse.lib.msgpack as s_msgpack
3
+ import synapse.lib.spooled as s_spooled
4
+ import synapse.lib.stormtypes as s_stormtypes
5
+
6
+ @s_stormtypes.registry.registerLib
7
+ class LibSpooled(s_stormtypes.Lib):
8
+ '''
9
+ A Storm Library for interacting with Spooled Objects.
10
+ '''
11
+ _storm_locals = (
12
+ {'name': 'set', 'desc': '''
13
+ Get a Spooled Storm Set object.
14
+
15
+ A Spooled Storm Set object is memory-safe to grow to extraordinarily large sizes,
16
+ as it will fallback to file backed storage, with two restrictions. First
17
+ is that all items in the set can be serialized to a file if the set grows too large,
18
+ so all items added must be a serializable Storm primitive. Second is that when an
19
+ item is added to the Set, because it could be immediately written disk,
20
+ do not hold any references to it outside of the Set itself, as the two objects could
21
+ differ.
22
+ ''',
23
+ 'type': {'type': 'function', '_funcname': '_methSet',
24
+ 'args': (
25
+ {'name': '*vals', 'type': 'any', 'desc': 'Initial values to place in the set.', },
26
+ ),
27
+ 'returns': {'type': 'set', 'desc': 'The new set.'}}},
28
+ )
29
+ _storm_lib_path = ('spooled',)
30
+
31
+ def getObjLocals(self):
32
+ return {
33
+ 'set': self._methSet,
34
+ }
35
+
36
+ @s_stormtypes.stormfunc(readonly=True)
37
+ async def _methSet(self, *vals):
38
+ core = self.runt.snap.core
39
+ spool = await s_spooled.Set.anit(dirn=core.dirn, cell=core, size=1000)
40
+
41
+ valu = list(vals)
42
+ for item in valu:
43
+ if s_stormtypes.ismutable(item):
44
+ mesg = f'{await s_stormtypes.torepr(item)} is mutable and cannot be used in a set.'
45
+ raise s_exc.StormRuntimeError(mesg=mesg)
46
+
47
+ if not s_msgpack.isok(item):
48
+ mesg = f'{await s_stormtypes.torepr(item)} is not safe to be used in a SpooledSet.'
49
+ raise s_exc.StormRuntimeError(mesg=mesg)
50
+
51
+ await spool.add(item)
52
+
53
+ return SpooledSet(spool)
54
+
55
+ @s_stormtypes.registry.registerType
56
+ class SpooledSet(s_stormtypes.Set):
57
+ '''
58
+ A StormLib API instance of a Storm Set object that can fallback to lmdb.
59
+ '''
60
+ _storm_typename = 'spooled:set'
61
+ _ismutable = True
62
+
63
+ def __init__(self, valu, path=None):
64
+
65
+ s_stormtypes.Prim.__init__(self, valu, path=path)
66
+ self.locls.update(self.getObjLocals())
67
+
68
+ async def iter(self):
69
+ async for item in self.valu:
70
+ yield item
71
+
72
+ @s_stormtypes.stormfunc(readonly=True)
73
+ async def _methSetAdd(self, *items):
74
+ for i in items:
75
+ if s_stormtypes.ismutable(i):
76
+ mesg = f'{await s_stormtypes.torepr(i)} is mutable and cannot be used in a set.'
77
+ raise s_exc.StormRuntimeError(mesg=mesg)
78
+
79
+ if not s_msgpack.isok(i):
80
+ mesg = f'{await s_stormtypes.torepr(i)} is not safe to be used in a SpooledSet.'
81
+ raise s_exc.StormRuntimeError(mesg=mesg)
82
+
83
+ await self.valu.add(i)
84
+
85
+ @s_stormtypes.stormfunc(readonly=True)
86
+ async def _methSetAdds(self, *items):
87
+ for item in items:
88
+ async for i in s_stormtypes.toiter(item):
89
+ if s_stormtypes.ismutable(i):
90
+ mesg = f'{await s_stormtypes.torepr(i)} is mutable and cannot be used in a set.'
91
+ raise s_exc.StormRuntimeError(mesg=mesg)
92
+
93
+ if not s_msgpack.isok(i):
94
+ mesg = f'{await s_stormtypes.torepr(i)} is not safe to be used in a SpooledSet.'
95
+ raise s_exc.StormRuntimeError(mesg=mesg)
96
+
97
+ await self.valu.add(i)
98
+
99
+ @s_stormtypes.stormfunc(readonly=True)
100
+ async def _methSetList(self):
101
+ return [x async for x in self.valu]
102
+
103
+ async def stormrepr(self):
104
+ reprs = [await s_stormtypes.torepr(k) async for k in self.valu]
105
+ rval = ', '.join(reprs)
106
+ return f'{{{rval}}}'
107
+
108
+ async def value(self):
109
+ return set([x async for x in self.valu])
@@ -585,7 +585,7 @@ def validateStix(bundle, version='2.1'):
585
585
  'result': {},
586
586
  }
587
587
  bundle = json.loads(json.dumps(bundle))
588
- opts = stix2validator.ValidationOptions(strict=True, version=version, no_cache=True)
588
+ opts = stix2validator.ValidationOptions(strict=True, version=version)
589
589
  try:
590
590
  results = stix2validator.validate_parsed_json(bundle, options=opts)
591
591
  except stix2validator.ValidationError as e: