synapse 2.162.0__py311-none-any.whl → 2.163.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/axon.py +48 -40
- synapse/cortex.py +4 -0
- synapse/lib/cell.py +63 -0
- synapse/lib/oauth.py +1 -7
- synapse/lib/schemas.py +10 -0
- synapse/lib/stormhttp.py +52 -28
- synapse/lib/stormtypes.py +188 -7
- synapse/lib/version.py +2 -2
- synapse/tests/test_cortex.py +2 -2
- synapse/tests/test_lib_ast.py +1 -1
- synapse/tests/test_lib_storm.py +1 -1
- synapse/tests/test_lib_stormhttp.py +164 -1
- synapse/tests/test_lib_stormtypes.py +32 -7
- {synapse-2.162.0.dist-info → synapse-2.163.0.dist-info}/METADATA +1 -1
- {synapse-2.162.0.dist-info → synapse-2.163.0.dist-info}/RECORD +18 -18
- {synapse-2.162.0.dist-info → synapse-2.163.0.dist-info}/LICENSE +0 -0
- {synapse-2.162.0.dist-info → synapse-2.163.0.dist-info}/WHEEL +0 -0
- {synapse-2.162.0.dist-info → synapse-2.163.0.dist-info}/top_level.txt +0 -0
synapse/lib/stormtypes.py
CHANGED
|
@@ -34,11 +34,15 @@ import synapse.lib.scope as s_scope
|
|
|
34
34
|
import synapse.lib.msgpack as s_msgpack
|
|
35
35
|
import synapse.lib.trigger as s_trigger
|
|
36
36
|
import synapse.lib.urlhelp as s_urlhelp
|
|
37
|
+
import synapse.lib.version as s_version
|
|
37
38
|
import synapse.lib.stormctrl as s_stormctrl
|
|
38
39
|
import synapse.lib.provenance as s_provenance
|
|
39
40
|
|
|
40
41
|
logger = logging.getLogger(__name__)
|
|
41
42
|
|
|
43
|
+
AXON_MINVERS_PROXY = (2, 97, 0)
|
|
44
|
+
AXON_MINVERS_SSLOPTS = '>=2.162.0'
|
|
45
|
+
|
|
42
46
|
class Undef:
|
|
43
47
|
_storm_typename = 'undef'
|
|
44
48
|
async def stormrepr(self):
|
|
@@ -1097,6 +1101,8 @@ class LibBase(Lib):
|
|
|
1097
1101
|
'type': {'type': 'function', '_funcname': '_guid',
|
|
1098
1102
|
'args': (
|
|
1099
1103
|
{'name': '*args', 'type': 'prim', 'desc': 'Arguments which are hashed to create a guid.', },
|
|
1104
|
+
{'name': 'valu', 'type': 'prim', 'default': '$lib.undef',
|
|
1105
|
+
'desc': 'Create a guid from a single value (no positional arguments can be specified).', },
|
|
1100
1106
|
),
|
|
1101
1107
|
'returns': {'type': 'str', 'desc': 'A guid.', }}},
|
|
1102
1108
|
{'name': 'fire', 'desc': '''
|
|
@@ -1543,10 +1549,17 @@ class LibBase(Lib):
|
|
|
1543
1549
|
return Text(valu)
|
|
1544
1550
|
|
|
1545
1551
|
@stormfunc(readonly=True)
|
|
1546
|
-
async def _guid(self, *args):
|
|
1552
|
+
async def _guid(self, *args, valu=undef):
|
|
1547
1553
|
if args:
|
|
1554
|
+
if valu is not undef:
|
|
1555
|
+
raise s_exc.BadArg(mesg='Valu cannot be specified if positional arguments are provided')
|
|
1548
1556
|
args = await toprim(args)
|
|
1549
1557
|
return s_common.guid(args)
|
|
1558
|
+
|
|
1559
|
+
if valu is not undef:
|
|
1560
|
+
valu = await toprim(valu)
|
|
1561
|
+
return s_common.guid(valu)
|
|
1562
|
+
|
|
1550
1563
|
return s_common.guid()
|
|
1551
1564
|
|
|
1552
1565
|
@stormfunc(readonly=True)
|
|
@@ -1916,6 +1929,14 @@ class LibStr(Lib):
|
|
|
1916
1929
|
class LibAxon(Lib):
|
|
1917
1930
|
'''
|
|
1918
1931
|
A Storm library for interacting with the Cortex's Axon.
|
|
1932
|
+
|
|
1933
|
+
For APIs that accept an ssl_opts argument, the dictionary may contain the following values::
|
|
1934
|
+
|
|
1935
|
+
{
|
|
1936
|
+
'verify': <bool> - Perform SSL/TLS verification. Is overridden by the ssl argument.
|
|
1937
|
+
'client_cert': <str> - PEM encoded full chain certificate for use in mTLS.
|
|
1938
|
+
'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
|
|
1939
|
+
}
|
|
1919
1940
|
'''
|
|
1920
1941
|
_storm_locals = (
|
|
1921
1942
|
{'name': 'wget', 'desc': """
|
|
@@ -1951,6 +1972,9 @@ class LibAxon(Lib):
|
|
|
1951
1972
|
'default': None},
|
|
1952
1973
|
{'name': 'proxy', 'type': ['bool', 'null', 'str'],
|
|
1953
1974
|
'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
|
|
1975
|
+
{'name': 'ssl_opts', 'type': 'dict',
|
|
1976
|
+
'desc': 'Optional SSL/TLS options. See $lib.axon help for additional details.',
|
|
1977
|
+
'default': None},
|
|
1954
1978
|
),
|
|
1955
1979
|
'returns': {'type': 'dict', 'desc': 'A status dictionary of metadata.'}}},
|
|
1956
1980
|
{'name': 'wput', 'desc': """
|
|
@@ -1971,6 +1995,9 @@ class LibAxon(Lib):
|
|
|
1971
1995
|
'default': None},
|
|
1972
1996
|
{'name': 'proxy', 'type': ['bool', 'null', 'str'],
|
|
1973
1997
|
'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
|
|
1998
|
+
{'name': 'ssl_opts', 'type': 'dict',
|
|
1999
|
+
'desc': 'Optional SSL/TLS options. See $lib.axon help for additional details.',
|
|
2000
|
+
'default': None},
|
|
1974
2001
|
),
|
|
1975
2002
|
'returns': {'type': 'dict', 'desc': 'A status dictionary of metadata.'}}},
|
|
1976
2003
|
{'name': 'urlfile', 'desc': '''
|
|
@@ -2121,6 +2148,82 @@ class LibAxon(Lib):
|
|
|
2121
2148
|
''',
|
|
2122
2149
|
'type': {'type': 'function', '_funcname': 'metrics',
|
|
2123
2150
|
'returns': {'type': 'dict', 'desc': 'A dictionary containing runtime data about the Axon.'}}},
|
|
2151
|
+
{'name': 'put', 'desc': '''
|
|
2152
|
+
Save the given bytes variable to the Axon the Cortex is configured to use.
|
|
2153
|
+
|
|
2154
|
+
Examples:
|
|
2155
|
+
Save a base64 encoded buffer to the Axon::
|
|
2156
|
+
|
|
2157
|
+
cli> storm $s='dGVzdA==' $buf=$lib.base64.decode($s) ($size, $sha256)=$lib.axon.put($buf)
|
|
2158
|
+
$lib.print('size={size} sha256={sha256}', size=$size, sha256=$sha256)
|
|
2159
|
+
|
|
2160
|
+
size=4 sha256=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08''',
|
|
2161
|
+
'type': {'type': 'function', '_funcname': 'put',
|
|
2162
|
+
'args': (
|
|
2163
|
+
{'name': 'byts', 'type': 'bytes', 'desc': 'The bytes to save.', },
|
|
2164
|
+
),
|
|
2165
|
+
'returns': {'type': 'list', 'desc': 'A tuple of the file size and sha256 value.', }}},
|
|
2166
|
+
{'name': 'has', 'desc': '''
|
|
2167
|
+
Check if the Axon the Cortex is configured to use has a given sha256 value.
|
|
2168
|
+
|
|
2169
|
+
Examples:
|
|
2170
|
+
Check if the Axon has a given file::
|
|
2171
|
+
|
|
2172
|
+
# This example assumes the Axon does have the bytes
|
|
2173
|
+
cli> storm if $lib.axon.has(9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08) {
|
|
2174
|
+
$lib.print("Has bytes")
|
|
2175
|
+
} else {
|
|
2176
|
+
$lib.print("Does not have bytes")
|
|
2177
|
+
}
|
|
2178
|
+
|
|
2179
|
+
Has bytes
|
|
2180
|
+
''',
|
|
2181
|
+
'type': {'type': 'function', '_funcname': 'has',
|
|
2182
|
+
'args': (
|
|
2183
|
+
{'name': 'sha256', 'type': 'str', 'desc': 'The sha256 value to check.', },
|
|
2184
|
+
),
|
|
2185
|
+
'returns': {'type': 'boolean', 'desc': 'True if the Axon has the file, false if it does not.', }}},
|
|
2186
|
+
{'name': 'size', 'desc': '''
|
|
2187
|
+
Return the size of the bytes stored in the Axon for the given sha256.
|
|
2188
|
+
|
|
2189
|
+
Examples:
|
|
2190
|
+
Get the size for a file given a variable named ``$sha256``::
|
|
2191
|
+
|
|
2192
|
+
$size = $lib.axon.size($sha256)
|
|
2193
|
+
''',
|
|
2194
|
+
'type': {'type': 'function', '_funcname': 'size',
|
|
2195
|
+
'args': (
|
|
2196
|
+
{'name': 'sha256', 'type': 'str', 'desc': 'The sha256 value to check.', },
|
|
2197
|
+
),
|
|
2198
|
+
'returns': {'type': ['int', 'null'],
|
|
2199
|
+
'desc': 'The size of the file or ``null`` if the file is not found.', }}},
|
|
2200
|
+
{'name': 'hashset', 'desc': '''
|
|
2201
|
+
Return additional hashes of the bytes stored in the Axon for the given sha256.
|
|
2202
|
+
|
|
2203
|
+
Examples:
|
|
2204
|
+
Get the md5 hash for a file given a variable named ``$sha256``::
|
|
2205
|
+
|
|
2206
|
+
$hashset = $lib.axon.hashset($sha256)
|
|
2207
|
+
$md5 = $hashset.md5
|
|
2208
|
+
''',
|
|
2209
|
+
'type': {'type': 'function', '_funcname': 'hashset',
|
|
2210
|
+
'args': (
|
|
2211
|
+
{'name': 'sha256', 'type': 'str', 'desc': 'The sha256 value to calculate hashes for.', },
|
|
2212
|
+
),
|
|
2213
|
+
'returns': {'type': 'dict', 'desc': 'A dictionary of additional hashes.', }}},
|
|
2214
|
+
{'name': 'upload', 'desc': '''
|
|
2215
|
+
Upload a stream of bytes to the Axon as a file.
|
|
2216
|
+
|
|
2217
|
+
Examples:
|
|
2218
|
+
Upload bytes from a generator::
|
|
2219
|
+
|
|
2220
|
+
($size, $sha256) = $lib.axon.upload($getBytesChunks())
|
|
2221
|
+
''',
|
|
2222
|
+
'type': {'type': 'function', '_funcname': 'upload',
|
|
2223
|
+
'args': (
|
|
2224
|
+
{'name': 'genr', 'type': 'generator', 'desc': 'A generator which yields bytes.', },
|
|
2225
|
+
),
|
|
2226
|
+
'returns': {'type': 'list', 'desc': 'A tuple of the file size and sha256 value.', }}},
|
|
2124
2227
|
)
|
|
2125
2228
|
_storm_lib_path = ('axon',)
|
|
2126
2229
|
_storm_lib_perms = (
|
|
@@ -2148,6 +2251,11 @@ class LibAxon(Lib):
|
|
|
2148
2251
|
'jsonlines': self.jsonlines,
|
|
2149
2252
|
'csvrows': self.csvrows,
|
|
2150
2253
|
'metrics': self.metrics,
|
|
2254
|
+
'put': self.put,
|
|
2255
|
+
'has': self.has,
|
|
2256
|
+
'size': self.size,
|
|
2257
|
+
'upload': self.upload,
|
|
2258
|
+
'hashset': self.hashset,
|
|
2151
2259
|
}
|
|
2152
2260
|
|
|
2153
2261
|
def strify(self, item):
|
|
@@ -2207,7 +2315,8 @@ class LibAxon(Lib):
|
|
|
2207
2315
|
axon = self.runt.snap.core.axon
|
|
2208
2316
|
return await axon.del_(sha256b)
|
|
2209
2317
|
|
|
2210
|
-
async def wget(self, url, headers=None, params=None, method='GET', json=None, body=None,
|
|
2318
|
+
async def wget(self, url, headers=None, params=None, method='GET', json=None, body=None,
|
|
2319
|
+
ssl=True, timeout=None, proxy=None, ssl_opts=None):
|
|
2211
2320
|
|
|
2212
2321
|
if not self.runt.allowed(('axon', 'wget')):
|
|
2213
2322
|
self.runt.confirm(('storm', 'lib', 'axon', 'wget'))
|
|
@@ -2222,6 +2331,7 @@ class LibAxon(Lib):
|
|
|
2222
2331
|
headers = await toprim(headers)
|
|
2223
2332
|
timeout = await toprim(timeout)
|
|
2224
2333
|
proxy = await toprim(proxy)
|
|
2334
|
+
ssl_opts = await toprim(ssl_opts)
|
|
2225
2335
|
|
|
2226
2336
|
if proxy is not None:
|
|
2227
2337
|
self.runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy'))
|
|
@@ -2233,16 +2343,23 @@ class LibAxon(Lib):
|
|
|
2233
2343
|
|
|
2234
2344
|
kwargs = {}
|
|
2235
2345
|
axonvers = self.runt.snap.core.axoninfo['synapse']['version']
|
|
2236
|
-
if axonvers >=
|
|
2346
|
+
if axonvers >= AXON_MINVERS_PROXY:
|
|
2237
2347
|
kwargs['proxy'] = proxy
|
|
2238
2348
|
|
|
2349
|
+
if ssl_opts is not None:
|
|
2350
|
+
mesg = f'The ssl_opts argument requires an Axon Synapse version {AXON_MINVERS_SSLOPTS}, ' \
|
|
2351
|
+
f'but the Axon is running {axonvers}'
|
|
2352
|
+
s_version.reqVersion(axonvers, AXON_MINVERS_SSLOPTS, mesg=mesg)
|
|
2353
|
+
kwargs['ssl_opts'] = ssl_opts
|
|
2354
|
+
|
|
2239
2355
|
axon = self.runt.snap.core.axon
|
|
2240
2356
|
resp = await axon.wget(url, headers=headers, params=params, method=method, ssl=ssl, body=body, json=json,
|
|
2241
2357
|
timeout=timeout, **kwargs)
|
|
2242
2358
|
resp['original_url'] = url
|
|
2243
2359
|
return resp
|
|
2244
2360
|
|
|
2245
|
-
async def wput(self, sha256, url, headers=None, params=None, method='PUT',
|
|
2361
|
+
async def wput(self, sha256, url, headers=None, params=None, method='PUT',
|
|
2362
|
+
ssl=True, timeout=None, proxy=None, ssl_opts=None):
|
|
2246
2363
|
|
|
2247
2364
|
if not self.runt.allowed(('axon', 'wput')):
|
|
2248
2365
|
self.runt.confirm(('storm', 'lib', 'axon', 'wput'))
|
|
@@ -2256,6 +2373,7 @@ class LibAxon(Lib):
|
|
|
2256
2373
|
params = await toprim(params)
|
|
2257
2374
|
headers = await toprim(headers)
|
|
2258
2375
|
timeout = await toprim(timeout)
|
|
2376
|
+
ssl_opts = await toprim(ssl_opts)
|
|
2259
2377
|
|
|
2260
2378
|
params = self.strify(params)
|
|
2261
2379
|
headers = self.strify(headers)
|
|
@@ -2268,10 +2386,17 @@ class LibAxon(Lib):
|
|
|
2268
2386
|
|
|
2269
2387
|
kwargs = {}
|
|
2270
2388
|
axonvers = self.runt.snap.core.axoninfo['synapse']['version']
|
|
2271
|
-
if axonvers >=
|
|
2389
|
+
if axonvers >= AXON_MINVERS_PROXY:
|
|
2272
2390
|
kwargs['proxy'] = proxy
|
|
2273
2391
|
|
|
2274
|
-
|
|
2392
|
+
if ssl_opts is not None:
|
|
2393
|
+
mesg = f'The ssl_opts argument requires an Axon Synapse version {AXON_MINVERS_SSLOPTS}, ' \
|
|
2394
|
+
f'but the Axon is running {axonvers}'
|
|
2395
|
+
s_version.reqVersion(axonvers, AXON_MINVERS_SSLOPTS, mesg=mesg)
|
|
2396
|
+
kwargs['ssl_opts'] = ssl_opts
|
|
2397
|
+
|
|
2398
|
+
return await axon.wput(sha256byts, url, headers=headers, params=params, method=method,
|
|
2399
|
+
ssl=ssl, timeout=timeout, **kwargs)
|
|
2275
2400
|
|
|
2276
2401
|
async def urlfile(self, *args, **kwargs):
|
|
2277
2402
|
gateiden = self.runt.snap.wlyr.iden
|
|
@@ -2372,10 +2497,62 @@ class LibAxon(Lib):
|
|
|
2372
2497
|
self.runt.confirm(('storm', 'lib', 'axon', 'has'))
|
|
2373
2498
|
return await self.runt.snap.core.axon.metrics()
|
|
2374
2499
|
|
|
2500
|
+
async def upload(self, genr):
|
|
2501
|
+
|
|
2502
|
+
self.runt.confirm(('axon', 'upload'))
|
|
2503
|
+
|
|
2504
|
+
await self.runt.snap.core.getAxon()
|
|
2505
|
+
async with await self.runt.snap.core.axon.upload() as upload:
|
|
2506
|
+
async for byts in s_coro.agen(genr):
|
|
2507
|
+
await upload.write(byts)
|
|
2508
|
+
size, sha256 = await upload.save()
|
|
2509
|
+
return size, s_common.ehex(sha256)
|
|
2510
|
+
|
|
2511
|
+
@stormfunc(readonly=True)
|
|
2512
|
+
async def has(self, sha256):
|
|
2513
|
+
sha256 = await tostr(sha256, noneok=True)
|
|
2514
|
+
if sha256 is None:
|
|
2515
|
+
return None
|
|
2516
|
+
|
|
2517
|
+
self.runt.confirm(('axon', 'has'))
|
|
2518
|
+
|
|
2519
|
+
await self.runt.snap.core.getAxon()
|
|
2520
|
+
return await self.runt.snap.core.axon.has(s_common.uhex(sha256))
|
|
2521
|
+
|
|
2522
|
+
@stormfunc(readonly=True)
|
|
2523
|
+
async def size(self, sha256):
|
|
2524
|
+
sha256 = await tostr(sha256)
|
|
2525
|
+
|
|
2526
|
+
self.runt.confirm(('axon', 'has'))
|
|
2527
|
+
|
|
2528
|
+
await self.runt.snap.core.getAxon()
|
|
2529
|
+
return await self.runt.snap.core.axon.size(s_common.uhex(sha256))
|
|
2530
|
+
|
|
2531
|
+
async def put(self, byts):
|
|
2532
|
+
if not isinstance(byts, bytes):
|
|
2533
|
+
mesg = '$lib.axon.put() requires a bytes argument'
|
|
2534
|
+
raise s_exc.BadArg(mesg=mesg)
|
|
2535
|
+
|
|
2536
|
+
self.runt.confirm(('axon', 'upload'))
|
|
2537
|
+
|
|
2538
|
+
await self.runt.snap.core.getAxon()
|
|
2539
|
+
size, sha256 = await self.runt.snap.core.axon.put(byts)
|
|
2540
|
+
|
|
2541
|
+
return (size, s_common.ehex(sha256))
|
|
2542
|
+
|
|
2543
|
+
@stormfunc(readonly=True)
|
|
2544
|
+
async def hashset(self, sha256):
|
|
2545
|
+
sha256 = await tostr(sha256)
|
|
2546
|
+
|
|
2547
|
+
self.runt.confirm(('axon', 'has'))
|
|
2548
|
+
|
|
2549
|
+
await self.runt.snap.core.getAxon()
|
|
2550
|
+
return await self.runt.snap.core.axon.hashset(s_common.uhex(sha256))
|
|
2551
|
+
|
|
2375
2552
|
@registry.registerLib
|
|
2376
2553
|
class LibBytes(Lib):
|
|
2377
2554
|
'''
|
|
2378
|
-
A Storm Library for interacting with bytes storage.
|
|
2555
|
+
A Storm Library for interacting with bytes storage. This Library is deprecated; use ``$lib.axon.*`` instead.
|
|
2379
2556
|
'''
|
|
2380
2557
|
_storm_locals = (
|
|
2381
2558
|
{'name': 'put', 'desc': '''
|
|
@@ -2479,6 +2656,7 @@ class LibBytes(Lib):
|
|
|
2479
2656
|
|
|
2480
2657
|
@stormfunc(readonly=True)
|
|
2481
2658
|
async def _libBytesHas(self, sha256):
|
|
2659
|
+
|
|
2482
2660
|
sha256 = await tostr(sha256, noneok=True)
|
|
2483
2661
|
if sha256 is None:
|
|
2484
2662
|
return None
|
|
@@ -2492,6 +2670,7 @@ class LibBytes(Lib):
|
|
|
2492
2670
|
|
|
2493
2671
|
@stormfunc(readonly=True)
|
|
2494
2672
|
async def _libBytesSize(self, sha256):
|
|
2673
|
+
|
|
2495
2674
|
sha256 = await tostr(sha256)
|
|
2496
2675
|
|
|
2497
2676
|
self.runt.confirm(('axon', 'has'), default=True)
|
|
@@ -2502,6 +2681,7 @@ class LibBytes(Lib):
|
|
|
2502
2681
|
return ret
|
|
2503
2682
|
|
|
2504
2683
|
async def _libBytesPut(self, byts):
|
|
2684
|
+
|
|
2505
2685
|
if not isinstance(byts, bytes):
|
|
2506
2686
|
mesg = '$lib.bytes.put() requires a bytes argument'
|
|
2507
2687
|
raise s_exc.BadArg(mesg=mesg)
|
|
@@ -2516,6 +2696,7 @@ class LibBytes(Lib):
|
|
|
2516
2696
|
|
|
2517
2697
|
@stormfunc(readonly=True)
|
|
2518
2698
|
async def _libBytesHashset(self, sha256):
|
|
2699
|
+
|
|
2519
2700
|
sha256 = await tostr(sha256)
|
|
2520
2701
|
|
|
2521
2702
|
self.runt.confirm(('axon', 'has'), default=True)
|
synapse/lib/version.py
CHANGED
|
@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
|
|
|
223
223
|
##############################################################################
|
|
224
224
|
# The following are touched during the release process by bumpversion.
|
|
225
225
|
# Do not modify these directly.
|
|
226
|
-
version = (2,
|
|
226
|
+
version = (2, 163, 0)
|
|
227
227
|
verstring = '.'.join([str(x) for x in version])
|
|
228
|
-
commit = '
|
|
228
|
+
commit = '72acabe1afe8661274c9e09284ab1726994fa41b'
|
synapse/tests/test_cortex.py
CHANGED
|
@@ -6312,11 +6312,11 @@ class CortexBasicTest(s_t_utils.SynTest):
|
|
|
6312
6312
|
|
|
6313
6313
|
# Use dyncalls, not direct object access.
|
|
6314
6314
|
asdfhash_h = '2413fb3709b05939f04cf2e92f7d0897fc2596f9ad0b8a9ea855c7bfebaae892'
|
|
6315
|
-
size, sha2 = await core.callStorm('return( $lib.
|
|
6315
|
+
size, sha2 = await core.callStorm('return( $lib.axon.put($buf) )',
|
|
6316
6316
|
{'vars': {'buf': b'asdfasdf'}})
|
|
6317
6317
|
self.eq(size, 8)
|
|
6318
6318
|
self.eq(sha2, asdfhash_h)
|
|
6319
|
-
self.true(await core.callStorm('return( $lib.
|
|
6319
|
+
self.true(await core.callStorm('return( $lib.axon.has($hash) )',
|
|
6320
6320
|
{'vars': {'hash': asdfhash_h}}))
|
|
6321
6321
|
|
|
6322
6322
|
unset = False
|
synapse/tests/test_lib_ast.py
CHANGED
|
@@ -2754,7 +2754,7 @@ class AstTest(s_test.SynTest):
|
|
|
2754
2754
|
opts = {'vars': {'asdf': b'asdf'}}
|
|
2755
2755
|
await core.nodes('[ file:bytes=$asdf ]', opts=opts)
|
|
2756
2756
|
await core.axon.put(b'asdf')
|
|
2757
|
-
self.len(1, await core.nodes('file:bytes +$lib.
|
|
2757
|
+
self.len(1, await core.nodes('file:bytes +$lib.axon.has(:sha256)'))
|
|
2758
2758
|
|
|
2759
2759
|
async def test_ast_walkcond(self):
|
|
2760
2760
|
|
synapse/tests/test_lib_storm.py
CHANGED
|
@@ -4513,7 +4513,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
4513
4513
|
visi = await core.auth.addUser('visi')
|
|
4514
4514
|
await visi.addRule((True, ('node',)))
|
|
4515
4515
|
|
|
4516
|
-
size, sha256 = await core.callStorm('return($lib.
|
|
4516
|
+
size, sha256 = await core.callStorm('return($lib.axon.put($buf))', {'vars': {'buf': b'asdfasdf'}})
|
|
4517
4517
|
|
|
4518
4518
|
self.len(1, await core.nodes(f'[ file:bytes={sha256} ]'))
|
|
4519
4519
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import os
|
|
2
|
+
import ssl
|
|
2
3
|
import json
|
|
3
4
|
import shutil
|
|
4
5
|
|
|
@@ -466,7 +467,7 @@ class StormHttpTest(s_test.SynTest):
|
|
|
466
467
|
await root.setPasswd('root')
|
|
467
468
|
text = '''
|
|
468
469
|
$url = $lib.str.format("https://root:root@127.0.0.1:{port}/api/v1/storm", port=$port)
|
|
469
|
-
$stormq = "($size, $sha2) = $lib.
|
|
470
|
+
$stormq = "($size, $sha2) = $lib.axon.put($lib.base64.decode('dmVydGV4')) [ test:str = $sha2 ] [ test:int = $size ]"
|
|
470
471
|
$json = ({"query": $stormq})
|
|
471
472
|
$bytez = $lib.inet.http.post($url, json=$json, ssl_verify=$(0))
|
|
472
473
|
'''
|
|
@@ -672,3 +673,165 @@ class StormHttpTest(s_test.SynTest):
|
|
|
672
673
|
with self.raises(s_stormctrl.StormExit) as cm:
|
|
673
674
|
await core.callStorm(query, opts=opts)
|
|
674
675
|
self.isin('connect to proxy 127.0.0.1:1', str(cm.exception))
|
|
676
|
+
|
|
677
|
+
async def test_storm_http_mtls(self):
|
|
678
|
+
|
|
679
|
+
with self.getTestDir() as dirn:
|
|
680
|
+
|
|
681
|
+
cdir = s_common.gendir(dirn, 'certs')
|
|
682
|
+
cadir = s_common.gendir(cdir, 'cas')
|
|
683
|
+
tdir = s_certdir.CertDir(cdir)
|
|
684
|
+
tdir.genCaCert('somelocalca')
|
|
685
|
+
tdir.genHostCert('localhost', signas='somelocalca')
|
|
686
|
+
|
|
687
|
+
localkeyfp = tdir.getHostKeyPath('localhost')
|
|
688
|
+
localcertfp = tdir.getHostCertPath('localhost')
|
|
689
|
+
pkeypath = shutil.copyfile(localkeyfp, s_common.genpath(dirn, 'sslkey.pem'))
|
|
690
|
+
certpath = shutil.copyfile(localcertfp, s_common.genpath(dirn, 'sslcert.pem'))
|
|
691
|
+
|
|
692
|
+
tlscadir = s_common.gendir(dirn, 'cadir')
|
|
693
|
+
cacertpath = shutil.copyfile(os.path.join(cadir, 'somelocalca.crt'), os.path.join(tlscadir, 'somelocalca.crt'))
|
|
694
|
+
|
|
695
|
+
pkey, cert = tdir.genUserCert('someuser', signas='somelocalca')
|
|
696
|
+
user_pkey = tdir._pkeyToByts(pkey).decode()
|
|
697
|
+
user_cert = tdir._certToByts(cert).decode()
|
|
698
|
+
|
|
699
|
+
user_fullchain = user_cert + s_common.getbytes(cacertpath).decode()
|
|
700
|
+
user_fullchain_key = user_fullchain + user_pkey
|
|
701
|
+
|
|
702
|
+
conf = {'tls:ca:dir': tlscadir}
|
|
703
|
+
async with self.getTestCore(dirn=dirn, conf=conf) as core:
|
|
704
|
+
|
|
705
|
+
sslctx = core.initSslCtx(certpath, pkeypath)
|
|
706
|
+
sslctx.load_verify_locations(cafile=cacertpath)
|
|
707
|
+
|
|
708
|
+
addr, port = await core.addHttpsPort(0, sslctx=sslctx)
|
|
709
|
+
root = await core.auth.getUserByName('root')
|
|
710
|
+
await root.setPasswd('root')
|
|
711
|
+
|
|
712
|
+
core.addHttpApi('/api/v0/test', s_test.HttpReflector, {'cell': core})
|
|
713
|
+
core.addHttpApi('/test/ws', TstWebSock, {})
|
|
714
|
+
|
|
715
|
+
sslopts = {}
|
|
716
|
+
|
|
717
|
+
opts = {
|
|
718
|
+
'vars': {
|
|
719
|
+
'url': f'https://root:root@localhost:{port}/api/v0/test',
|
|
720
|
+
'ws': f'https://localhost:{port}/test/ws',
|
|
721
|
+
'verify': True,
|
|
722
|
+
'sslopts': sslopts,
|
|
723
|
+
},
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
q = 'return($lib.inet.http.get($url, ssl_verify=$verify, ssl_opts=$sslopts))'
|
|
727
|
+
|
|
728
|
+
size, sha256 = await core.callStorm('return($lib.bytes.put($lib.base64.decode(Zm9v)))')
|
|
729
|
+
opts['vars']['sha256'] = sha256
|
|
730
|
+
|
|
731
|
+
# mtls required
|
|
732
|
+
|
|
733
|
+
sslctx.verify_mode = ssl.CERT_REQUIRED
|
|
734
|
+
|
|
735
|
+
## no client cert provided
|
|
736
|
+
resp = await core.callStorm(q, opts=opts)
|
|
737
|
+
self.eq(-1, resp['code'])
|
|
738
|
+
self.isin('tlsv13 alert certificate required', resp['reason'])
|
|
739
|
+
|
|
740
|
+
## full chain cert w/key
|
|
741
|
+
sslopts['client_cert'] = user_fullchain_key
|
|
742
|
+
resp = await core.callStorm(q, opts=opts)
|
|
743
|
+
self.eq(200, resp['code'])
|
|
744
|
+
|
|
745
|
+
## separate cert and key
|
|
746
|
+
sslopts['client_cert'] = user_fullchain
|
|
747
|
+
sslopts['client_key'] = user_pkey
|
|
748
|
+
resp = await core.callStorm(q, opts=opts)
|
|
749
|
+
self.eq(200, resp['code'])
|
|
750
|
+
|
|
751
|
+
## sslctx's are cached
|
|
752
|
+
self.len(3, core._sslctx_cache)
|
|
753
|
+
resp = await core.callStorm(q, opts=opts)
|
|
754
|
+
self.eq(200, resp['code'])
|
|
755
|
+
self.len(3, core._sslctx_cache)
|
|
756
|
+
|
|
757
|
+
## remaining methods
|
|
758
|
+
self.eq(200, await core.callStorm('return($lib.inet.http.post($url, ssl_opts=$sslopts).code)', opts=opts))
|
|
759
|
+
self.eq(200, await core.callStorm('return($lib.inet.http.head($url, ssl_opts=$sslopts).code)', opts=opts))
|
|
760
|
+
self.eq(200, await core.callStorm('return($lib.inet.http.request(get, $url, ssl_opts=$sslopts).code)', opts=opts))
|
|
761
|
+
|
|
762
|
+
## connect
|
|
763
|
+
ret = await core.callStorm('''
|
|
764
|
+
($ok, $sock) = $lib.inet.http.connect($ws, ssl_opts=$sslopts)
|
|
765
|
+
if (not $ok) { return(($ok, $sock)) }
|
|
766
|
+
($ok, $mesg) = $sock.rx()
|
|
767
|
+
return(($ok, $mesg))
|
|
768
|
+
''', opts=opts)
|
|
769
|
+
self.true(ret[0])
|
|
770
|
+
self.eq('woot', ret[1]['hi'])
|
|
771
|
+
|
|
772
|
+
# Axon APIs
|
|
773
|
+
|
|
774
|
+
axon_queries = {
|
|
775
|
+
'postfile': '''
|
|
776
|
+
$fields = ([{"name": "file", "sha256": $sha256}])
|
|
777
|
+
return($lib.inet.http.post($url, fields=$fields, ssl_opts=$sslopts).code)
|
|
778
|
+
''',
|
|
779
|
+
'wget': 'return($lib.axon.wget($url, ssl_opts=$sslopts).code)',
|
|
780
|
+
'wput': 'return($lib.axon.wput($sha256, $url, method=POST, ssl_opts=$sslopts).code)',
|
|
781
|
+
'urlfile': 'yield $lib.axon.urlfile($url, ssl_opts=$sslopts)',
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
## version check fails
|
|
785
|
+
try:
|
|
786
|
+
oldv = core.axoninfo['synapse']['version']
|
|
787
|
+
core.axoninfo['synapse']['version'] = (2, 161, 0)
|
|
788
|
+
await self.asyncraises(s_exc.BadVersion, core.callStorm(axon_queries['postfile'], opts=opts))
|
|
789
|
+
await self.asyncraises(s_exc.BadVersion, core.callStorm(axon_queries['wget'], opts=opts))
|
|
790
|
+
await self.asyncraises(s_exc.BadVersion, core.callStorm(axon_queries['wput'], opts=opts))
|
|
791
|
+
await self.asyncraises(s_exc.BadVersion, core.nodes(axon_queries['urlfile'], opts=opts))
|
|
792
|
+
finally:
|
|
793
|
+
core.axoninfo['synapse']['version'] = oldv
|
|
794
|
+
|
|
795
|
+
## version check succeeds
|
|
796
|
+
# todo: setting the synapse version can be removed once ssl_opts is released
|
|
797
|
+
try:
|
|
798
|
+
oldv = core.axoninfo['synapse']['version']
|
|
799
|
+
core.axoninfo['synapse']['version'] = (oldv[0], oldv[1] + 1, oldv[2])
|
|
800
|
+
self.eq(200, await core.callStorm(axon_queries['postfile'], opts=opts))
|
|
801
|
+
self.eq(200, await core.callStorm(axon_queries['wget'], opts=opts))
|
|
802
|
+
self.eq(200, await core.callStorm(axon_queries['wput'], opts=opts))
|
|
803
|
+
self.len(1, await core.nodes(axon_queries['urlfile'], opts=opts))
|
|
804
|
+
finally:
|
|
805
|
+
core.axoninfo['synapse']['version'] = oldv
|
|
806
|
+
|
|
807
|
+
# verify arg precedence
|
|
808
|
+
|
|
809
|
+
core.conf.pop('tls:ca:dir')
|
|
810
|
+
core._sslctx_cache.clear()
|
|
811
|
+
|
|
812
|
+
## fail w/o ca
|
|
813
|
+
resp = await core.callStorm(q, opts=opts)
|
|
814
|
+
self.eq(-1, resp['code'])
|
|
815
|
+
self.isin('self-signed certificate', resp['reason'])
|
|
816
|
+
|
|
817
|
+
## verify arg wins
|
|
818
|
+
opts['vars']['verify'] = False
|
|
819
|
+
sslopts['verify'] = True
|
|
820
|
+
resp = await core.callStorm(q, opts=opts)
|
|
821
|
+
self.eq(200, resp['code'])
|
|
822
|
+
|
|
823
|
+
# bad opts
|
|
824
|
+
|
|
825
|
+
## schema violation
|
|
826
|
+
sslopts['newp'] = 'wut'
|
|
827
|
+
await self.asyncraises(s_exc.SchemaViolation, core.callStorm(q, opts=opts))
|
|
828
|
+
sslopts.pop('newp')
|
|
829
|
+
|
|
830
|
+
## missing key
|
|
831
|
+
sslopts['client_cert'] = user_fullchain
|
|
832
|
+
sslopts['client_key'] = None
|
|
833
|
+
await self.asyncraises(s_exc.BadArg, core.callStorm(q, opts=opts))
|
|
834
|
+
|
|
835
|
+
## bad cert
|
|
836
|
+
sslopts['client_cert'] = 'not-gonna-work'
|
|
837
|
+
await self.asyncraises(s_exc.BadArg, core.callStorm(q, opts=opts))
|