synapse 2.162.0__py311-none-any.whl → 2.163.0__py311-none-any.whl

synapse/axon.py

@@ -593,7 +593,8 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
         await self._reqUserAllowed(('axon', 'del'))
         return await self.cell.dels(sha256s)
 
-    async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET', ssl=True, timeout=None, proxy=None):
+    async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET',
+                   ssl=True, timeout=None, proxy=None, ssl_opts=None):
         '''
         Stream a file download directly into the Axon.
 
@@ -606,11 +607,20 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
             method (str): The HTTP method to use.
             ssl (bool): Perform SSL verification.
             timeout (int): The timeout of the request, in seconds.
+            ssl_opts (dict): Additional SSL/TLS options.
 
         Notes:
             The response body will be stored, regardless of the response code. The ``ok`` value in the reponse does not
             reflect that a status code, such as a 404, was encountered when retrieving the URL.
 
+            The ssl_opts dictionary may contain the following values::
+
+                {
+                    'verify': <bool> - Perform SSL/TLS verification. Is overridden by the ssl argument.
+                    'client_cert': <str> - PEM encoded full chain certificate for use in mTLS.
+                    'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
+                }
+
             The dictionary returned by this may contain the following values::
 
                 {
@@ -640,18 +650,20 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
             dict: An information dictionary containing the results of the request.
         '''
         await self._reqUserAllowed(('axon', 'wget'))
-        return await self.cell.wget(url, params=params, headers=headers, json=json, body=body, method=method, ssl=ssl,
-                                    timeout=timeout, proxy=proxy)
+        return await self.cell.wget(url, params=params, headers=headers, json=json, body=body, method=method,
+                                    ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts)
 
-    async def postfiles(self, fields, url, params=None, headers=None, method='POST', ssl=True, timeout=None, proxy=None):
+    async def postfiles(self, fields, url, params=None, headers=None, method='POST',
+                        ssl=True, timeout=None, proxy=None, ssl_opts=None):
         await self._reqUserAllowed(('axon', 'wput'))
-        return await self.cell.postfiles(fields, url, params=params, headers=headers,
-                                         method=method, ssl=ssl, timeout=timeout, proxy=proxy)
+        return await self.cell.postfiles(fields, url, params=params, headers=headers, method=method,
+                                         ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts)
 
-    async def wput(self, sha256, url, params=None, headers=None, method='PUT', ssl=True, timeout=None, proxy=None):
+    async def wput(self, sha256, url, params=None, headers=None, method='PUT',
+                   ssl=True, timeout=None, proxy=None, ssl_opts=None):
         await self._reqUserAllowed(('axon', 'wput'))
-        return await self.cell.wput(sha256, url, params=params, headers=headers, method=method, ssl=ssl,
-                                    timeout=timeout, proxy=proxy)
+        return await self.cell.wput(sha256, url, params=params, headers=headers, method=method,
+                                    ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts)
 
     async def metrics(self):
         '''
@@ -1434,7 +1446,8 @@ class Axon(s_cell.Cell):
                 raise s_exc.BadJsonText(mesg=f'Bad json line encountered while processing {sha256}, ({e})',
                                         sha256=sha256) from None
 
-    async def postfiles(self, fields, url, params=None, headers=None, method='POST', ssl=True, timeout=None, proxy=None):
+    async def postfiles(self, fields, url, params=None, headers=None, method='POST',
+                        ssl=True, timeout=None, proxy=None, ssl_opts=None):
         '''
         Send files from the axon as fields in a multipart/form-data HTTP request.
 
@@ -1447,6 +1460,7 @@ class Axon(s_cell.Cell):
             ssl (bool): Perform SSL verification.
             timeout (int): The timeout of the request, in seconds.
             proxy (bool|str|null): Use a specific proxy or disable proxy use.
+            ssl_opts (dict): Additional SSL/TLS options.
 
         Notes:
             The dictionaries in the fields list may contain the following values::
@@ -1460,6 +1474,14 @@ class Axon(s_cell.Cell):
                     'content_transfer_encoding': <str> - Optional content-transfer-encoding header for the field.
                 }
 
+            The ssl_opts dictionary may contain the following values::
+
+                {
+                    'verify': <bool> - Perform SSL/TLS verification. Is overridden by the ssl argument.
+                    'client_cert': <str> - PEM encoded full chain certificate for use in mTLS.
+                    'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
+                }
+
             The dictionary returned by this may contain the following values::
 
                 {
@@ -1478,20 +1500,12 @@ class Axon(s_cell.Cell):
         if proxy is None:
             proxy = self.conf.get('http:proxy')
 
-        cadir = self.conf.get('tls:ca:dir')
+        ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl)
 
         connector = None
         if proxy:
             connector = aiohttp_socks.ProxyConnector.from_url(proxy)
 
-        if ssl is False:
-            pass
-        elif cadir:
-            ssl = s_common.getSslCtx(cadir)
-        else:
-            # default aiohttp behavior
-            ssl = None
-
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
@@ -1556,27 +1570,19 @@ class Axon(s_cell.Cell):
                 }
 
     async def wput(self, sha256, url, params=None, headers=None, method='PUT', ssl=True, timeout=None,
-                   filename=None, filemime=None, proxy=None):
+                   filename=None, filemime=None, proxy=None, ssl_opts=None):
         '''
         Stream a blob from the axon as the body of an HTTP request.
         '''
         if proxy is None:
-            prox = self.conf.get('http:proxy')
+            proxy = self.conf.get('http:proxy')
 
-        cadir = self.conf.get('tls:ca:dir')
+        ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl)
 
         connector = None
         if proxy:
             connector = aiohttp_socks.ProxyConnector.from_url(proxy)
 
-        if ssl is False:
-            pass
-        elif cadir:
-            ssl = s_common.getSslCtx(cadir)
-        else:
-            # default aiohttp behavior
-            ssl = None
-
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
@@ -1638,7 +1644,8 @@ class Axon(s_cell.Cell):
 
         return info
 
-    async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET', ssl=True, timeout=None, proxy=None):
+    async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET',
+                   ssl=True, timeout=None, proxy=None, ssl_opts=None):
         '''
         Stream a file download directly into the Axon.
 
@@ -1652,11 +1659,20 @@ class Axon(s_cell.Cell):
             ssl (bool): Perform SSL verification.
             timeout (int): The timeout of the request, in seconds.
             proxy (bool|str|null): Use a specific proxy or disable proxy use.
+            ssl_opts (dict): Additional SSL/TLS options.
 
         Notes:
             The response body will be stored, regardless of the response code. The ``ok`` value in the reponse does not
             reflect that a status code, such as a 404, was encountered when retrieving the URL.
 
+            The ssl_opts dictionary may contain the following values::
+
+                {
+                    'verify': <bool> - Perform SSL/TLS verification. Is overridden by the ssl argument.
+                    'client_cert': <str> - PEM encoded full chain certificate for use in mTLS.
+                    'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
+                }
+
             The dictionary returned by this may contain the following values::
 
                 {
@@ -1690,7 +1706,7 @@ class Axon(s_cell.Cell):
         if proxy is None:
             proxy = self.conf.get('http:proxy')
 
-        cadir = self.conf.get('tls:ca:dir')
+        ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl)
 
         connector = None
         if proxy:
@@ -1698,14 +1714,6 @@ class Axon(s_cell.Cell):
 
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
-        if ssl is False:
-            pass
-        elif cadir:
-            ssl = s_common.getSslCtx(cadir)
-        else:
-            # default aiohttp behavior
-            ssl = None
-
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
 
             try:

synapse/cortex.py

@@ -3879,6 +3879,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             if proxyurl is not None:
                 conf['http:proxy'] = proxyurl
 
+            cadir = self.conf.get('tls:ca:dir')
+            if cadir is not None:
+                conf['tls:ca:dir'] = cadir
+
             self.axon = await s_axon.Axon.anit(path, conf=conf, parent=self)
             self.axoninfo = await self.axon.getCellInfo()
             self.axon.onfini(self.axready.clear)

synapse/lib/cell.py

@@ -1,5 +1,6 @@
 import gc
 import os
+import ssl
 import copy
 import time
 import fcntl
@@ -12,6 +13,7 @@ import tarfile
 import argparse
 import datetime
 import platform
+import tempfile
 import functools
 import contextlib
 import multiprocessing
@@ -32,6 +34,7 @@ import synapse.lib.boss as s_boss
 import synapse.lib.coro as s_coro
 import synapse.lib.hive as s_hive
 import synapse.lib.link as s_link
+import synapse.lib.cache as s_cache
 import synapse.lib.const as s_const
 import synapse.lib.nexus as s_nexus
 import synapse.lib.queue as s_queue
@@ -58,6 +61,7 @@ import synapse.tools.backup as s_t_backup
 logger = logging.getLogger(__name__)
 
 SLAB_MAP_SIZE = 128 * s_const.mebibyte
+SSLCTX_CACHE_SIZE = 64
 
 '''
 Base classes for the synapse "cell" microservice architecture.
@@ -1151,6 +1155,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         self.usermetadb = self.slab.initdb('user:meta')  # useriden + <valu> -> dict valu
         self.rolemetadb = self.slab.initdb('role:meta')  # roleiden + <valu> -> dict valu
 
+        self._sslctx_cache = s_cache.FixedCache(self._makeCachedSslCtx, size=SSLCTX_CACHE_SIZE)
+
         self.hive = await self._initCellHive()
 
         # self.cellinfo, a HiveDict for general purpose persistent storage
@@ -4342,3 +4348,60 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 self.slab.delete(key_iden, db=self.apikeydb)
             self.slab.delete(lkey, db=self.usermetadb)
             await asyncio.sleep(0)
+
+    def _makeCachedSslCtx(self, opts):
+
+        opts = dict(opts)
+
+        cadir = self.conf.get('tls:ca:dir')
+
+        if cadir is not None:
+            sslctx = s_common.getSslCtx(cadir, purpose=ssl.Purpose.SERVER_AUTH)
+        else:
+            sslctx = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH)
+
+        if not opts['verify']:
+            sslctx.check_hostname = False
+            sslctx.verify_mode = ssl.CERT_NONE
+
+        if not opts['client_cert']:
+            return sslctx
+
+        client_cert = opts['client_cert'].encode()
+
+        if opts['client_key']:
+            client_key = opts['client_key'].encode()
+        else:
+            client_key = None
+            client_key_path = None
+
+        with self.getTempDir() as tmpdir:
+
+            with tempfile.NamedTemporaryFile(dir=tmpdir, mode='wb', delete=False) as fh:
+                fh.write(client_cert)
+                client_cert_path = fh.name
+
+            if client_key:
+                with tempfile.NamedTemporaryFile(dir=tmpdir, mode='wb', delete=False) as fh:
+                    fh.write(client_key)
+                    client_key_path = fh.name
+
+            try:
+                sslctx.load_cert_chain(client_cert_path, keyfile=client_key_path)
+            except ssl.SSLError as e:
+                raise s_exc.BadArg(mesg=f'Error loading client cert: {str(e)}') from None
+
+        return sslctx
+
+    def getCachedSslCtx(self, opts=None, verify=None):
+
+        if opts is None:
+            opts = {}
+
+        if verify is not None:
+            opts['verify'] = verify
+
+        opts = s_schemas.reqValidSslCtxOpts(opts)
+
+        key = tuple(sorted(opts.items()))
+        return self._sslctx_cache.get(key)

synapse/lib/oauth.py

@@ -230,13 +230,7 @@ class OAuthMixin(s_nexus.Pusher):
 
         timeout = aiohttp.ClientTimeout(total=DEFAULT_TIMEOUT)
 
-        cadir = self.conf.get('tls:ca:dir')
-        if ssl_verify is False:
-            ssl = False
-        elif cadir:
-            ssl = s_common.getSslCtx(cadir)
-        else:
-            ssl = None
+        ssl = self.getCachedSslCtx(verify=ssl_verify)
 
         async with aiohttp.ClientSession(timeout=timeout) as sess:
 

synapse/lib/schemas.py

@@ -263,3 +263,13 @@ _cellUserApiKeySchema = {
     ],
 }
 reqValidUserApiKeyDef = s_config.getJsValidator(_cellUserApiKeySchema)
+
+reqValidSslCtxOpts = s_config.getJsValidator({
+    'type': 'object',
+    'properties': {
+        'verify': {'type': 'boolean', 'default': True},
+        'client_cert': {'type': ['string', 'null'], 'default': None},
+        'client_key': {'type': ['string', 'null'], 'default': None},
+    },
+    'additionalProperties': False,
+})

synapse/lib/stormhttp.py

@@ -13,6 +13,7 @@ import synapse.common as s_common
 
 import synapse.lib.base as s_base
 import synapse.lib.msgpack as s_msgpack
+import synapse.lib.version as s_version
 import synapse.lib.stormtypes as s_stormtypes
 
 @s_stormtypes.registry.registerType
@@ -87,6 +88,14 @@ class WebSocket(s_base.Base, s_stormtypes.StormType):
 class LibHttp(s_stormtypes.Lib):
     '''
     A Storm Library exposing an HTTP client API.
+
+    For APIs that accept an ssl_opts argument, the dictionary may contain the following values::
+
+        {
+            'verify': <bool> - Perform SSL/TLS verification. Is overridden by the ssl_verify argument.
+            'client_cert': <str> - PEM encoded full chain certificate for use in mTLS.
+            'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
+        }
     '''
     _storm_locals = (
         {'name': 'get', 'desc': 'Get the contents of a given URL.',
@@ -105,6 +114,9 @@ class LibHttp(s_stormtypes.Lib):
                        'default': True},
                       {'name': 'proxy', 'type': ['bool', 'null', 'str'],
                        'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
+                      {'name': 'ssl_opts', 'type': 'dict',
+                       'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
+                       'default': None},
                   ),
                   'returns': {'type': 'inet:http:resp', 'desc': 'The response object.'}}},
         {'name': 'post', 'desc': 'Post data to a given URL.',
@@ -134,6 +146,9 @@ class LibHttp(s_stormtypes.Lib):
                        'default': None},
                       {'name': 'proxy', 'type': ['bool', 'null', 'str'],
                        'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
+                      {'name': 'ssl_opts', 'type': 'dict',
+                       'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
+                       'default': None},
                   ),
                   'returns': {'type': 'inet:http:resp', 'desc': 'The response object.'}}},
         {'name': 'head', 'desc': 'Get the HEAD response for a URL.',
@@ -153,6 +168,9 @@ class LibHttp(s_stormtypes.Lib):
                        'default': False},
                       {'name': 'proxy', 'type': ['bool', 'null', 'str'],
                        'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
+                      {'name': 'ssl_opts', 'type': 'dict',
+                       'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
+                       'default': None},
                   ),
                   'returns': {'type': 'inet:http:resp', 'desc': 'The response object.'}}},
         {'name': 'request', 'desc': 'Make an HTTP request using the given HTTP method to the url.',
@@ -183,6 +201,9 @@ class LibHttp(s_stormtypes.Lib):
                        'default': None},
                       {'name': 'proxy', 'type': ['bool', 'null', 'str'],
                        'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
+                      {'name': 'ssl_opts', 'type': 'dict',
+                       'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
+                       'default': None},
                    ),
                   'returns': {'type': 'inet:http:resp', 'desc': 'The response object.'}
                   }
@@ -201,6 +222,9 @@ class LibHttp(s_stormtypes.Lib):
                        'default': None},
                       {'name': 'proxy', 'type': ['bool', 'null', 'str'],
                        'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
+                      {'name': 'ssl_opts', 'type': 'dict',
+                       'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
+                       'default': None},
                   ),
                   'returns': {'type': 'inet:http:socket', 'desc': 'A websocket object.'}}},
         {'name': 'urlencode', 'desc': '''
@@ -290,28 +314,31 @@ class LibHttp(s_stormtypes.Lib):
         return s_common.httpcodereason(code)
 
     async def _httpEasyHead(self, url, headers=None, ssl_verify=True, params=None, timeout=300,
-                            allow_redirects=False, proxy=None):
+                            allow_redirects=False, proxy=None, ssl_opts=None):
         return await self._httpRequest('HEAD', url, headers=headers, ssl_verify=ssl_verify, params=params,
-                                       timeout=timeout, allow_redirects=allow_redirects, proxy=proxy)
+                                       timeout=timeout, allow_redirects=allow_redirects, proxy=proxy, ssl_opts=ssl_opts)
 
     async def _httpEasyGet(self, url, headers=None, ssl_verify=True, params=None, timeout=300,
-                           allow_redirects=True, proxy=None):
+                           allow_redirects=True, proxy=None, ssl_opts=None):
         return await self._httpRequest('GET', url, headers=headers, ssl_verify=ssl_verify, params=params,
-                                       timeout=timeout, allow_redirects=allow_redirects, proxy=proxy)
+                                       timeout=timeout, allow_redirects=allow_redirects, proxy=proxy, ssl_opts=ssl_opts)
 
     async def _httpPost(self, url, headers=None, json=None, body=None, ssl_verify=True,
-                        params=None, timeout=300, allow_redirects=True, fields=None, proxy=None):
+                        params=None, timeout=300, allow_redirects=True, fields=None, proxy=None, ssl_opts=None):
         return await self._httpRequest('POST', url, headers=headers, json=json, body=body,
                                        ssl_verify=ssl_verify, params=params, timeout=timeout,
-                                       allow_redirects=allow_redirects, fields=fields, proxy=proxy)
+                                       allow_redirects=allow_redirects, fields=fields, proxy=proxy, ssl_opts=ssl_opts)
 
-    async def inetHttpConnect(self, url, headers=None, ssl_verify=True, timeout=300, params=None, proxy=None):
+    async def inetHttpConnect(self, url, headers=None, ssl_verify=True, timeout=300,
+                              params=None, proxy=None, ssl_opts=None):
 
         url = await s_stormtypes.tostr(url)
         headers = await s_stormtypes.toprim(headers)
         timeout = await s_stormtypes.toint(timeout, noneok=True)
         params = await s_stormtypes.toprim(params)
         proxy = await s_stormtypes.toprim(proxy)
+        ssl_verify = await s_stormtypes.tobool(ssl_verify, noneok=True)
+        ssl_opts = await s_stormtypes.toprim(ssl_opts)
 
         headers = self.strify(headers)
 
@@ -332,15 +359,7 @@ class LibHttp(s_stormtypes.Lib):
         if params:
             kwargs['params'] = params
 
-        cadir = self.runt.snap.core.conf.get('tls:ca:dir')
-
-        if ssl_verify is False:
-            kwargs['ssl'] = False
-        elif cadir:
-            kwargs['ssl'] = s_common.getSslCtx(cadir)
-        else:
-            # default aiohttp behavior
-            kwargs['ssl'] = None
+        kwargs['ssl'] = self.runt.snap.core.getCachedSslCtx(opts=ssl_opts, verify=ssl_verify)
 
         try:
             sess = await sock.enter_context(aiohttp.ClientSession(connector=connector, timeout=timeout))
@@ -374,7 +393,7 @@ class LibHttp(s_stormtypes.Lib):
 
     async def _httpRequest(self, meth, url, headers=None, json=None, body=None,
                            ssl_verify=True, params=None, timeout=300, allow_redirects=True,
-                           fields=None, proxy=None):
+                           fields=None, proxy=None, ssl_opts=None):
         meth = await s_stormtypes.tostr(meth)
         url = await s_stormtypes.tostr(url)
         json = await s_stormtypes.toprim(json)
@@ -386,6 +405,7 @@ class LibHttp(s_stormtypes.Lib):
         ssl_verify = await s_stormtypes.tobool(ssl_verify, noneok=True)
         allow_redirects = await s_stormtypes.tobool(allow_redirects)
         proxy = await s_stormtypes.toprim(proxy)
+        ssl_opts = await s_stormtypes.toprim(ssl_opts)
 
         kwargs = {'allow_redirects': allow_redirects}
         if params:
@@ -399,12 +419,24 @@ class LibHttp(s_stormtypes.Lib):
         if fields:
             if any(['sha256' in field for field in fields]):
                 self.runt.confirm(('storm', 'lib', 'axon', 'wput'))
+
+                kwargs = {}
+                axonvers = self.runt.snap.core.axoninfo['synapse']['version']
+                if axonvers >= s_stormtypes.AXON_MINVERS_PROXY:
+                    kwargs['proxy'] = proxy
+
+                if ssl_opts is not None:
+                    mesg = f'The ssl_opts argument requires an Axon Synapse version {s_stormtypes.AXON_MINVERS_SSLOPTS}, ' \
+                           f'but the Axon is running {axonvers}'
+                    s_version.reqVersion(axonvers, s_stormtypes.AXON_MINVERS_SSLOPTS, mesg=mesg)
+                    kwargs['ssl_opts'] = ssl_opts
+
                 axon = self.runt.snap.core.axon
-                info = await axon.postfiles(fields, url, headers=headers, params=params,
-                                            method=meth, ssl=ssl_verify, timeout=timeout, proxy=proxy)
+                info = await axon.postfiles(fields, url, headers=headers, params=params, method=meth,
+                                            ssl=ssl_verify, timeout=timeout, **kwargs)
                 return HttpResp(info)
 
-        cadir = self.runt.snap.core.conf.get('tls:ca:dir')
+        kwargs['ssl'] = self.runt.snap.core.getCachedSslCtx(opts=ssl_opts, verify=ssl_verify)
 
         if proxy is None:
             proxy = await self.runt.snap.core.getConfOpt('http:proxy')
@@ -413,14 +445,6 @@ class LibHttp(s_stormtypes.Lib):
         if proxy:
             connector = aiohttp_socks.ProxyConnector.from_url(proxy)
 
-        if ssl_verify is False:
-            kwargs['ssl'] = False
-        elif cadir:
-            kwargs['ssl'] = s_common.getSslCtx(cadir)
-        else:
-            # default aiohttp behavior
-            kwargs['ssl'] = None
-
         timeout = aiohttp.ClientTimeout(total=timeout)
 
         async with aiohttp.ClientSession(connector=connector, timeout=timeout) as sess: