synapse 2.161.0__py311-none-any.whl → 2.163.0__py311-none-any.whl

synapse/axon.py

@@ -593,7 +593,8 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
         await self._reqUserAllowed(('axon', 'del'))
         return await self.cell.dels(sha256s)
 
-    async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET', ssl=True, timeout=None, proxy=None):
+    async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET',
+                   ssl=True, timeout=None, proxy=None, ssl_opts=None):
         '''
         Stream a file download directly into the Axon.
 
@@ -606,11 +607,20 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
             method (str): The HTTP method to use.
             ssl (bool): Perform SSL verification.
             timeout (int): The timeout of the request, in seconds.
+            ssl_opts (dict): Additional SSL/TLS options.
 
         Notes:
             The response body will be stored, regardless of the response code. The ``ok`` value in the reponse does not
             reflect that a status code, such as a 404, was encountered when retrieving the URL.
 
+            The ssl_opts dictionary may contain the following values::
+
+                {
+                    'verify': <bool> - Perform SSL/TLS verification. Is overridden by the ssl argument.
+                    'client_cert': <str> - PEM encoded full chain certificate for use in mTLS.
+                    'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
+                }
+
             The dictionary returned by this may contain the following values::
 
                 {
@@ -640,18 +650,20 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
             dict: An information dictionary containing the results of the request.
         '''
         await self._reqUserAllowed(('axon', 'wget'))
-        return await self.cell.wget(url, params=params, headers=headers, json=json, body=body, method=method, ssl=ssl,
-                                    timeout=timeout, proxy=proxy)
+        return await self.cell.wget(url, params=params, headers=headers, json=json, body=body, method=method,
+                                    ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts)
 
-    async def postfiles(self, fields, url, params=None, headers=None, method='POST', ssl=True, timeout=None, proxy=None):
+    async def postfiles(self, fields, url, params=None, headers=None, method='POST',
+                        ssl=True, timeout=None, proxy=None, ssl_opts=None):
         await self._reqUserAllowed(('axon', 'wput'))
-        return await self.cell.postfiles(fields, url, params=params, headers=headers,
-                                         method=method, ssl=ssl, timeout=timeout, proxy=proxy)
+        return await self.cell.postfiles(fields, url, params=params, headers=headers, method=method,
+                                         ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts)
 
-    async def wput(self, sha256, url, params=None, headers=None, method='PUT', ssl=True, timeout=None, proxy=None):
+    async def wput(self, sha256, url, params=None, headers=None, method='PUT',
+                   ssl=True, timeout=None, proxy=None, ssl_opts=None):
         await self._reqUserAllowed(('axon', 'wput'))
-        return await self.cell.wput(sha256, url, params=params, headers=headers, method=method, ssl=ssl,
-                                    timeout=timeout, proxy=proxy)
+        return await self.cell.wput(sha256, url, params=params, headers=headers, method=method,
+                                    ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts)
 
     async def metrics(self):
         '''
@@ -1434,7 +1446,8 @@ class Axon(s_cell.Cell):
                 raise s_exc.BadJsonText(mesg=f'Bad json line encountered while processing {sha256}, ({e})',
                                         sha256=sha256) from None
 
-    async def postfiles(self, fields, url, params=None, headers=None, method='POST', ssl=True, timeout=None, proxy=None):
+    async def postfiles(self, fields, url, params=None, headers=None, method='POST',
+                        ssl=True, timeout=None, proxy=None, ssl_opts=None):
         '''
         Send files from the axon as fields in a multipart/form-data HTTP request.
 
@@ -1447,6 +1460,7 @@ class Axon(s_cell.Cell):
             ssl (bool): Perform SSL verification.
             timeout (int): The timeout of the request, in seconds.
             proxy (bool|str|null): Use a specific proxy or disable proxy use.
+            ssl_opts (dict): Additional SSL/TLS options.
 
         Notes:
             The dictionaries in the fields list may contain the following values::
@@ -1460,6 +1474,14 @@ class Axon(s_cell.Cell):
                     'content_transfer_encoding': <str> - Optional content-transfer-encoding header for the field.
                 }
 
+            The ssl_opts dictionary may contain the following values::
+
+                {
+                    'verify': <bool> - Perform SSL/TLS verification. Is overridden by the ssl argument.
+                    'client_cert': <str> - PEM encoded full chain certificate for use in mTLS.
+                    'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
+                }
+
             The dictionary returned by this may contain the following values::
 
                 {
@@ -1478,20 +1500,12 @@ class Axon(s_cell.Cell):
         if proxy is None:
             proxy = self.conf.get('http:proxy')
 
-        cadir = self.conf.get('tls:ca:dir')
+        ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl)
 
         connector = None
         if proxy:
             connector = aiohttp_socks.ProxyConnector.from_url(proxy)
 
-        if ssl is False:
-            pass
-        elif cadir:
-            ssl = s_common.getSslCtx(cadir)
-        else:
-            # default aiohttp behavior
-            ssl = None
-
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
@@ -1556,27 +1570,19 @@ class Axon(s_cell.Cell):
                 }
 
     async def wput(self, sha256, url, params=None, headers=None, method='PUT', ssl=True, timeout=None,
-                   filename=None, filemime=None, proxy=None):
+                   filename=None, filemime=None, proxy=None, ssl_opts=None):
         '''
         Stream a blob from the axon as the body of an HTTP request.
         '''
         if proxy is None:
-            prox = self.conf.get('http:proxy')
+            proxy = self.conf.get('http:proxy')
 
-        cadir = self.conf.get('tls:ca:dir')
+        ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl)
 
         connector = None
         if proxy:
             connector = aiohttp_socks.ProxyConnector.from_url(proxy)
 
-        if ssl is False:
-            pass
-        elif cadir:
-            ssl = s_common.getSslCtx(cadir)
-        else:
-            # default aiohttp behavior
-            ssl = None
-
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
@@ -1638,7 +1644,8 @@ class Axon(s_cell.Cell):
 
         return info
 
-    async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET', ssl=True, timeout=None, proxy=None):
+    async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET',
+                   ssl=True, timeout=None, proxy=None, ssl_opts=None):
         '''
         Stream a file download directly into the Axon.
 
@@ -1652,11 +1659,20 @@ class Axon(s_cell.Cell):
             ssl (bool): Perform SSL verification.
             timeout (int): The timeout of the request, in seconds.
             proxy (bool|str|null): Use a specific proxy or disable proxy use.
+            ssl_opts (dict): Additional SSL/TLS options.
 
         Notes:
             The response body will be stored, regardless of the response code. The ``ok`` value in the reponse does not
             reflect that a status code, such as a 404, was encountered when retrieving the URL.
 
+            The ssl_opts dictionary may contain the following values::
+
+                {
+                    'verify': <bool> - Perform SSL/TLS verification. Is overridden by the ssl argument.
+                    'client_cert': <str> - PEM encoded full chain certificate for use in mTLS.
+                    'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
+                }
+
             The dictionary returned by this may contain the following values::
 
                 {
@@ -1690,7 +1706,7 @@ class Axon(s_cell.Cell):
         if proxy is None:
             proxy = self.conf.get('http:proxy')
 
-        cadir = self.conf.get('tls:ca:dir')
+        ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl)
 
         connector = None
         if proxy:
@@ -1698,14 +1714,6 @@ class Axon(s_cell.Cell):
 
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
-        if ssl is False:
-            pass
-        elif cadir:
-            ssl = s_common.getSslCtx(cadir)
-        else:
-            # default aiohttp behavior
-            ssl = None
-
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
 
             try:

synapse/cortex.py

@@ -3879,6 +3879,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             if proxyurl is not None:
                 conf['http:proxy'] = proxyurl
 
+            cadir = self.conf.get('tls:ca:dir')
+            if cadir is not None:
+                conf['tls:ca:dir'] = cadir
+
             self.axon = await s_axon.Axon.anit(path, conf=conf, parent=self)
             self.axoninfo = await self.axon.getCellInfo()
             self.axon.onfini(self.axready.clear)

synapse/daemon.py

@@ -218,7 +218,7 @@ async def t2call(link, meth, args, kwargs):
 
 class Daemon(s_base.Base):
 
-    async def __anit__(self, certdir=None):
+    async def __anit__(self, certdir=None, ahainfo=None):
 
         await s_base.Base.__anit__(self)
 
@@ -227,6 +227,8 @@ class Daemon(s_base.Base):
         if certdir is None:
             certdir = s_certdir.getCertDir()
 
+        self.ahainfo = ahainfo
+
         self.certdir = certdir
         self.televers = s_telepath.televers
 
@@ -414,13 +416,16 @@ class Daemon(s_base.Base):
     async def _getSharedItem(self, name):
         return self.shared.get(name)
 
-    async def _onTeleSyn(self, link, mesg):
+    async def _onTeleSyn(self, link: s_link.Link, mesg):
 
         reply = ('tele:syn', {
             'vers': self.televers,
             'retn': (True, None),
         })
 
+        if self.ahainfo is not None:
+            reply[1]['ahainfo'] = self.ahainfo
+
         try:
 
             vers = mesg[1].get('vers')

synapse/lib/cell.py

@@ -1,5 +1,6 @@
 import gc
 import os
+import ssl
 import copy
 import time
 import fcntl
@@ -12,6 +13,7 @@ import tarfile
 import argparse
 import datetime
 import platform
+import tempfile
 import functools
 import contextlib
 import multiprocessing
@@ -32,6 +34,7 @@ import synapse.lib.boss as s_boss
 import synapse.lib.coro as s_coro
 import synapse.lib.hive as s_hive
 import synapse.lib.link as s_link
+import synapse.lib.cache as s_cache
 import synapse.lib.const as s_const
 import synapse.lib.nexus as s_nexus
 import synapse.lib.queue as s_queue
@@ -58,6 +61,7 @@ import synapse.tools.backup as s_t_backup
 logger = logging.getLogger(__name__)
 
 SLAB_MAP_SIZE = 128 * s_const.mebibyte
+SSLCTX_CACHE_SIZE = 64
 
 '''
 Base classes for the synapse "cell" microservice architecture.
@@ -1151,6 +1155,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         self.usermetadb = self.slab.initdb('user:meta')  # useriden + <valu> -> dict valu
         self.rolemetadb = self.slab.initdb('role:meta')  # roleiden + <valu> -> dict valu
 
+        self._sslctx_cache = s_cache.FixedCache(self._makeCachedSslCtx, size=SSLCTX_CACHE_SIZE)
+
         self.hive = await self._initCellHive()
 
         # self.cellinfo, a HiveDict for general purpose persistent storage
@@ -2647,8 +2653,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         for name, valu in vals.items():
             self.sessstor.set(iden, name, valu)
         sess = self.sessions.get(iden)
-        for name, valu in vals.items():
-            sess.info[name] = valu
+        if sess is not None:
+            sess.info.update(vals)
 
     @contextlib.contextmanager
     def getTempDir(self):
@@ -2840,7 +2846,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
     async def _initCellDmon(self):
 
-        self.dmon = await s_daemon.Daemon.anit()
+        ahainfo = {
+            'name': self.ahasvcname
+        }
+
+        self.dmon = await s_daemon.Daemon.anit(ahainfo=ahainfo)
         self.dmon.share('*', self)
 
         self.onfini(self.dmon.fini)
@@ -4338,3 +4348,60 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 self.slab.delete(key_iden, db=self.apikeydb)
             self.slab.delete(lkey, db=self.usermetadb)
             await asyncio.sleep(0)
+
+    def _makeCachedSslCtx(self, opts):
+
+        opts = dict(opts)
+
+        cadir = self.conf.get('tls:ca:dir')
+
+        if cadir is not None:
+            sslctx = s_common.getSslCtx(cadir, purpose=ssl.Purpose.SERVER_AUTH)
+        else:
+            sslctx = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH)
+
+        if not opts['verify']:
+            sslctx.check_hostname = False
+            sslctx.verify_mode = ssl.CERT_NONE
+
+        if not opts['client_cert']:
+            return sslctx
+
+        client_cert = opts['client_cert'].encode()
+
+        if opts['client_key']:
+            client_key = opts['client_key'].encode()
+        else:
+            client_key = None
+            client_key_path = None
+
+        with self.getTempDir() as tmpdir:
+
+            with tempfile.NamedTemporaryFile(dir=tmpdir, mode='wb', delete=False) as fh:
+                fh.write(client_cert)
+                client_cert_path = fh.name
+
+            if client_key:
+                with tempfile.NamedTemporaryFile(dir=tmpdir, mode='wb', delete=False) as fh:
+                    fh.write(client_key)
+                    client_key_path = fh.name
+
+            try:
+                sslctx.load_cert_chain(client_cert_path, keyfile=client_key_path)
+            except ssl.SSLError as e:
+                raise s_exc.BadArg(mesg=f'Error loading client cert: {str(e)}') from None
+
+        return sslctx
+
+    def getCachedSslCtx(self, opts=None, verify=None):
+
+        if opts is None:
+            opts = {}
+
+        if verify is not None:
+            opts['verify'] = verify
+
+        opts = s_schemas.reqValidSslCtxOpts(opts)
+
+        key = tuple(sorted(opts.items()))
+        return self._sslctx_cache.get(key)

synapse/lib/layer.py

@@ -1529,12 +1529,14 @@ class Layer(s_nexus.Pusher):
 
         mirror = self.layrinfo.get('mirror')
         if mirror is not None:
+            s_common.deprecated('mirror layer configuration option', curv='2.162.0')
             conf = {'retrysleep': 2}
             self.leader = await s_telepath.Client.anit(mirror, conf=conf)
             self.leadtask = self.schedCoro(self._runMirrorLoop())
 
         uplayr = self.layrinfo.get('upstream')
         if uplayr is not None:
+            s_common.deprecated('upstream layer configuration option', curv='2.162.0')
             if isinstance(uplayr, (tuple, list)):
                 for layr in uplayr:
                     await self.initUpstreamSync(layr)
@@ -4382,9 +4384,14 @@ def getNodeEditPerms(nodeedits):
     '''
     Yields (offs, perm) tuples that can be used in user.allowed()
     '''
+    tags = []
+    tagadds = []
 
     for nodeoffs, (buid, form, edits) in enumerate(nodeedits):
 
+        tags.clear()
+        tagadds.clear()
+
         for editoffs, (edit, info, _) in enumerate(edits):
 
             permoffs = (nodeoffs, editoffs)
@@ -4406,7 +4413,11 @@ def getNodeEditPerms(nodeedits):
                 continue
 
             if edit == EDIT_TAG_SET:
-                yield (permoffs, ('node', 'tag', 'add', *info[0].split('.')))
+                if info[1] != (None, None):
+                    tagadds.append(info[0])
+                    yield (permoffs, ('node', 'tag', 'add', *info[0].split('.')))
+                else:
+                    tags.append((len(info[0]), editoffs, info[0]))
                 continue
 
             if edit == EDIT_TAG_DEL:
@@ -4436,3 +4447,11 @@ def getNodeEditPerms(nodeedits):
             if edit == EDIT_EDGE_DEL:
                 yield (permoffs, ('node', 'edge', 'del', info[0]))
                 continue
+
+        for _, editoffs, tag in sorted(tags, reverse=True):
+            look = tag + '.'
+            if any([tagadd.startswith(look) for tagadd in tagadds]):
+                continue
+
+            yield ((nodeoffs, editoffs), ('node', 'tag', 'add', *tag.split('.')))
+            tagadds.append(tag)

synapse/lib/oauth.py

@@ -230,13 +230,7 @@ class OAuthMixin(s_nexus.Pusher):
 
         timeout = aiohttp.ClientTimeout(total=DEFAULT_TIMEOUT)
 
-        cadir = self.conf.get('tls:ca:dir')
-        if ssl_verify is False:
-            ssl = False
-        elif cadir:
-            ssl = s_common.getSslCtx(cadir)
-        else:
-            ssl = None
+        ssl = self.getCachedSslCtx(verify=ssl_verify)
 
         async with aiohttp.ClientSession(timeout=timeout) as sess:
 

synapse/lib/rstorm.py

@@ -29,6 +29,7 @@ re_directive = regex.compile(r'^\.\.\s(storm.*|[^:])::(?:\s(.*)$|$)')
 
 logger = logging.getLogger(__name__)
 
+ONLOAD_TIMEOUT = int(os.getenv('SYNDEV_PKG_LOAD_TIMEOUT', 30))  # seconds
 
 class OutPutRst(s_output.OutPutStr):
     '''
@@ -413,8 +414,17 @@ class StormRst(s_base.Base):
         core = self._reqCore()
 
         pkg = s_genpkg.loadPkgProto(text)
+
+        if pkg.get('onload') is not None:
+            waiter = core.waiter(1, 'core:pkg:onload:complete')
+        else:
+            waiter = None
+
         await core.addStormPkg(pkg)
 
+        if waiter is not None and not await waiter.wait(timeout=ONLOAD_TIMEOUT):
+            raise s_exc.SynErr(mesg=f'Package onload failed to run for {pkg.get("name")}')
+
     async def _handleStormPre(self, text):
         '''
         Run a Storm query to prepare the Cortex without output.
@@ -453,6 +463,9 @@ class StormRst(s_base.Base):
 
         svc = await self._getCell(ctor, conf=svcconf)
 
+        onloadcnt = len([p for p in svc.cellapi._storm_svc_pkgs if p.get('onload') is not None])
+        waiter = core.waiter(onloadcnt, 'core:pkg:onload:complete') if onloadcnt else None
+
         svc.dmon.share('svc', svc)
         root = await svc.auth.getUserByName('root')
         await root.setPasswd('root')
@@ -463,6 +476,9 @@ class StormRst(s_base.Base):
         await core.nodes(f'service.add {svcname} {surl}')
         await core.nodes(f'$lib.service.wait({svcname})')
 
+        if waiter is not None and not await waiter.wait(timeout=ONLOAD_TIMEOUT):
+            raise s_exc.SynErr(mesg=f'Package onload failed to run for service {svcname}')
+
     async def _handleStormFail(self, text):
         valu = json.loads(text)
         assert valu in (True, False), f'storm-fail must be a boolean: {text}'

synapse/lib/schemas.py

@@ -263,3 +263,13 @@ _cellUserApiKeySchema = {
     ],
 }
 reqValidUserApiKeyDef = s_config.getJsValidator(_cellUserApiKeySchema)
+
+reqValidSslCtxOpts = s_config.getJsValidator({
+    'type': 'object',
+    'properties': {
+        'verify': {'type': 'boolean', 'default': True},
+        'client_cert': {'type': ['string', 'null'], 'default': None},
+        'client_key': {'type': ['string', 'null'], 'default': None},
+    },
+    'additionalProperties': False,
+})

synapse/lib/storm.py

@@ -181,7 +181,7 @@ wgetdescr = '''Retrieve bytes from a URL and store them in the axon. Yields inet
 Examples:
 
     # Specify custom headers and parameters
-    inet:url=https://vertex.link/foo.bar.txt | wget --headers $lib.dict("User-Agent"="Foo/Bar") --params $lib.dict("clientid"="42")
+    inet:url=https://vertex.link/foo.bar.txt | wget --headers ({"User-Agent": "Foo/Bar"}) --params ({"clientid": "42"})
 
     # Download multiple URL targets without inbound nodes
     wget https://vertex.link https://vtx.lk
@@ -3787,7 +3787,23 @@ class MergeCmd(Cmd):
             runt.confirmPropDel(prop, layriden=layr0)
             runt.confirmPropSet(prop, layriden=layr1)
 
+        tags = []
+        tagadds = []
         for tag, valu in sode.get('tags', {}).items():
+            if valu != (None, None):
+                tagadds.append(tag)
+                tagperm = tuple(tag.split('.'))
+                runt.confirm(('node', 'tag', 'del') + tagperm, gateiden=layr0)
+                runt.confirm(('node', 'tag', 'add') + tagperm, gateiden=layr1)
+            else:
+                tags.append((len(tag), tag))
+
+        for _, tag in sorted(tags, reverse=True):
+            look = tag + '.'
+            if any([tagadd.startswith(look) for tagadd in tagadds]):
+                continue
+
+            tagadds.append(tag)
             tagperm = tuple(tag.split('.'))
             runt.confirm(('node', 'tag', 'del') + tagperm, gateiden=layr0)
             runt.confirm(('node', 'tag', 'add') + tagperm, gateiden=layr1)