synapse 2.191.0__py311-none-any.whl → 2.193.0__py311-none-any.whl

synapse/axon.py

@@ -594,7 +594,7 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
         return await self.cell.dels(sha256s)
 
     async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET',
-                   ssl=True, timeout=None, proxy=None, ssl_opts=None):
+                   ssl=True, timeout=None, proxy=True, ssl_opts=None):
         '''
         Stream a file download directly into the Axon.
 
@@ -607,6 +607,7 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
             method (str): The HTTP method to use.
             ssl (bool): Perform SSL verification.
             timeout (int): The timeout of the request, in seconds.
+            proxy (str|bool): The proxy value.
             ssl_opts (dict): Additional SSL/TLS options.
 
         Notes:
@@ -621,6 +622,13 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
                     'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
                 }
 
+            The following proxy arguments are supported::
+
+                None: Deprecated - Use the proxy defined by the http:proxy configuration option if set.
+                True: Use the proxy defined by the http:proxy configuration option if set.
+                False: Do not use the proxy defined by the http:proxy configuration option if set.
+                <str>: A proxy URL string.
+
             The dictionary returned by this may contain the following values::
 
                 {
@@ -654,13 +662,13 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
                                     ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts)
 
     async def postfiles(self, fields, url, params=None, headers=None, method='POST',
-                        ssl=True, timeout=None, proxy=None, ssl_opts=None):
+                        ssl=True, timeout=None, proxy=True, ssl_opts=None):
         await self._reqUserAllowed(('axon', 'wput'))
         return await self.cell.postfiles(fields, url, params=params, headers=headers, method=method,
                                          ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts)
 
     async def wput(self, sha256, url, params=None, headers=None, method='PUT',
-                   ssl=True, timeout=None, proxy=None, ssl_opts=None):
+                   ssl=True, timeout=None, proxy=True, ssl_opts=None):
         await self._reqUserAllowed(('axon', 'wput'))
         return await self.cell.wput(sha256, url, params=params, headers=headers, method=method,
                                     ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts)
@@ -935,6 +943,24 @@ class Axon(s_cell.Cell):
         self.axonhist.add(item, tick=tick)
         self.axonseqn.add(item)
 
+    async def _resolveProxyUrl(self, valu):
+        match valu:
+            case None:
+                s_common.deprecated('Setting the Axon HTTP proxy argument to None', curv='2.192.0')
+                return await self.getConfOpt('http:proxy')
+
+            case True:
+                return await self.getConfOpt('http:proxy')
+
+            case False:
+                return None
+
+            case str():
+                return valu
+
+            case _:
+                raise s_exc.BadArg(mesg='HTTP proxy argument must be a string or bool.')
+
     async def _reqHas(self, sha256):
         '''
         Ensure a file exists; and return its size if so.
@@ -1462,7 +1488,7 @@ class Axon(s_cell.Cell):
                                         sha256=sha256) from None
 
     async def postfiles(self, fields, url, params=None, headers=None, method='POST',
-                        ssl=True, timeout=None, proxy=None, ssl_opts=None):
+                        ssl=True, timeout=None, proxy=True, ssl_opts=None):
         '''
         Send files from the axon as fields in a multipart/form-data HTTP request.
 
@@ -1474,7 +1500,7 @@ class Axon(s_cell.Cell):
             method (str): The HTTP method to use.
             ssl (bool): Perform SSL verification.
             timeout (int): The timeout of the request, in seconds.
-            proxy (bool|str|null): Use a specific proxy or disable proxy use.
+            proxy (str|bool): The proxy value.
             ssl_opts (dict): Additional SSL/TLS options.
 
         Notes:
@@ -1497,6 +1523,13 @@ class Axon(s_cell.Cell):
                     'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
                 }
 
+            The following proxy arguments are supported::
+
+                None: Deprecated - Use the proxy defined by the http:proxy configuration option if set.
+                True: Use the proxy defined by the http:proxy configuration option if set.
+                False: Do not use the proxy defined by the http:proxy configuration option if set.
+                <str>: A proxy URL string.
+
             The dictionary returned by this may contain the following values::
 
                 {
@@ -1512,14 +1545,11 @@ class Axon(s_cell.Cell):
         Returns:
             dict: An information dictionary containing the results of the request.
         '''
-        if proxy is None:
-            proxy = self.conf.get('http:proxy')
-
         ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl)
 
         connector = None
-        if proxy:
-            connector = aiohttp_socks.ProxyConnector.from_url(proxy)
+        if proxyurl := await self._resolveProxyUrl(proxy):
+            connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
 
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
@@ -1583,18 +1613,15 @@ class Axon(s_cell.Cell):
                 }
 
     async def wput(self, sha256, url, params=None, headers=None, method='PUT', ssl=True, timeout=None,
-                   filename=None, filemime=None, proxy=None, ssl_opts=None):
+                   filename=None, filemime=None, proxy=True, ssl_opts=None):
         '''
         Stream a blob from the axon as the body of an HTTP request.
         '''
-        if proxy is None:
-            proxy = self.conf.get('http:proxy')
-
         ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl)
 
         connector = None
-        if proxy:
-            connector = aiohttp_socks.ProxyConnector.from_url(proxy)
+        if proxyurl := await self._resolveProxyUrl(proxy):
+            connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
 
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
@@ -1654,7 +1681,7 @@ class Axon(s_cell.Cell):
         return info
 
     async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET',
-                   ssl=True, timeout=None, proxy=None, ssl_opts=None):
+                   ssl=True, timeout=None, proxy=True, ssl_opts=None):
         '''
         Stream a file download directly into the Axon.
 
@@ -1667,7 +1694,7 @@ class Axon(s_cell.Cell):
             method (str): The HTTP method to use.
             ssl (bool): Perform SSL verification.
             timeout (int): The timeout of the request, in seconds.
-            proxy (bool|str|null): Use a specific proxy or disable proxy use.
+            proxy (str|bool): The proxy value.
             ssl_opts (dict): Additional SSL/TLS options.
 
         Notes:
@@ -1682,6 +1709,13 @@ class Axon(s_cell.Cell):
                     'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
                 }
 
+            The following proxy arguments are supported::
+
+                None: Deprecated - Use the proxy defined by the http:proxy configuration option if set.
+                True: Use the proxy defined by the http:proxy configuration option if set.
+                False: Do not use the proxy defined by the http:proxy configuration option if set.
+                <str>: A proxy URL string.
+
             The dictionary returned by this may contain the following values::
 
                 {
@@ -1712,14 +1746,11 @@ class Axon(s_cell.Cell):
         '''
         logger.debug(f'Wget called for [{url}].', extra=await self.getLogExtra(url=s_urlhelp.sanitizeUrl(url)))
 
-        if proxy is None:
-            proxy = self.conf.get('http:proxy')
-
         ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl)
 
         connector = None
-        if proxy:
-            connector = aiohttp_socks.ProxyConnector.from_url(proxy)
+        if proxyurl := await self._resolveProxyUrl(proxy):
+            connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
 
         atimeout = aiohttp.ClientTimeout(total=timeout)
 

synapse/common.py

@@ -29,6 +29,8 @@ import traceback
 import contextlib
 import collections
 
+import http.cookies
+
 import yaml
 import regex
 
@@ -38,6 +40,8 @@ import synapse.lib.msgpack as s_msgpack
 import synapse.lib.structlog as s_structlog
 
 import synapse.vendor.cpython.lib.ipaddress as ipaddress
+import synapse.vendor.cpython.lib.http.cookies as v_cookies
+
 
 try:
     from yaml import CSafeLoader as Loader
@@ -1218,6 +1222,17 @@ def trimText(text: str, n: int = 256, placeholder: str = '...') -> str:
     assert n > plen
     return f'{text[:mlen]}{placeholder}'
 
+def _patch_http_cookies():
+    '''
+    Patch stdlib http.cookies._unquote from the 3.11.10 implementation if
+    the interpreter we are using is not patched for CVE-2024-7592.
+    '''
+    if not hasattr(http.cookies, '_QuotePatt'):
+        return
+    http.cookies._unquote = v_cookies._unquote
+
+_patch_http_cookies()
+
 # TODO:  Switch back to using asyncio.wait_for when we are using py 3.12+
 # This is a workaround for a race where asyncio.wait_for can end up
 # ignoring cancellation https://github.com/python/cpython/issues/86296

synapse/cortex.py

@@ -1739,7 +1739,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         for filt in gdef.get('filters', ()):
             await self.getStormQuery(filt)
 
-        for pivo in gdef.get('filters', ()):
+        for pivo in gdef.get('pivots', ()):
             await self.getStormQuery(pivo)
 
         for form, rule in gdef.get('forms', {}).items():
@@ -1749,7 +1749,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             for filt in rule.get('filters', ()):
                 await self.getStormQuery(filt)
 
-            for pivo in rule.get('filters', ()):
+            for pivo in rule.get('pivots', ()):
                 await self.getStormQuery(pivo)
 
     async def addStormGraph(self, gdef, user=None):
@@ -2240,7 +2240,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         '''
         name = cdef.get('name')
-        await self._setStormCmd(cdef)
+        self._setStormCmd(cdef)
         self.cmddefs.set(name, cdef)
 
     async def _reqStormCmd(self, cdef):
@@ -2483,7 +2483,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             async for sodes in self._mergeSodes(layers, genrs, cmprkey_indx, filtercmpr, reverse=reverse):
                 yield sodes
 
-    async def _setStormCmd(self, cdef):
+    def _setStormCmd(self, cdef):
         '''
         Note:
             No change control or persistence
@@ -2543,13 +2543,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         name = cdef.get('name')
         self.stormcmds[name] = ctor
 
-        await self.fire('core:cmd:change', cmd=name, act='add')
-
-    async def _popStormCmd(self, name):
+    def _popStormCmd(self, name):
         self.stormcmds.pop(name, None)
 
-        await self.fire('core:cmd:change', cmd=name, act='del')
-
     async def delStormCmd(self, name):
         '''
         Remove a previously set pure storm command.
@@ -2575,8 +2571,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         self.cmddefs.pop(name)
         self.stormcmds.pop(name, None)
 
-        await self.fire('core:cmd:change', cmd=name, act='del')
-
     async def addStormPkg(self, pkgdef, verify=False):
         '''
         Add the given storm package to the cortex.
@@ -2630,11 +2624,11 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         olddef = self.pkgdefs.get(name, None)
         if olddef is not None:
             if s_hashitem.hashitem(pkgdef) != s_hashitem.hashitem(olddef):
-                await self._dropStormPkg(olddef)
+                self._dropStormPkg(olddef)
             else:
                 return
 
-        await self.loadStormPkg(pkgdef)
+        self.loadStormPkg(pkgdef)
         self.pkgdefs.set(name, pkgdef)
 
         self._clearPermDefs()
@@ -2664,7 +2658,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         if pkgdef is None:
             return
 
-        await self._dropStormPkg(pkgdef)
+        self._dropStormPkg(pkgdef)
 
         self._clearPermDefs()
 
@@ -2713,7 +2707,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     async def _tryLoadStormPkg(self, pkgdef):
         try:
             await self._normStormPkg(pkgdef, validstorm=False)
-            await self.loadStormPkg(pkgdef)
+            self.loadStormPkg(pkgdef)
 
         except asyncio.CancelledError:  # pragma: no cover  TODO:  remove once >= py 3.8 only
             raise
@@ -2881,7 +2875,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         for configvar in pkgdef.get('configvars', ()):
             self._reqStormPkgVarType(pkgname, configvar.get('type'))
 
-    async def loadStormPkg(self, pkgdef):
+    # N.B. This function is intentionally not async in order to prevent possible user race conditions for code
+    # executing outside of the nexus lock.
+    def loadStormPkg(self, pkgdef):
         '''
         Load a storm package into the storm library for this cortex.
 
@@ -2911,7 +2907,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         self.stormmods = stormmods
 
         for cdef in cmds:
-            await self._setStormCmd(cdef)
+            self._setStormCmd(cdef)
 
         for gdef in pkgdef.get('graphs', ()):
             gdef = copy.deepcopy(gdef)
@@ -2937,7 +2933,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                 await self.fire('core:pkg:onload:complete', pkg=name)
             self.schedCoro(_onload())
 
-    async def _dropStormPkg(self, pkgdef):
+    # N.B. This function is intentionally not async in order to prevent possible user race conditions for code
+    # executing outside of the nexus lock.
+    def _dropStormPkg(self, pkgdef):
         '''
         Reverse the process of loadStormPkg()
         '''
@@ -2948,7 +2946,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         for cdef in pkgdef.get('commands', ()):
             name = cdef.get('name')
-            await self._popStormCmd(name)
+            self._popStormCmd(name)
 
         pkgname = pkgdef.get('name')
 
@@ -4435,7 +4433,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
     async def _trySetStormCmd(self, name, cdef):
         try:
-            await self._setStormCmd(cdef)
+            self._setStormCmd(cdef)
         except (asyncio.CancelledError, Exception):
             logger.exception(f'Storm command load failed: {name}')
 

synapse/exc.py

@@ -56,9 +56,14 @@ class SynErr(Exception):
         self.errinfo[name] = valu
         self._setExcMesg()
 
+    def update(self, items: dict):
+        '''Update multiple items in the errinfo dict at once.'''
+        self.errinfo.update(**items)
+        self._setExcMesg()
+
 class StormRaise(SynErr):
     '''
-    This represents a user provided exception inside of a Storm runtime. It requires a errname key.
+    This represents a user provided exception raised in the Storm runtime. It requires a errname key.
     '''
     def __init__(self, *args, **info):
         SynErr.__init__(self, *args, **info)

synapse/lib/agenda.py

@@ -5,9 +5,7 @@ import asyncio
 import logging
 import calendar
 import datetime
-import functools
 import itertools
-import collections
 from datetime import timezone as tz
 from collections.abc import Iterable, Mapping
 

synapse/lib/ast.py

@@ -59,7 +59,8 @@ class AstNode:
         }
 
     def addExcInfo(self, exc):
-        exc.errinfo['highlight'] = self.getPosInfo()
+        if 'highlight' not in exc.errinfo:
+            exc.set('highlight', self.getPosInfo())
         return exc
 
     def repr(self):
@@ -3554,7 +3555,10 @@ class VarDeref(Value):
 
         valu = s_stormtypes.fromprim(base, path=path)
         with s_scope.enter({'runt': runt}):
-            return await valu.deref(name)
+            try:
+                return await valu.deref(name)
+            except s_exc.SynErr as e:
+                raise self.kids[1].addExcInfo(e)
 
 class FuncCall(Value):
 
@@ -3575,10 +3579,23 @@ class FuncCall(Value):
         kwargs = {k: v for (k, v) in await self.kids[2].compute(runt, path)}
 
         with s_scope.enter({'runt': runt}):
-            retn = func(*argv, **kwargs)
-            if s_coro.iscoro(retn):
-                return await retn
-            return retn
+            try:
+                retn = func(*argv, **kwargs)
+                if s_coro.iscoro(retn):
+                    return await retn
+                return retn
+
+            except TypeError as e:
+                mesg = str(e)
+                if (funcpath := getattr(func, '_storm_funcpath', None)) is not None:
+                    mesg = f"{funcpath}(){mesg.split(')', 1)[1]}"
+
+                raise self.addExcInfo(s_exc.StormRuntimeError(mesg=mesg))
+
+            except s_exc.SynErr as e:
+                if getattr(func, '_storm_runtime_lib_func', None) is not None:
+                    e.errinfo.pop('highlight', None)
+                raise self.addExcInfo(e)
 
 class DollarExpr(Value):
     '''
@@ -4887,8 +4904,9 @@ class Function(AstNode):
 
             @s_stormtypes.stormfunc(readonly=True)
             async def realfunc(*args, **kwargs):
-                return await self.callfunc(runt, argdefs, args, kwargs)
+                return await self.callfunc(runt, argdefs, args, kwargs, realfunc._storm_funcpath)
 
+            realfunc._storm_funcpath = self.name
             await runt.setVar(self.name, realfunc)
 
         count = 0
@@ -4910,7 +4928,7 @@ class Function(AstNode):
         # var scope validation occurs in the sub-runtime
         pass
 
-    async def callfunc(self, runt, argdefs, args, kwargs):
+    async def callfunc(self, runt, argdefs, args, kwargs, funcpath):
         '''
         Execute a function call using the given runtime.
 
@@ -4921,7 +4939,7 @@ class Function(AstNode):
 
         argcount = len(args) + len(kwargs)
         if argcount > len(argdefs):
-            mesg = f'{self.name}() takes {len(argdefs)} arguments but {argcount} were provided'
+            mesg = f'{funcpath}() takes {len(argdefs)} arguments but {argcount} were provided'
             raise self.kids[1].addExcInfo(s_exc.StormRuntimeError(mesg=mesg))
 
         # Fill in the positional arguments
@@ -4935,7 +4953,7 @@ class Function(AstNode):
             valu = kwargs.pop(name, s_common.novalu)
             if valu is s_common.novalu:
                 if defv is s_common.novalu:
-                    mesg = f'{self.name}() missing required argument {name}'
+                    mesg = f'{funcpath}() missing required argument {name}'
                     raise self.kids[1].addExcInfo(s_exc.StormRuntimeError(mesg=mesg))
                 valu = defv
 
@@ -4946,11 +4964,11 @@ class Function(AstNode):
             # used a kwarg not defined.
             kwkeys = list(kwargs.keys())
             if kwkeys[0] in posnames:
-                mesg = f'{self.name}() got multiple values for parameter {kwkeys[0]}'
+                mesg = f'{funcpath}() got multiple values for parameter {kwkeys[0]}'
                 raise self.kids[1].addExcInfo(s_exc.StormRuntimeError(mesg=mesg))
 
             plural = 's' if len(kwargs) > 1 else ''
-            mesg = f'{self.name}() got unexpected keyword argument{plural}: {",".join(kwkeys)}'
+            mesg = f'{funcpath}() got unexpected keyword argument{plural}: {",".join(kwkeys)}'
             raise self.kids[1].addExcInfo(s_exc.StormRuntimeError(mesg=mesg))
 
         assert len(mergargs) == len(argdefs)