synapse 2.191.0__py311-none-any.whl → 2.192.0__py311-none-any.whl

synapse/axon.py

@@ -594,7 +594,7 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
         return await self.cell.dels(sha256s)
 
     async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET',
-                   ssl=True, timeout=None, proxy=None, ssl_opts=None):
+                   ssl=True, timeout=None, proxy=True, ssl_opts=None):
         '''
         Stream a file download directly into the Axon.
 
@@ -607,6 +607,7 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
             method (str): The HTTP method to use.
             ssl (bool): Perform SSL verification.
             timeout (int): The timeout of the request, in seconds.
+            proxy (str|bool): The proxy value.
             ssl_opts (dict): Additional SSL/TLS options.
 
         Notes:
@@ -621,6 +622,13 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
                     'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
                 }
 
+            The following proxy arguments are supported::
+
+                None: Deprecated - Use the proxy defined by the http:proxy configuration option if set.
+                True: Use the proxy defined by the http:proxy configuration option if set.
+                False: Do not use the proxy defined by the http:proxy configuration option if set.
+                <str>: A proxy URL string.
+
             The dictionary returned by this may contain the following values::
 
                 {
@@ -654,13 +662,13 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
                                     ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts)
 
     async def postfiles(self, fields, url, params=None, headers=None, method='POST',
-                        ssl=True, timeout=None, proxy=None, ssl_opts=None):
+                        ssl=True, timeout=None, proxy=True, ssl_opts=None):
         await self._reqUserAllowed(('axon', 'wput'))
         return await self.cell.postfiles(fields, url, params=params, headers=headers, method=method,
                                          ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts)
 
     async def wput(self, sha256, url, params=None, headers=None, method='PUT',
-                   ssl=True, timeout=None, proxy=None, ssl_opts=None):
+                   ssl=True, timeout=None, proxy=True, ssl_opts=None):
         await self._reqUserAllowed(('axon', 'wput'))
         return await self.cell.wput(sha256, url, params=params, headers=headers, method=method,
                                     ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts)
@@ -935,6 +943,24 @@ class Axon(s_cell.Cell):
         self.axonhist.add(item, tick=tick)
         self.axonseqn.add(item)
 
+    async def _resolveProxyUrl(self, valu):
+        match valu:
+            case None:
+                s_common.deprecated('Setting the Axon HTTP proxy argument to None', curv='2.192.0')
+                return await self.getConfOpt('http:proxy')
+
+            case True:
+                return await self.getConfOpt('http:proxy')
+
+            case False:
+                return None
+
+            case str():
+                return valu
+
+            case _:
+                raise s_exc.BadArg(mesg='HTTP proxy argument must be a string or bool.')
+
     async def _reqHas(self, sha256):
         '''
         Ensure a file exists; and return its size if so.
@@ -1462,7 +1488,7 @@ class Axon(s_cell.Cell):
                                         sha256=sha256) from None
 
     async def postfiles(self, fields, url, params=None, headers=None, method='POST',
-                        ssl=True, timeout=None, proxy=None, ssl_opts=None):
+                        ssl=True, timeout=None, proxy=True, ssl_opts=None):
         '''
         Send files from the axon as fields in a multipart/form-data HTTP request.
 
@@ -1474,7 +1500,7 @@ class Axon(s_cell.Cell):
             method (str): The HTTP method to use.
             ssl (bool): Perform SSL verification.
             timeout (int): The timeout of the request, in seconds.
-            proxy (bool|str|null): Use a specific proxy or disable proxy use.
+            proxy (str|bool): The proxy value.
             ssl_opts (dict): Additional SSL/TLS options.
 
         Notes:
@@ -1497,6 +1523,13 @@ class Axon(s_cell.Cell):
                     'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
                 }
 
+            The following proxy arguments are supported::
+
+                None: Deprecated - Use the proxy defined by the http:proxy configuration option if set.
+                True: Use the proxy defined by the http:proxy configuration option if set.
+                False: Do not use the proxy defined by the http:proxy configuration option if set.
+                <str>: A proxy URL string.
+
             The dictionary returned by this may contain the following values::
 
                 {
@@ -1512,14 +1545,11 @@ class Axon(s_cell.Cell):
         Returns:
             dict: An information dictionary containing the results of the request.
         '''
-        if proxy is None:
-            proxy = self.conf.get('http:proxy')
-
         ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl)
 
         connector = None
-        if proxy:
-            connector = aiohttp_socks.ProxyConnector.from_url(proxy)
+        if proxyurl := await self._resolveProxyUrl(proxy):
+            connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
 
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
@@ -1583,18 +1613,15 @@ class Axon(s_cell.Cell):
                 }
 
     async def wput(self, sha256, url, params=None, headers=None, method='PUT', ssl=True, timeout=None,
-                   filename=None, filemime=None, proxy=None, ssl_opts=None):
+                   filename=None, filemime=None, proxy=True, ssl_opts=None):
         '''
         Stream a blob from the axon as the body of an HTTP request.
         '''
-        if proxy is None:
-            proxy = self.conf.get('http:proxy')
-
         ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl)
 
         connector = None
-        if proxy:
-            connector = aiohttp_socks.ProxyConnector.from_url(proxy)
+        if proxyurl := await self._resolveProxyUrl(proxy):
+            connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
 
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
@@ -1654,7 +1681,7 @@ class Axon(s_cell.Cell):
         return info
 
     async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET',
-                   ssl=True, timeout=None, proxy=None, ssl_opts=None):
+                   ssl=True, timeout=None, proxy=True, ssl_opts=None):
         '''
         Stream a file download directly into the Axon.
 
@@ -1667,7 +1694,7 @@ class Axon(s_cell.Cell):
             method (str): The HTTP method to use.
             ssl (bool): Perform SSL verification.
             timeout (int): The timeout of the request, in seconds.
-            proxy (bool|str|null): Use a specific proxy or disable proxy use.
+            proxy (str|bool): The proxy value.
             ssl_opts (dict): Additional SSL/TLS options.
 
         Notes:
@@ -1682,6 +1709,13 @@ class Axon(s_cell.Cell):
                     'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
                 }
 
+            The following proxy arguments are supported::
+
+                None: Deprecated - Use the proxy defined by the http:proxy configuration option if set.
+                True: Use the proxy defined by the http:proxy configuration option if set.
+                False: Do not use the proxy defined by the http:proxy configuration option if set.
+                <str>: A proxy URL string.
+
             The dictionary returned by this may contain the following values::
 
                 {
@@ -1712,14 +1746,11 @@ class Axon(s_cell.Cell):
         '''
         logger.debug(f'Wget called for [{url}].', extra=await self.getLogExtra(url=s_urlhelp.sanitizeUrl(url)))
 
-        if proxy is None:
-            proxy = self.conf.get('http:proxy')
-
         ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl)
 
         connector = None
-        if proxy:
-            connector = aiohttp_socks.ProxyConnector.from_url(proxy)
+        if proxyurl := await self._resolveProxyUrl(proxy):
+            connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
 
         atimeout = aiohttp.ClientTimeout(total=timeout)
 

synapse/cortex.py

@@ -1739,7 +1739,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         for filt in gdef.get('filters', ()):
             await self.getStormQuery(filt)
 
-        for pivo in gdef.get('filters', ()):
+        for pivo in gdef.get('pivots', ()):
             await self.getStormQuery(pivo)
 
         for form, rule in gdef.get('forms', {}).items():
@@ -1749,7 +1749,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             for filt in rule.get('filters', ()):
                 await self.getStormQuery(filt)
 
-            for pivo in rule.get('filters', ()):
+            for pivo in rule.get('pivots', ()):
                 await self.getStormQuery(pivo)
 
     async def addStormGraph(self, gdef, user=None):

synapse/lib/ast.py

@@ -59,7 +59,8 @@ class AstNode:
         }
 
     def addExcInfo(self, exc):
-        exc.errinfo['highlight'] = self.getPosInfo()
+        if 'highlight' not in exc.errinfo:
+            exc.errinfo['highlight'] = self.getPosInfo()
         return exc
 
     def repr(self):
@@ -3554,7 +3555,10 @@ class VarDeref(Value):
 
         valu = s_stormtypes.fromprim(base, path=path)
         with s_scope.enter({'runt': runt}):
-            return await valu.deref(name)
+            try:
+                return await valu.deref(name)
+            except s_exc.SynErr as e:
+                raise self.kids[1].addExcInfo(e)
 
 class FuncCall(Value):
 

synapse/lib/cell.py

@@ -459,6 +459,43 @@ class CellApi(s_base.Base):
     async def delRole(self, iden):
         return await self.cell.delRole(iden)
 
+    async def addUserApiKey(self, name, duration=None, useriden=None):
+        if useriden is None:
+            useriden = self.user.iden
+
+        if self.user.iden == useriden:
+            self.user.confirm(('auth', 'self', 'set', 'apikey'), default=True)
+        else:
+            self.user.confirm(('auth', 'user', 'set', 'apikey'))
+
+        return await self.cell.addUserApiKey(useriden, name, duration=duration)
+
+    async def listUserApiKeys(self, useriden=None):
+        if useriden is None:
+            useriden = self.user.iden
+
+        if self.user.iden == useriden:
+            self.user.confirm(('auth', 'self', 'set', 'apikey'), default=True)
+        else:
+            self.user.confirm(('auth', 'user', 'set', 'apikey'))
+
+        return await self.cell.listUserApiKeys(useriden)
+
+    async def delUserApiKey(self, iden):
+        apikey = await self.cell.getUserApiKey(iden)
+        if apikey is None:
+            mesg = f'User API key with {iden=} does not exist.'
+            raise s_exc.NoSuchIden(mesg=mesg, iden=iden)
+
+        useriden = apikey.get('user')
+
+        if self.user.iden == useriden:
+            self.user.confirm(('auth', 'self', 'set', 'apikey'), default=True)
+        else:
+            self.user.confirm(('auth', 'user', 'set', 'apikey'))
+
+        return await self.cell.delUserApiKey(iden)
+
     @adminapi()
     async def dyncall(self, iden, todo, gatekeys=()):
         return await self.cell.dyncall(iden, todo, gatekeys=gatekeys)
@@ -1095,6 +1132,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
     }
     SYSCTL_CHECK_FREQ = 60.0
 
+    LOGGED_HTTPAPI_HEADERS = ('User-Agent',)
+
     async def __anit__(self, dirn, conf=None, readonly=False, parent=None):
 
         # phase 1
@@ -3248,6 +3287,15 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 'remoteip': remote_ip,
                 }
 
+        headers = {}
+
+        for header in self.LOGGED_HTTPAPI_HEADERS:
+            if (valu := handler.request.headers.get(header)) is not None:
+                headers[header.lower()] = valu
+
+        if headers:
+            enfo['headers'] = headers
+
         extra = {'synapse': enfo}
 
         # It is possible that a Cell implementor may register handlers which

synapse/lib/nexus.py

@@ -505,6 +505,7 @@ class NexsRoot(s_base.Base):
             if features.get('dynmirror'):
                 await proxy.readyToMirror()
 
+            synvers = cellinfo['synapse']['version']
             cellvers = cellinfo['cell']['version']
             if cellvers > self.cell.VERSION:
                 logger.error('Leader is a higher version than we are. Mirrors must be updated first. Entering read-only mode.')
@@ -537,7 +538,7 @@ class NexsRoot(s_base.Base):
                 offs = self.nexslog.index()
 
                 opts = {}
-                if cellvers >= (2, 95, 0):
+                if synvers >= (2, 95, 0):
                     opts['tellready'] = True
 
                 genr = proxy.getNexusChanges(offs, **opts)

synapse/lib/stormhttp.py

@@ -91,12 +91,19 @@ class LibHttp(s_stormtypes.Lib):
 
     For APIs that accept an ssl_opts argument, the dictionary may contain the following values::
 
-        {
+        ({
             'verify': <bool> - Perform SSL/TLS verification. Is overridden by the ssl_verify argument.
             'client_cert': <str> - PEM encoded full chain certificate for use in mTLS.
             'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
             'ca_cert': <str> - A PEM encoded full chain CA certificate for use when verifying the request.
-        }
+        })
+
+    For APIs that accept a proxy argument, the following values are supported::
+
+        $lib.null: Deprecated - Use the proxy defined by the http:proxy configuration option if set.
+        $lib.true: Use the proxy defined by the http:proxy configuration option if set.
+        $lib.false: Do not use the proxy defined by the http:proxy configuration option if set.
+        <str>: A proxy URL string.
     '''
     _storm_locals = (
         {'name': 'get', 'desc': 'Get the contents of a given URL.',
@@ -113,8 +120,8 @@ class LibHttp(s_stormtypes.Lib):
                        'default': 300},
                       {'name': 'allow_redirects', 'type': 'bool', 'desc': 'If set to false, do not follow redirects.',
                        'default': True},
-                      {'name': 'proxy', 'type': ['bool', 'null', 'str'],
-                       'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
+                      {'name': 'proxy', 'type': ['bool', 'str'],
+                       'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
                       {'name': 'ssl_opts', 'type': 'dict',
                        'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
                        'default': None},
@@ -145,8 +152,8 @@ class LibHttp(s_stormtypes.Lib):
                                'and the corresponding file will be uploaded as the value for '
                                'the field.',
                        'default': None},
-                      {'name': 'proxy', 'type': ['bool', 'null', 'str'],
-                       'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
+                      {'name': 'proxy', 'type': ['bool', 'str'],
+                       'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
                       {'name': 'ssl_opts', 'type': 'dict',
                        'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
                        'default': None},
@@ -167,8 +174,8 @@ class LibHttp(s_stormtypes.Lib):
                        'default': 300, },
                       {'name': 'allow_redirects', 'type': 'bool', 'desc': 'If set to true, follow redirects.',
                        'default': False},
-                      {'name': 'proxy', 'type': ['bool', 'null', 'str'],
-                       'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
+                      {'name': 'proxy', 'type': ['bool', 'str'],
+                       'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
                       {'name': 'ssl_opts', 'type': 'dict',
                        'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
                        'default': None},
@@ -200,8 +207,8 @@ class LibHttp(s_stormtypes.Lib):
                                'and the corresponding file will be uploaded as the value for '
                                'the field.',
                        'default': None},
-                      {'name': 'proxy', 'type': ['bool', 'null', 'str'],
-                       'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
+                      {'name': 'proxy', 'type': ['bool', 'str'],
+                       'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
                       {'name': 'ssl_opts', 'type': 'dict',
                        'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
                        'default': None},
@@ -221,8 +228,8 @@ class LibHttp(s_stormtypes.Lib):
                        'default': 300},
                       {'name': 'params', 'type': 'dict', 'desc': 'Optional parameters which may be passed to the connection request.',
                        'default': None},
-                      {'name': 'proxy', 'type': ['bool', 'null', 'str'],
-                       'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
+                      {'name': 'proxy', 'type': ['bool', 'str'],
+                       'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
                       {'name': 'ssl_opts', 'type': 'dict',
                        'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
                        'default': None},
@@ -308,23 +315,23 @@ class LibHttp(s_stormtypes.Lib):
         return s_common.httpcodereason(code)
 
     async def _httpEasyHead(self, url, headers=None, ssl_verify=True, params=None, timeout=300,
-                            allow_redirects=False, proxy=None, ssl_opts=None):
+                            allow_redirects=False, proxy=True, ssl_opts=None):
         return await self._httpRequest('HEAD', url, headers=headers, ssl_verify=ssl_verify, params=params,
                                        timeout=timeout, allow_redirects=allow_redirects, proxy=proxy, ssl_opts=ssl_opts)
 
     async def _httpEasyGet(self, url, headers=None, ssl_verify=True, params=None, timeout=300,
-                           allow_redirects=True, proxy=None, ssl_opts=None):
+                           allow_redirects=True, proxy=True, ssl_opts=None):
         return await self._httpRequest('GET', url, headers=headers, ssl_verify=ssl_verify, params=params,
                                        timeout=timeout, allow_redirects=allow_redirects, proxy=proxy, ssl_opts=ssl_opts)
 
     async def _httpPost(self, url, headers=None, json=None, body=None, ssl_verify=True,
-                        params=None, timeout=300, allow_redirects=True, fields=None, proxy=None, ssl_opts=None):
+                        params=None, timeout=300, allow_redirects=True, fields=None, proxy=True, ssl_opts=None):
         return await self._httpRequest('POST', url, headers=headers, json=json, body=body,
                                        ssl_verify=ssl_verify, params=params, timeout=timeout,
                                        allow_redirects=allow_redirects, fields=fields, proxy=proxy, ssl_opts=ssl_opts)
 
     async def inetHttpConnect(self, url, headers=None, ssl_verify=True, timeout=300,
-                              params=None, proxy=None, ssl_opts=None):
+                              params=None, proxy=True, ssl_opts=None):
 
         url = await s_stormtypes.tostr(url)
         headers = await s_stormtypes.toprim(headers)
@@ -338,15 +345,9 @@ class LibHttp(s_stormtypes.Lib):
 
         sock = await WebSocket.anit()
 
-        if proxy is not None:
-            self.runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy'))
-
-        if proxy is None:
-            proxy = await self.runt.snap.core.getConfOpt('http:proxy')
-
         connector = None
-        if proxy:
-            connector = aiohttp_socks.ProxyConnector.from_url(proxy)
+        if proxyurl := await s_stormtypes.resolveCoreProxyUrl(proxy):
+            connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
 
         timeout = aiohttp.ClientTimeout(total=timeout)
         kwargs = {'timeout': timeout}
@@ -387,7 +388,7 @@ class LibHttp(s_stormtypes.Lib):
 
     async def _httpRequest(self, meth, url, headers=None, json=None, body=None,
                            ssl_verify=True, params=None, timeout=300, allow_redirects=True,
-                           fields=None, proxy=None, ssl_opts=None):
+                           fields=None, proxy=True, ssl_opts=None):
         meth = await s_stormtypes.tostr(meth)
         url = await s_stormtypes.tostr(url)
         json = await s_stormtypes.toprim(json)
@@ -407,19 +408,18 @@ class LibHttp(s_stormtypes.Lib):
 
         headers = s_stormtypes.strifyHttpArg(headers)
 
-        if proxy is not None:
-            self.runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy'))
-
         if fields:
             if any(['sha256' in field for field in fields]):
                 self.runt.confirm(('storm', 'lib', 'axon', 'wput'))
 
                 kwargs = {}
-                axonvers = self.runt.snap.core.axoninfo['synapse']['version']
-                if axonvers >= s_stormtypes.AXON_MINVERS_PROXY:
+
+                ok, proxy = await s_stormtypes.resolveAxonProxyArg(proxy)
+                if ok:
                     kwargs['proxy'] = proxy
 
                 if ssl_opts is not None:
+                    axonvers = self.runt.snap.core.axoninfo['synapse']['version']
                     mesg = f'The ssl_opts argument requires an Axon Synapse version {s_stormtypes.AXON_MINVERS_SSLOPTS}, ' \
                            f'but the Axon is running {axonvers}'
                     s_version.reqVersion(axonvers, s_stormtypes.AXON_MINVERS_SSLOPTS, mesg=mesg)
@@ -432,12 +432,9 @@ class LibHttp(s_stormtypes.Lib):
 
         kwargs['ssl'] = self.runt.snap.core.getCachedSslCtx(opts=ssl_opts, verify=ssl_verify)
 
-        if proxy is None:
-            proxy = await self.runt.snap.core.getConfOpt('http:proxy')
-
         connector = None
-        if proxy:
-            connector = aiohttp_socks.ProxyConnector.from_url(proxy)
+        if proxyurl := await s_stormtypes.resolveCoreProxyUrl(proxy):
+            connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
 
         timeout = aiohttp.ClientTimeout(total=timeout)