synapse 2.191.0__py311-none-any.whl → 2.193.0__py311-none-any.whl

synapse/lib/stormtypes.py

@@ -3,6 +3,7 @@ import copy
 import gzip
 import json
 import time
+
 import regex
 import types
 import base64
@@ -39,6 +40,7 @@ import synapse.lib.stormctrl as s_stormctrl
 logger = logging.getLogger(__name__)
 
 AXON_MINVERS_PROXY = (2, 97, 0)
+AXON_MINVERS_PROXYTRUE = (2, 192, 0)
 AXON_MINVERS_SSLOPTS = '>=2.162.0'
 
 class Undef:
@@ -73,6 +75,79 @@ def strifyHttpArg(item, multi=False):
         return retn
     return item
 
+async def resolveCoreProxyUrl(valu):
+    '''
+    Resolve a proxy value to a proxy URL.
+
+    Args:
+        valu (str|None|bool): The proxy value.
+
+    Returns:
+        (str|None): A proxy URL string or None.
+    '''
+    runt = s_scope.get('runt')
+
+    match valu:
+        case None:
+            s_common.deprecated('Setting the HTTP proxy argument $lib.null', curv='2.192.0')
+            await runt.snap.warnonce('Setting the HTTP proxy argument to $lib.null is deprecated. Use $lib.true instead.')
+            return await runt.snap.core.getConfOpt('http:proxy')
+
+        case True:
+            return await runt.snap.core.getConfOpt('http:proxy')
+
+        case False:
+            runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy'))
+            return None
+
+        case str():
+            runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy'))
+            return valu
+
+        case _:
+            raise s_exc.BadArg(mesg='HTTP proxy argument must be a string or bool.')
+
+async def resolveAxonProxyArg(valu):
+    '''
+    Resolve a proxy value to the kwarg to set for an Axon HTTP call.
+
+    Args:
+        valu (str|null|bool): The proxy value.
+
+    Returns:
+        tuple: A retn tuple where the proxy kwarg should not be set if ok=False, otherwise a proxy URL or None.
+    '''
+    runt = s_scope.get('runt')
+
+    axonvers = runt.snap.core.axoninfo['synapse']['version']
+    if axonvers < AXON_MINVERS_PROXY:
+        await runt.snap.warnonce(f'Axon version does not support proxy argument: {axonvers} < {AXON_MINVERS_PROXY}')
+        return False, None
+
+    match valu:
+        case None:
+            s_common.deprecated('Setting the Storm HTTP proxy argument $lib.null', curv='2.192.0')
+            await runt.snap.warnonce('Setting the Storm HTTP proxy argument to $lib.null is deprecated. Use $lib.true instead.')
+            if axonvers >= AXON_MINVERS_PROXYTRUE:
+                return True, True
+            return True, None
+
+        case True:
+            if axonvers < AXON_MINVERS_PROXYTRUE:
+                return True, None
+            return True, True
+
+        case False:
+            runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy'))
+            return True, False
+
+        case str():
+            runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy'))
+            return True, valu
+
+        case _:
+            raise s_exc.BadArg(mesg='HTTP proxy argument must be a string or bool.')
+
 class StormTypesRegistry:
     # The following types are currently undefined.
     base_undefined_types = (
@@ -126,11 +201,29 @@ class StormTypesRegistry:
             raise Exception('no key!')
         self.addStormLib(path, ctor)
 
+        for info in ctor._storm_locals:
+            rtype = info.get('type')
+            if isinstance(rtype, dict) and rtype.get('type') == 'function':
+                if (fname := rtype.get('_funcname')) == '_storm_query':
+                    continue
+
+                if (func := getattr(ctor, fname, None)) is not None:
+                    funcpath = '.'.join(('lib',) + ctor._storm_lib_path + (info['name'],))
+                    func._storm_funcpath = f"${funcpath}"
+
         return ctor
 
     def registerType(self, ctor):
         '''Decorator to register a StormPrim'''
         self.addStormType(ctor.__name__, ctor)
+
+        for info in ctor._storm_locals:
+            rtype = info.get('type')
+            if isinstance(rtype, dict) and rtype.get('type') == 'function':
+                fname = rtype.get('_funcname')
+                if (func := getattr(ctor, fname, None)) is not None:
+                    func._storm_funcpath = f"{ctor._storm_typename}.{info['name']}"
+
         return ctor
 
     def iterLibs(self):
@@ -553,6 +646,7 @@ class Lib(StormType):
                 if callable(v) and v.__name__ == 'realfunc':
                     v._storm_runtime_lib = self
                     v._storm_runtime_lib_func = k
+                    v._storm_funcpath = f'${".".join(("lib",) + self.name + (k,))}'
 
                 self.locls[k] = v
 
@@ -1993,11 +2087,18 @@ class LibAxon(Lib):
 
     For APIs that accept an ssl_opts argument, the dictionary may contain the following values::
 
-        {
+        ({
             'verify': <bool> - Perform SSL/TLS verification. Is overridden by the ssl argument.
             'client_cert': <str> - PEM encoded full chain certificate for use in mTLS.
             'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
-        }
+        })
+
+    For APIs that accept a proxy argument, the following values are supported::
+
+        $lib.null: Deprecated - Use the proxy defined by the http:proxy configuration option if set.
+        $lib.true: Use the proxy defined by the http:proxy configuration option if set.
+        $lib.false: Do not use the proxy defined by the http:proxy configuration option if set.
+        <str>: A proxy URL string.
     '''
     _storm_locals = (
         {'name': 'wget', 'desc': """
@@ -2031,8 +2132,8 @@ class LibAxon(Lib):
                        'desc': 'Set to False to disable SSL/TLS certificate verification.', 'default': True},
                       {'name': 'timeout', 'type': 'int', 'desc': 'Timeout for the download operation.',
                        'default': None},
-                      {'name': 'proxy', 'type': ['boolean', 'null', 'str'],
-                       'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
+                      {'name': 'proxy', 'type': ['bool', 'str'],
+                       'desc': 'Configure proxy usage. See $lib.axon help for additional details.', 'default': True},
                       {'name': 'ssl_opts', 'type': 'dict',
                        'desc': 'Optional SSL/TLS options. See $lib.axon help for additional details.',
                        'default': None},
@@ -2054,8 +2155,8 @@ class LibAxon(Lib):
                        'desc': 'Set to False to disable SSL/TLS certificate verification.', 'default': True},
                       {'name': 'timeout', 'type': 'int', 'desc': 'Timeout for the download operation.',
                        'default': None},
-                      {'name': 'proxy', 'type': ['boolean', 'null', 'str'],
-                       'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
+                      {'name': 'proxy', 'type': ['bool', 'str'],
+                       'desc': 'Configure proxy usage. See $lib.axon help for additional details.', 'default': True},
                       {'name': 'ssl_opts', 'type': 'dict',
                        'desc': 'Optional SSL/TLS options. See $lib.axon help for additional details.',
                        'default': None},
@@ -2370,7 +2471,7 @@ class LibAxon(Lib):
         return await axon.del_(sha256b)
 
     async def wget(self, url, headers=None, params=None, method='GET', json=None, body=None,
-                   ssl=True, timeout=None, proxy=None, ssl_opts=None):
+                   ssl=True, timeout=None, proxy=True, ssl_opts=None):
 
         if not self.runt.allowed(('axon', 'wget')):
             self.runt.confirm(('storm', 'lib', 'axon', 'wget'))
@@ -2387,20 +2488,19 @@ class LibAxon(Lib):
         proxy = await toprim(proxy)
         ssl_opts = await toprim(ssl_opts)
 
-        if proxy is not None:
-            self.runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy'))
-
         params = strifyHttpArg(params, multi=True)
         headers = strifyHttpArg(headers)
 
         await self.runt.snap.core.getAxon()
 
         kwargs = {}
-        axonvers = self.runt.snap.core.axoninfo['synapse']['version']
-        if axonvers >= AXON_MINVERS_PROXY:
+
+        ok, proxy = await resolveAxonProxyArg(proxy)
+        if ok:
             kwargs['proxy'] = proxy
 
         if ssl_opts is not None:
+            axonvers = self.runt.snap.core.axoninfo['synapse']['version']
             mesg = f'The ssl_opts argument requires an Axon Synapse version {AXON_MINVERS_SSLOPTS}, ' \
                    f'but the Axon is running {axonvers}'
             s_version.reqVersion(axonvers, AXON_MINVERS_SSLOPTS, mesg=mesg)
@@ -2413,7 +2513,7 @@ class LibAxon(Lib):
         return resp
 
     async def wput(self, sha256, url, headers=None, params=None, method='PUT',
-                   ssl=True, timeout=None, proxy=None, ssl_opts=None):
+                   ssl=True, timeout=None, proxy=True, ssl_opts=None):
 
         if not self.runt.allowed(('axon', 'wput')):
             self.runt.confirm(('storm', 'lib', 'axon', 'wput'))
@@ -2432,23 +2532,24 @@ class LibAxon(Lib):
         params = strifyHttpArg(params, multi=True)
         headers = strifyHttpArg(headers)
 
-        if proxy is not None:
-            self.runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy'))
-
-        axon = self.runt.snap.core.axon
-        sha256byts = s_common.uhex(sha256)
+        await self.runt.snap.core.getAxon()
 
         kwargs = {}
-        axonvers = self.runt.snap.core.axoninfo['synapse']['version']
-        if axonvers >= AXON_MINVERS_PROXY:
+
+        ok, proxy = await resolveAxonProxyArg(proxy)
+        if ok:
             kwargs['proxy'] = proxy
 
         if ssl_opts is not None:
+            axonvers = self.runt.snap.core.axoninfo['synapse']['version']
             mesg = f'The ssl_opts argument requires an Axon Synapse version {AXON_MINVERS_SSLOPTS}, ' \
                    f'but the Axon is running {axonvers}'
             s_version.reqVersion(axonvers, AXON_MINVERS_SSLOPTS, mesg=mesg)
             kwargs['ssl_opts'] = ssl_opts
 
+        axon = self.runt.snap.core.axon
+        sha256byts = s_common.uhex(sha256)
+
         return await axon.wput(sha256byts, url, headers=headers, params=params, method=method,
                                ssl=ssl, timeout=timeout, **kwargs)
 

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 191, 0)
+version = (2, 193, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'f6b07ada7cb68701e769e155735c9c004a404454'
+commit = 'ad17bf2e740ba11453b7827ce70c140f969a1430'

synapse/models/inet.py

@@ -1442,7 +1442,11 @@ class InetModule(s_module.CoreModule):
                         'doc': 'A channel within a web service or instance such as slack or discord.'
                     }),
 
-                    ('inet:web:hashtag', ('str', {'lower': True, 'regex': r'^#\w[\w·]*(?<!·)$'}), {
+                    ('inet:web:hashtag', ('str', {'lower': True, 'strip': True, 'regex': r'^#[^\p{Z}#]+$'}), {
+                        # regex explanation:
+                        # - starts with pound
+                        # - one or more non-whitespace/non-pound character
+                        # The minimum hashtag is a pound with a single non-whitespace character
                         'doc': 'A hashtag used in a web post.',
                     }),
 
@@ -1728,6 +1732,9 @@ class InetModule(s_module.CoreModule):
                         'template': {'service:base': 'object'},
                         'props': (
 
+                            ('url', ('inet:url', {}), {
+                                'doc': 'The primary URL associated with the {service:base}.'}),
+
                             ('status', ('inet:service:object:status', {}), {
                                 'doc': 'The status of the {service:base}.'}),
 
@@ -1811,6 +1818,9 @@ class InetModule(s_module.CoreModule):
 
                     ('inet:email:message', {}, (
 
+                        ('id', ('str', {'strip': True}), {
+                            'doc': 'The ID parsed from the "message-id" header.'}),
+
                         ('to', ('inet:email', {}), {
                             'doc': 'The email address of the recipient.'}),
 
@@ -2171,6 +2181,9 @@ class InetModule(s_module.CoreModule):
                         ('dst:ssh:key', ('crypto:key', {}), {
                             'doc': 'The key sent by the server as part of an SSH session setup.'}),
 
+                        ('capture:host', ('it:host', {}), {
+                            'doc': 'The host which captured the flow.'}),
+
                         ('raw', ('data', {}), {
                             'doc': 'A raw record used to create the flow which may contain additional protocol details.'}),
                     )),
@@ -2194,6 +2207,9 @@ class InetModule(s_module.CoreModule):
                         ('host', ('it:host', {}), {
                             'doc': 'The host that used the network egress.'}),
 
+                        ('host:iface', ('inet:iface', {}), {
+                            'doc': 'The interface which the host used to connect out via the egress.'}),
+
                         ('account', ('inet:service:account', {}), {
                             'doc': 'The service account which used the client address to egress.'}),
 

synapse/models/infotech.py

@@ -655,6 +655,7 @@ class ItModule(s_module.CoreModule):
 
                 ('it:host', ('guid', {}), {
                     'interfaces': ('inet:service:object',),
+                    'template': {'service:base': 'host'},
                     'doc': 'A GUID that represents a host or system.'}),
 
                 ('it:log:event:type:taxonomy', ('taxonomy', {}), {
@@ -784,6 +785,7 @@ class ItModule(s_module.CoreModule):
                 }),
                 ('it:dev:repo', ('guid', {}), {
                     'interfaces': ('inet:service:object',),
+                    'template': {'service:base': 'repository'},
                     'doc': 'A version control system instance.',
                 }),
                 ('it:dev:repo:remote', ('guid', {}), {
@@ -791,10 +793,12 @@ class ItModule(s_module.CoreModule):
                 }),
                 ('it:dev:repo:branch', ('guid', {}), {
                     'interfaces': ('inet:service:object',),
+                    'template': {'service:base': 'repository branch'},
                     'doc': 'A branch in a version control system instance.',
                 }),
                 ('it:dev:repo:commit', ('guid', {}), {
                     'interfaces': ('inet:service:object',),
+                    'template': {'service:base': 'repository commit'},
                     'doc': 'A commit to a repository.',
                 }),
                 ('it:dev:repo:diff', ('guid', {}), {
@@ -802,18 +806,22 @@ class ItModule(s_module.CoreModule):
                 }),
                 ('it:dev:repo:issue:label', ('guid', {}), {
                     'interfaces': ('inet:service:object',),
+                    'template': {'service:base': 'repository issue label'},
                     'doc': 'A label applied to a repository issue.',
                 }),
                 ('it:dev:repo:issue', ('guid', {}), {
                     'interfaces': ('inet:service:object',),
+                    'template': {'service:base': 'repository issue'},
                     'doc': 'An issue raised in a repository.',
                 }),
                 ('it:dev:repo:issue:comment', ('guid', {}), {
                     'interfaces': ('inet:service:object',),
+                    'template': {'service:base': 'repository issue comment'},
                     'doc': 'A comment on an issue in a repository.',
                 }),
                 ('it:dev:repo:diff:comment', ('guid', {}), {
                     'interfaces': ('inet:service:object',),
+                    'template': {'service:base': 'repository diff comment'},
                     'doc': 'A comment on a diff in a repository.',
                 }),
                 ('it:prod:soft', ('guid', {}), {
@@ -963,12 +971,12 @@ class ItModule(s_module.CoreModule):
                 }),
                 ('it:exec:pipe', ('guid', {}), {
                     'interfaces': ('it:host:activity',),
-                    'doc': 'A named pipe created by a process at runtime.',
-                }),
+                    'doc': 'A named pipe created by a process at runtime.'}),
+
                 ('it:exec:url', ('guid', {}), {
                     'interfaces': ('it:host:activity',),
-                    'doc': 'An instance of a host requesting a URL.',
-                }),
+                    'doc': 'An instance of a host requesting a URL using any protocol scheme.'}),
+
                 ('it:exec:bind', ('guid', {}), {
                     'interfaces': ('it:host:activity',),
                     'doc': 'An instance of a host binding a listening port.',
@@ -1046,6 +1054,7 @@ class ItModule(s_module.CoreModule):
 
                 ('it:host:tenancy', ('guid', {}), {
                     'interfaces': ('inet:service:object',),
+                    'template': {'service:base': 'host tenancy'},
                     'doc': 'A time window where a host was a tenant run by another host.'}),
 
                 ('it:software:image:type:taxonomy', ('taxonomy', {}), {
@@ -1054,6 +1063,7 @@ class ItModule(s_module.CoreModule):
 
                 ('it:software:image', ('guid', {}), {
                     'interfaces': ('inet:service:object',),
+                    'template': {'service:base': 'software image'},
                     'doc': 'The base image used to create a container or OS.'}),
 
                 ('it:storage:mount', ('guid', {}), {

synapse/models/risk.py

@@ -242,9 +242,18 @@ class RiskModule(s_module.CoreModule):
                 (('risk:mitigation', 'uses', 'inet:service:rule'), {
                     'doc': 'The mitigation uses the service rule.'}),
 
+                (('risk:mitigation', 'uses', 'it:prod:softver'), {
+                    'doc': 'The mitigation uses the software version.'}),
+
+                (('risk:mitigation', 'uses', 'it:prod:hardware'), {
+                    'doc': 'The mitigation uses the hardware.'}),
+
                 (('risk:leak', 'leaked', None), {
                     'doc': 'The leak included the disclosure of the target node.'}),
 
+                (('risk:leak', 'enabled', 'risk:leak'), {
+                    'doc': 'The source leak enabled the target leak to occur.'}),
+
                 (('risk:extortion', 'leveraged', None), {
                     'doc': 'The extortion event was based on attacker access to the target node.'}),
 
@@ -407,10 +416,12 @@ class RiskModule(s_module.CoreModule):
                         'doc': 'A description of the mitigation approach for the vulnerability.'}),
 
                     ('software', ('it:prod:softver', {}), {
-                        'doc': 'A software version which implements a fix for the vulnerability.'}),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use risk:mitigation -(uses)> it:prod:softver.'}),
 
                     ('hardware', ('it:prod:hardware', {}), {
-                        'doc': 'A hardware version which implements a fix for the vulnerability.'}),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use risk:mitigation -(uses)> it:prod:hardware.'}),
 
                     ('reporter', ('ou:org', {}), {
                         'doc': 'The organization reporting on the mitigation.'}),
@@ -1034,6 +1045,9 @@ class RiskModule(s_module.CoreModule):
                     ('leaker', ('ps:contact', {}), {
                         'doc': 'The identity which leaked the information.'}),
 
+                    ('recipient', ('ps:contact', {}), {
+                        'doc': 'The identity which received the leaked information.'}),
+
                     ('type', ('risk:leak:type:taxonomy', {}), {
                         'doc': 'A type taxonomy for the leak.'}),
 

synapse/tests/test_axon.py

@@ -945,10 +945,16 @@ bar baz",vv
                 self.false(resp.get('ok'))
                 self.isin('connect to proxy 127.0.0.1:1', resp.get('mesg', ''))
 
+                resp = await proxy.wget('http://vertex.link/', proxy=None)
+                self.false(resp.get('ok'))
+                self.isin('connect to proxy 127.0.0.1:1', resp.get('mesg', ''))
+
             resp = await proxy.wget('vertex.link')
             self.false(resp.get('ok'))
             self.isin('InvalidUrlClientError: vertex.link', resp.get('mesg', ''))
 
+            await self.asyncraises(s_exc.BadArg, proxy.wget('http://vertex.link', proxy=1.1))
+
     async def test_axon_wput(self):
 
         async with self.getTestCore() as core:
@@ -1025,6 +1031,10 @@ bar baz",vv
                 self.false(resp.get('ok'))
                 self.isin('connect to proxy 127.0.0.1:1', resp.get('reason'))
 
+                resp = await proxy.postfiles(fields, f'https://127.0.0.1:{port}/api/v1/pushfile', ssl=False, proxy=None)
+                self.false(resp.get('ok'))
+                self.isin('connect to proxy 127.0.0.1:1', resp.get('reason'))
+
             resp = await proxy.wput(sha256, 'vertex.link')
             self.false(resp.get('ok'))
             self.isin('InvalidUrlClientError: vertex.link', resp.get('mesg', ''))

synapse/tests/test_cortex.py

@@ -484,7 +484,7 @@ class CortexTest(s_t_utils.SynTest):
             self.len(0, mods)
             self.len(0, core.modsbyiface.get('lookup'))
 
-            await core.loadStormPkg(pkgdef)
+            core.loadStormPkg(pkgdef)
 
             mods = await core.getStormIfaces('lookup')
             self.len(1, mods)
@@ -513,7 +513,7 @@ class CortexTest(s_t_utils.SynTest):
             vals = [r async for r in core.view.callStormIface('boom', todo)]
             self.eq((), vals)
 
-            await core._dropStormPkg(pkgdef)
+            core._dropStormPkg(pkgdef)
             self.none(core.modsbyiface.get('lookup'))
 
             mods = await core.getStormIfaces('lookup')
@@ -558,7 +558,7 @@ class CortexTest(s_t_utils.SynTest):
             nodes = await core.nodes('foo@bar.com foo@bar.com', opts={'mode': 'lookup'})
             self.eq(['inet:email', 'inet:email'], [n.ndef[0] for n in nodes])
 
-            await core.loadStormPkg(pkgdef)
+            core.loadStormPkg(pkgdef)
             self.len(1, await core.getStormIfaces('search'))
 
             todo = s_common.todo('search', ('foo@bar.com',))
@@ -4023,6 +4023,58 @@ class CortexBasicTest(s_t_utils.SynTest):
                 gdef = await core.callStorm('return($lib.graph.add(({"name": "def", "permissions": {"default": 0}})))')
                 self.eq(0, gdef['permissions']['default'])
 
+    async def test_graph_projection_query_validation(self):
+        async with self.getTestCore() as core:
+            valid = {
+                'name': 'valid',
+                'forms': {
+                    'inet:fqdn': {
+                        'pivots': ['<- *'],
+                        'filters': []
+                    }
+                }
+            }
+
+            self.nn(await core.addStormGraph(valid))
+
+            bad_form_pivot = {
+                'name': 'bad form pivot',
+                'forms': {
+                    'inet:fqdn': {
+                        'pivots': ['<- * |||'],
+                        'filters': []
+                    }
+                }
+            }
+
+            await self.asyncraises(s_exc.BadSyntax, core.addStormGraph(bad_form_pivot))
+
+            bad_form_filter = {
+                'name': 'bad form filter',
+                'forms': {
+                    'inet:fqdn': {
+                        'pivots': [],
+                        'filters': ['+++:wat']
+                    }
+                }
+            }
+
+            await self.asyncraises(s_exc.BadSyntax, core.addStormGraph(bad_form_filter))
+
+            bad_global_filter = {
+                'name': 'bad global filter',
+                'filters': ['+++:wat']
+            }
+
+            await self.asyncraises(s_exc.BadSyntax, core.addStormGraph(bad_global_filter))
+
+            bad_global_pivot = {
+                'name': 'bad global pivot',
+                'pivots': ['-> * |||']
+            }
+
+            await self.asyncraises(s_exc.BadSyntax, core.addStormGraph(bad_global_pivot))
+
     async def test_storm_two_level_assignment(self):
         async with self.getTestCore() as core:
             q = '$foo=baz $bar=$foo [test:str=$bar]'

synapse/tests/test_exc.py

@@ -27,6 +27,9 @@ class ExcTest(s_t_utils.SynTest):
         e.setdefault('defv', 2)
         self.eq("SynErr: defv=1 foo='words' hehe=1234 mesg='words'", str(e))
 
+        e.update({'foo': 'newwords', 'bar': 'baz'})
+        self.eq("SynErr: bar='baz' defv=1 foo='newwords' hehe=1234 mesg='words'", str(e))
+
         self.eq(e.errname, 'SynErr')
 
         e2 = s_exc.BadTypeValu(mesg='haha')