synapse 2.201.0__py311-none-any.whl → 2.203.0__py311-none-any.whl

synapse/lib/schemas.py

@@ -383,6 +383,7 @@ _changelogTypes = {'migration': 'Automatic Migrations',
                    'model': 'Model Changes',
                    'feat': 'Features and Enhancements',
                    'bug': 'Bugfixes',
+                   'note': 'Notes',
                    'doc': 'Improved documentation',
                    'deprecation': 'Deprecations'}
 
@@ -407,7 +408,7 @@ _changelogSchema = {
     'additionalProperties': False,
     'required': ['type', 'desc']
 }
-_reqChanglogSchema = s_config.getJsValidator(_changelogSchema)
+_reqChangelogSchema = s_config.getJsValidator(_changelogSchema)
 
 tabularConfSchema = {
     'type': 'object',
@@ -1042,3 +1043,88 @@ _reqValidDdefSchema = {
     }
 }
 reqValidDdef = s_config.getJsValidator(_reqValidDdefSchema)
+
+_client_assertion_schema = {
+    'type': 'object',
+    'oneOf': [
+        {
+            'required': ['cortex:callstorm'],
+            'properties': {
+                'cortex:callstorm': {
+                    'type': 'object',
+                    'properties': {
+                        'query': {'type': 'string'},
+                        'vars': {'type': 'object'},
+                        'view': {'type': 'string', 'pattern': s_config.re_iden},
+                    },
+                    'required': ['query', 'view'],
+                    'additionalProperties': False,
+                },
+            },
+            'additionalProperties': False,
+            'not': {
+                'required': ['msft:azure:workloadidentity'],
+            }
+        },
+        {
+            'required': ['msft:azure:workloadidentity'],
+            'properties': {
+                'msft:azure:workloadidentity': {
+                    'type': 'object',
+                    'properties': {
+                        'token': {'type': 'boolean'},
+                        'client_id': {'type': 'boolean'},
+                    },
+                    'required': ['token'],
+                    'additionalProperties': False,
+                }
+            },
+            'additionalProperties': False,
+            'not': {
+                'required': ['cortex:callstorm'],
+            }
+        }
+    ]
+}
+_reqValidOauth2ProviderSchema = {
+    'type': 'object',
+    'properties': {
+        'iden': {'type': 'string', 'pattern': s_config.re_iden},
+        'name': {'type': 'string'},
+        'flow_type': {'type': 'string', 'default': 'authorization_code', 'enum': ['authorization_code']},
+        'auth_scheme': {'type': 'string', 'default': 'basic', 'enum': ['basic', 'client_assertion']},
+        'client_id': {'type': 'string'},
+        'client_secret': {'type': 'string'},
+        'client_assertion': _client_assertion_schema,
+        'scope': {'type': 'string'},
+        'ssl_verify': {'type': 'boolean', 'default': True},
+        'auth_uri': {'type': 'string'},
+        'token_uri': {'type': 'string'},
+        'redirect_uri': {'type': 'string'},
+        'extensions': {
+            'type': 'object',
+            'properties': {
+                'pkce': {'type': 'boolean'},
+            },
+            'additionalProperties': False,
+        },
+        'extra_auth_params': {
+            'type': 'object',
+            'additionalProperties': {'type': 'string'},
+        },
+    },
+    'additionalProperties': False,
+    'required': ['iden', 'name', 'scope', 'auth_uri', 'token_uri', 'redirect_uri'],
+}
+reqValidOauth2Provider = s_config.getJsValidator(_reqValidOauth2ProviderSchema)
+
+_reqValidOauth2TokenResponseSchema = {
+    'type': 'object',
+    'properties': {
+        'access_token': {'type': 'string'},
+        'expires_in': {'type': 'number', 'exclusiveMinimum': 0},
+    },
+    'additionalProperties': True,
+    'required': ['access_token', 'expires_in'],
+}
+reqValidOauth2TokenResponse = s_config.getJsValidator(_reqValidOauth2TokenResponseSchema)

synapse/lib/scrape.py

@@ -24,6 +24,22 @@ ipaddress = s_common.ipaddress
 
 logger = logging.getLogger(__name__)
 
+urilist = set(s_data.get('iana.uris'))
+urilist.update([
+    'aha',
+    'ftps',
+    'mysql',
+    'postgresql',
+    'slack',
+    'socks4',
+    'socks4a',
+    'socks5',
+    'socks5h',
+    'tcp',
+    'unk',
+    'webpack'
+])
+
 tldlist = list(s_data.get('iana.tlds'))
 tldlist.extend([
     'bit',
@@ -53,18 +69,6 @@ inverse_prefixs = {
 
 cve_dashes = ''.join(('-',) + s_chop.unicode_dashes)
 
-def fqdn_prefix_check(match: regex.Match):
-    mnfo = match.groupdict()
-    valu = mnfo.get('valu')
-    prefix = mnfo.get('prefix')
-    cbfo = {}
-    if prefix is not None:
-        new_valu = valu.rstrip(inverse_prefixs.get(prefix))
-        if new_valu != valu:
-            valu = new_valu
-            cbfo['match'] = valu
-    return valu, cbfo
-
 def fqdn_check(match: regex.Match):
     mnfo = match.groupdict()
     valu = mnfo.get('valu')
@@ -261,12 +265,30 @@ def unc_path_check(match: regex.Match):
 
     return valu, {}
 
+def url_scheme_check(match: regex.Match):
+    mnfo = match.groupdict()
+    valu = mnfo.get('valu')
+    prefix = mnfo.get('prefix')
+
+    cbfo = {}
+    if prefix is not None:
+        new_valu = valu.rstrip(inverse_prefixs.get(prefix))
+        if new_valu != valu:
+            valu = new_valu
+            cbfo['match'] = valu
+
+    scheme = valu.split('://')[0].lower()
+    if scheme not in urilist:
+        return None, {}
+
+    return valu, cbfo
+
 # these must be ordered from most specific to least specific to allow first=True to work
 scrape_types = [  # type: ignore
     ('file:path', linux_path_regex, {'callback': linux_path_check, 'flags': regex.VERBOSE}),
     ('file:path', windows_path_regex, {'callback': windows_path_check, 'flags': regex.VERBOSE}),
     ('inet:url', r'(?P<prefix>[\\{<\(\[]?)(?P<valu>[a-zA-Z][a-zA-Z0-9]*://(?(?=[,.]+[ \'\"\t\n\r\f\v])|[^ \'\"\t\n\r\f\v])+)',
-     {'callback': fqdn_prefix_check}),
+     {'callback': url_scheme_check}),
     ('inet:url', r'(["\'])?(?P<valu>\\[^\n]+?)(?(1)\1|\s)', {'callback': unc_path_check}),
     ('inet:email', r'(?=(?:[^a-z0-9_.+-]|^)(?P<valu>[a-z0-9_\.\-+]{1,256}@(?:[a-z0-9_-]{1,63}\.){1,10}(?:%s))(?:[^a-z0-9_.-]|[.\s]|$))' % tldcat, {}),
     ('inet:server', fr'(?P<valu>(?:(?<!\d|\d\.|[0-9a-f:]:)((?P<addr>{ipv4_match})|\[(?P<v6addr>{ipv6_match})\]):(?P<port>\d{{1,5}})(?!\d|\.\d)))',

synapse/lib/snap.py

@@ -11,6 +11,7 @@ import synapse.exc as s_exc
 import synapse.common as s_common
 
 import synapse.lib.base as s_base
+import synapse.lib.json as s_json
 import synapse.lib.node as s_node
 import synapse.lib.time as s_time
 import synapse.lib.cache as s_cache
@@ -218,7 +219,7 @@ class ProtoNode:
             return
 
         try:
-            s_common.reqjsonsafe(valu)
+            s_json.reqjsonsafe(valu)
         except s_exc.MustBeJsonSafe as e:
             if self.ctx.snap.strict:
                 raise e

synapse/lib/storm.lark

@@ -437,7 +437,8 @@ _cond: notcond | "(" _condexpr ")"
     | condsubq | arraycond
     | _varvalu | _reqdollarexprs
 
-notcond: "not" _cond
+_NOT: "not"
+notcond: _NOT _cond
 
 hasrelpropcond: relprop | univprop
 relpropcond: relpropvalue _cmpr _valu
@@ -627,15 +628,15 @@ OCTNUMBER.1: /
 /x
 
 BOOL.2: /(true|false)(?=$|[\s\),\]}\|\=])/
-NULL.2: "null"
-NOT.2: "not"
+NULL.2: /null(?=$|[\s\),\]}\|\=])/
+NOTOP.2: /not(?=$|[\s\),\]}\|\=])/
 OR.2: "or"
 AND.2: "and"
 
 // $ expression rules in increasing order of precedence (modeled on Python's order)
 ?expror: exprand | expror OR exprand
 ?exprand: exprnot | exprand AND exprnot
-?exprnot: exprcmp | NOT exprcmp
+?exprnot: exprcmp | NOTOP exprcmp
 ?exprcmp: exprsum | exprcmp (CMPR | EQSPACE | EQNOSPACE) exprsum
 ?exprsum: exprproduct | exprsum (EXPRPLUS | EXPRMINUS) exprproduct
 ?exprproduct: exprunary | exprproduct (EXPRTIMES | EXPRDIVIDE | EXPRMODULO) exprunary

synapse/lib/storm.py

@@ -2592,7 +2592,7 @@ class Cmd:
 
     async def execStormCmd(self, runt, genr):  # pragma: no cover
         ''' Abstract base method '''
-        raise s_exc.NoSuchImpl('Subclass must implement execStormCmd')
+        raise s_exc.NoSuchImpl(mesg='Subclass must implement execStormCmd')
         for item in genr:
             yield item
 
@@ -4768,7 +4768,7 @@ class GraphCmd(Cmd):
             inet:fqdn | graph
                         --degrees 2
                         --filter { -#nope }
-                        --pivot { -> meta:seen }
+                        --pivot { <(seen)- meta:source }
                         --form-pivot inet:fqdn {<- * | limit 20}
                         --form-pivot inet:fqdn {-> * | limit 20}
                         --form-filter inet:fqdn {-inet:fqdn:issuffix=1}

synapse/lib/storm_format.py

@@ -57,7 +57,7 @@ TerminalPygMap = {
     'MODSET': p_t.Operator,
     'MODSETMULTI': p_t.Operator,
     'NONQUOTEWORD': p_t.Literal,
-    'NOT': p_t.Keyword,
+    'NOTOP': p_t.Operator,
     'NULL': p_t.Keyword,
     'NUMBER': p_t.Literal.Number,
     'OCTNUMBER': p_t.Literal.Number,
@@ -115,6 +115,7 @@ TerminalPygMap = {
     '_LPARNOSPACE': p_t.Punctuation,
     '_MATCHHASH': p_t.Punctuation,
     '_MATCHHASHWILD': p_t.Punctuation,
+    '_NOT': p_t.Keyword,
     '_RETURN': p_t.Keyword,
     '_REVERSE': p_t.Keyword,
     '_RIGHTJOIN': p_t.Punctuation,

synapse/lib/stormhttp.py

@@ -1,4 +1,3 @@
-import json
 import asyncio
 import logging
 import urllib.parse
@@ -12,6 +11,7 @@ import synapse.exc as s_exc
 import synapse.common as s_common
 
 import synapse.lib.base as s_base
+import synapse.lib.json as s_json
 import synapse.lib.msgpack as s_msgpack
 import synapse.lib.version as s_version
 import synapse.lib.stormtypes as s_stormtypes
@@ -56,7 +56,7 @@ class WebSocket(s_base.Base, s_stormtypes.StormType):
         try:
 
             mesg = await s_stormtypes.toprim(mesg)
-            await self.resp.send_bytes(json.dumps(mesg).encode())
+            await self.resp.send_bytes(s_json.dumps(mesg))
             return (True, None)
 
         except asyncio.CancelledError:  # pragma: no cover
@@ -69,10 +69,8 @@ class WebSocket(s_base.Base, s_stormtypes.StormType):
 
         try:
             _type, data, extra = await s_common.wait_for(self.resp.receive(), timeout=timeout)
-            if _type == aiohttp.WSMsgType.BINARY:
-                return (True, json.loads(data))
-            if _type == aiohttp.WSMsgType.TEXT:
-                return (True, json.loads(data.encode()))
+            if _type in (aiohttp.WSMsgType.BINARY, aiohttp.WSMsgType.TEXT):
+                return (True, s_json.loads(data))
             if _type == aiohttp.WSMsgType.CLOSED:  # pragma: no cover
                 return (True, None)
             return (False, ('BadMesgFormat', {'mesg': f'WebSocket RX unhandled type: {_type.name}'}))  # pragma: no cover
@@ -100,9 +98,9 @@ class LibHttp(s_stormtypes.Lib):
 
     For APIs that accept a proxy argument, the following values are supported::
 
-        $lib.null: Deprecated - Use the proxy defined by the http:proxy configuration option if set.
-        $lib.true: Use the proxy defined by the http:proxy configuration option if set.
-        $lib.false: Do not use the proxy defined by the http:proxy configuration option if set.
+        (null): Deprecated - Use the proxy defined by the http:proxy configuration option if set.
+        (true): Use the proxy defined by the http:proxy configuration option if set.
+        (false): Do not use the proxy defined by the http:proxy configuration option if set.
         <str>: A proxy URL string.
     '''
     _storm_locals = (
@@ -563,17 +561,17 @@ class HttpResp(s_stormtypes.Prim):
             errors = await s_stormtypes.tostr(errors)
 
             if encoding is None:
-                encoding = json.detect_encoding(valu)
+                encoding = s_json.detect_encoding(valu)
             else:
                 encoding = await s_stormtypes.tostr(encoding)
 
-            return json.loads(valu.decode(encoding, errors))
+            return s_json.loads(valu.decode(encoding, errors))
 
         except UnicodeDecodeError as e:
             raise s_exc.StormRuntimeError(mesg=f'{e}: {s_common.trimText(repr(valu))}') from None
 
-        except json.JSONDecodeError as e:
-            mesg = f'Unable to decode HTTP response as json: {e.args[0]}'
+        except s_exc.BadJsonText as e:
+            mesg = f'Unable to decode HTTP response as json: {e.get("mesg")}'
             raise s_exc.BadJsonText(mesg=mesg)
 
     async def _httpRespMsgpack(self):

synapse/lib/stormlib/aha.py

@@ -47,7 +47,7 @@ class AhaLib(s_stormtypes.Lib):
                        'desc': 'An optional dictionary of filters to use when resolving the AHA service.'}
                   ),
                   'returns': {'type': ('null', 'dict'),
-                              'desc': 'The AHA service information dictionary, or $lib.null.', }}},
+                              'desc': 'The AHA service information dictionary, or ``(null))``.', }}},
         {'name': 'list', 'desc': 'Enumerate all of the AHA services.',
          'type': {'type': 'function', '_funcname': '_methAhaList', 'args': (),
                   'returns': {'name': 'Yields', 'type': 'list',
@@ -97,14 +97,14 @@ class AhaLib(s_stormtypes.Lib):
         Examples:
             Call getNexusChanges on an AHA service::
 
-                $todo = $lib.utils.todo('getNexusChanges', (0), wait=$lib.false)
+                $todo = $lib.utils.todo('getNexusChanges', (0), wait=(false))
                 for $info in $lib.aha.callPeerGenr(cortex..., $todo) {
                     $lib.print($info)
                 }
 
             Call getNexusChanges on an AHA service, skipping the invoking service::
 
-                $todo = $lib.utils.todo('getNexusChanges', (0), wait=$lib.false)
+                $todo = $lib.utils.todo('getNexusChanges', (0), wait=(false))
                 for $info in $lib.aha.callPeerGenr(cortex..., $todo, skiprun=$lib.cell.getCellInfo().cell.run) {
                     $lib.print($info)
                 }
@@ -259,7 +259,7 @@ class AhaPoolLib(s_stormtypes.Lib):
                       {'name': 'name', 'type': 'str',
                        'desc': 'The name of the pool to get. It is easiest to use the relative name of a pool, ending with "...".', },
                   ),
-                  'returns': {'type': ['null', 'aha:pool'], 'desc': 'The pool if it exists, or $lib.null.'}}},
+                  'returns': {'type': ['null', 'aha:pool'], 'desc': 'The pool if it exists, or ``(null)``.'}}},
         {'name': 'list', 'desc': 'Enumerate all of the AHA service pools.',
          'type': {'type': 'function', '_funcname': '_methPoolList',
                   'returns': {'name': 'yields', 'type': 'aha:pool'}}},

synapse/lib/stormlib/auth.py

@@ -684,7 +684,7 @@ class UserJson(s_stormtypes.Prim):
                         {'name': 'path', 'type': 'str|list', 'desc': 'A path string or list of path parts.'},
                         {'name': 'prop', 'type': 'str|list', 'desc': 'A property name or list of name parts.', 'default': None},
                     ),
-                    'returns': {'type': 'prim', 'desc': 'The previously stored value or $lib.null'}}},
+                    'returns': {'type': 'prim', 'desc': 'The previously stored value or ``(null)``.'}}},
 
         {'name': 'set', 'desc': 'Set a JSON object or object property for the user.',
          'type': {'type': 'function', '_funcname': 'set',

synapse/lib/stormlib/cache.py

@@ -24,7 +24,7 @@ class LibCache(s_stormtypes.Lib):
             to the key argument provided to .get().
 
             The callback Storm query must contain a return statement, and if it does not return a value
-            when executed with the input, $lib.null will be set as the value.
+            when executed with the input, ``(null)`` will be set as the value.
 
             The fixed cache uses FIFO to evict items once the maximum size is reached.
 
@@ -114,7 +114,7 @@ class FixedCache(s_stormtypes.StormType):
                       {'name': 'key', 'type': 'any', 'desc': 'The key to pop.'},
                   ),
                   'returns': {'type': 'any',
-                              'desc': 'The value from the cache, or $lib.null if it does not exist', }}},
+                              'desc': 'The value from the cache, or ``(null)`` if it does not exist', }}},
         {'name': 'put', 'desc': 'Put an item into the cache.',
          'type': {'type': 'function', '_funcname': '_methPut',
                   'args': (

synapse/lib/stormlib/cortex.py

@@ -1,10 +1,10 @@
 import copy
-import json
 import logging
 
 import synapse.exc as s_exc
 import synapse.telepath as s_telepath
 
+import synapse.lib.json as s_json
 import synapse.lib.storm as s_storm
 import synapse.lib.stormtypes as s_stormtypes
 import synapse.lib.stormlib.auth as slib_auth
@@ -990,8 +990,8 @@ class HttpReq(s_stormtypes.StormType):
     @s_stormtypes.stormfunc(readonly=True)
     def _ctorJson(self, path=None):
         try:
-            return json.loads(self.rnfo.get('body'))
-        except (UnicodeDecodeError, json.JSONDecodeError) as e:
+            return s_json.loads(self.rnfo.get('body'))
+        except (UnicodeDecodeError, s_exc.BadJsonText) as e:
             raise s_exc.StormRuntimeError(mesg=f'Failed to decode request body as JSON: {e}') from None
 
     @s_stormtypes.stormfunc(readonly=True)
@@ -1028,7 +1028,7 @@ class HttpReq(s_stormtypes.StormType):
         if body is not s_stormtypes.undef:
             if not isinstance(body, bytes):
                 body = await s_stormtypes.toprim(body)
-                body = json.dumps(body).encode('utf-8', 'surrogatepass')
+                body = s_json.dumps(body)
                 headers['Content-Type'] = 'application/json; charset=utf8"'
                 headers['Content-Length'] = len(body)
 
@@ -1126,7 +1126,7 @@ class CortexHttpApi(s_stormtypes.Lib):
                       {'name': 'path', 'type': 'string',
                        'desc': 'Path to use to retrieve an object.'},
                   ),
-                  'returns': {'type': ['http:api', 'null'], 'desc': 'The ``http:api`` object or ``$lib.null`` if there is no match.'}}},
+                  'returns': {'type': ['http:api', 'null'], 'desc': 'The ``http:api`` object or ``(null)`` if there is no match.'}}},
         {'name': 'list', 'desc': 'Get all the Extended HTTP APIs on the Cortex',
          'type': {'type': 'function', '_funcname': 'listHttpApis', 'args': (),
                  'returns': {'type': 'list', 'desc': 'A list of ``http:api`` objects'}}},

synapse/lib/stormlib/graph.py

@@ -36,7 +36,7 @@ class GraphLib(s_stormtypes.Lib):
                         "name": "Test Projection",
                         "desc": "My test projection",
                         "degrees": 2,
-                        "pivots": ["-> meta:seen"],
+                        "pivots": [" <(seen)- meta:source "],
                         "filters": ["-#nope"],
                         "forms": {
                             "inet:fqdn": {

synapse/lib/stormlib/imap.py

@@ -193,7 +193,7 @@ class ImapServer(s_stormtypes.StormType):
                     {'type': 'str', 'name': '*args',
                      'desc': 'A set of search criteria to use.'},
                     {'type': ['str', 'null'], 'name': 'charset', 'default': 'utf-8',
-                     'desc': 'The CHARSET used for the search. May be set to $lib.null to disable CHARSET.'},
+                     'desc': 'The CHARSET used for the search. May be set to ``(null)`` to disable CHARSET.'},
                 ),
                 'returns': {
                     'type': 'list',

synapse/lib/stormlib/json.py

@@ -1,11 +1,11 @@
 import copy
-import json
 import asyncio
 import logging
 
 import synapse.exc as s_exc
 
 import synapse.lib.coro as s_coro
+import synapse.lib.json as s_json
 import synapse.lib.config as s_config
 import synapse.lib.stormtypes as s_stormtypes
 
@@ -93,7 +93,7 @@ class JsonLib(s_stormtypes.Lib):
          'type': {'type': 'function', '_funcname': '_jsonSave',
                   'args': (
                       {'name': 'item', 'type': 'any', 'desc': 'The item to be serialized as a JSON string.', },
-                      {'name': 'indent', 'type': 'int', 'desc': 'Specify a number of spaces to indent with.', 'default': None},
+                      {'name': 'indent', 'type': 'boolean', 'desc': 'Indent serialized data with two spaces.', 'default': False},
                   ),
                   'returns': {'type': 'str', 'desc': 'The JSON serialized object.', }}},
         {'name': 'schema', 'desc': 'Get a JS schema validation object.',
@@ -116,24 +116,21 @@ class JsonLib(s_stormtypes.Lib):
         }
 
     @s_stormtypes.stormfunc(readonly=True)
-    async def _jsonSave(self, item, indent=None):
-        indent = await s_stormtypes.toint(indent, noneok=True)
+    async def _jsonSave(self, item, indent=False):
+        indent = await s_stormtypes.tobool(indent)
 
         try:
             item = await s_stormtypes.toprim(item)
-            return json.dumps(item, indent=indent)
-        except Exception as e:
+        except Exception:
             mesg = f'Argument is not JSON compatible: {item}'
             raise s_exc.MustBeJsonSafe(mesg=mesg)
 
+        return s_json.dumps(item, indent=indent).decode()
+
     @s_stormtypes.stormfunc(readonly=True)
     async def _jsonLoad(self, text):
         text = await s_stormtypes.tostr(text)
-        try:
-            return json.loads(text, strict=True)
-        except Exception as e:
-            mesg = f'Text is not valid JSON: {text}'
-            raise s_exc.BadJsonText(mesg=mesg)
+        return s_json.loads(text)
 
     @s_stormtypes.stormfunc(readonly=True)
     async def _jsonSchema(self, schema, use_default=True):

synapse/lib/stormlib/model.py

@@ -200,7 +200,7 @@ class LibModelTags(s_stormtypes.Lib):
         Examples:
             Create a tag model for the ``cno.cve`` tag::
 
-                $regx = ($lib.null, $lib.null, "[0-9]{4}", "[0-9]{5}")
+                $regx = ([null, null, "[0-9]{4}", "[0-9]{5}"])
                 $lib.model.tags.set(cno.cve, regex, $regx)''',
          'type': {'type': 'function', '_funcname': '_setTagModel',
                   'args': (

synapse/lib/stormlib/notifications.py

@@ -41,7 +41,7 @@ class NotifyLib(s_stormtypes.Lib):
         {
             'name': 'get',
             'desc': '''
-            Return a notification by ID (or $lib.null).
+            Return a notification by ID (or ``(null)`` ).
 
             ''',
             'type': {
@@ -51,7 +51,7 @@ class NotifyLib(s_stormtypes.Lib):
                 ),
                 'returns': {
                     'name': 'retn', 'type': 'dict',
-                    'desc': 'The requested notification or $lib.null.'},
+                    'desc': 'The requested notification or ``(null)``.'},
             },
         },
     )