synapse 2.201.0__py311-none-any.whl → 2.203.0__py311-none-any.whl

synapse/lib/stormlib/oauth.py

@@ -145,15 +145,118 @@ class OAuthV2Lib(s_stormtypes.Lib):
                         })
 
                         // Optionally enable PKCE
-                        $conf.extensions = ({"pkce": $lib.true})
+                        $conf.extensions = ({"pkce": true})
 
                         // Optionally disable SSL verification
-                        $conf.ssl_verify = $lib.false
+                        $conf.ssl_verify = (false)
 
                         // Optionally provide additional key-val parameters
                         // to include when calling the auth URI
                         $conf.extra_auth_params = ({"customparam": "foo"})
 
+                        $lib.inet.http.oauth.v2.addProvider($conf)
+
+                    Add a new provider which uses the Microsoft Azure Federated Workflow Identify token credentials.
+                    This resolves the client_assertion from the AZURE_FEDERATED_TOKEN_FILE environment variable::
+
+                        $iden = $lib.guid(azureexample, provider, oauth)
+                        $authority_id = '4b70ee6f-d47b-3262-baa1-41cd7faed71b'
+                        $conf = ({
+                            "iden": $iden,
+                            "name": "example_provider",
+                            "auth_scheme": "client_assertion",
+                            "client_id": "yourclientid",
+                            "client_assertion": {
+                                "msft:azure:workloadidentity": {
+                                    "token": true,
+                                },
+                            },
+                            "scope": "first_scope second_scope",
+                            "auth_uri": `https://login.microsoftonline.com/{$authority_id}/oauth2/v2.0/authorize`,
+                            "token_uri": `https://login.microsoftonline.com/{$authority_id}/oauth2/v2.0/token`,
+                            "redirect_uri": "https://local.redirect.com/oauth",
+                        })
+
+                        // Optionally enable PKCE
+                        $conf.extensions = ({"pkce": true})
+
+                        // Optionally disable SSL verification
+                        $conf.ssl_verify = (false)
+
+                        // Optionally provide additional key-val parameters
+                        // to include when calling the auth URI
+                        $conf.extra_auth_params = ({"customparam": "foo"})
+
+                        $lib.inet.http.oauth.v2.addProvider($conf)
+
+                    If the ``client_id`` value should come from the AZURE_CLIENT_ID environment variable, use the
+                    following configuration::
+
+                        $conf = ({
+                            "iden": $iden,
+                            "name": "example_provider",
+                            "auth_scheme": "client_assertion",
+                            "client_assertion": {
+                                "msft:azure:workloadidentity": {
+                                    "token": true,
+                                    "client_id": true,
+                                },
+                            },
+                            "scope": "first_scope second_scope",
+                            "auth_uri": `https://login.microsoftonline.com/{$authority_id}/oauth2/v2.0/authorize`,
+                            "token_uri": `https://login.microsoftonline.com/{$authority_id}/oauth2/v2.0/token`,
+                            "redirect_uri": "https://local.redirect.com/oauth",
+                        })
+
+                    Add a new provider which uses a custom Storm callback to obtain the client_assertion data. These
+                    callbacks are executed as the user who is performing the authorization_code workflow. The Storm
+                    callback must return data in a tuple of ``bool`` and a dictionary containing the assertion in the
+                    key ``token``. Error messages should be in the key ``error``::
+
+                        $iden = $lib.guid(callstormexample, provider, oauth)
+
+                        // Example callback
+                        $callbackQuery = ${
+                            $url = `{$baseurl}/api/oauth/getAssertion`
+                            $resp = $lib.inet.http.get($url, ssl_verify=$ssl_verify)
+                            if ($resp.code = 200) {
+                                $resp = ([true, {'token': $resp.json().assertion}])
+                            } else {
+                                $resp = ([false, {"error": `Failed to get assertion from {$url}`}])
+                            }
+                            return ( $resp )
+                        }
+
+                        // Specify any variables that need to be provided to $callbackQuery
+                        $myCallbackVars = ({
+                            'baseurl': 'https://local.assertion.provider.corp',
+                            'ssl_verify': true,
+                        })
+
+                        // Specify the view the callback is run in.
+                        $view = $lib.view.get().iden
+
+                        $conf = ({
+                            "iden": $iden,
+                            "name": "example_provider",
+                            "auth_scheme": "client_assertion",
+                            "client_id": "yourclientid",
+                            "client_assertion": {
+                                "cortex:callstorm": {
+                                    "query": $callbackQuery,
+                                    "view": $view,
+                                    "vars": $myCallbackVars,
+                                },
+                            },
+                            "scope": "first_scope second_scope",
+                            "auth_uri": "https://provider.com/auth",
+                            "token_uri": "https://provider.com/token",
+                            "redirect_uri": "https://local.redirect.com/oauth",
+                        })
+
+                        // Optionally enable PKCE
+                        $conf.extensions = ({"pkce": true})
+
                         $lib.inet.http.oauth.v2.addProvider($conf)
             ''',
             'type': {

synapse/lib/stormlib/stats.py

@@ -220,11 +220,13 @@ class StatTally(s_stormtypes.Prim):
 
     @s_stormtypes.stormfunc(readonly=True)
     async def inc(self, name, valu=1):
+        name = await s_stormtypes.tostr(name)
         valu = await s_stormtypes.toint(valu)
         self.counters[name] += valu
 
     @s_stormtypes.stormfunc(readonly=True)
     async def get(self, name):
+        name = await s_stormtypes.tostr(name)
         return self.counters.get(name, 0)
 
     def value(self):
@@ -236,6 +238,8 @@ class StatTally(s_stormtypes.Prim):
 
     @s_stormtypes.stormfunc(readonly=True)
     async def sorted(self, byname=False, reverse=False):
+        byname = await s_stormtypes.tobool(byname)
+        reverse = await s_stormtypes.tobool(reverse)
         if byname:
             return list(sorted(self.counters.items(), reverse=reverse))
         else:

synapse/lib/stormlib/stix.py

@@ -1,4 +1,3 @@
-import json
 import uuid
 import asyncio
 import logging
@@ -589,7 +588,7 @@ def validateStix(bundle, version='2.1'):
         'mesg': '',
         'result': {},
     }
-    bundle = json.loads(json.dumps(bundle))
+    bundle = s_msgpack.deepcopy(bundle, use_list=True)
     opts = stix2validator.ValidationOptions(strict=True, version=version)
     try:
         results = stix2validator.validate_parsed_json(bundle, options=opts)
@@ -1172,7 +1171,7 @@ class LibStixExport(s_stormtypes.Lib):
     @s_stormtypes.stormfunc(readonly=True)
     async def config(self):
         # make a new mutable config
-        return json.loads(json.dumps(_DefaultConfig))
+        return s_msgpack.deepcopy(_DefaultConfig, use_list=True)
 
     @s_stormtypes.stormfunc(readonly=True)
     async def bundle(self, config=None):
@@ -1372,7 +1371,7 @@ class StixBundle(s_stormtypes.Prim):
         return stixid
 
     def _initStixItem(self, stixid, stixtype, node):
-        ndef = json.loads(json.dumps(node.ndef))
+        ndef = s_msgpack.deepcopy(node.ndef, use_list=True)
         retn = {
             'id': stixid,
             'type': stixtype,

synapse/lib/stormlib/vault.py

@@ -321,7 +321,7 @@ class LibVault(s_stormtypes.Lib):
                       {'name': 'vtype', 'type': 'str',
                        'desc': 'The type of this vault.'},
                       {'name': 'scope', 'type': 'str',
-                       'desc': 'Scope for this vault. One of "user", "role", "global", or $lib.null for unscoped vaults.'},
+                       'desc': 'Scope for this vault. One of "user", "role", "global", or ``(null)`` for unscoped vaults.'},
                       {'name': 'owner', 'type': 'str',
                        'desc': 'User/role iden for this vault if scope is "user" or "role". None for "global" scope vaults.'},
                       {'name': 'secrets', 'type': 'dict',
@@ -359,9 +359,9 @@ class LibVault(s_stormtypes.Lib):
                   'args': (
                       {'name': 'vtype', 'type': 'str', 'desc': 'The vault type to retrieved.'},
                       {'name': 'scope', 'type': 'str', 'default': None,
-                       'desc': 'The scope for the specified type. If $lib.null, then getByType will search.'},
+                       'desc': 'The scope for the specified type. If ``(null)``, then getByType will search.'},
                   ),
-                  'returns': {'type': 'vault', 'desc': 'Vault or $lib.null if the vault could not be retrieved.'}}},
+                  'returns': {'type': 'vault', 'desc': 'Vault or ``(null)`` if the vault could not be retrieved.'}}},
         {'name': 'list', 'desc': 'List vaults accessible to the current user.',
          'type': {'type': 'function', '_funcname': '_listVaults',
                   'args': (),
@@ -649,14 +649,14 @@ class Vault(s_stormtypes.Prim):
          'type': {'type': 'function', '_funcname': '_methSetPerm',
                   'args': (
                       {'name': 'iden', 'type': 'str', 'desc': 'The user or role to modify.'},
-                      {'name': 'level', 'type': 'str', 'desc': 'The easyperm level for the iden. $lib.null to remove an existing permission.'},
+                      {'name': 'level', 'type': 'str', 'desc': 'The easyperm level for the iden. ``(null)`` to remove an existing permission.'},
                   ),
-                  'returns': {'type': 'boolean', 'desc': '$lib.true if the permission was set, $lib.false otherwise.', }}},
+                  'returns': {'type': 'boolean', 'desc': '``(true)`` if the permission was set, ``(false)`` otherwise.', }}},
 
         {'name': 'delete', 'desc': 'Delete the Vault.',
          'type': {'type': 'function', '_funcname': '_methDelete',
                   'args': (),
-                  'returns': {'type': 'boolean', 'desc': '$lib.true if the vault was deleted, $lib.false otherwise.', }}},
+                  'returns': {'type': 'boolean', 'desc': '``(true)`` if the vault was deleted, ``(false)`` otherwise.', }}},
     )
     _storm_typename = 'vault'
     _ismutable = False

synapse/lib/stormlib/xml.py

@@ -21,7 +21,7 @@ class XmlElement(s_stormtypes.Prim):
                   'args': (
                       {'name': 'name', 'type': 'str', 'desc': 'The name of the XML tag.'},
                       {'name': 'nested', 'type': 'bool', 'default': True,
-                        'desc': 'Set to $lib.false to only find direct children.'},
+                        'desc': 'Set to ``(false)`` to only find direct children.'},
                   ),
                   'returns': {'type': 'generator', 'desc': 'A generator which yields xml:elements.'}}},
 
@@ -30,7 +30,7 @@ class XmlElement(s_stormtypes.Prim):
                   'args': (
                       {'name': 'name', 'type': 'str', 'desc': 'The name of the child XML element tag.'},
                   ),
-                  'returns': {'type': 'xml:element', 'desc': 'The child XML element or $lib.null'}}},
+                  'returns': {'type': 'xml:element', 'desc': 'The child XML element or ``(null)``.'}}},
     )
 
     def __init__(self, runt, elem):

synapse/lib/stormtypes.py

@@ -1,7 +1,6 @@
 import bz2
 import copy
 import gzip
-import json
 import time
 
 import regex
@@ -25,6 +24,7 @@ import synapse.common as s_common
 import synapse.telepath as s_telepath
 
 import synapse.lib.coro as s_coro
+import synapse.lib.json as s_json
 import synapse.lib.node as s_node
 import synapse.lib.time as s_time
 import synapse.lib.cache as s_cache
@@ -837,14 +837,14 @@ class LibDmon(Lib):
                       {'name': 'iden', 'type': 'str', 'desc': 'The GUID of the Dmon to stop.'},
                   ),
                   'returns': {'type': 'boolean',
-                              'desc': '$lib.true unless the dmon does not exist or was already stopped.'}}},
+                              'desc': '``(true)`` unless the dmon does not exist or was already stopped.'}}},
         {'name': 'start', 'desc': 'Start a storm dmon.',
          'type': {'type': 'function', '_funcname': '_libDmonStart',
                   'args': (
                       {'name': 'iden', 'type': 'str', 'desc': 'The GUID of the dmon to start.'},
                   ),
                   'returns': {'type': 'boolean',
-                              'desc': '$lib.true unless the dmon does not exist or was already started.'}}},
+                              'desc': '``(true)`` unless the dmon does not exist or was already started.'}}},
     )
     _storm_lib_path = ('dmon',)
 
@@ -1469,7 +1469,7 @@ class LibBase(Lib):
 
                 Update the current runtime to enable debugging::
 
-                    $lib.debug = $lib.true''',
+                    $lib.debug = (true)''',
          'type': {
              'type': ['gtor', 'stor'],
              '_storfunc': '_setRuntDebug',
@@ -1782,7 +1782,7 @@ class LibBase(Lib):
         name = await tostr(name)
         mesg = await tostr(mesg)
         info = await toprim(info)
-        s_common.reqjsonsafe(info)
+        s_json.reqjsonsafe(info)
 
         ctor = getattr(s_exc, name, None)
         if ctor is not None:
@@ -1841,7 +1841,7 @@ class LibBase(Lib):
     @stormfunc(readonly=True)
     async def _fire(self, name, **info):
         info = await toprim(info)
-        s_common.reqjsonsafe(info)
+        s_json.reqjsonsafe(info)
         await self.runt.snap.fire('storm:fire', type=name, data=info)
 
 @registry.registerLib
@@ -2103,9 +2103,9 @@ class LibAxon(Lib):
 
     For APIs that accept a proxy argument, the following values are supported::
 
-        $lib.null: Deprecated - Use the proxy defined by the http:proxy configuration option if set.
-        $lib.true: Use the proxy defined by the http:proxy configuration option if set.
-        $lib.false: Do not use the proxy defined by the http:proxy configuration option if set.
+        ``(null)``: Deprecated - Use the proxy defined by the http:proxy configuration option if set.
+        ``(true)``: Use the proxy defined by the http:proxy configuration option if set.
+        ``(false)``: Do not use the proxy defined by the http:proxy configuration option if set.
         <str>: A proxy URL string.
     '''
     _storm_locals = (
@@ -3025,7 +3025,7 @@ class LibTime(Lib):
                       {'name': 'valu', 'type': 'str', 'desc': 'The timestamp string to parse.', },
                       {'name': 'format', 'type': 'str', 'desc': 'The format string to use for parsing.', },
                       {'name': 'errok', 'type': 'boolean', 'default': False,
-                       'desc': 'If set, parsing errors will return ``$lib.null`` instead of raising an exception.'}
+                       'desc': 'If set, parsing errors will return ``(null)`` instead of raising an exception.'}
                   ),
                   'returns': {'type': 'int', 'desc': 'The epoch timestamp for the string.', }}},
         {'name': 'format', 'desc': '''
@@ -3365,7 +3365,7 @@ class LibRegx(Lib):
                 In order to get the matching groups, patterns must use parentheses
                 to indicate the start and stop of the regex to return portions of.
                 If groups are not used, a successful match will return a empty list
-                and a unsuccessful match will return ``$lib.null``.
+                and a unsuccessful match will return ``(null)``.
 
             Example:
                 Extract the matching groups from a piece of text::
@@ -3407,7 +3407,6 @@ class LibRegx(Lib):
                   'returns': {'type': 'list', 'desc': 'A list of lists of strings for the matching groups in the pattern.', }}},
         {'name': 'matches', 'desc': '''
             Check if text matches a pattern.
-            Returns $lib.true if the text matches the pattern, otherwise $lib.false.
 
             Notes:
                 This API requires the pattern to match at the start of the string.
@@ -4797,11 +4796,7 @@ class Str(Prim):
 
     @stormfunc(readonly=True)
     async def _methStrJson(self):
-        try:
-            return json.loads(self.valu, strict=True)
-        except Exception as e:
-            mesg = f'Text is not valid JSON: {self.valu}'
-            raise s_exc.BadJsonText(mesg=mesg)
+        return s_json.loads(self.valu)
 
 @registry.registerType
 class Bytes(Prim):
@@ -4990,19 +4985,15 @@ class Bytes(Prim):
             errors = await tostr(errors)
 
             if encoding is None:
-                encoding = json.detect_encoding(valu)
+                encoding = s_json.detect_encoding(valu)
             else:
                 encoding = await tostr(encoding)
 
-            return json.loads(valu.decode(encoding, errors))
+            return s_json.loads(valu.decode(encoding, errors))
 
         except UnicodeDecodeError as e:
             raise s_exc.StormRuntimeError(mesg=f'{e}: {s_common.trimText(repr(valu))}') from None
 
-        except json.JSONDecodeError as e:
-            mesg = f'Unable to decode bytes as json: {e.args[0]}'
-            raise s_exc.BadJsonText(mesg=mesg)
-
 @registry.registerType
 class Dict(Prim):
     '''
@@ -6262,7 +6253,7 @@ class NodeData(Prim):
         gateiden = self.valu.snap.wlyr.iden
         confirm(('node', 'data', 'set', name), gateiden=gateiden)
         valu = await toprim(valu)
-        s_common.reqjsonsafe(valu)
+        s_json.reqjsonsafe(valu)
         return await self.valu.setData(name, valu)
 
     async def _popNodeData(self, name):
@@ -7790,7 +7781,7 @@ class View(Prim):
     _storm_locals = (
         {'name': 'iden', 'desc': 'The iden of the View.', 'type': 'str', },
         {'name': 'layers', 'desc': 'The ``layer`` objects associated with the ``view``.', 'type': 'list', },
-        {'name': 'parent', 'desc': 'The parent View. Will be ``$lib.null`` if the view is not a fork.', 'type': 'str'},
+        {'name': 'parent', 'desc': 'The parent View. Will be ``(null)`` if the view is not a fork.', 'type': 'str'},
         {'name': 'triggers', 'desc': 'The ``trigger`` objects associated with the ``view``.',
          'type': 'list', },
         {'name': 'children', 'desc': 'Yield Views which are children of this View.',
@@ -7815,7 +7806,7 @@ class View(Prim):
                     the protected option (below) until this option is removed.
 
                 protected (bool)
-                    Setting to $lib.true will prevent the layer from being merged or deleted.
+                    Setting to ``(true)`` will prevent the layer from being merged or deleted.
 
                 layers (list(str))
                     Set the list of layer idens for a non-forked view. Layers are specified
@@ -8752,7 +8743,7 @@ class LibTrigger(Lib):
                     pass
 
         if trigger is None:
-            raise s_exc.NoSuchIden('Trigger not found')
+            raise s_exc.NoSuchIden(mesg='Trigger not found', iden=iden)
 
         self.runt.confirm(('trigger', 'get'), gateiden=iden)
 
@@ -8906,7 +8897,7 @@ class LibJsonStor(Lib):
                         {'name': 'path', 'type': 'str|list', 'desc': 'A path string or list of path parts.'},
                         {'name': 'prop', 'type': 'str|list', 'desc': 'A property name or list of name parts.', 'default': None},
                     ),
-                    'returns': {'type': 'prim', 'desc': 'The previously stored value or $lib.null'}}},
+                    'returns': {'type': 'prim', 'desc': 'The previously stored value or ``(null)``.'}}},
 
         {'name': 'set', 'desc': 'Set a JSON object or object property.',
          'type': {'type': 'function', '_funcname': 'set',

synapse/lib/structlog.py

@@ -1,9 +1,9 @@
-import json
-
 import logging
 
 import synapse.common as s_common
 
+import synapse.lib.json as s_json
+
 class JsonFormatter(logging.Formatter):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -37,4 +37,4 @@ class JsonFormatter(logging.Formatter):
         if extras:
             ret.update({k: v for k, v in extras.items() if k not in ret})
 
-        return json.dumps(ret, default=str)
+        return s_json.dumps(ret, default=str).decode()

synapse/lib/types.py

@@ -11,6 +11,7 @@ import synapse.exc as s_exc
 import synapse.common as s_common
 
 import synapse.lib.chop as s_chop
+import synapse.lib.json as s_json
 import synapse.lib.node as s_node
 import synapse.lib.time as s_time
 import synapse.lib.cache as s_cache
@@ -1601,7 +1602,7 @@ class Data(Type):
 
     def norm(self, valu):
         try:
-            s_common.reqjsonsafe(valu)
+            s_json.reqjsonsafe(valu)
             if self.validator is not None:
                 self.validator(valu)
         except (s_exc.MustBeJsonSafe, s_exc.SchemaViolation) as e:

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 201, 0)
+version = (2, 203, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'eed8c13ff316a77ae835539215c862d052d31994'
+commit = '809ecdca2e2b05f1cd111bb23c237127b2b9c177'

synapse/lib/view.py

@@ -1360,17 +1360,21 @@ class View(s_nexus.Pusher):  # type: ignore
             'readonly': False
         }
 
+        if name is None:
+            if (pname := self.parent.info.get('name')) is not None:
+                name = f'inserted fork of {pname}'
+            else:
+                name = f'inserted fork of {self.parent.iden}'
+
         vdef = {
             'iden': s_common.guid(),
+            'name': name,
             'created': ctime,
             'creator': useriden,
             'parent': self.parent.iden,
             'layers': [layriden] + [lyr.iden for lyr in self.parent.layers]
         }
 
-        if name is not None:
-            vdef['name'] = name
-
         s_layer.reqValidLdef(ldef)
         s_schemas.reqValidView(vdef)
 

synapse/models/base.py

@@ -29,10 +29,18 @@ class BaseModule(s_module.CoreModule):
 
             'types': (
 
+                ('meta:feed', ('guid', {}), {
+                    'doc': 'A data feed provided by a specific source.'}),
+
+                ('meta:feed:type:taxonomy', ('taxonomy', {}), {
+                    'interfaces': ('meta:taxonomy',),
+                    'doc': 'A data feed type taxonomy.'}),
+
                 ('meta:source', ('guid', {}), {
                     'doc': 'A data source unique identifier.'}),
 
                 ('meta:seen', ('comp', {'fields': (('source', 'meta:source'), ('node', 'ndef'))}), {
+                    'deprecated': True,
                     'doc': 'Annotates that the data in a node was obtained from or observed by a given source.'}),
 
                 ('meta:note', ('guid', {}), {
@@ -167,15 +175,22 @@ class BaseModule(s_module.CoreModule):
             'edges': (
                 ((None, 'refs', None), {
                     'doc': 'The source node contains a reference to the target node.'}),
+
                 (('meta:source', 'seen', None), {
                     'doc': 'The meta:source observed the target node.'}),
+
+                (('meta:feed', 'found', None), {
+                    'doc': 'The meta:feed produced the target node.'}),
+
                 (('meta:note', 'about', None), {
-                    'doc': 'The meta:note is about the target node.'
-                }),
+                    'doc': 'The meta:note is about the target node.'}),
+
                 (('meta:ruleset', 'has', 'meta:rule'), {
                     'doc': 'The meta:ruleset includes the meta:rule.'}),
+
                 (('meta:rule', 'matches', None), {
                     'doc': 'The meta:rule has matched on target node.'}),
+
                 (('meta:rule', 'detects', None), {
                     'doc': 'The meta:rule is designed to detect instances of the target node.'}),
             ),
@@ -213,6 +228,40 @@ class BaseModule(s_module.CoreModule):
 
                 )),
 
+                ('meta:feed:type:taxonomy', {}, ()),
+                ('meta:feed', {}, (
+
+                    ('name', ('str', {'lower': True, 'onespace': True}), {
+                        'doc': 'A name for the feed.'}),
+
+                    ('type', ('meta:feed:type:taxonomy', {}), {
+                        'doc': 'The type of data feed.'}),
+
+                    ('source', ('meta:source', {}), {
+                        'doc': 'The meta:source which provides the feed.'}),
+
+                    ('url', ('inet:url', {}), {
+                        'doc': 'The URL of the feed API endpoint.'}),
+
+                    ('query', ('str', {}), {
+                        'doc': 'The query logic associated with generating the feed output.'}),
+
+                    ('opts', ('data', {}), {
+                        'doc': 'An opaque JSON object containing feed parameters and options.'}),
+
+                    ('period', ('ival', {}), {
+                        'doc': 'The time window over which results have been ingested.'}),
+
+                    ('latest', ('time', {}), {
+                        'doc': 'The time of the last record consumed from the feed.'}),
+
+                    ('offset', ('int', {}), {
+                        'doc': 'The offset of the last record consumed from the feed.'}),
+
+                    ('cursor', ('str', {'strip': True}), {
+                        'doc': 'A cursor used to track ingest offset within the feed.'}),
+                )),
+
                 ('meta:note:type:taxonomy', {}, ()),
                 ('meta:note', {}, (