synapse 2.164.0__py311-none-any.whl → 2.166.0__py311-none-any.whl

synapse/lib/stormlib/spooled.py

@@ -0,0 +1,109 @@
+import synapse.exc as s_exc
+import synapse.lib.msgpack as s_msgpack
+import synapse.lib.spooled as s_spooled
+import synapse.lib.stormtypes as s_stormtypes
+
+@s_stormtypes.registry.registerLib
+class LibSpooled(s_stormtypes.Lib):
+    '''
+    A Storm Library for interacting with Spooled Objects.
+    '''
+    _storm_locals = (
+        {'name': 'set', 'desc': '''
+            Get a Spooled Storm Set object.
+
+            A Spooled Storm Set object is memory-safe to grow to extraordinarily large sizes,
+            as it will fallback to file backed storage, with two restrictions. First
+            is that all items in the set can be serialized to a file if the set grows too large,
+            so all items added must be a serializable Storm primitive. Second is that when an
+            item is added to the Set, because it could be immediately written disk,
+            do not hold any references to it outside of the Set itself, as the two objects could
+            differ.
+            ''',
+         'type': {'type': 'function', '_funcname': '_methSet',
+                  'args': (
+                      {'name': '*vals', 'type': 'any', 'desc': 'Initial values to place in the set.', },
+                  ),
+                  'returns': {'type': 'set', 'desc': 'The new set.'}}},
+    )
+    _storm_lib_path = ('spooled',)
+
+    def getObjLocals(self):
+        return {
+            'set': self._methSet,
+        }
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _methSet(self, *vals):
+        core = self.runt.snap.core
+        spool = await s_spooled.Set.anit(dirn=core.dirn, cell=core, size=1000)
+
+        valu = list(vals)
+        for item in valu:
+            if s_stormtypes.ismutable(item):
+                mesg = f'{await s_stormtypes.torepr(item)} is mutable and cannot be used in a set.'
+                raise s_exc.StormRuntimeError(mesg=mesg)
+
+            if not s_msgpack.isok(item):
+                mesg = f'{await s_stormtypes.torepr(item)} is not safe to be used in a SpooledSet.'
+                raise s_exc.StormRuntimeError(mesg=mesg)
+
+            await spool.add(item)
+
+        return SpooledSet(spool)
+
+@s_stormtypes.registry.registerType
+class SpooledSet(s_stormtypes.Set):
+    '''
+    A StormLib API instance of a Storm Set object that can fallback to lmdb.
+    '''
+    _storm_typename = 'spooled:set'
+    _ismutable = True
+
+    def __init__(self, valu, path=None):
+
+        s_stormtypes.Prim.__init__(self, valu, path=path)
+        self.locls.update(self.getObjLocals())
+
+    async def iter(self):
+        async for item in self.valu:
+            yield item
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _methSetAdd(self, *items):
+        for i in items:
+            if s_stormtypes.ismutable(i):
+                mesg = f'{await s_stormtypes.torepr(i)} is mutable and cannot be used in a set.'
+                raise s_exc.StormRuntimeError(mesg=mesg)
+
+            if not s_msgpack.isok(i):
+                mesg = f'{await s_stormtypes.torepr(i)} is not safe to be used in a SpooledSet.'
+                raise s_exc.StormRuntimeError(mesg=mesg)
+
+            await self.valu.add(i)
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _methSetAdds(self, *items):
+        for item in items:
+            async for i in s_stormtypes.toiter(item):
+                if s_stormtypes.ismutable(i):
+                    mesg = f'{await s_stormtypes.torepr(i)} is mutable and cannot be used in a set.'
+                    raise s_exc.StormRuntimeError(mesg=mesg)
+
+                if not s_msgpack.isok(i):
+                    mesg = f'{await s_stormtypes.torepr(i)} is not safe to be used in a SpooledSet.'
+                    raise s_exc.StormRuntimeError(mesg=mesg)
+
+                await self.valu.add(i)
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _methSetList(self):
+        return [x async for x in self.valu]
+
+    async def stormrepr(self):
+        reprs = [await s_stormtypes.torepr(k) async for k in self.valu]
+        rval = ', '.join(reprs)
+        return f'{{{rval}}}'
+
+    async def value(self):
+        return set([x async for x in self.valu])

synapse/lib/stormlib/vault.py

@@ -279,7 +279,7 @@ stormcmds = (
             }
 
             if $cmdopts.role {
-                $iden = $lib.auth.roles.byname($cmdopts.user).iden
+                $iden = $lib.auth.roles.byname($cmdopts.role).iden
                 $type = "Role"
             } else {
                 $iden = $lib.auth.users.byname($cmdopts.user).iden

synapse/lib/stormtypes.py

@@ -36,7 +36,6 @@ import synapse.lib.trigger as s_trigger
 import synapse.lib.urlhelp as s_urlhelp
 import synapse.lib.version as s_version
 import synapse.lib.stormctrl as s_stormctrl
-import synapse.lib.provenance as s_provenance
 
 logger = logging.getLogger(__name__)
 
@@ -907,6 +906,18 @@ class LibService(Lib):
                   'returns': {'type': 'boolean', 'desc': 'Returns true if the service is available, false on a '
                                                          'timeout waiting for the service to be ready.', }}},
     )
+    _storm_lib_perms = (
+        {'perm': ('service', 'add'), 'gate': 'cortex',
+            'desc': 'Controls the ability to add a Storm Service to the Cortex.'},
+        {'perm': ('service', 'del'), 'gate': 'cortex',
+            'desc': 'Controls the ability to delete a Storm Service from the Cortex'},
+        {'perm': ('service', 'get'), 'gate': 'cortex',
+            'desc': 'Controls the ability to get the Service object for any Storm Service.'},
+        {'perm': ('service', 'get', '<iden>'), 'gate': 'cortex',
+            'desc': 'Controls the ability to get the Service object for a Storm Service by iden.'},
+        {'perm': ('service', 'list'), 'gate': 'cortex',
+         'desc': 'Controls the ability to list all available Storm Services and their service definitions.'},
+    )
     _storm_lib_path = ('service',)
 
     def getObjLocals(self):
@@ -931,6 +942,7 @@ class LibService(Lib):
             except s_exc.AuthDeny:
                 raise e from None
             else:
+                # TODO: Remove support for this permission in 3.0.0
                 mesg = 'Use of service.get.<servicename> permissions are deprecated.'
                 await self.runt.warnonce(mesg, svcname=ssvc.name, svciden=ssvc.iden)
 
@@ -3192,6 +3204,26 @@ class LibRegx(Lib):
                        'default': 0, },
                   ),
                   'returns': {'type': 'str', 'desc': 'The new string with matches replaced.', }}},
+        {'name': 'escape', 'desc': '''
+            Escape arbitrary strings for use in a regular expression pattern.
+
+            Example:
+
+                Escape node values for use in a regex pattern::
+
+                    for $match in $lib.regex.findall($lib.regex.escape($node.repr()), $mydocument) {
+                        // do something with $match
+                    }
+
+                Escape node values for use in regular expression filters::
+
+                    it:dev:str~=$lib.regex.escape($node.repr())
+                    ''',
+         'type': {'type': 'function', '_funcname': 'escape',
+                  'args': (
+                      {'name': 'text', 'type': 'str', 'desc': 'The text to escape.', },
+                  ),
+                  'returns': {'type': 'str', 'desc': 'Input string with special characters escaped.', }}},
         {'name': 'flags.i', 'desc': 'Regex flag to indicate that case insensitive matches are allowed.',
          'type': 'int', },
         {'name': 'flags.m', 'desc': 'Regex flag to indicate that multiline matches are allowed.', 'type': 'int', },
@@ -3208,6 +3240,7 @@ class LibRegx(Lib):
             'matches': self.matches,
             'findall': self.findall,
             'replace': self.replace,
+            'escape': self.escape,
             'flags': {'i': regex.IGNORECASE,
                       'm': regex.MULTILINE,
                       },
@@ -3258,6 +3291,11 @@ class LibRegx(Lib):
         regx = await self._getRegx(pattern, flags)
         return regx.findall(text)
 
+    @stormfunc(readonly=True)
+    async def escape(self, text):
+        text = await tostr(text)
+        return regex.escape(text)
+
 @registry.registerLib
 class LibCsv(Lib):
     '''
@@ -3399,13 +3437,14 @@ class LibFeed(Lib):
         data = await toprim(data)
 
         self.runt.layerConfirm(('feed:data', *name.split('.')))
-        with s_provenance.claim('feed:data', name=name):
-            #  small work around for the feed API consistency
-            if name == 'syn.nodes':
-                async for node in self.runt.snap.addNodes(data):
-                    yield node
-                return
-            await self.runt.snap.addFeedData(name, data)
+
+        #  small work around for the feed API consistency
+        if name == 'syn.nodes':
+            async for node in self.runt.snap.addNodes(data):
+                yield node
+            return
+
+        await self.runt.snap.addFeedData(name, data)
 
     @stormfunc(readonly=True)
     async def _libList(self):
@@ -3417,11 +3456,12 @@ class LibFeed(Lib):
         data = await toprim(data)
 
         self.runt.layerConfirm(('feed:data', *name.split('.')))
-        with s_provenance.claim('feed:data', name=name):
-            strict = self.runt.snap.strict
-            self.runt.snap.strict = False
-            await self.runt.snap.addFeedData(name, data)
-            self.runt.snap.strict = strict
+
+        # TODO this should be a reentrent safe with block
+        strict = self.runt.snap.strict
+        self.runt.snap.strict = False
+        await self.runt.snap.addFeedData(name, data)
+        self.runt.snap.strict = strict
 
 @registry.registerLib
 class LibPipe(Lib):
@@ -6047,6 +6087,8 @@ class Node(Prim):
                        'desc': 'An optional prefix to match tags under.', },
                       {'name': 'apply', 'type': 'boolean', 'desc': 'If true, apply the diff.',
                        'default': False, },
+                      {'name': 'norm', 'type': 'boolean', 'default': False,
+                       'desc': 'Optionally norm the list of tags. If a prefix is provided, it will not be normed.'},
                   ),
                   'returns':
                       {'type': 'dict',
@@ -6198,8 +6240,23 @@ class Node(Prim):
                     ret.append(groups)
         return ret
 
-    async def _methNodeDiffTags(self, tags, prefix=None, apply=False):
-        tags = set(await toprim(tags))
+    async def _methNodeDiffTags(self, tags, prefix=None, apply=False, norm=False):
+        norm = await tobool(norm)
+        apply = await tobool(apply)
+
+        if norm:
+            normtags = set()
+            tagpart = self.valu.snap.core.model.type('syn:tag:part')
+
+            async for part in toiter(tags):
+                try:
+                    normtags.add(tagpart.norm(part)[0])
+                except s_exc.BadTypeValu:
+                    pass
+
+            tags = normtags
+        else:
+            tags = set(await toprim(tags))
 
         if prefix:
             prefix = tuple((await tostr(prefix)).split('.'))
@@ -7443,7 +7500,7 @@ class View(Prim):
 
         {'name': 'getPropArrayCount',
          'desc': '''
-            Get the number of invidivual array property values in the View for the given array property name.
+            Get the number of individual array property values in the View for the given array property name.
 
             Notes:
                This is a fast approximate count calculated by summing the number of
@@ -7537,6 +7594,18 @@ class View(Prim):
                   'args': (),
                   'returns': {'name': 'Yields', 'type': 'dict',
                               'desc': 'Yields previously successful merges into the view.'}}},
+        {'name': 'getMergingViews', 'desc': 'Get a list of idens of Views that have open merge requests to this View.',
+         'type': {'type': 'function', '_funcname': 'getMergingViews',
+                  'args': (),
+                  'returns': {'name': 'idens', 'type': 'list', 'desc': 'The list of View idens that have an open merge request into this View.'}}},
+        {'name': 'setMergeVoteComment', 'desc': 'Set the comment associated with your vote on a merge request.',
+         'type': {'type': 'function', '_funcname': 'setMergeVoteComment',
+                  'args': ({'name': 'comment', 'type': 'str', 'desc': 'The text comment to set for the merge vote'},),
+                  'returns': {'type': 'dict', 'desc': 'The fully updated vote record.'}}},
+        {'name': 'setMergeComment', 'desc': 'Set the main comment/description of a merge request.',
+         'type': {'type': 'function', '_funcname': 'setMergeComment',
+                  'args': ({'name': 'comment', 'type': 'str', 'desc': 'The text comment to set for the merge request'}, ),
+                  'returns': {'type': 'dict', 'desc': 'The updated merge request.'}}},
     )
     _storm_typename = 'view'
     _ismutable = False
@@ -7577,10 +7646,13 @@ class View(Prim):
             'getMerges': self.getMerges,
             'delMergeVote': self.delMergeVote,
             'setMergeVote': self.setMergeVote,
+            'setMergeVoteComment': self.setMergeVoteComment,
             'getMergeRequest': self.getMergeRequest,
             'getMergeRequestSummary': self.getMergeRequestSummary,
             'delMergeRequest': self.delMergeRequest,
             'setMergeRequest': self.setMergeRequest,
+            'setMergeComment': self.setMergeComment,
+            'getMergingViews': self.getMergingViews,
         }
 
     async def addNode(self, form, valu, props=None):
@@ -7888,6 +7960,22 @@ class View(Prim):
 
         return await view.setMergeRequest(mreq)
 
+    async def setMergeComment(self, comment):
+        view = self._reqView()
+        quorum = view.reqParentQuorum()
+
+        if not self.runt.isAdmin(gateiden=view.iden):
+            mesg = 'Editing a merge request requires admin permissions on the view.'
+            raise s_exc.AuthDeny(mesg=mesg)
+
+        return await view.setMergeComment((await tostr(comment)))
+
+    async def getMergingViews(self):
+        view = self._reqView()
+        self.runt.confirm(('view', 'read'), gateiden=view.iden)
+
+        return await view.getMergingViews()
+
     async def setMergeVote(self, approved=True, comment=None):
         view = self._reqView()
         quorum = view.reqParentQuorum()
@@ -7908,6 +7996,11 @@ class View(Prim):
 
         return await view.setMergeVote(vote)
 
+    async def setMergeVoteComment(self, comment):
+        view = self._reqView()
+
+        return await view.setMergeVoteComment(self.runt.user.iden, (await tostr(comment)))
+
     async def delMergeVote(self, useriden=None):
         view = self._reqView()
         quorum = view.reqParentQuorum()
@@ -8596,7 +8689,7 @@ class LibCron(Lib):
                   'returns': {'type': 'str', 'desc': 'The iden of the CronJob which was moved.'}}},
         {'name': 'list', 'desc': 'List CronJobs in the Cortex.',
          'type': {'type': 'function', '_funcname': '_methCronList',
-                  'returns': {'type': 'list', 'desc': 'A list of ``cronjob`` objects..', }}},
+                  'returns': {'type': 'list', 'desc': 'A list of ``cronjob`` objects.', }}},
         {'name': 'enable', 'desc': 'Enable a CronJob in the Cortex.',
          'type': {'type': 'function', '_funcname': '_methCronEnable',
                   'args': (
@@ -8787,6 +8880,7 @@ class LibCron(Lib):
         incunit = None
         incval = None
         reqdict = {}
+        pool = await tobool(kwargs.get('pool', False))
         valinfo = {  # unit: (minval, next largest unit)
             'month': (1, 'year'),
             'dayofmonth': (1, 'month'),
@@ -8892,6 +8986,7 @@ class LibCron(Lib):
 
         cdef = {'storm': query,
                 'reqs': reqdict,
+                'pool': pool,
                 'incunit': incunit,
                 'incvals': incval,
                 'creator': self.runt.user.iden
@@ -9140,6 +9235,7 @@ class CronJob(Prim):
             'view': view,
             'viewshort': view[:8] + '..',
             'query': self.valu.get('query') or '<missing>',
+            'pool': self.valu.get('pool', False),
             'isrecur': 'Y' if self.valu.get('recur') else 'N',
             'isrunning': 'Y' if self.valu.get('isrunning') else 'N',
             'enabled': 'Y' if self.valu.get('enabled', True) else 'N',

synapse/lib/trigger.py

@@ -11,7 +11,6 @@ import synapse.lib.chop as s_chop
 import synapse.lib.cache as s_cache
 import synapse.lib.config as s_config
 import synapse.lib.grammar as s_grammar
-import synapse.lib.provenance as s_provenance
 
 logger = logging.getLogger(__name__)
 
@@ -137,49 +136,47 @@ class Triggers:
         finally:
             RecursionDepth.reset(token)
 
-    async def runNodeAdd(self, node, view=None):
+    async def runNodeAdd(self, node):
         with self._recursion_check():
-            [await trig.execute(node, view=view) for trig in self.nodeadd.get(node.form.name, ())]
+            [await trig.execute(node) for trig in self.nodeadd.get(node.form.name, ())]
 
-    async def runNodeDel(self, node, view=None):
+    async def runNodeDel(self, node):
         with self._recursion_check():
-            [await trig.execute(node, view=view) for trig in self.nodedel.get(node.form.name, ())]
+            [await trig.execute(node) for trig in self.nodedel.get(node.form.name, ())]
 
-    async def runPropSet(self, node, prop, oldv, view=None):
+    async def runPropSet(self, node, prop, oldv):
         vars = {'propname': prop.name, 'propfull': prop.full,
                 'auto': {'opts': {'propname': prop.name, 'propfull': prop.full, }},
                 }
         with self._recursion_check():
-            [await trig.execute(node, vars=vars, view=view) for trig in self.propset.get(prop.full, ())]
+            [await trig.execute(node, vars=vars) for trig in self.propset.get(prop.full, ())]
             if prop.univ is not None:
-                [await trig.execute(node, vars=vars, view=view) for trig in self.propset.get(prop.univ.full, ())]
+                [await trig.execute(node, vars=vars) for trig in self.propset.get(prop.univ.full, ())]
 
-    async def runTagAdd(self, node, tag, view=None):
+    async def runTagAdd(self, node, tag):
 
-        vars = {'tag': tag,
-                'auto': {'opts': {'tag': tag}},
-                }
+        vars = {'tag': tag, 'auto': {'opts': {'tag': tag}}}
         with self._recursion_check():
 
             for trig in self.tagadd.get((node.form.name, tag), ()):
-                await trig.execute(node, vars=vars, view=view)
+                await trig.execute(node, vars=vars)
 
             for trig in self.tagadd.get((None, tag), ()):
-                await trig.execute(node, vars=vars, view=view)
+                await trig.execute(node, vars=vars)
 
             # check for form specific globs
             globs = self.tagaddglobs.get(node.form.name)
             if globs is not None:
                 for _, trig in globs.get(tag):
-                    await trig.execute(node, vars=vars, view=view)
+                    await trig.execute(node, vars=vars)
 
             # check for form agnostic globs
             globs = self.tagaddglobs.get(None)
             if globs is not None:
                 for _, trig in globs.get(tag):
-                    await trig.execute(node, vars=vars, view=view)
+                    await trig.execute(node, vars=vars)
 
-    async def runTagDel(self, node, tag, view=None):
+    async def runTagDel(self, node, tag):
 
         vars = {'tag': tag,
                 'auto': {'opts': {'tag': tag}},
@@ -187,24 +184,24 @@ class Triggers:
         with self._recursion_check():
 
             for trig in self.tagdel.get((node.form.name, tag), ()):
-                await trig.execute(node, vars=vars, view=view)
+                await trig.execute(node, vars=vars)
 
             for trig in self.tagdel.get((None, tag), ()):
-                await trig.execute(node, vars=vars, view=view)
+                await trig.execute(node, vars=vars)
 
             # check for form specific globs
             globs = self.tagdelglobs.get(node.form.name)
             if globs is not None:
                 for _, trig in globs.get(tag):
-                    await trig.execute(node, vars=vars, view=view)
+                    await trig.execute(node, vars=vars)
 
             # check for form agnostic globs
             globs = self.tagdelglobs.get(None)
             if globs is not None:
                 for _, trig in globs.get(tag):
-                    await trig.execute(node, vars=vars, view=view)
+                    await trig.execute(node, vars=vars)
 
-    async def runEdgeAdd(self, n1, verb, n2, view=None):
+    async def runEdgeAdd(self, n1, verb, n2):
         n1form = n1.form.name if n1 else None
         n2form = n2.form.name if n2 else None
         n2iden = n2.iden() if n2 else None
@@ -252,9 +249,9 @@ class Triggers:
                 self.edgeaddcache[cachekey] = cached
 
             for trig in cached:
-                await trig.execute(n1, vars=varz, view=view)
+                await trig.execute(n1, vars=varz)
 
-    async def runEdgeDel(self, n1, verb, n2, view=None):
+    async def runEdgeDel(self, n1, verb, n2):
         n1form = n1.form.name if n1 else None
         n2form = n2.form.name if n2 else None
         n2iden = n2.iden() if n2 else None
@@ -302,7 +299,7 @@ class Triggers:
                 self.edgedelcache[cachekey] = cached
 
             for trig in cached:
-                await trig.execute(n1, vars=varz, view=view)
+                await trig.execute(n1, vars=varz)
 
     async def load(self, tdef):
 
@@ -502,7 +499,7 @@ class Trigger:
     def get(self, name):
         return self.tdef.get(name)
 
-    async def execute(self, node, vars=None, view=None):
+    async def execute(self, node, vars=None):
         '''
         Actually execute the query
         '''
@@ -514,9 +511,9 @@ class Trigger:
             await self.view.addTrigQueue(triginfo)
             return
 
-        return await self._execute(node, vars=vars, view=view)
+        return await self._execute(node, vars=vars)
 
-    async def _execute(self, node, vars=None, view=None):
+    async def _execute(self, node, vars=None):
 
         locked = self.user.info.get('locked')
         if locked:
@@ -533,9 +530,6 @@ class Trigger:
 
         query = await self.view.core.getStormQuery(storm)
 
-        if view is None:
-            view = self.view.iden
-
         if vars is None:
             vars = {}
         else:
@@ -547,28 +541,23 @@ class Trigger:
         optvars['form'] = node.ndef[0]
         optvars['valu'] = node.ndef[1]
 
-        opts = {
-            'vars': vars,
-            'view': view,
-            'user': self.user.iden,
-        }
+        opts = {'vars': vars, 'user': self.user.iden, 'view': self.view.iden}
 
         self.startcount += 1
 
-        with s_provenance.claim('trig', cond=cond, form=form, tag=tag, prop=prop):
-            try:
-                async with self.view.core.getStormRuntime(query, opts=opts) as runt:
+        try:
+            async with self.view.core.getStormRuntime(query, opts=opts) as runt:
 
-                    runt.addInput(node)
-                    await s_common.aspin(runt.execute())
+                runt.addInput(node)
+                await s_common.aspin(runt.execute())
 
-            except (asyncio.CancelledError, s_exc.RecursionLimitHit):
-                raise
+        except (asyncio.CancelledError, s_exc.RecursionLimitHit):
+            raise
 
-            except Exception as e:
-                self.errcount += 1
-                self.lasterrs.append(str(e))
-                logger.exception('Trigger encountered exception running storm query %s', storm)
+        except Exception as e:
+            self.errcount += 1
+            self.lasterrs.append(str(e))
+            logger.exception('Trigger encountered exception running storm query %s', storm)
 
     def pack(self):
         tdef = self.tdef.copy()