synapse 2.165.0__py311-none-any.whl → 2.166.0__py311-none-any.whl

synapse/lib/cell.py

@@ -1560,32 +1560,22 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         self.ahasvcname = f'{ahaname}.{ahanetw}'
 
-        async def onlink(proxy):
-            while not proxy.isfini:
-                info = await self.getAhaInfo()
+        async def _runAhaRegLoop():
+
+            while not self.isfini:
                 try:
+                    proxy = await self.ahaclient.proxy()
+                    info = await self.getAhaInfo()
                     await proxy.addAhaSvc(ahaname, info, network=ahanetw)
                     if self.isactive and ahalead is not None:
                         await proxy.addAhaSvc(ahalead, info, network=ahanetw)
+                except Exception as e:
+                    logger.exception(f'Error registering service {self.ahasvcname} with AHA: {e}')
+                    await self.waitfini(1)
+                else:
+                    await proxy.waitfini()
 
-                    return
-
-                except asyncio.CancelledError:  # pragma: no cover
-                    raise
-
-                except Exception:
-                    logger.exception('Error in _initAhaService() onlink')
-
-                await proxy.waitfini(1)
-
-        async def fini():
-            await self.ahaclient.offlink(onlink)
-
-        async def init():
-            await self.ahaclient.onlink(onlink)
-            self.onfini(fini)
-
-        self.schedCoro(init())
+        self.schedCoro(_runAhaRegLoop())
 
     async def initServiceRuntime(self):
         pass
@@ -3694,7 +3684,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         await self.ahaclient.waitready()
 
-        mirrors = await self.ahaclient.getAhaSvcMirrors(self.ahasvcname)
+        proxy = await self.ahaclient.proxy(timeout=5)
+        mirrors = await proxy.getAhaSvcMirrors(self.ahasvcname)
         if mirrors is None:
             mesg = 'Service must be configured with AHA to enumerate mirror URLs'
             raise s_exc.NoSuchName(mesg=mesg, name=self.ahasvcname)

synapse/lib/httpapi.py

@@ -1344,6 +1344,7 @@ class ExtApiHandler(StormHandler):
         varz['_http_request_info'] = info
 
         opts = {
+            'mirror': adef.get('pool', False),
             'readonly': adef.get('readonly'),
             'show': (
                 'http:resp:body',

synapse/lib/nexus.py

@@ -589,10 +589,11 @@ class NexsRoot(s_base.Base):
 
         try:
             await self.cell.ahaclient.waitready(timeout=5)
-            ahainfo = await self.cell.ahaclient.getCellInfo()
+            proxy = await self.cell.ahaclient.proxy(timeout=5)
+            ahainfo = await proxy.getCellInfo()
             ahavers = ahainfo['synapse']['version']
             if self.cell.ahasvcname is not None and ahavers >= (2, 95, 0):
-                await self.cell.ahaclient.modAhaSvcInfo(self.cell.ahasvcname, {'ready': status})
+                await proxy.modAhaSvcInfo(self.cell.ahasvcname, {'ready': status})
 
         except asyncio.CancelledError:  # pragma: no cover  TODO:  remove once >= py 3.8 only
             raise

synapse/lib/schemas.py

@@ -40,6 +40,7 @@ _HttpExtAPIConfSchema = {
         'name': {'type': 'string', 'default': ''},
         'desc': {'type': 'string', 'default': ''},
         'path': {'type': 'string', 'minlen': 1},
+        'pool': {'type': 'boolean', 'default': False},
         'view': {'type': 'string', 'pattern': s_config.re_iden},
         'runas': {'type': 'string', 'pattern': '^(owner|user)$'},
         'owner': {'type': 'string', 'pattern': s_config.re_iden},
@@ -93,6 +94,7 @@ _CronJobSchema = {
         'iden': {'type': 'string', 'pattern': s_config.re_iden},
         'view': {'type': 'string', 'pattern': s_config.re_iden},
         'name': {'type': 'string'},
+        'pool': {'type': 'boolean'},
         'doc': {'type': 'string'},
         'incunit': {
             'oneOf': [

synapse/lib/snap.py

@@ -209,6 +209,27 @@ class ProtoNode:
         if tagnode is not s_common.novalu:
             return self.ctx.loadNode(tagnode)
 
+        # check for an :isnow tag redirection in our hierarchy...
+        toks = info.get('toks')
+        for i in range(len(toks)):
+
+            toktag = '.'.join(toks[:i + 1])
+            toknode = await self.ctx.snap.getTagNode(toktag)
+            if toknode is s_common.novalu:
+                continue
+
+            tokvalu = toknode.ndef[1]
+            if tokvalu == toktag:
+                continue
+
+            realnow = tokvalu + norm[len(toktag):]
+            tagnode = await self.ctx.snap.getTagNode(realnow)
+            if tagnode is not s_common.novalu:
+                return self.ctx.loadNode(tagnode)
+
+            norm, info = await self.ctx.snap.getTagNorm(realnow)
+            break
+
         return await self.ctx.addNode('syn:tag', norm, norminfo=info)
 
     def getTag(self, tag):
@@ -657,6 +678,33 @@ class Snap(s_base.Base):
         self.onfini(runt)
         return runt
 
+    async def _joinEmbedStor(self, storage, embeds):
+        for nodePath, relProps in embeds.items():
+            await asyncio.sleep(0)
+            iden = relProps.get('*')
+            if not iden:
+                continue
+
+            stor = await self.view.getStorNodes(s_common.uhex(iden))
+            for relProp in relProps.keys():
+                await asyncio.sleep(0)
+                if relProp == '*':
+                    continue
+
+                for idx, layrstor in enumerate(stor):
+                    await asyncio.sleep(0)
+                    props = layrstor.get('props')
+                    if not props:
+                        continue
+
+                    if relProp not in props:
+                        continue
+
+                    if 'embeds' not in storage[idx]:
+                        storage[idx]['embeds'] = {}
+
+                    storage[idx]['embeds'][f'{nodePath}::{relProp}'] = props[relProp]
+
     async def iterStormPodes(self, text, opts, user=None):
         '''
         Yield packed node tuples for the given storm query text.
@@ -699,6 +747,8 @@ class Snap(s_base.Base):
                 embdef = embeds.get(node.form.name)
                 if embdef is not None:
                     pode[1]['embeds'] = await node.getEmbeds(embdef)
+                    if show_storage:
+                        await self._joinEmbedStor(pode[1]['storage'], pode[1]['embeds'])
 
             yield pode
 

synapse/lib/storm.py

@@ -29,7 +29,6 @@ import synapse.lib.version as s_version
 import synapse.lib.hashitem as s_hashitem
 import synapse.lib.hiveauth as s_hiveauth
 import synapse.lib.stormctrl as s_stormctrl
-import synapse.lib.provenance as s_provenance
 import synapse.lib.stormtypes as s_stormtypes
 
 import synapse.lib.stormlib.graph as s_stormlib_graph
@@ -1208,6 +1207,8 @@ stormcmds = (
         'descr': addcrondescr,
         'cmdargs': (
             ('query', {'help': 'Query for the cron job to execute.'}),
+            ('--pool', {'action': 'store_true', 'default': False,
+                'help': 'Allow the cron job to be run by a mirror from the query pool.'}),
             ('--minute', {'help': 'Minute value for job or recurrence period.'}),
             ('--name', {'help': 'An optional name for the cron job.'}),
             ('--doc', {'help': 'An optional doc string for the cron job.'}),
@@ -1227,6 +1228,7 @@ stormcmds = (
                                   minute=$cmdopts.minute,
                                   hour=$cmdopts.hour,
                                   day=$cmdopts.day,
+                                  pool=$cmdopts.pool,
                                   month=$cmdopts.month,
                                   year=$cmdopts.year,
                                   hourly=$cmdopts.hourly,
@@ -1374,6 +1376,7 @@ stormcmds = (
                 $lib.print('iden:            {iden}', iden=$job.iden)
                 $lib.print('user:            {user}', user=$job.user)
                 $lib.print('enabled:         {enabled}', enabled=$job.enabled)
+                $lib.print(`pool:            {$job.pool}`)
                 $lib.print('recurring:       {isrecur}', isrecur=$job.isrecur)
                 $lib.print('# starts:        {startcount}', startcount=$job.startcount)
                 $lib.print('# errors:        {errcount}', errcount=$job.errcount)
@@ -2310,27 +2313,26 @@ class Runtime(s_base.Base):
 
     async def execute(self, genr=None):
         try:
-            with s_provenance.claim('storm', q=self.query.text, user=self.user.iden):
 
-                async with contextlib.aclosing(self.query.iterNodePaths(self, genr=genr)) as nodegenr:
-                    nodegenr, empty = await s_ast.pullone(nodegenr)
+            async with contextlib.aclosing(self.query.iterNodePaths(self, genr=genr)) as nodegenr:
+                nodegenr, empty = await s_ast.pullone(nodegenr)
 
-                    if empty:
-                        return
+                if empty:
+                    return
 
-                    rules = self.opts.get('graph')
-                    if rules not in (False, None):
-                        if rules is True:
-                            rules = {'degrees': None, 'refs': True}
-                        elif isinstance(rules, str):
-                            rules = await self.snap.core.getStormGraph(rules)
+                rules = self.opts.get('graph')
+                if rules not in (False, None):
+                    if rules is True:
+                        rules = {'degrees': None, 'refs': True}
+                    elif isinstance(rules, str):
+                        rules = await self.snap.core.getStormGraph(rules)
 
-                        subgraph = s_ast.SubGraph(rules)
-                        nodegenr = subgraph.run(self, nodegenr)
+                    subgraph = s_ast.SubGraph(rules)
+                    nodegenr = subgraph.run(self, nodegenr)
 
-                    async for item in nodegenr:
-                        self.tick()
-                        yield item
+                async for item in nodegenr:
+                    self.tick()
+                    yield item
 
         except RecursionError:
             mesg = 'Maximum Storm pipeline depth exceeded.'

synapse/lib/stormlib/aha.py

@@ -35,6 +35,7 @@ class AhaPoolLib(s_stormtypes.Lib):
         if poolinfo is not None:
             return AhaPool(self.runt, poolinfo)
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def list(self):
 
         self.runt.reqAdmin()
@@ -60,6 +61,9 @@ class AhaPool(s_stormtypes.StormType):
             'del': self._del,
         })
 
+    async def stormrepr(self):
+        return f'{self._storm_typename}: {self.poolinfo.get("name")}'
+
     async def _derefGet(self, name):
         return self.poolinfo.get(name)
 
@@ -97,7 +101,6 @@ stormcmds = (
         for $pool in $lib.aha.pool.list() {
             $count = ($count + 1)
             $lib.print(`Pool: {$pool.name}`)
-            $lib.print($pool)
             for ($svcname, $svcinfo) in $pool.services {
                 $lib.print(`    {$svcname}`)
             }

synapse/lib/stormlib/auth.py

@@ -1227,6 +1227,11 @@ class User(s_stormtypes.Prim):
         indx = await s_stormtypes.toint(indx, noneok=True)
         gateiden = await s_stormtypes.tostr(gateiden, noneok=True)
         self.runt.confirm(('auth', 'user', 'set', 'rules'), gateiden=gateiden)
+        # TODO: Remove me in 3.0.0
+        if gateiden == 'cortex':
+            mesg = f'Adding rule on the "cortex" authgate. This authgate is not used ' \
+                   f'for permission checks and will be removed in Synapse v3.0.0.'
+            await self.runt.snap.warn(mesg, log=False)
         await self.runt.snap.core.addUserRule(self.valu, rule, indx=indx, gateiden=gateiden)
 
     async def _methUserDelRule(self, rule, gateiden=None):
@@ -1480,6 +1485,11 @@ class Role(s_stormtypes.Prim):
         indx = await s_stormtypes.toint(indx, noneok=True)
         gateiden = await s_stormtypes.tostr(gateiden, noneok=True)
         self.runt.confirm(('auth', 'role', 'set', 'rules'), gateiden=gateiden)
+        # TODO: Remove me in 3.0.0
+        if gateiden == 'cortex':
+            mesg = f'Adding rule on the "cortex" authgate. This authgate is not used ' \
+                   f'for permission checks and will be removed in Synapse v3.0.0.'
+            await self.runt.snap.warn(mesg, log=False)
         await self.runt.snap.core.addRoleRule(self.valu, rule, indx=indx, gateiden=gateiden)
 
     async def _methRoleDelRule(self, rule, gateiden=None):
@@ -1558,10 +1568,7 @@ class LibAuth(s_stormtypes.Lib):
     @s_stormtypes.stormfunc(readonly=True)
     async def textFromRule(self, rule):
         rule = await s_stormtypes.toprim(rule)
-        text = '.'.join(rule[1])
-        if not rule[0]:
-            text = '!' + text
-        return text
+        return s_common.reprauthrule(rule)
 
     @s_stormtypes.stormfunc(readonly=True)
     async def getPermDefs(self):

synapse/lib/stormlib/cache.py

@@ -0,0 +1,202 @@
+import asyncio
+
+import synapse.exc as s_exc
+
+import synapse.lib.ast as s_ast
+import synapse.lib.cache as s_cache
+import synapse.lib.stormctrl as s_stormctrl
+import synapse.lib.stormtypes as s_stormtypes
+
+CACHE_SIZE_MAX = 10_000
+CACHE_SIZE_DEFAULT = 10_000
+
+@s_stormtypes.registry.registerLib
+class LibCache(s_stormtypes.Lib):
+    '''
+    A Storm Library for interacting with Cache Objects.
+    '''
+    _storm_locals = (
+        {'name': 'fixed', 'desc': '''
+            Get a new Fixed Cache object.
+
+            On a cache-miss when calling .get(), the callback Storm query is executed in a sub-runtime
+            in the current execution context. A special variable, $cache_key, will be set
+            to the key argument provided to .get().
+
+            The callback Storm query must contain a return statement, and if it does not return a value
+            when executed with the input, $lib.null will be set as the value.
+
+            The fixed cache uses FIFO to evict items once the maximum size is reached.
+
+            Examples:
+
+                // Use a callback query with a function that modifies the outer runtime,
+                // since it will run in the scope where it was defined.
+                $test = foo
+
+                function callback(key) {
+                    $test = $key // this will modify $test in the outer runtime
+                    return(`{$key}-val`)
+                }
+
+                $cache = $lib.cache.fixed(${ return($callback($cache_key)) })
+                $value = $cache.get(bar)
+                $lib.print($test) // this will equal "bar"
+
+                // Use a callback query that will not modify the outer runtime,
+                // except for variables accessible as references.
+                $test = foo
+                $tests = ([])
+
+                $cache = $lib.cache.fixed(${
+                    $test = $cache_key        // this will *not* modify $test in the outer runtime
+                    $tests.append($cache_key) // this will modify $tests in the outer runtime
+                    return(`{$cache_key}-val`)
+                })
+
+                $value = $cache.get(bar)
+                $lib.print($test)  // this will equal "foo"
+                $lib.print($tests) // this will equal (foo,)
+            ''',
+         'type': {'type': 'function', '_funcname': '_methFixedCache',
+                  'args': (
+                      {'name': 'callback', 'type': ['str', 'storm:query'],
+                       'desc': 'A Storm query that will return a value for $cache_key on a cache miss.', },
+                      {'name': 'size', 'type': 'int', 'default': CACHE_SIZE_DEFAULT,
+                       'desc': 'The maximum size of the cache.', },
+                  ),
+                  'returns': {'type': 'cache:fixed', 'desc': 'A new ``cache:fixed`` object.'}}},
+    )
+    _storm_lib_path = ('cache',)
+
+    def getObjLocals(self):
+        return {
+            'fixed': self._methFixedCache,
+        }
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _methFixedCache(self, callback, size=CACHE_SIZE_DEFAULT):
+        size = await s_stormtypes.toint(size)
+        callback = await s_stormtypes.tostr(callback)
+
+        if size < 1 or size > CACHE_SIZE_MAX:
+            raise s_exc.BadArg(mesg=f'Cache size must be between 1-{CACHE_SIZE_MAX}')
+
+        try:
+            query = await self.runt.getStormQuery(callback)
+        except s_exc.BadSyntax as e:
+            raise s_exc.BadArg(mesg=f'Invalid callback query: {e.errinfo.get("mesg")}')
+
+        if not query.hasAstClass(s_ast.Return):
+            raise s_exc.BadArg(mesg='Callback query must return a value')
+
+        return FixedCache(self.runt, query, size=size)
+
+@s_stormtypes.registry.registerType
+class FixedCache(s_stormtypes.StormType):
+    '''
+    A StormLib API instance of a Storm Fixed Cache.
+    '''
+    _storm_locals = (
+        {'name': 'query', 'desc': 'Get the callback Storm query as string.',
+         'type': {'type': 'gtor', '_gtorfunc': '_gtorQuery',
+                  'returns': {'type': 'str', 'desc': 'The callback Storm query text.', }}},
+        {'name': 'get', 'desc': 'Get an item from the cache by key.',
+         'type': {'type': 'function', '_funcname': '_methGet',
+                  'args': (
+                      {'name': 'key', 'type': 'any', 'desc': 'The key to lookup.'},
+                  ),
+                  'returns': {'type': 'any',
+                              'desc': 'The value from the cache, or the callback query if it does not exist', }}},
+        {'name': 'pop', 'desc': 'Pop an item from the cache.',
+         'type': {'type': 'function', '_funcname': '_methPop',
+                  'args': (
+                      {'name': 'key', 'type': 'any', 'desc': 'The key to pop.'},
+                  ),
+                  'returns': {'type': 'any',
+                              'desc': 'The value from the cache, or $lib.null if it does not exist', }}},
+        {'name': 'put', 'desc': 'Put an item into the cache.',
+         'type': {'type': 'function', '_funcname': '_methPut',
+                  'args': (
+                      {'name': 'key', 'type': 'any', 'desc': 'The key put in the cache.'},
+                      {'name': 'value', 'type': 'any', 'desc': 'The value to assign to the key.'},
+                  ),
+                  'returns': {'type': 'null', }}},
+        {'name': 'clear', 'desc': 'Clear all items from the cache.',
+         'type': {'type': 'function', '_funcname': '_methClear',
+                  'returns': {'type': 'null', }}},
+    )
+    _storm_typename = 'cache:fixed'
+    _ismutable = False
+
+    def __init__(self, runt, query, size=CACHE_SIZE_DEFAULT):
+        s_stormtypes.StormType.__init__(self)
+        self.runt = runt
+        self.size = size
+        self.query = query
+        self.locls.update(self.getObjLocals())
+        self.gtors.update({
+            'query': self._gtorQuery,
+        })
+
+        self.cache = s_cache.FixedCache(self._runCallback, size=size)
+
+    def __len__(self):
+        return len(self.cache)
+
+    async def stormrepr(self):
+        if len(qtext := self.query.text) > 100:
+            qtext = qtext[:100] + '...'
+        return f'{self._storm_typename}: size={self.size} query="{qtext}"'
+
+    def getObjLocals(self):
+        return {
+            'pop': self._methPop,
+            'put': self._methPut,
+            'get': self._methGet,
+            'clear': self._methClear,
+        }
+
+    async def _gtorQuery(self):
+        return self.query.text
+
+    async def _runCallback(self, key):
+
+        varz = self.runt.getScopeVars()
+        varz['cache_key'] = key
+
+        opts = {'vars': varz}
+        async with self.runt.getCmdRuntime(self.query, opts=opts) as runt:
+            try:
+                async for _ in runt.execute():
+                    await asyncio.sleep(0)
+            except s_stormctrl.StormReturn as e:
+                return await s_stormtypes.toprim(e.item)
+            except s_stormctrl.StormCtrlFlow:
+                pass
+
+    async def _reqKey(self, key):
+        if s_stormtypes.ismutable(key):
+            mesg = 'Mutable values are not allowed as cache keys'
+            raise s_exc.BadArg(mesg=mesg, name=await s_stormtypes.torepr(key))
+        return await s_stormtypes.toprim(key)
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _methPop(self, key):
+        key = await self._reqKey(key)
+        return self.cache.pop(key)
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _methPut(self, key, value):
+        key = await self._reqKey(key)
+        val = await s_stormtypes.toprim(value)
+        self.cache.put(key, val)
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _methGet(self, key):
+        key = await self._reqKey(key)
+        return await self.cache.aget(key)
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _methClear(self):
+        self.cache.clear()