synapse 2.165.0__py311-none-any.whl → 2.167.0__py311-none-any.whl

synapse/lib/storm.py

@@ -29,7 +29,6 @@ import synapse.lib.version as s_version
 import synapse.lib.hashitem as s_hashitem
 import synapse.lib.hiveauth as s_hiveauth
 import synapse.lib.stormctrl as s_stormctrl
-import synapse.lib.provenance as s_provenance
 import synapse.lib.stormtypes as s_stormtypes
 
 import synapse.lib.stormlib.graph as s_stormlib_graph
@@ -1208,6 +1207,8 @@ stormcmds = (
         'descr': addcrondescr,
         'cmdargs': (
             ('query', {'help': 'Query for the cron job to execute.'}),
+            ('--pool', {'action': 'store_true', 'default': False,
+                'help': 'Allow the cron job to be run by a mirror from the query pool.'}),
             ('--minute', {'help': 'Minute value for job or recurrence period.'}),
             ('--name', {'help': 'An optional name for the cron job.'}),
             ('--doc', {'help': 'An optional doc string for the cron job.'}),
@@ -1227,6 +1228,7 @@ stormcmds = (
                                   minute=$cmdopts.minute,
                                   hour=$cmdopts.hour,
                                   day=$cmdopts.day,
+                                  pool=$cmdopts.pool,
                                   month=$cmdopts.month,
                                   year=$cmdopts.year,
                                   hourly=$cmdopts.hourly,
@@ -1374,6 +1376,7 @@ stormcmds = (
                 $lib.print('iden:            {iden}', iden=$job.iden)
                 $lib.print('user:            {user}', user=$job.user)
                 $lib.print('enabled:         {enabled}', enabled=$job.enabled)
+                $lib.print(`pool:            {$job.pool}`)
                 $lib.print('recurring:       {isrecur}', isrecur=$job.isrecur)
                 $lib.print('# starts:        {startcount}', startcount=$job.startcount)
                 $lib.print('# errors:        {errcount}', errcount=$job.errcount)
@@ -2310,27 +2313,26 @@ class Runtime(s_base.Base):
 
     async def execute(self, genr=None):
         try:
-            with s_provenance.claim('storm', q=self.query.text, user=self.user.iden):
 
-                async with contextlib.aclosing(self.query.iterNodePaths(self, genr=genr)) as nodegenr:
-                    nodegenr, empty = await s_ast.pullone(nodegenr)
+            async with contextlib.aclosing(self.query.iterNodePaths(self, genr=genr)) as nodegenr:
+                nodegenr, empty = await s_ast.pullone(nodegenr)
 
-                    if empty:
-                        return
+                if empty:
+                    return
 
-                    rules = self.opts.get('graph')
-                    if rules not in (False, None):
-                        if rules is True:
-                            rules = {'degrees': None, 'refs': True}
-                        elif isinstance(rules, str):
-                            rules = await self.snap.core.getStormGraph(rules)
+                rules = self.opts.get('graph')
+                if rules not in (False, None):
+                    if rules is True:
+                        rules = {'degrees': None, 'refs': True}
+                    elif isinstance(rules, str):
+                        rules = await self.snap.core.getStormGraph(rules)
 
-                        subgraph = s_ast.SubGraph(rules)
-                        nodegenr = subgraph.run(self, nodegenr)
+                    subgraph = s_ast.SubGraph(rules)
+                    nodegenr = subgraph.run(self, nodegenr)
 
-                    async for item in nodegenr:
-                        self.tick()
-                        yield item
+                async for item in nodegenr:
+                    self.tick()
+                    yield item
 
         except RecursionError:
             mesg = 'Maximum Storm pipeline depth exceeded.'

synapse/lib/stormlib/aha.py

@@ -1,42 +1,169 @@
-import synapse.common as s_common
+import synapse.exc as s_exc
 import synapse.lib.stormtypes as s_stormtypes
 
+@s_stormtypes.registry.registerLib
+class AhaLib(s_stormtypes.Lib):
+    '''
+    A Storm Library for interacting with AHA.
+    '''
+
+    _storm_locals = (
+        {'name': 'del', 'desc': '''Delete a service from AHA.
+
+        Examples:
+            Deleting a service with its relative name::
+
+                $lib.aha.del(00.mysvc...)
+
+            Deleting a service with its full name::
+
+                $lib.aha.del(00.mysvc.loop.vertex.link)
+        ''',
+         'type': {'type': 'function', '_funcname': '_methAhaDel',
+                  'args': (
+                      {'name': 'svcname', 'type': 'str',
+                       'desc': 'The name of the service to delete. It is easiest to use the relative name of a service, ending with "...".', },
+                  ),
+                  'returns': {'type': 'null'}}},
+        {'name': 'get', 'desc': '''Get information about an AHA service.
+
+        Examples:
+            Getting service information with a relative name::
+
+                $lib.aha.get(00.cortex...)
+
+            Getting service information with its full name::
+
+                $lib.aha.get(00.cortex.loop.vertex.link)
+        ''',
+         'type': {'type': 'function', '_funcname': '_methAhaGet',
+                  'args': (
+                      {'name': 'svcname', 'type': 'str',
+                       'desc': 'The name of the AHA service to look up. It is easiest to use the relative name of a service, ending with "...".', },
+                      {'name': 'filters', 'type': 'dict', 'default': None,
+                       'desc': 'An optional dictionary of filters to use when resolving the AHA service.'}
+                  ),
+                  'returns': {'type': ('null', 'dict'),
+                              'desc': 'The AHA service information dictionary, or $lib.null.', }}},
+        {'name': 'list', 'desc': 'Enumerate all of the AHA services.',
+         'type': {'type': 'function', '_funcname': '_methAhaList', 'args': (),
+                  'returns': {'name': 'Yields', 'type': 'list',
+                              'desc': 'The AHA service dictionaries.', }}},
+    )
+    _storm_lib_path = ('aha',)
+    def getObjLocals(self):
+        return {
+            'del': self._methAhaDel,
+            'get': self._methAhaGet,
+            'list': self._methAhaList,
+        }
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _methAhaList(self):
+        self.runt.reqAdmin()
+        proxy = await self.runt.snap.core.reqAhaProxy()
+        async for info in proxy.getAhaSvcs():
+            yield info
+
+    async def _methAhaDel(self, svcname):
+        self.runt.reqAdmin()
+        svcname = await s_stormtypes.tostr(svcname)
+        proxy = await self.runt.snap.core.reqAhaProxy()
+        svc = await proxy.getAhaSvc(svcname)
+        if svc is None:
+            raise s_exc.NoSuchName(mesg=f'No AHA service for {svcname=}')
+        if svc.get('services'):  # It is an AHA Pool!
+            mesg = f'Cannot use $lib.aha.del() to remove an AHA Pool. Use $lib.aha.pool.del(); {svcname=}'
+            raise s_exc.BadArg(mesg=mesg)
+        return await proxy.delAhaSvc(svc.get('svcname'), network=svc.get('svcnetw'))
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _methAhaGet(self, svcname, filters=None):
+        self.runt.reqAdmin()
+        svcname = await s_stormtypes.tostr(svcname)
+        filters = await s_stormtypes.toprim(filters)
+        proxy = await self.runt.snap.core.reqAhaProxy()
+        return await proxy.getAhaSvc(svcname, filters=filters)
+
 @s_stormtypes.registry.registerLib
 class AhaPoolLib(s_stormtypes.Lib):
     '''
     A Storm Library for interacting with AHA service pools.
     '''
+
+    _storm_locals = (
+        {'name': 'add', 'desc': '''Add a new AHA service pool.
+
+        Examples:
+            Add a pool via its relative name::
+
+                $lib.aha.pool.add(pool00.cortex...)
+        ''',
+         'type': {'type': 'function', '_funcname': '_methPoolAdd',
+                  'args': (
+                      {'name': 'name', 'type': 'str',
+                       'desc': 'The name of the pool to add. It is easiest to use the relative name of a pool, ending with "...".', },
+                  ),
+                  'returns': {'type': 'aha:pool'}}},
+        {'name': 'del', 'desc': '''Delete an existing AHA service pool.
+
+        Examples:
+            Delete a pool via its relative name::
+
+                $lib.aha.pool.del(pool00.cortex...)
+        ''',
+         'type': {'type': 'function', '_funcname': '_methPoolDel',
+                  'args': (
+                      {'name': 'name', 'type': 'str',
+                       'desc': 'The name of the pool to delete. It is easiest to use the relative name of a pool, ending with "...".', },
+                  ),
+                  'returns': {'type': 'dict', 'desc': 'The AHA pool definition that was deleted.'}}},
+        {'name': 'get', 'desc': 'Get an existing AHA service pool.',
+         'type': {'type': 'function', '_funcname': '_methPoolGet',
+                  'args': (
+                      {'name': 'name', 'type': 'str',
+                       'desc': 'The name of the pool to get. It is easiest to use the relative name of a pool, ending with "...".', },
+                  ),
+                  'returns': {'type': ['null', 'aha:pool'], 'desc': 'The pool if it exists, or $lib.null.'}}},
+        {'name': 'list', 'desc': 'Enumerate all of the AHA service pools.',
+         'type': {'type': 'function', '_funcname': '_methPoolList',
+                  'returns': {'name': 'yields', 'type': 'aha:pool'}}},
+    )
     _storm_lib_path = ('aha', 'pool')
 
     def getObjLocals(self):
         return {
-            'add': self.add,
-            'del': self._del,
-            'get': self.get,
-            'list': self.list,
+            'add': self._methPoolAdd,
+            'del': self._methPoolDel,
+            'get': self._methPoolGet,
+            'list': self._methPoolList,
         }
 
-    async def add(self, name):
+    async def _methPoolAdd(self, name):
         self.runt.reqAdmin()
+        name = await s_stormtypes.tostr(name)
         proxy = await self.runt.snap.core.reqAhaProxy()
         poolinfo = {'creator': self.runt.user.iden}
         poolinfo = await proxy.addAhaPool(name, poolinfo)
         return AhaPool(self.runt, poolinfo)
 
-    async def _del(self, name):
+    async def _methPoolDel(self, name):
         self.runt.reqAdmin()
+        name = await s_stormtypes.tostr(name)
         proxy = await self.runt.snap.core.reqAhaProxy()
         return await proxy.delAhaPool(name)
 
-    async def get(self, name):
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _methPoolGet(self, name):
         self.runt.reqAdmin()
+        name = await s_stormtypes.tostr(name)
         proxy = await self.runt.snap.core.reqAhaProxy()
         poolinfo = await proxy.getAhaPool(name)
         if poolinfo is not None:
             return AhaPool(self.runt, poolinfo)
 
-    async def list(self):
-
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _methPoolList(self):
         self.runt.reqAdmin()
         proxy = await self.runt.snap.core.reqAhaProxy()
 
@@ -48,6 +175,36 @@ class AhaPool(s_stormtypes.StormType):
     '''
     Implements the Storm API for an AHA pool.
     '''
+    _storm_locals = (
+        {'name': 'add', 'desc': '''Add a service to the AHA pool
+
+        Examples:
+            Add a service to a pool with its relative name::
+
+                $pool = $lib.aha.pool.get(pool00.cortex...)
+                $pool.add(00.cortex...)
+        ''',
+         'type': {'type': 'function', '_funcname': '_methPoolSvcAdd',
+                  'args': (
+                      {'name': 'svcname', 'type': 'str',
+                       'desc': 'The name of the AHA service to add. It is easiest to use the relative name of a service, ending with "...".', },
+                  ),
+                  'returns': {'type': 'null', }}},
+        {'name': 'add', 'desc': '''Remove a service from the AHA pool.
+
+        Examples:
+            Remove a service from a pool with its relative name::
+
+                $pool = $lib.aha.pool.get(pool00.cortex...)
+                $pool.del(00.cortex...)
+        ''',
+         'type': {'type': 'function', '_funcname': '_methPoolSvcDel',
+                  'args': (
+                      {'name': 'svcname', 'type': 'str',
+                       'desc': 'The name of the AHA service to remove. It is easiest to use the relative name of a service, ending with "...".', },
+                  ),
+                  'returns': {'type': 'null', }}},
+    )
     _storm_typename = 'aha:pool'
 
     def __init__(self, runt, poolinfo):
@@ -56,14 +213,17 @@ class AhaPool(s_stormtypes.StormType):
         self.poolinfo = poolinfo
 
         self.locls.update({
-            'add': self.add,
-            'del': self._del,
+            'add': self._methPoolSvcAdd,
+            'del': self._methPoolSvcDel,
         })
 
+    async def stormrepr(self):
+        return f'{self._storm_typename}: {self.poolinfo.get("name")}'
+
     async def _derefGet(self, name):
         return self.poolinfo.get(name)
 
-    async def add(self, svcname):
+    async def _methPoolSvcAdd(self, svcname):
         self.runt.reqAdmin()
         svcname = await s_stormtypes.tostr(svcname)
 
@@ -76,7 +236,7 @@ class AhaPool(s_stormtypes.StormType):
 
         self.poolinfo.update(poolinfo)
 
-    async def _del(self, svcname):
+    async def _methPoolSvcDel(self, svcname):
         self.runt.reqAdmin()
         svcname = await s_stormtypes.tostr(svcname)
 
@@ -97,7 +257,6 @@ stormcmds = (
         for $pool in $lib.aha.pool.list() {
             $count = ($count + 1)
             $lib.print(`Pool: {$pool.name}`)
-            $lib.print($pool)
             for ($svcname, $svcinfo) in $pool.services {
                 $lib.print(`    {$svcname}`)
             }
@@ -143,7 +302,7 @@ stormcmds = (
         ),
         'storm': '''
             $pool = $lib.aha.pool.get($cmdopts.poolname)
-            if (not $pool) { $lib.exit(`No AHA serivce pool named: {$cmdopts.poolname}`) }
+            if (not $pool) { $lib.exit(`No AHA service pool named: {$cmdopts.poolname}`) }
 
             $pool.add($cmdopts.svcname)
             $lib.print(`AHA service ({$cmdopts.svcname}) added to service pool ({$pool.name})`)
@@ -158,10 +317,204 @@ stormcmds = (
         ),
         'storm': '''
             $pool = $lib.aha.pool.get($cmdopts.poolname)
-            if (not $pool) { $lib.exit(`No AHA serivce pool named: {$cmdopts.poolname}`) }
+            if (not $pool) { $lib.exit(`No AHA service pool named: {$cmdopts.poolname}`) }
 
             $pool.del($cmdopts.svcname)
             $lib.print(`AHA service ({$cmdopts.svcname}) removed from service pool ({$pool.name})`)
         ''',
     },
+    {
+        'name': 'aha.svc.stat',
+        'descr': '''Show all information for a specific AHA service.
+
+If the --nexus argument is given, the Cortex will attempt to connect the service and report the Nexus offset of the service.
+
+The ready value indicates that a service has entered into the realtime change window for synchronizing changes from its leader.
+        ''',
+        'cmdargs': (
+            ('svc', {'help': 'The service to inspect.'}),
+            ('--nexus', {'help': 'Try to connect to online services and report their nexus offset.',
+                         'default': False, 'action': 'store_true'}),
+        ),
+        'storm': '''
+        function _getNexus(svcname) {
+            $_url = `aha://{$svcname}/`
+            try {
+                $_prox = $lib.telepath.open($_url)
+                $_info = $_prox.getCellInfo()
+                return ( $_info.cell.nexsindx )
+            } catch * as _err {
+                $_emsg = $_err.mesg
+                if ($_emsg = null ) {
+                    $_emsg = `{$_err}`
+                }
+                return ( $_emsg )
+            }
+        }
+
+        $svc = $lib.aha.get($cmdopts.svc)
+        if ($svc = null) {
+            $lib.print(`No service found for: "{$cmdopts.svc}"`)
+        } else {
+            $services = $svc.services
+            if $services {
+                $lib.print(`Resolved {$cmdopts.svc} to an AHA Pool.\n`)
+                $lib.print(`The pool currently has {$lib.len($services)} members.`)
+
+                $lib.print(`AHA Pool:   {$svc.name}`)
+                for ($_svcname, $_svcinfo) in $services {
+                    $lib.print(`Member:     {$_svcname}`)
+                }
+            } else {
+                $lib.print(`Resolved {$cmdopts.svc} to an AHA Service.\n`)
+                $svcinfo = $svc.svcinfo
+                $leader = $svcinfo.leader
+                if ($leader = null) {
+                    $leader = 'Service did not register itself with a leader name.'
+                }
+                $online = false
+                if $svcinfo.online {
+                    $online = true
+                }
+                $ready = 'null'
+                if $lib.dict.has($svcinfo, ready) {
+                    $ready = `{$svcinfo.ready}`
+                }
+                $lib.print(`Name:       {$svc.name}`)
+                $lib.print(`Online:     {$online}`)
+                $lib.print(`Ready:      {$ready}`)
+                $lib.print(`Run iden:   {$svcinfo.run}`)
+                $lib.print(`Cell iden:  {$svcinfo.iden}`)
+                $lib.print(`Leader:     {$leader}`)
+
+                if $cmdopts.nexus {
+                    if $svcinfo.online {
+                        $nexusOffset = $_getNexus($svc.name)
+                    } else {
+                        $nexusOffset = 'Service is not online. Will not attempt to retrieve its nexus offset.'
+                    }
+                    $lib.print(`Nexus:      {$nexusOffset}`)
+                }
+
+                $lib.print('Connection information:')
+                $urlinfo = $svcinfo.urlinfo
+                $keys = $lib.dict.keys($urlinfo)
+                $keys.sort()
+                for $k in $keys {
+                    $dk = `{$k}:`
+                    $dk = $dk.ljust(12)
+                    $lib.print(`    {$dk}{$urlinfo.$k}`)
+                }
+            }
+        }
+        '''
+    },
+    {
+        'name': 'aha.svc.list',
+        'descr': '''List AHA services.
+
+If the --nexus argument is given, the Cortex will attempt to connect to each service and report the Nexus offset of the service.
+
+The ready column indicates that a service has entered into the realtime change window for synchronizing changes from its leader.''',
+        'cmdargs': (
+            ('--nexus', {'help': 'Try to connect to online services and report their nexus offset.',
+                         'default': False, 'action': 'store_true'}),
+        ),
+        'storm': '''
+        function _getNexus(svcname) {
+            $_url = `aha://{$svcname}/`
+            try {
+                $_prox = $lib.telepath.open($_url)
+                $_info = $_prox.getCellInfo()
+                return ( $_info.cell.nexsindx )
+            } catch * as _err {
+                $_emsg = $_err.mesg
+                if ($_emsg = null ) {
+                    $_emsg = `{$_err}`
+                }
+                return ( $_emsg )
+            }
+        }
+
+        $svcs = ()
+        for $svc in $lib.aha.list() {
+            $svcs.append($svc)
+        }
+
+        if ($lib.len($svcs) = 0) {
+            $lib.print('No AHA services registered.')
+        }
+        else {
+            $columns = 'Name                                          Leader Online Ready Host            Port '
+            if $cmdopts.nexus {
+                $columns = `{$columns} Nexus`
+            }
+
+            $leaders = $lib.set()
+            for $info in $svcs {
+                $svcinfo = $info.svcinfo
+                if $svcinfo {
+                    if ($info.svcname = $svcinfo.leader) {
+                        $leaders.add($svcinfo.run)
+                    }
+                }
+            }
+
+            $lib.print($columns)
+
+            for $info in $svcs {
+                $name = $info.name
+                $nexusOffset = (null)
+                $svcinfo = $info.svcinfo
+
+                if $cmdopts.nexus {
+                    if $svcinfo.online {
+                        $nexusOffset = $_getNexus($name)
+                    } else {
+                        $nexusOffset = 'Service is not online. Will not attempt to retrieve its nexus offset.'
+                    }
+                }
+                $name=$name.ljust(45)
+
+                $online = false
+                if $svcinfo.online {
+                    $online = true
+                }
+                $online = $online.ljust(6)
+
+                $urlinfo = $svcinfo.urlinfo
+
+                $host = $urlinfo.host
+                $host = $host.ljust(15)
+
+                $port = $lib.cast(str, $urlinfo.port)  // Cast to str
+                $port = $port.ljust(5)
+
+                $ready = 'null'
+                if $lib.dict.has($svcinfo, ready) {
+                    $ready = `{$svcinfo.ready}`
+                }
+                $ready = $ready.ljust(5)
+
+                $leader = null
+                if ( $svcinfo.leader != null ) {
+                    if $leaders.has($svcinfo.run) {
+                        $leader = true
+                    } else {
+                        $leader = false
+                    }
+                }
+                $leader = $leader.ljust(6)
+
+                if $info {
+                    $s = `{$name} {$leader} {$online} {$ready} {$host} {$port}`
+                    if ($nexusOffset != null) {
+                        $s = `{$s} {$nexusOffset}`
+                    }
+                    $lib.print($s)
+                }
+            }
+        }
+        '''
+    }
 )

synapse/lib/stormlib/auth.py

@@ -1227,6 +1227,11 @@ class User(s_stormtypes.Prim):
         indx = await s_stormtypes.toint(indx, noneok=True)
         gateiden = await s_stormtypes.tostr(gateiden, noneok=True)
         self.runt.confirm(('auth', 'user', 'set', 'rules'), gateiden=gateiden)
+        # TODO: Remove me in 3.0.0
+        if gateiden == 'cortex':
+            mesg = f'Adding rule on the "cortex" authgate. This authgate is not used ' \
+                   f'for permission checks and will be removed in Synapse v3.0.0.'
+            await self.runt.snap.warn(mesg, log=False)
         await self.runt.snap.core.addUserRule(self.valu, rule, indx=indx, gateiden=gateiden)
 
     async def _methUserDelRule(self, rule, gateiden=None):
@@ -1480,6 +1485,11 @@ class Role(s_stormtypes.Prim):
         indx = await s_stormtypes.toint(indx, noneok=True)
         gateiden = await s_stormtypes.tostr(gateiden, noneok=True)
         self.runt.confirm(('auth', 'role', 'set', 'rules'), gateiden=gateiden)
+        # TODO: Remove me in 3.0.0
+        if gateiden == 'cortex':
+            mesg = f'Adding rule on the "cortex" authgate. This authgate is not used ' \
+                   f'for permission checks and will be removed in Synapse v3.0.0.'
+            await self.runt.snap.warn(mesg, log=False)
         await self.runt.snap.core.addRoleRule(self.valu, rule, indx=indx, gateiden=gateiden)
 
     async def _methRoleDelRule(self, rule, gateiden=None):
@@ -1558,10 +1568,7 @@ class LibAuth(s_stormtypes.Lib):
     @s_stormtypes.stormfunc(readonly=True)
     async def textFromRule(self, rule):
         rule = await s_stormtypes.toprim(rule)
-        text = '.'.join(rule[1])
-        if not rule[0]:
-            text = '!' + text
-        return text
+        return s_common.reprauthrule(rule)
 
     @s_stormtypes.stormfunc(readonly=True)
     async def getPermDefs(self):