synapse 2.164.0__py311-none-any.whl → 2.166.0__py311-none-any.whl

synapse/datamodel.py

@@ -116,6 +116,9 @@ class Prop:
             self.setperms.append(('node', 'prop', 'set', form.name, self.name))
             self.delperms.append(('node', 'prop', 'del', form.name, self.name))
 
+        self.setperms.reverse()  # Make them in precedence order
+        self.delperms.reverse()  # Make them in precedence order
+
         self.form = form
         self.type = None
         self.typedef = typedef
@@ -870,10 +873,42 @@ class Model:
         for ifname in form.type.info.get('interfaces', ()):
             self._addFormIface(form, ifname)
 
+        self._checkFormDisplay(form)
+
         self.formprefixcache.clear()
 
         return form
 
+    def _checkFormDisplay(self, form):
+
+        formtype = self.types.get(form.full)
+
+        display = formtype.info.get('display')
+        if display is None:
+            return
+
+        for column in display.get('columns', ()):
+            coltype = column.get('type')
+            colopts = column.get('opts')
+
+            if coltype == 'prop':
+                curf = form
+                propname = colopts.get('name')
+                parts = propname.split('::')
+
+                for partname in parts:
+                    prop = curf.prop(partname)
+                    if prop is None:
+                        mesg = (f'Form {form.name} defines prop column {propname}'
+                               f' but {curf.full} has no property named {partname}.')
+                        raise s_exc.BadFormDef(mesg=mesg)
+
+                    curf = self.form(prop.type.name)
+
+            else:
+                mesg = f'Form {form.name} defines column with invalid type ({coltype}).'
+                raise s_exc.BadFormDef(mesg=mesg)
+
     def delForm(self, formname):
 
         form = self.forms.get(formname)
@@ -960,7 +995,7 @@ class Model:
         iface = self.ifaces.get(name)
 
         if iface is None:
-            mesg = f'Form {form.name} depends on non-existant interface: {name}'
+            mesg = f'Form {form.name} depends on non-existent interface: {name}'
             raise s_exc.NoSuchName(mesg=mesg)
 
         if iface.get('deprecated'):

synapse/lib/agenda.py

@@ -16,7 +16,6 @@ import synapse.common as s_common
 
 import synapse.lib.base as s_base
 import synapse.lib.coro as s_coro
-import synapse.lib.provenance as s_provenance
 
 # Agenda: manages running one-shot and periodic tasks in the future ("appointments")
 
@@ -259,6 +258,7 @@ class _Appt:
     _synced_attrs = {
         'doc',
         'name',
+        'pool',
         'created',
         'enabled',
         'errcount',
@@ -271,10 +271,11 @@ class _Appt:
         'lastfinishtime',
     }
 
-    def __init__(self, stor, iden, recur, indx, query, creator, recs, nexttime=None, view=None, created=None):
+    def __init__(self, stor, iden, recur, indx, query, creator, recs, nexttime=None, view=None, created=None, pool=False):
         self.doc = ''
         self.name = ''
         self.stor = stor
+        self.pool = pool
         self.iden = iden
         self.recur = recur  # does this appointment repeat
         self.indx = indx  # incremented for each appt added ever.  Used for nexttime tiebreaking for stable ordering
@@ -339,6 +340,7 @@ class _Appt:
             'ver': 1,
             'doc': self.doc,
             'name': self.name,
+            'pool': self.pool,
             'enabled': self.enabled,
             'recur': self.recur,
             'iden': self.iden,
@@ -366,6 +368,7 @@ class _Appt:
         appt = cls(stor, val['iden'], val['recur'], val['indx'], val['query'], val['creator'], recs, nexttime=val['nexttime'], view=val.get('view'))
         appt.doc = val.get('doc', '')
         appt.name = val.get('name', '')
+        appt.pool = val.get('pool', False)
         appt.created = val.get('created', None)
         appt.laststarttime = val['laststarttime']
         appt.lastfinishtime = val['lastfinishtime']
@@ -449,46 +452,9 @@ class Agenda(s_base.Base):
         self.onfini(self._wake_event.set)
 
         self._hivenode = await self.core.hive.open(('agenda', 'appts'))  # Persistent storage
-        self.onfini(self.stop)
 
-        self.enabled = False
-        self._schedtask = None  # The task of the scheduler loop.  Doesn't run until we're enabled
-
-        self._running_tasks = []  # The actively running cron job tasks
         await self._load_all()
 
-    async def start(self):
-        '''
-        Enable cron jobs to start running, start the scheduler loop
-
-        Go through all the appointments, making sure the query is valid, and remove the ones that aren't.  (We can't
-        evaluate queries until enabled because not all the modules are loaded yet.)
-        '''
-        if self.enabled:
-            return
-
-        await self._load_all()
-        for iden, appt in self.appts.items():
-            try:
-                await self.core.getStormQuery(appt.query)
-            except Exception as e:
-                logger.exception(f'Invalid appointment {iden} {appt.name} found in storage. Disabling. {e}',
-                                 extra={'synapse': {'iden': iden, 'name': appt.name, 'text': appt.query}})
-                appt.enabled = False
-
-        self._schedtask = self.schedCoro(self._scheduleLoop())
-        self.enabled = True
-
-    async def stop(self):
-        "Cancel the scheduler loop, and set self.enabled to False."
-        if not self.enabled:
-            return
-        self._schedtask.cancel()
-        for task in self._running_tasks:
-            await task.fini()
-
-        self.enabled = False
-
     async def _load_all(self):
         '''
         Load all the appointments from persistent storage
@@ -599,6 +565,8 @@ class Agenda(s_base.Base):
         view = cdef.get('view')
         created = cdef.get('created')
 
+        pool = cdef.get('pool', False)
+
         recur = incunit is not None
         indx = self._next_indx
         self._next_indx += 1
@@ -639,7 +607,7 @@ class Agenda(s_base.Base):
                 incvals = (incvals, )
             recs.extend(ApptRec(rd, incunit, v) for (rd, v) in itertools.product(reqdicts, incvals))
 
-        appt = _Appt(self, iden, recur, indx, query, creator, recs, nexttime=nexttime, view=view, created=created)
+        appt = _Appt(self, iden, recur, indx, query, creator, recs, nexttime=nexttime, view=view, created=created, pool=pool)
         self._addappt(iden, appt)
 
         appt.doc = cdef.get('doc', '')
@@ -683,8 +651,7 @@ class Agenda(s_base.Base):
         if not query:
             raise ValueError('empty query')
 
-        if self.enabled:
-            await self.core.getStormQuery(query)
+        await self.core.getStormQuery(query)
 
         appt.query = query
         appt.enabled = True  # in case it was disabled for a bad query
@@ -736,7 +703,14 @@ class Agenda(s_base.Base):
         self.tickoff += offs
         self._wake_event.set()
 
-    async def _scheduleLoop(self):
+    async def clearRunningStatus(self):
+        '''Used for clearing the running state at startup or change of leadership.'''
+        for appt in list(self.appts.values()):
+            if appt.isrunning:
+                logger.debug(f'Clearing the isrunning flag for {appt.iden}')
+                await self.core.addCronEdits(appt.iden, {'isrunning': False})
+
+    async def runloop(self):
         '''
         Task loop to issue query tasks at the right times.
         '''
@@ -766,7 +740,7 @@ class Agenda(s_base.Base):
                 if appt.nexttime:
                     heapq.heappush(self.apptheap, appt)
 
-                if not appt.enabled or not self.enabled:
+                if not appt.enabled:
                     continue
 
                 if appt.isrunning:  # pragma: no cover
@@ -818,12 +792,10 @@ class Agenda(s_base.Base):
             return
 
         info = {'iden': appt.iden, 'query': appt.query, 'view': view.iden}
-        task = await self.core.boss.execute(self._runJob(user, appt), f'Cron {appt.iden}', user, info=info)
 
-        appt.task = task
-        self._running_tasks.append(task)
-
-        task.onfini(functools.partial(self._running_tasks.remove, task))
+        coro = self._runJob(user, appt)
+        task = self.core.runActiveTask(coro)
+        appt.task = await self.core.boss.promotetask(task, f'Cron {appt.iden}', user, info=info)
 
     async def _markfailed(self, appt, reason):
         now = self._getNowTick()
@@ -848,57 +820,75 @@ class Agenda(s_base.Base):
         }
         await self.core.addCronEdits(appt.iden, edits)
 
-        with s_provenance.claim('cron', iden=appt.iden):
-            logger.info(f'Agenda executing for iden={appt.iden}, name={appt.name} user={user.name}, view={appt.view}, query={appt.query}',
-                        extra={'synapse': {'iden': appt.iden, 'name': appt.name, 'user': user.iden, 'text': appt.query,
-                                           'username': user.name, 'view': appt.view}})
-            starttime = self._getNowTick()
-            success = False
-            try:
-                opts = {'user': user.iden, 'view': appt.view, 'vars': {'auto': {'iden': appt.iden, 'type': 'cron'}}}
-                opts = self.core._initStormOpts(opts)
-                view = self.core._viewFromOpts(opts)
-                # Yes, this isn't technically on the bottom half of a nexus transaction
-                # But because the scheduling loop only runs on non-mirrors, we can kinda skirt by all that
-                # and be relatively okay. The only catch is that the nexus offset will correspond to the
-                # last nexus transaction, and not the start/stop
-                await self.core.feedBeholder('cron:start', {'iden': appt.iden})
-                async for node in view.eval(appt.query, opts=opts, log_info={'cron': appt.iden}):
+        logger.info(f'Agenda executing for iden={appt.iden}, name={appt.name} user={user.name}, view={appt.view}, query={appt.query}',
+                    extra={'synapse': {'iden': appt.iden, 'name': appt.name, 'user': user.iden, 'text': appt.query,
+                                       'username': user.name, 'view': appt.view}})
+        starttime = self._getNowTick()
+        success = False
+        try:
+            opts = {
+                'user': user.iden,
+                'view': appt.view,
+                'mirror': appt.pool,
+                'vars': {'auto': {'iden': appt.iden, 'type': 'cron'}},
+            }
+            opts = self.core._initStormOpts(opts)
+
+            await self.core.feedBeholder('cron:start', {'iden': appt.iden})
+
+            async for mesg in self.core.storm(appt.query, opts=opts):
+
+                if mesg[0] == 'node':
                     count += 1
-            except asyncio.CancelledError:
-                result = 'cancelled'
-                raise
-            except Exception as e:
-                result = f'raised exception {e}'
-                logger.exception(f'Agenda job {appt.iden} {appt.name} raised exception',
-                                 extra={'synapse': {'iden': appt.iden, 'name': appt.name}}
-                                 )
-            else:
-                success = True
-                result = f'finished successfully with {count} nodes'
-            finally:
-                finishtime = self._getNowTick()
-                if not success:
-                    appt.lasterrs.append(result)
-                    edits = {
-                        'errcount': appt.errcount + 1,
-                        # we only care about the last five errors
-                        'lasterrs': list(appt.lasterrs[-5:]),
-                    }
-                    await self.core.addCronEdits(appt.iden, edits)
 
-                took = finishtime - starttime
-                mesg = f'Agenda completed query for iden={appt.iden} name={appt.name} with result "{result}" ' \
-                       f'took {took:.3f}s'
-                logger.info(mesg, extra={'synapse': {'iden': appt.iden, 'name': appt.name, 'user': user.iden,
-                                                     'result': result, 'username': user.name, 'took': took}})
+                elif mesg[0] == 'err':
+                    excname, errinfo = mesg[1]
+                    errinfo.pop('eline', None)
+                    errinfo.pop('efile', None)
+                    excctor = getattr(s_exc, excname, s_exc.SynErr)
+                    raise excctor(**errinfo)
+
+        except asyncio.CancelledError:
+            result = 'cancelled'
+            raise
+
+        except Exception as e:
+            result = f'raised exception {e}'
+            logger.exception(f'Agenda job {appt.iden} {appt.name} raised exception',
+                             extra={'synapse': {'iden': appt.iden, 'name': appt.name}}
+                             )
+        else:
+            success = True
+            result = f'finished successfully with {count} nodes'
+
+        finally:
+            finishtime = self._getNowTick()
+            if not success:
+                appt.lasterrs.append(result)
                 edits = {
-                    'lastfinishtime': finishtime,
-                    'isrunning': False,
-                    'lastresult': result,
+                    'errcount': appt.errcount + 1,
+                    # we only care about the last five errors
+                    'lasterrs': list(appt.lasterrs[-5:]),
                 }
+
+                if self.core.isactive:
+                    await self.core.addCronEdits(appt.iden, edits)
+
+            took = finishtime - starttime
+            mesg = f'Agenda completed query for iden={appt.iden} name={appt.name} with result "{result}" ' \
+                   f'took {took:.3f}s'
+            if not self.core.isactive:
+                mesg = mesg + ' Agenda status will not be saved since the Cortex is no longer the leader.'
+            logger.info(mesg, extra={'synapse': {'iden': appt.iden, 'name': appt.name, 'user': user.iden,
+                                                 'result': result, 'username': user.name, 'took': took}})
+            edits = {
+                'lastfinishtime': finishtime,
+                'isrunning': False,
+                'lastresult': result,
+            }
+            if self.core.isactive:
                 await self.core.addCronEdits(appt.iden, edits)
 
-                if not self.isfini:
-                    # fire beholder event before invoking nexus change (in case readonly)
-                    await self.core.feedBeholder('cron:stop', {'iden': appt.iden})
+            if not self.isfini:
+                # fire beholder event before invoking nexus change (in case readonly)
+                await self.core.feedBeholder('cron:stop', {'iden': appt.iden})

synapse/lib/aha.py

@@ -327,6 +327,20 @@ class AhaApi(s_cell.CellApi):
         '''
         return await self.cell.delAhaUserEnroll(iden)
 
+    @s_cell.adminapi()
+    async def clearAhaSvcProvs(self):
+        '''
+        Remove all unused service provisioning values.
+        '''
+        return await self.cell.clearAhaSvcProvs()
+
+    @s_cell.adminapi()
+    async def clearAhaUserEnrolls(self):
+        '''
+        Remove all unused user enrollment provisioning values.
+        '''
+        return await self.cell.clearAhaUserEnrolls()
+
 class ProvDmon(s_daemon.Daemon):
 
     async def __anit__(self, aha):
@@ -386,6 +400,9 @@ class EnrollApi:
             mesg = f'Invalid user CSR CN={name}.'
             raise s_exc.BadArg(mesg=mesg)
 
+        logger.info(f'Signing user CSR for [{username}], signas={ahanetw}',
+                   extra=await self.aha.getLogExtra(name=username, signas=ahanetw))
+
         pkey, cert = self.aha.certdir.signUserCsr(xcsr, ahanetw, save=False)
         return self.aha.certdir._certToByts(cert)
 
@@ -415,6 +432,9 @@ class ProvApi:
             mesg = f'Invalid host CSR CN={name}.'
             raise s_exc.BadArg(mesg=mesg)
 
+        logger.info(f'Signing host CSR for [{hostname}], signas={ahanetw}',
+                    extra=await self.aha.getLogExtra(name=hostname, signas=ahanetw))
+
         pkey, cert = self.aha.certdir.signHostCsr(xcsr, ahanetw, save=False)
         return self.aha.certdir._certToByts(cert)
 
@@ -431,6 +451,9 @@ class ProvApi:
             mesg = f'Invalid user CSR CN={name}.'
             raise s_exc.BadArg(mesg=mesg)
 
+        logger.info(f'Signing user CSR for [{username}], signas={ahanetw}',
+                    extra=await self.aha.getLogExtra(name=username, signas=ahanetw))
+
         pkey, cert = self.aha.certdir.signUserCsr(xcsr, ahanetw, save=False)
         return self.aha.certdir._certToByts(cert)
 
@@ -1010,6 +1033,9 @@ class AhaCell(s_cell.Cell):
             mesg = 'The AHA server does not have a provision:listen URL!'
             raise s_exc.NeedConfValu(mesg=mesg)
 
+        if not name:
+            raise s_exc.BadArg(mesg='Empty name values are not allowed for provisioning.')
+
         if provinfo is None:
             provinfo = {}
 
@@ -1039,6 +1065,10 @@ class AhaCell(s_cell.Cell):
 
         hostname = f'{name}.{netw}'
 
+        if len(hostname) > 64:
+            mesg = f'Hostname value must not exceed 64 characters in length. {hostname=}, len={len(hostname)}'
+            raise s_exc.BadArg(mesg=mesg)
+
         conf.setdefault('aha:name', name)
         dmon_port = provinfo.get('dmon:port', 0)
         dmon_listen = f'ssl://0.0.0.0:{dmon_port}?hostname={hostname}&ca={netw}'
@@ -1124,6 +1154,20 @@ class AhaCell(s_cell.Cell):
         self.slab.put(iden.encode(), s_msgpack.en(provinfo), db='aha:provs')
         return iden
 
+    @s_nexus.Pusher.onPushAuto('aha:svc:prov:clear')
+    async def clearAhaSvcProvs(self):
+        for iden, byts in self.slab.scanByFull(db='aha:provs'):
+            self.slab.delete(iden, db='aha:provs')
+            provinfo = s_msgpack.un(byts)
+            logger.info(f'Deleted service provisioning service={provinfo.get("conf").get("aha:name")}, iden={iden.decode()}')
+
+    @s_nexus.Pusher.onPushAuto('aha:enroll:clear')
+    async def clearAhaUserEnrolls(self):
+        for iden, byts in self.slab.scanByFull(db='aha:enrolls'):
+            self.slab.delete(iden, db='aha:enrolls')
+            userinfo = s_msgpack.un(byts)
+            logger.info(f'Deleted user enrollment username={userinfo.get("name")}, iden={iden.decode()}')
+
     @s_nexus.Pusher.onPushAuto('aha:svc:prov:del')
     async def delAhaSvcProv(self, iden):
         self.slab.delete(iden.encode(), db='aha:provs')
@@ -1140,8 +1184,15 @@ class AhaCell(s_cell.Cell):
             mesg = 'AHA server requires aha:network configuration.'
             raise s_exc.NeedConfValu(mesg=mesg)
 
+        if not name:
+            raise s_exc.BadArg(mesg='Empty name values are not allowed for provisioning.')
+
         username = f'{name}@{ahanetw}'
 
+        if len(username) > 64:
+            mesg = f'Username value must not exceed 64 characters in length. username={username}, len={len(username)}'
+            raise s_exc.BadArg(mesg=mesg)
+
         user = await self.auth.getUserByName(username)
 
         if user is not None:

synapse/lib/ast.py

@@ -24,7 +24,6 @@ import synapse.lib.scrape as s_scrape
 import synapse.lib.msgpack as s_msgpack
 import synapse.lib.spooled as s_spooled
 import synapse.lib.stormctrl as s_stormctrl
-import synapse.lib.provenance as s_provenance
 import synapse.lib.stormtypes as s_stormtypes
 
 from synapse.lib.stormtypes import tobool, toint, toprim, tostr, tonumber, tocmprvalu, undef
@@ -1180,33 +1179,32 @@ class CmdOper(Oper):
             mesg = f'Command ({name}) is not marked safe for readonly use.'
             raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
 
-        with s_provenance.claim('stormcmd', name=name):
-            async def genx():
+        async def genx():
 
-                async for node, path in genr:
-                    argv = await self.kids[1].compute(runt, path)
-                    if not await scmd.setArgv(argv):
-                        raise s_stormctrl.StormExit()
+            async for node, path in genr:
+                argv = await self.kids[1].compute(runt, path)
+                if not await scmd.setArgv(argv):
+                    raise s_stormctrl.StormExit()
 
-                    yield node, path
+                yield node, path
 
-            # must pull through the genr to get opts set
-            # ( many commands expect self.opts is set at run() )
-            genr, empty = await pullone(genx())
+        # must pull through the genr to get opts set
+        # ( many commands expect self.opts is set at run() )
+        genr, empty = await pullone(genx())
 
-            try:
-                if runtsafe:
-                    argv = await self.kids[1].compute(runt, None)
-                    if not await scmd.setArgv(argv):
-                        raise s_stormctrl.StormExit()
-
-                if runtsafe or not empty:
-                    async with contextlib.aclosing(scmd.execStormCmd(runt, genr)) as agen:
-                        async for item in agen:
-                            yield item
+        try:
+            if runtsafe:
+                argv = await self.kids[1].compute(runt, None)
+                if not await scmd.setArgv(argv):
+                    raise s_stormctrl.StormExit()
+
+            if runtsafe or not empty:
+                async with contextlib.aclosing(scmd.execStormCmd(runt, genr)) as agen:
+                    async for item in agen:
+                        yield item
 
-            finally:
-                await genr.aclose()
+        finally:
+            await genr.aclose()
 
 class SetVarOper(Oper):
 
@@ -3359,6 +3357,7 @@ class VarValue(Value):
     def prepare(self):
         assert isinstance(self.kids[0], Const)
         self.name = self.kids[0].value()
+        self.isconst = False
 
     def isRuntSafe(self, runt):
         return runt.isRuntVar(self.name)

synapse/lib/base.py

@@ -477,8 +477,6 @@ class Base:
             asyncio.Task: An asyncio.Task object.
 
         '''
-        import synapse.lib.provenance as s_provenance  # avoid import cycle
-
         if __debug__:
             assert inspect.isawaitable(coro)
             import synapse.lib.threads as s_threads  # avoid import cycle
@@ -491,10 +489,6 @@ class Base:
 
         s_scope.clone(task)
 
-        # In rare cases, (Like this function being triggered from call_soon_threadsafe), there's no task context
-        if asyncio.current_task():
-            s_provenance.dupstack(task)
-
         def taskDone(task):
             self._active_tasks.remove(task)
             try:

synapse/lib/boss.py

@@ -41,6 +41,9 @@ class Boss(s_base.Base):
             s_task.Task: The Synapse Task object.
         '''
         task = asyncio.current_task()
+        return await self.promotetask(task, name, user, info=info, taskiden=taskiden)
+
+    async def promotetask(self, task, name, user, info=None, taskiden=None):
 
         synt = getattr(task, '_syn_task', None)