synapse 2.213.0__tar.gz → 2.215.0__tar.gz

{synapse-2.213.0/synapse.egg-info → synapse-2.215.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 2.213.0
+Version: 2.215.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0

{synapse-2.213.0 → synapse-2.215.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.213.0'
+version = '2.215.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]
@@ -95,3 +95,15 @@ include-package-data = true
 
 [tool.setuptools.packages.find]
 include = ["synapse*"]
+
+[tool.ruff]
+include = [
+    'synapse/*.py',
+    'examples/*.py',
+]
+target-version = "py311"
+
+[tool.ruff.lint]
+select = [
+    "F821",  # Check for undefined names - this is generally fatal
+]

{synapse-2.213.0 → synapse-2.215.0}/setup.cfg

@@ -12,7 +12,7 @@ select = E111,E101,E114,
 	W191,
 	W291,W293,W292,
 	W391,
-	W601,W602,W603,W604,W605,W606,
+	W605,
 
 [vtx_common:github_release]
 release-name = Synapse

{synapse-2.213.0 → synapse-2.215.0}/synapse/cortex.py

@@ -957,6 +957,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         await self._bumpCellVers('cortex:storage', (
             (1, self._storUpdateMacros),
             (4, self._storCortexHiveMigration),
+            (5, self._storCleanQueueAuthGates),
         ), nexs=False)
 
         # Perform module loading
@@ -1095,6 +1096,23 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             await view.setViewInfo('protected', nomerge)
             await view.setViewInfo('nomerge', None)
 
+    async def _storCleanQueueAuthGates(self):
+
+        logger.warning('removing AuthGates for Queues which no longer exist')
+
+        path = os.path.join(self.dirn, 'slabs', 'queues.lmdb')
+
+        async with await s_lmdbslab.Slab.anit(path) as slab:
+            async with await slab.getMultiQueue('cortex:queue', nexsroot=self.nexsroot) as multiqueue:
+                for info in self.auth.getAuthGates():
+                    if info.type == 'queue':
+                        iden = info.iden
+                        name = info.iden.split(':', 1)[1]
+                        if not multiqueue.exists(name):
+                            await self.auth.delAuthGate(info.iden)
+
+        logger.warning('...Queue AuthGate cleanup complete!')
+
     async def _storUpdateMacros(self):
         for name, node in await self.hive.open(('cortex', 'storm', 'macros')):
 
@@ -1933,13 +1951,17 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             raise s_exc.NoSuchName(mesg=mesg)
 
         await self._push('queue:del', name)
-        await self.auth.delAuthGate(f'queue:{name}')
 
     @s_nexus.Pusher.onPush('queue:del')
     async def _delCoreQueue(self, name):
         if not self.multiqueue.exists(name):
             return
 
+        try:
+            await self.auth.delAuthGate(f'queue:{name}')
+        except s_exc.NoSuchAuthGate:
+            pass
+
         await self.multiqueue.rem(name)
 
     async def coreQueueGet(self, name, offs=0, cull=True, wait=False):
@@ -5435,7 +5457,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         # push() will refire as needed
 
         async def push():
-            async with await self.boss.promote(f'layer push: {layr.iden} {iden}', self.auth.rootuser):
+            taskname = f'layer push: {layr.iden} {iden}'
+            async with await self.boss.promote(taskname, self.auth.rootuser, background=True):
                 async with await s_telepath.openurl(url) as proxy:
                     await self._pushBulkEdits(layr, proxy, pdef)
 
@@ -5447,7 +5470,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         # pull() will refire as needed
 
         async def pull():
-            async with await self.boss.promote(f'layer pull: {layr.iden} {iden}', self.auth.rootuser):
+            taskname = f'layer pull: {layr.iden} {iden}'
+            async with await self.boss.promote(taskname, self.auth.rootuser, background=True):
                 async with await s_telepath.openurl(url) as proxy:
                     await self._pushBulkEdits(proxy, layr, pdef)
 
@@ -5898,15 +5922,17 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             logger.warning('Storm query mirror pool is empty, running query locally.')
             return None
 
+        timeout = self.stormpoolopts.get('timeout:connection')
+
         for _ in range(size):
 
             try:
-                timeout = self.stormpoolopts.get('timeout:connection')
                 proxy = await self.stormpool.proxy(timeout=timeout)
+
                 proxyname = proxy._ahainfo.get('name')
                 if proxyname is not None and proxyname == self.ahasvcname:
-                    # we are part of the pool and were selected. Convert to local use.
-                    return None
+                    # we are part of the pool and were selected. Skip.
+                    continue
 
             except TimeoutError:
                 logger.warning('Timeout waiting for pool mirror proxy.')
@@ -5922,6 +5948,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                 mesg = f'Pool mirror [{proxyname}] is too far out of sync. Skipping.'
                 logger.warning(mesg, extra=await self.getLogExtra(delta=delta, mirror=proxyname, mirror_offset=miroffs))
 
+            except s_exc.ShuttingDown:
+                mesg = f'Proxy for pool mirror [{proxyname}] is shutting down. Skipping.'
+                logger.warning(mesg, extra=await self.getLogExtra(mirror=proxyname))
+
             except s_exc.IsFini:
                 mesg = f'Proxy for pool mirror [{proxyname}] was shutdown. Skipping.'
                 logger.warning(mesg, extra=await self.getLogExtra(mirror=proxyname))
@@ -5998,6 +6028,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         return await view.callStorm(text, opts=opts)
 
     async def exportStorm(self, text, opts=None):
+
         opts = self._initStormOpts(opts)
 
         if self.stormpool is not None and opts.get('mirror', True):

{synapse-2.213.0 → synapse-2.215.0}/synapse/daemon.py

@@ -177,6 +177,8 @@ async def t2call(link, meth, args, kwargs):
 
             if isinstance(e, asyncio.CancelledError):
                 logger.info('t2call task %s cancelled', meth.__name__)
+            elif isinstance(e, (BrokenPipeError, ConnectionResetError)):
+                logger.debug(f'tx closed unexpectedly {e} link={link.getAddrInfo()} meth={meth.__name__}')
             else:
                 logger.exception(f'error during task {meth.__name__} {e}')
 
@@ -527,12 +529,11 @@ class Daemon(s_base.Base):
             methname, args, kwargs = todo
 
             if methname[0] == '_':
-                raise s_exc.NoSuchMeth(name=methname)
+                raise s_exc.NoSuchMeth.init(methname, item)
 
             meth = getattr(item, methname, None)
             if meth is None:
-                logger.warning('%r has no method: %r', item, methname)
-                raise s_exc.NoSuchMeth(name=methname)
+                raise s_exc.NoSuchMeth.init(methname, item)
 
             sessitem = await t2call(link, meth, args, kwargs)
             if sessitem is not None:
@@ -562,12 +563,11 @@ class Daemon(s_base.Base):
             methname, args, kwargs = mesg[1].get('todo')
 
             if methname[0] == '_':
-                raise s_exc.NoSuchMeth(name=methname)
+                raise s_exc.NoSuchMeth.init(methname, item)
 
             meth = getattr(item, methname, None)
             if meth is None:
-                logger.warning('%r has no method: %s', item, methname)
-                raise s_exc.NoSuchMeth(name=methname)
+                raise s_exc.NoSuchMeth.init(methname, item)
 
             valu = await self._runTodoMeth(link, meth, args, kwargs)
 

{synapse-2.213.0 → synapse-2.215.0}/synapse/exc.py

@@ -226,6 +226,7 @@ class IsFini(SynErr): pass
 class IsReadOnly(SynErr): pass
 class IsDeprLocked(SynErr): pass
 class IsRuntForm(SynErr): pass
+class ShuttingDown(SynErr): pass
 
 class LayerInUse(SynErr): pass
 
@@ -300,7 +301,11 @@ class NoSuchImpl(SynErr): pass
 class NoSuchIndx(SynErr): pass
 class NoSuchLayer(SynErr): pass
 class NoSuchLift(SynErr): pass
-class NoSuchMeth(SynErr): pass
+class NoSuchMeth(SynErr):
+    @classmethod
+    def init(cls, name, item):
+        return cls(mesg=f'{item.__class__.__name__} has no method: {name}.', name=name)
+
 class NoSuchName(SynErr): pass
 class NoSuchObj(SynErr): pass
 class NoSuchOpt(SynErr): pass
@@ -361,3 +366,10 @@ class FatalErr(SynErr):
     pass
 
 class LmdbLock(SynErr): pass
+
+def reprexc(e):
+    if isinstance(e, SynErr):
+        text = e.get('mesg')
+        if text is not None:
+            return text
+    return repr(e)

{synapse-2.213.0 → synapse-2.215.0}/synapse/lib/aha.py

@@ -178,6 +178,10 @@ class AhaApi(s_cell.CellApi):
         async for info in self.cell.getAhaSvcs(network=network):
             yield info
 
+    async def getAhaSvcsByIden(self, iden, online=True, skiprun=None):
+        async for svcdef in self.cell.getAhaSvcsByIden(iden, online=online, skiprun=skiprun):
+            yield svcdef
+
     async def addAhaSvc(self, name, info, network=None):
         '''
         Register a service with the AHA discovery server.
@@ -584,6 +588,7 @@ class AhaCell(s_cell.Cell):
     async def initServiceStorage(self):
 
         self.features['callpeers'] = 1
+        self.features['getAhaSvcsByIden'] = 1
 
         dirn = s_common.gendir(self.dirn, 'slabs', 'jsonstor')
 

{synapse-2.213.0 → synapse-2.215.0}/synapse/lib/ast.py

@@ -444,8 +444,8 @@ class SubGraph:
 
         for pivq in self.rules.get('pivots'):
             indx = 0
-            async for node, path in node.storm(runt, pivq):
-                yield node, path, {'type': 'rules', 'scope': 'global', 'index': indx}
+            async for n, p in node.storm(runt, pivq):
+                yield n, p, {'type': 'rules', 'scope': 'global', 'index': indx}
                 indx += 1
 
         scope = node.form.name
@@ -885,7 +885,6 @@ class InitBlock(AstNode):
     async def run(self, runt, genr):
 
         subq = self.kids[0]
-        self.reqRuntSafe(runt, 'Init block query must be runtsafe')
 
         once = False
         async for item in genr:
@@ -923,7 +922,6 @@ class EmptyBlock(AstNode):
     async def run(self, runt, genr):
 
         subq = self.kids[0]
-        self.reqRuntSafe(runt, 'Empty block query must be runtsafe')
 
         empty = True
         async for item in genr:
@@ -955,8 +953,6 @@ class FiniBlock(AstNode):
 
         subq = self.kids[0]
 
-        self.reqRuntSafe(runt, 'Fini block query must be runtsafe')
-
         async for item in genr:
             yield item
 

{synapse-2.213.0 → synapse-2.215.0}/synapse/lib/boss.py

@@ -1,7 +1,10 @@
 import asyncio
 import logging
 
+import synapse.exc as s_exc
+
 import synapse.lib.base as s_base
+import synapse.lib.coro as s_coro
 import synapse.lib.task as s_task
 
 logger = logging.getLogger(__name__)
@@ -18,8 +21,44 @@ class Boss(s_base.Base):
     async def __anit__(self):
         await s_base.Base.__anit__(self)
         self.tasks = {}
+        self.is_shutdown = False
+        self.shutdown_lock = asyncio.Lock()
         self.onfini(self._onBossFini)
 
+    async def shutdown(self, timeout=None):
+        # when a boss is "shutting down" it should not promote any new tasks,
+        # but await the completion of any which are already underway...
+
+        self.reqNotShut()
+
+        async with self.shutdown_lock:
+
+            for task in list(self.tasks.values()):
+
+                # do not wait on child tasks
+                if task.root is not None:
+                    continue
+
+                # do not wait on background tasks
+                if task.background:
+                    continue
+
+                if not await s_coro.waittask(task.task, timeout=timeout):
+                    return False
+
+            self.is_shutdown = True
+            return True
+
+    def reqNotShut(self, mesg=None):
+        if self.shutdown_lock.locked():
+            if mesg is None:
+                mesg = 'The service is shutting down.'
+            raise s_exc.ShuttingDown(mesg=mesg)
+        if self.is_shutdown:
+            if mesg is None:
+                mesg = 'The service is shut down.'
+            raise s_exc.ShuttingDown(mesg=mesg)
+
     async def _onBossFini(self):
         for task in list(self.tasks.values()):
             await task.kill()
@@ -31,7 +70,7 @@ class Boss(s_base.Base):
     def get(self, iden):
         return self.tasks.get(iden)
 
-    async def promote(self, name, user, info=None, taskiden=None):
+    async def promote(self, name, user, info=None, taskiden=None, background=False):
         '''
         Promote the currently running task.
 
@@ -45,10 +84,15 @@ class Boss(s_base.Base):
             s_task.Task: The Synapse Task object.
         '''
         task = asyncio.current_task()
-        return await self.promotetask(task, name, user, info=info, taskiden=taskiden)
+
+        syntask = await self.promotetask(task, name, user, info=info, taskiden=taskiden)
+        syntask.background = background
+
+        return syntask
 
     async def promotetask(self, task, name, user, info=None, taskiden=None):
 
+        self.reqNotShut()
         synt = s_task.syntask(task)
 
         if synt is not None:
@@ -69,5 +113,6 @@ class Boss(s_base.Base):
         '''
         Create a synapse task from the given coroutine.
         '''
+        self.reqNotShut()
         task = self.schedCoro(coro)
         return await s_task.Task.anit(self, task, name, user, info=info, iden=iden)

{synapse-2.213.0 → synapse-2.215.0}/synapse/lib/cell.py

@@ -209,6 +209,10 @@ class CellApi(s_base.Base):
     async def initCellApi(self):
         pass
 
+    @adminapi(log=True)
+    async def shutdown(self, timeout=None):
+        return await self.cell.shutdown(timeout=timeout)
+
     @adminapi(log=True)
     async def freeze(self, timeout=30):
         return await self.cell.freeze(timeout=timeout)
@@ -299,6 +303,7 @@ class CellApi(s_base.Base):
 
     @adminapi()
     def getNexsIndx(self):
+        self.cell.boss.reqNotShut()
         return self.cell.getNexsIndx()
 
     @adminapi()
@@ -381,6 +386,10 @@ class CellApi(s_base.Base):
     async def handoff(self, turl, timeout=30):
         return await self.cell.handoff(turl, timeout=timeout)
 
+    @adminapi(log=True)
+    async def demote(self, timeout=None):
+        return await self.cell.demote(timeout=timeout)
+
     def getCellUser(self):
         return self.user.pack()
 
@@ -1024,6 +1033,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             'description': 'The username of this service when connecting to others.',
             'type': 'string',
         },
+        'aha:promotable': {
+            'description': 'Set to false to prevent this service from being promoted to leader.',
+            'type': 'boolean',
+            'default': True,
+        },
         'aha:leader': {
             'description': 'The AHA service name to claim as the active instance of a storm service.',
             'type': 'string',
@@ -1493,6 +1507,16 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         logger.warning(f'...Cell ({self.getCellType()}) auth migration complete!')
 
+    async def _drivePermMigration(self):
+        for lkey, lval in self.slab.scanByPref(s_drive.LKEY_INFO, db=self.drive.dbname):
+            info = s_msgpack.un(lval)
+            perm = info.pop('perm', None)
+            if perm is not None:
+                perm.setdefault('users', {})
+                perm.setdefault('roles', {})
+                info['permissions'] = perm
+                self.slab.put(lkey, s_msgpack.en(info), db=self.drive.dbname)
+
     def getPermDef(self, perm):
         perm = tuple(perm)
         if self.permlook is None:
@@ -1533,6 +1557,33 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             self._onFiniCellGuid()
         return retn
 
+    async def shutdown(self, timeout=None):
+        '''
+        Execute a graceful shutdown by allowing any promoted boss tasks to complete
+        and prevents the boss from accepting additional tasks.
+        '''
+        extra = await self.getLogExtra()
+        logger.warning('Graceful shutdown initiated...', extra=extra)
+
+        # if we're the leader, lets see if we can handoff...
+        if self.isactive and self.ahaclient is not None:
+            peers = await self._getDemotePeers(timeout=timeout)
+            if peers:
+                if not await self.demote(peers=peers, timeout=timeout): # pragma: no cover
+                    logger.warning('...we are the leader and failed to demote. Aborting shutdown.', extra=extra)
+                    return False
+
+        if not await self.boss.shutdown(timeout=timeout):
+            logger.warning('...tasks did not complete within timeout. Aborting shutdown.', extra=extra)
+            return False
+
+        def syncfini(futu):
+            logger.warning('...all tasks exited. Shutting down.')
+            s_coro.create_task(self.fini())
+
+        asyncio.current_task().add_done_callback(syncfini)
+        return True
+
     def _onFiniCellGuid(self):
         fcntl.lockf(self._cellguidfd, fcntl.LOCK_UN)
         self._cellguidfd.close()
@@ -1831,6 +1882,10 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
     async def initCellStorage(self):
         self.drive = await s_drive.Drive.anit(self.slab, 'celldrive')
+        await self._bumpCellVers('drive:storage', (
+            (1, self._drivePermMigration),
+        ), nexs=False)
+
         self.onfini(self.drive.fini)
 
     async def addDriveItem(self, info, path=None, reldir=s_drive.rootdir):
@@ -1852,6 +1907,13 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         if self.drive.hasItemInfo(iden): # pragma: no cover
             return await self.drive.getItemPath(iden)
 
+        # TODO: Remove this in synapse-3xx
+        perm = info.pop('perm', None)
+        if perm:
+            perm.setdefault('users', {})
+            perm.setdefault('roles', {})
+            info['permissions'] = perm
+
         return await self.drive.addItemInfo(info, path=path, reldir=reldir)
 
     async def getDriveInfo(self, iden, typename=None):
@@ -1897,7 +1959,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
             info = {
                 'name': name,
-                'perm': perm,
+                'permissions': perm,
                 'iden': s_common.guid(),
                 'created': tick,
                 'creator': user,
@@ -1927,6 +1989,50 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
     async def setDriveInfoPerm(self, iden, perm):
         return self.drive.setItemPerm(iden, perm)
 
+    @s_nexus.Pusher.onPushAuto('drive:data:path:set')
+    async def setDriveItemProp(self, iden, vers, path, valu):
+        if isinstance(path, str):
+            path = (path,)
+
+        data = await self.getDriveData(iden)
+        if data is None:
+            mesg = f'No drive item with ID {iden}.'
+            raise s_exc.NoSuchIden(mesg=mesg)
+
+        _, item = data
+
+        try:
+            step = item
+            for p in path[:-1]:
+                step = step[p]
+            step[path[-1]] = valu
+        except (KeyError, IndexError):
+            raise s_exc.BadArg(mesg=f'Invalid path {path}')
+
+        return await self.drive.setItemData(iden, vers, item)
+
+    @s_nexus.Pusher.onPushAuto('drive:data:path:del')
+    async def delDriveItemProp(self, iden, vers, path):
+        if isinstance(path, str):
+            path = (path,)
+
+        data = await self.getDriveData(iden)
+        if data is None:
+            mesg = f'No drive item with ID {iden}.'
+            raise s_exc.NoSuchIden(mesg=mesg)
+
+        _, item = data
+
+        try:
+            step = item
+            for p in path[:-1]:
+                step = step[p]
+            del step[path[-1]]
+        except (KeyError, IndexError):
+            return
+
+        return await self.drive.setItemData(iden, vers, item)
+
     @s_nexus.Pusher.onPushAuto('drive:set:path')
     async def setDriveInfoPath(self, iden, path):
 
@@ -2043,6 +2149,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             'leader': ahalead,
             'urlinfo': urlinfo,
             'ready': ready,
+            'promotable': self.conf.get('aha:promotable'),
         }
 
         return ahainfo
@@ -2267,11 +2374,92 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)}{_dispname}.')
 
-    async def reqAhaProxy(self, timeout=None):
+    async def _getDemotePeers(self, timeout=None):
+
+        ahaproxy = await self.reqAhaProxy(timeout=timeout, feats=[('getAhaSvcsByIden', 1)])
+
+        retn = []
+        async for svcdef in ahaproxy.getAhaSvcsByIden(self.iden, skiprun=self.runid):
+
+            if not svcdef['svcinfo'].get('promotable'): # pragma: no cover
+                continue
+
+            retn.append(svcdef)
+
+        return retn
+
+    async def demote(self, peers=None, timeout=None):
+
+        logger.warning('Service demotion requested. Locating a suitable service for promotion...')
+
+        extra = await self.getLogExtra()
+
+        if not self.isactive:
+            logger.warning('...service is not the leader. Aborting demotion.', extra=extra)
+            return False
+
+        user = self.conf.get('aha:user')
+
+        if peers is None:
+            peers = await self._getDemotePeers(timeout=timeout)
+
+        if not peers:
+            logger.warning('...no suitable services discovered. Aborting demotion.', extra=extra)
+            return False
+
+        try:
+
+            async with await s_base.Base.anit() as base:
+
+                cands = []
+
+                for svcdef in peers:
+
+                    name = svcdef.get('name')
+
+                    try:
+                        svcproxy = await base.enter_context(await s_telepath.openurl(f'aha://{user}@{name}'))
+
+                        svcindx = await svcproxy.getNexsIndx()
+                        cands.append((svcindx, svcproxy))
+
+                    except Exception as e:
+                        logger.warning(f'...error retrieving nexus index for {name}: {s_exc.reprexc(e)}. Skipping.', extra=extra)
+
+                if not cands:
+                    logger.warning('...no suitable services discovered. Aborting demotion.', extra=extra)
+                    return False
+
+                for svcindx, svcproxy in sorted(cands):
+
+                    try:
+                        await svcproxy.promote(graceful=True)
+
+                        logger.warning(f'... successfully promoted: {svcproxy._ahainfo["name"]}', extra=extra)
+                        return True
+
+                    except Exception as e:
+                        logger.warning(f'...error promoting {svcproxy._ahainfo["name"]}. Skipping.', extra=extra)
+
+        except Exception as e:
+            logger.exception(f'...error demoting service: {s_exc.reprexc(e)}', extra=extra)
+            return False
+
+    async def reqAhaProxy(self, timeout=None, feats=None):
+
         if self.ahaclient is None:
             mesg = 'AHA is not configured on this service.'
             raise s_exc.NeedConfValu(mesg=mesg)
-        return await self.ahaclient.proxy(timeout=timeout)
+
+        proxy = await self.ahaclient.proxy(timeout=timeout)
+
+        if feats is not None:
+            for name, vers in feats:
+                if not proxy._hasTeleFeat(name, vers):
+                    mesg = f'AHA server does not support feature: {name} >= {vers}'
+                    raise s_exc.BadVersion(mesg=mesg)
+
+        return proxy
 
     async def _setAhaActive(self):
 
@@ -4035,6 +4223,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 os.unlink(_csr)
             hostcsr = certdir.genHostCsr(hostname)
             certdir.saveHostCertByts(await prov.signHostCsr(hostcsr))
+            certdir.delHostCsr(hostname)
 
             userfull = f'{ahauser}@{ahanetw}'
             _crt = certdir.getUserCertPath(userfull)
@@ -4051,6 +4240,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 os.unlink(_csr)
             usercsr = certdir.genUserCsr(userfull)
             certdir.saveUserCertByts(await prov.signUserCsr(usercsr))
+            certdir.delUserCsr(userfull)
 
         with s_common.genfile(self.dirn, 'prov.done') as fd:
             fd.write(providen.encode())