synapse 2.177.0__py311-none-any.whl → 2.178.0__py311-none-any.whl

synapse/lib/base.py

@@ -179,6 +179,13 @@ class Base:
         '''
         Add a function/coroutine/Base to be called on fini().
         '''
+        if self.isfini:
+            if isinstance(func, Base):
+                s_coro.create_task(func.fini())
+            else:
+                s_coro.create_task(s_coro.ornot(func))
+            return
+
         if isinstance(func, Base):
             self.tofini.add(func)
             return
@@ -408,8 +415,6 @@ class Base:
         for fini in self._fini_funcs:
             try:
                 await s_coro.ornot(fini)
-            except asyncio.CancelledError:  # pragma: no cover  TODO:  remove once >= py 3.8 only
-                raise
             except Exception:
                 logger.exception(f'{self} - fini function failed: {fini}')
 
@@ -579,7 +584,7 @@ class Base:
         loop.add_signal_handler(signal.SIGINT, sigint)
         loop.add_signal_handler(signal.SIGTERM, sigterm)
 
-    async def main(self):
+    async def main(self): # pragma: no cover
         '''
         Helper function to setup signal handlers for this base as the main object.
         ( use base.waitfini() to block )
@@ -590,7 +595,7 @@ class Base:
         await self.addSignalHandlers()
         return await self.waitfini()
 
-    def waiter(self, count, *names):
+    def waiter(self, count, *names, timeout=None):
         '''
         Construct and return a new Waiter for events on this base.
 
@@ -615,16 +620,17 @@ class Base:
             race conditions with this mechanism ;)
 
         '''
-        return Waiter(self, count, *names)
+        return Waiter(self, count, *names, timeout=timeout)
 
 class Waiter:
     '''
     A helper to wait for a given number of events on a Base.
     '''
-    def __init__(self, base, count, *names):
+    def __init__(self, base, count, *names, timeout=None):
         self.base = base
         self.names = names
         self.count = count
+        self.timeout = timeout
         self.event = asyncio.Event()
 
         self.events = []
@@ -656,6 +662,9 @@ class Waiter:
                 doStuff(evnt)
 
         '''
+        if timeout is None:
+            timeout = self.timeout
+
         try:
 
             retn = await s_coro.event_wait(self.event, timeout)
@@ -676,6 +685,18 @@ class Waiter:
             self.base.unlink(self._onWaitEvent)
         del self.event
 
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc, cls, tb):
+        if exc is None:
+            if await self.wait() is None: # pragma: no cover
+                # these lines are 100% covered by the tests but
+                # the coverage plugin cannot seem to see them...
+                events = ','.join(self.names)
+                mesg = f'timeout waiting for {self.count} event(s): {events}'
+                raise s_exc.TimeOut(mesg=mesg)
+
 class BaseRef(Base):
     '''
     An object for managing multiple Base instances by name.
@@ -777,9 +798,6 @@ async def schedGenr(genr, maxsize=100):
 
             await q.put((False, None))
 
-        except asyncio.CancelledError:  # pragma: no cover  TODO:  remove once >= py 3.8 only
-            raise
-
         except Exception:
             if not base.isfini:
                 await q.put((False, None))

synapse/lib/cell.py

@@ -910,7 +910,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         },
         'nexslog:async': {
             'default': True,
-            'description': 'Set to false to disable async memory mapping of the nexus change log.',
+            'description': 'Deprecated. This option ignored.',
             'type': 'boolean',
             'hidedocs': True,
             'hidecmdl': True,
@@ -967,7 +967,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             'type': 'string',
         },
         'aha:network': {
-            'description': 'The AHA service network. This makes aha:name/aha:leader relative names.',
+            'description': 'The AHA service network.',
             'type': 'string',
         },
         'aha:registry': {
@@ -1111,6 +1111,9 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         self._checkspace = s_coro.Event()
         self._reloadfuncs = {}  # name -> func
 
+        self.nexslock = asyncio.Lock()
+        self.netready = asyncio.Event()
+
         self.conf = self._initCellConf(conf)
 
         self.minfree = self.conf.get('limit:disk:free')
@@ -1181,23 +1184,32 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         self.backlastexc = None  # err, errmsg, errtrace of last backup
 
         if self.conf.get('mirror') and not self.conf.get('nexslog:en'):
-            mesg = 'Mirror mode requires nexslog:en=True'
-            raise s_exc.BadConfValu(mesg=mesg)
+            self.modCellConf({'nexslog:en': True})
 
-        # construct our nexsroot instance ( but do not start it )
         await s_nexus.Pusher.__anit__(self, self.iden)
 
         self._initCertDir()
 
-        root = await self._ctorNexsRoot()
+        await self.enter_context(s_telepath.loadTeleCell(self.dirn))
+
+        await self._initCellSlab(readonly=readonly)
+
+        # initialize network daemons (but do not listen yet)
+        # to allow registration of callbacks and shared objects
+        await self._initCellHttp()
+        await self._initCellDmon()
 
-        # mutually assured destruction with our nexs root
-        self.onfini(root.fini)
-        root.onfini(self.fini)
+        await self.initServiceEarly()
 
-        self.setNexsRoot(root)
+        nexsroot = await self._ctorNexsRoot()
+
+        self.setNexsRoot(nexsroot)
+
+        async def fini():
+            await self.nexsroot.fini()
+
+        self.onfini(fini)
 
-        await self._initCellSlab(readonly=readonly)
         self.apikeydb = self.slab.initdb('user:apikeys')  # apikey -> useriden
         self.usermetadb = self.slab.initdb('user:meta')  # useriden + <valu> -> dict valu
         self.rolemetadb = self.slab.initdb('role:meta')  # roleiden + <valu> -> dict valu
@@ -1272,12 +1284,6 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         # initialize network backend infrastructure
         await self._initAhaRegistry()
 
-        # initialize network daemons (but do not listen yet)
-        # to allow registration of callbacks and shared objects
-        # within phase 2/4.
-        await self._initCellHttp()
-        await self._initCellDmon()
-
         # phase 2 - service storage
         await self.initServiceStorage()
         # phase 3 - nexus subsystem
@@ -1612,10 +1618,10 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
     async def _runFreeSpaceLoop(self):
 
-        nexsroot = self.getCellNexsRoot()
-
         while not self.isfini:
 
+            nexsroot = self.getCellNexsRoot()
+
             self._checkspace.clear()
 
             disk = shutil.disk_usage(self.dirn)
@@ -1662,37 +1668,61 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
             await self.waitfini(self.SYSCTL_CHECK_FREQ)
 
-    def _getAhaAdmin(self):
-        name = self.conf.get('aha:admin')
-        if name is not None:
-            return name
-
     async def _initAhaRegistry(self):
 
-        ahaurl = self.conf.get('aha:registry')
-        if ahaurl is not None:
+        ahaurls = self.conf.get('aha:registry')
+        if ahaurls is not None:
+
+            await s_telepath.addAhaUrl(ahaurls)
+            if self.ahaclient is not None:
+                await self.ahaclient.fini()
+
+            async def onlink(proxy):
+                ahauser = self.conf.get('aha:user', 'root')
+                newurls = await proxy.getAhaUrls(user=ahauser)
+                oldurls = self.conf.get('aha:registry')
+                if isinstance(oldurls, str):
+                    oldurls = (oldurls,)
+                elif isinstance(oldurls, list):
+                    oldurls = tuple(oldurls)
+                if newurls and newurls != oldurls:
+                    if oldurls[0].startswith('tcp://'):
+                        s_common.deprecated('aha:registry: tcp:// client values.')
+                        logger.warning('tcp:// based aha:registry options are deprecated and will be removed in Synapse v3.0.0')
+                        return
+
+                    self.modCellConf({'aha:registry': newurls})
+                    self.ahaclient.setBootUrls(newurls)
 
-            info = await s_telepath.addAhaUrl(ahaurl)
-            if self.ahaclient is None:
-                self.ahaclient = await s_telepath.Client.anit(info.get('url'))
-                self.ahaclient._fini_atexit = True
-                self.onfini(self.ahaclient)
+            self.ahaclient = await s_telepath.Client.anit(ahaurls, onlink=onlink)
+            self.onfini(self.ahaclient)
 
-            async def finiaha():
-                await s_telepath.delAhaUrl(ahaurl)
+            async def fini():
+                await s_telepath.delAhaUrl(ahaurls)
 
-            self.onfini(finiaha)
+            self.ahaclient.onfini(fini)
 
+        ahaadmin = self.conf.get('aha:admin')
         ahauser = self.conf.get('aha:user')
-        ahanetw = self.conf.get('aha:network')
 
-        ahaadmin = self._getAhaAdmin()
         if ahaadmin is not None:
             await self._addAdminUser(ahaadmin)
 
         if ahauser is not None:
             await self._addAdminUser(ahauser)
 
+    def _getDmonListen(self):
+
+        lisn = self.conf.get('dmon:listen', s_common.novalu)
+        if lisn is not s_common.novalu:
+            return lisn
+
+        ahaname = self.conf.get('aha:name')
+        ahanetw = self.conf.get('aha:network')
+        if ahaname is not None and ahanetw is not None:
+            hostname = f'{ahaname}.{ahanetw}'
+            return f'ssl://0.0.0.0:0?hostname={hostname}&ca={ahanetw}'
+
     async def _addAdminUser(self, username):
         # add the user in a pre-nexus compatible way
         user = await self.auth.getUserByName(username)
@@ -1708,6 +1738,9 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         if user.isLocked():
             await user.setLocked(False, logged=False)
 
+    async def initServiceEarly(self):
+        pass
+
     async def initServiceStorage(self):
         pass
 
@@ -1720,7 +1753,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             if self.minfree is not None:
                 self.schedCoro(self._runFreeSpaceLoop())
 
-    async def initServiceNetwork(self):
+    async def _bindDmonListen(self):
+
+        # functionalized so downstream code can bind early.
+        if self.sockaddr is not None:
+            return
 
         # start a unix local socket daemon listener
         sockpath = os.path.join(self.dirn, 'sock')
@@ -1728,24 +1765,25 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         try:
             await self.dmon.listen(sockurl)
-        except asyncio.CancelledError:  # pragma: no cover  TODO:  remove once >= py 3.8 only
-            raise
         except OSError as e:
             logger.error(f'Failed to listen on unix socket at: [{sockpath}][{e}]')
             logger.error('LOCAL UNIX SOCKET WILL BE UNAVAILABLE')
         except Exception:  # pragma: no cover
             logging.exception('Unknown dmon listen error.')
-            raise
 
-        self.sockaddr = None
-
-        turl = self.conf.get('dmon:listen')
+        turl = self._getDmonListen()
         if turl is not None:
-            self.sockaddr = await self.dmon.listen(turl)
             logger.info(f'dmon listening: {turl}')
+            self.sockaddr = await self.dmon.listen(turl)
+
+    async def initServiceNetwork(self):
+
+        await self._bindDmonListen()
 
         await self._initAhaService()
 
+        self.netready.set()
+
         port = self.conf.get('https:port')
         if port is not None:
             await self.addHttpsPort(port)
@@ -1804,27 +1842,34 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         if ahaname is None:
             return
 
-        ahalead = self.conf.get('aha:leader')
         ahanetw = self.conf.get('aha:network')
+        if ahanetw is None:
+            return
 
         ahainfo = await self.getAhaInfo()
         if ahainfo is None:
             return
 
+        ahalead = self.conf.get('aha:leader')
+
         self.ahasvcname = f'{ahaname}.{ahanetw}'
 
         async def _runAhaRegLoop():
 
             while not self.isfini:
+
                 try:
                     proxy = await self.ahaclient.proxy()
+
                     info = await self.getAhaInfo()
                     await proxy.addAhaSvc(ahaname, info, network=ahanetw)
                     if self.isactive and ahalead is not None:
                         await proxy.addAhaSvc(ahalead, info, network=ahanetw)
+
                 except Exception as e:
                     logger.exception(f'Error registering service {self.ahasvcname} with AHA: {e}')
                     await self.waitfini(1)
+
                 else:
                     await proxy.waitfini()
 
@@ -1905,6 +1950,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
     async def setNexsIndx(self, indx):
         return await self.nexsroot.setindex(indx)
 
+    def getMyUrl(self, user='root'):
+        host = self.conf.req('aha:name')
+        network = self.conf.req('aha:network')
+        return f'aha://{host}.{network}'
+
     async def promote(self, graceful=False):
         '''
         Transform this cell from a passive follower to
@@ -1915,48 +1965,39 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             mesg = 'promote() called on non-mirror'
             raise s_exc.BadConfValu(mesg=mesg)
 
-        ahaname = self.conf.get('aha:name')
-        logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful} ahaname={ahaname}')
+        _dispname = f' ahaname={self.conf.get("aha:name")}' if self.conf.get('aha:name') else ''
+        logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful}{_dispname}.')
 
         if graceful:
 
-            if ahaname is None: # pragma: no cover
-                mesg = 'Cannot gracefully promote without aha:name configured.'
-                raise s_exc.BadArg(mesg=mesg)
+            myurl = self.getMyUrl()
 
-            ahanetw = self.conf.get('aha:network')
-            if ahanetw is None: # pragma: no cover
-                mesg = 'Cannot gracefully promote without aha:network configured.'
-                raise s_exc.BadArg(mesg=mesg)
-
-            myurl = f'aha://{ahaname}.{ahanetw}'
-            logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff to ahaname={ahaname}')
+            logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff{_dispname}.')
             async with await s_telepath.openurl(mirurl) as lead:
-                logger.debug(f'PROMOTION: Requesting leadership handoff to ahaname={ahaname}')
                 await lead.handoff(myurl)
-                logger.warning(f'PROMOTION: Completed leadership handoff to ahaname={ahaname}')
+                logger.warning(f'PROMOTION: Completed leadership handoff to {myurl}{_dispname}')
                 return
 
-        logger.debug(f'PROMOTION: Clearing mirror configuration for ahaname={ahaname}')
+        logger.debug(f'PROMOTION: Clearing mirror configuration{_dispname}.')
         self.modCellConf({'mirror': None})
 
-        logger.debug(f'PROMOTION: Promoting the nexus root for ahaname={ahaname}')
+        logger.debug(f'PROMOTION: Promoting the nexus root{_dispname}.')
         await self.nexsroot.promote()
 
-        logger.debug(f'PROMOTION: Setting the cell as active ahaname={ahaname}')
+        logger.debug(f'PROMOTION: Setting the cell as active{_dispname}.')
         await self.setCellActive(True)
 
-        logger.warning(f'PROMOTION: Finished leadership promotion ahaname={ahaname}')
+        logger.warning(f'PROMOTION: Finished leadership promotion{_dispname}.')
 
     async def handoff(self, turl, timeout=30):
         '''
         Hand off leadership to a mirror in a transactional fashion.
         '''
-        ahaname = self.conf.get("aha:name")
-        logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)} from ahaname={ahaname}')
+        _dispname = f' ahaname={self.conf.get("aha:name")}' if self.conf.get('aha:name') else ''
+        logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)}{_dispname}.')
         async with await s_telepath.openurl(turl) as cell:
 
-            logger.debug(f'HANDOFF: Connected to {s_urlhelp.sanitizeUrl(turl)} from ahaname={ahaname}')
+            logger.debug(f'HANDOFF: Connected to {s_urlhelp.sanitizeUrl(turl)}{_dispname}.')
 
             if self.iden != await cell.getCellIden(): # pragma: no cover
                 mesg = 'Mirror handoff remote cell iden does not match!'
@@ -1966,33 +2007,34 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 mesg = 'Cannot handoff mirror leadership to myself!'
                 raise s_exc.BadArg(mesg=mesg)
 
-            logger.debug(f'HANDOFF: Obtaining nexus applylock ahaname={ahaname}')
+            logger.debug(f'HANDOFF: Obtaining nexus lock{_dispname}.')
 
-            async with self.nexsroot.applylock:
+            async with self.nexslock:
 
-                logger.debug(f'HANDOFF: Obtained nexus applylock ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Obtained nexus lock{_dispname}.')
                 indx = await self.getNexsIndx()
 
-                logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}, ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}{_dispname}.')
                 if not await cell.waitNexsOffs(indx - 1, timeout=timeout): # pragma: no cover
                     mndx = await cell.getNexsIndx()
                     mesg = f'Remote mirror did not catch up in time: {mndx}/{indx}.'
                     raise s_exc.NotReady(mesg=mesg)
 
-                logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion{_dispname}.')
                 await cell.promote()
 
-                logger.debug(f'HANDOFF: Setting the service as inactive ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Setting the service as inactive{_dispname}.')
                 await self.setCellActive(False)
 
-                logger.debug(f'HANDOFF: Configuring service to use the new leader as its mirror ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Configuring service to sync from new leader{_dispname}.')
                 self.modCellConf({'mirror': turl})
 
-                logger.debug(f'HANDOFF: Restarting the nexus ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Restarting the nexus{_dispname}.')
                 await self.nexsroot.startup()
 
-            logger.debug(f'HANDOFF: Released nexus applylock ahaname={ahaname}')
-        logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)} ahaname={self.conf.get("aha:name")}')
+            logger.debug(f'HANDOFF: Released nexus lock{_dispname}.')
+
+        logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)}{_dispname}.')
 
     async def reqAhaProxy(self, timeout=None):
         if self.ahaclient is None:
@@ -2025,7 +2067,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             await proxy.fini()
             return
 
-        ahanetw = self.conf.get('aha:network')
+        ahanetw = self.conf.req('aha:network')
         try:
             await proxy.addAhaSvc(ahalead, ahainfo, network=ahanetw)
         except asyncio.CancelledError:  # pragma: no cover
@@ -2292,7 +2334,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         try:
 
-            async with self.nexsroot.applylock:
+            async with self.nexslock:
 
                 logger.debug('Syncing LMDB Slabs')
 
@@ -3007,7 +3049,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             sslctx = self.initSslCtx(certpath, pkeypath)
 
         kwargs = {
-            'xheaders': self.conf.reqConfValu('https:parse:proxy:remoteip')
+            'xheaders': self.conf.req('https:parse:proxy:remoteip')
         }
         serv = self.wapp.listen(port, address=addr, ssl_options=sslctx, **kwargs)
         self.httpds.append(serv)
@@ -3158,9 +3200,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
     async def _initCellDmon(self):
 
-        ahainfo = {
-            'name': self.ahasvcname
-        }
+        ahainfo = {'name': self.ahasvcname}
 
         self.dmon = await s_daemon.Daemon.anit(ahainfo=ahainfo)
         self.dmon.share('*', self)
@@ -3174,6 +3214,12 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         return hive
 
+    async def _initSlabFile(self, path, readonly=False):
+        slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE, readonly=readonly)
+        slab.addResizeCallback(self.checkFreeSpace)
+        self.onfini(slab)
+        return slab
+
     async def _initCellSlab(self, readonly=False):
 
         s_common.gendir(self.dirn, 'slabs')
@@ -3185,10 +3231,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             _slab.initdb('hive')
             await _slab.fini()
 
-        self.slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE, readonly=readonly)
-        self.slab.addResizeCallback(self.checkFreeSpace)
-
-        self.onfini(self.slab.fini)
+        self.slab = await self._initSlabFile(path)
 
     async def _initCellAuth(self):
 
@@ -3598,13 +3641,24 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         return pars
 
     async def _initCellBoot(self):
+        # NOTE: best hook point for custom provisioning
 
-        pnfo = await self._bootCellProv()
+        isok, pnfo = await self._bootCellProv()
 
         # check this before we setup loadTeleCell()
         if not self._mustBootMirror():
             return
 
+        if not isok:
+            # The way that we get to this requires the following states to be true:
+            # 1. self.dirn/cell.guid file is NOT present in the service directory.
+            # 2. mirror config is present.
+            # 3. aha:provision config is not set OR the aha:provision guid matches the self.dirn/prov.done file.
+            mesg = 'Service has been configured to boot from an upstream mirror, but has entered into an invalid ' \
+                   'state. This may have been caused by manipulation of the service storage or an error during a ' \
+                   f'backup / restore operation. {pnfo.get("mesg")}'
+            raise s_exc.FatalErr(mesg=mesg)
+
         async with s_telepath.loadTeleCell(self.dirn):
             await self._bootCellMirror(pnfo)
 
@@ -3736,7 +3790,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         provurl = self.conf.get('aha:provision')
         if provurl is None:
-            return
+            return False, {'mesg': 'No aha:provision configuration has been provided to allow the service to '
+                                   'bootstrap via AHA.'}
 
         doneiden = None
 
@@ -3749,7 +3804,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         providen = urlinfo.get('path').strip('/')
 
         if doneiden == providen:
-            return
+            return False, {'mesg': f'The aha:provision URL guid matches the service prov.done guid, '
+                                   f'aha:provision={provurl}'}
 
         logger.info(f'Provisioning {self.getCellType()} from AHA service.')
 
@@ -3809,7 +3865,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         logger.info(f'Done provisioning {self.getCellType()} AHA service.')
 
-        return provconf, providen
+        return True, {'conf': provconf, 'iden': providen}
 
     async def _bootProvConf(self, provconf):
         '''
@@ -3891,23 +3947,14 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             await self.nexsroot.enNexsLog()
             await self.sync()
 
-    async def _bootCellMirror(self, pnfo):
-        # this function must assume almost nothing is initialized
-        # but that's ok since it will only run rarely.
-        # It assumes it has a tuple of (provisioning configuration, provisioning iden) available
-        murl = self.conf.reqConfValu('mirror')
-        provconf, providen = pnfo
-
-        logger.warning(f'Bootstrap mirror from: {murl} (this could take a while!)')
+    async def _initCloneCell(self, proxy):
 
         tarpath = s_common.genpath(self.dirn, 'tmp', 'bootstrap.tgz')
-
         try:
 
-            async with await s_telepath.openurl(murl) as cell:
-                await cell.readyToMirror()
+                await proxy.readyToMirror()
                 with s_common.genfile(tarpath) as fd:
-                    async for byts in cell.iterNewBackupArchive(remove=True):
+                    async for byts in proxy.iterNewBackupArchive(remove=True):
                         fd.write(byts)
 
                 with tarfile.open(tarpath) as tgz:
@@ -3922,6 +3969,18 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             if os.path.isfile(tarpath):
                 os.unlink(tarpath)
 
+    async def _bootCellMirror(self, pnfo):
+        # this function must assume almost nothing is initialized
+        # but that's ok since it will only run rarely.
+        # It assumes it has a tuple of (provisioning configuration, provisioning iden) available
+        murl = self.conf.req('mirror')
+        provconf, providen = pnfo.get('conf'), pnfo.get('iden')
+
+        logger.warning(f'Bootstrap mirror from: {murl} (this could take a while!)')
+
+        async with await s_telepath.openurl(murl) as proxy:
+            await self._initCloneCell(proxy)
+
         # Remove aha:provision from cell.yaml if it exists and the iden differs.
         mnfo = s_common.yamlload(self.dirn, 'cell.yaml')
         if mnfo:
@@ -4017,15 +4076,12 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         try:
 
-            if 'dmon:listen' not in cell.conf:
-                await cell.dmon.listen(opts.telepath)
-                logger.info(f'...{cell.getCellType()} API (telepath): {opts.telepath}')
-            else:
-                lisn = cell.conf.get('dmon:listen')
-                if lisn is None:
-                    lisn = cell.getLocalUrl()
+            turl = cell._getDmonListen()
+            if turl is None:
+                turl = opts.telepath
+                await cell.dmon.listen(turl)
 
-                logger.info(f'...{cell.getCellType()} API (telepath): {lisn}')
+            logger.info(f'...{cell.getCellType()} API (telepath): {turl}')
 
             if 'https:port' not in cell.conf:
                 await cell.addHttpsPort(opts.https)
@@ -4267,6 +4323,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 'type': self.getCellType(),
                 'iden': self.getCellIden(),
                 'active': self.isactive,
+                'started': self.startms,
                 'ready': self.nexsroot.ready.is_set(),
                 'commit': self.COMMIT,
                 'version': self.VERSION,