synapse 2.176.0__py311-none-any.whl → 2.178.0__py311-none-any.whl

synapse/lib/cell.py

@@ -29,6 +29,7 @@ import synapse.common as s_common
 import synapse.daemon as s_daemon
 import synapse.telepath as s_telepath
 
+import synapse.lib.auth as s_auth
 import synapse.lib.base as s_base
 import synapse.lib.boss as s_boss
 import synapse.lib.coro as s_coro
@@ -50,7 +51,6 @@ import synapse.lib.schemas as s_schemas
 import synapse.lib.spooled as s_spooled
 import synapse.lib.urlhelp as s_urlhelp
 import synapse.lib.version as s_version
-import synapse.lib.hiveauth as s_hiveauth
 import synapse.lib.lmdbslab as s_lmdbslab
 import synapse.lib.thisplat as s_thisplat
 
@@ -60,6 +60,8 @@ import synapse.tools.backup as s_t_backup
 
 logger = logging.getLogger(__name__)
 
+NEXUS_VERSION = (2, 177)
+
 SLAB_MAP_SIZE = 128 * s_const.mebibyte
 SSLCTX_CACHE_SIZE = 64
 
@@ -891,6 +893,10 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             'type': 'object',
             'hideconf': True,
         },
+        'auth:passwd:policy': {
+            'description': 'Specify password policy/complexity requirements.',
+            'type': 'object',
+        },
         'max:users': {
             'default': 0,
             'description': 'Maximum number of users allowed on system, not including root or locked/archived users (0 is no limit).',
@@ -904,7 +910,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         },
         'nexslog:async': {
             'default': True,
-            'description': 'Set to false to disable async memory mapping of the nexus change log.',
+            'description': 'Deprecated. This option ignored.',
             'type': 'boolean',
             'hidedocs': True,
             'hidecmdl': True,
@@ -961,7 +967,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             'type': 'string',
         },
         'aha:network': {
-            'description': 'The AHA service network. This makes aha:name/aha:leader relative names.',
+            'description': 'The AHA service network.',
             'type': 'string',
         },
         'aha:registry': {
@@ -1105,6 +1111,9 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         self._checkspace = s_coro.Event()
         self._reloadfuncs = {}  # name -> func
 
+        self.nexslock = asyncio.Lock()
+        self.netready = asyncio.Event()
+
         self.conf = self._initCellConf(conf)
 
         self.minfree = self.conf.get('limit:disk:free')
@@ -1175,23 +1184,32 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         self.backlastexc = None  # err, errmsg, errtrace of last backup
 
         if self.conf.get('mirror') and not self.conf.get('nexslog:en'):
-            mesg = 'Mirror mode requires nexslog:en=True'
-            raise s_exc.BadConfValu(mesg=mesg)
+            self.modCellConf({'nexslog:en': True})
 
-        # construct our nexsroot instance ( but do not start it )
         await s_nexus.Pusher.__anit__(self, self.iden)
 
         self._initCertDir()
 
-        root = await self._ctorNexsRoot()
+        await self.enter_context(s_telepath.loadTeleCell(self.dirn))
+
+        await self._initCellSlab(readonly=readonly)
+
+        # initialize network daemons (but do not listen yet)
+        # to allow registration of callbacks and shared objects
+        await self._initCellHttp()
+        await self._initCellDmon()
+
+        await self.initServiceEarly()
 
-        # mutually assured destruction with our nexs root
-        self.onfini(root.fini)
-        root.onfini(self.fini)
+        nexsroot = await self._ctorNexsRoot()
 
-        self.setNexsRoot(root)
+        self.setNexsRoot(nexsroot)
+
+        async def fini():
+            await self.nexsroot.fini()
+
+        self.onfini(fini)
 
-        await self._initCellSlab(readonly=readonly)
         self.apikeydb = self.slab.initdb('user:apikeys')  # apikey -> useriden
         self.usermetadb = self.slab.initdb('user:meta')  # useriden + <valu> -> dict valu
         self.rolemetadb = self.slab.initdb('role:meta')  # roleiden + <valu> -> dict valu
@@ -1203,10 +1221,15 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         self.hive = await self._initCellHive()
 
-        # self.cellinfo, a HiveDict for general purpose persistent storage
-        node = await self.hive.open(('cellinfo',))
-        self.cellinfo = await node.dict()
-        self.onfini(node)
+        self.cellinfo = self.slab.getSafeKeyVal('cell:info')
+        self.cellvers = self.slab.getSafeKeyVal('cell:vers')
+
+        await self._bumpCellVers('cell:storage', (
+            (1, self._storCellHiveMigration),
+        ), nexs=False)
+
+        if self.inaugural:
+            self.cellinfo.set('nexus:version', NEXUS_VERSION)
 
         # Check the cell version didn't regress
         if (lastver := self.cellinfo.get('cell:version')) is not None and self.VERSION < lastver:
@@ -1214,7 +1237,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             logger.error(mesg)
             raise s_exc.BadVersion(mesg=mesg, currver=self.VERSION, lastver=lastver)
 
-        await self.cellinfo.set('cell:version', self.VERSION)
+        self.cellinfo.set('cell:version', self.VERSION)
 
         # Check the synapse version didn't regress
         if (lastver := self.cellinfo.get('synapse:version')) is not None and s_version.version < lastver:
@@ -1222,10 +1245,14 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             logger.error(mesg)
             raise s_exc.BadVersion(mesg=mesg, currver=s_version.version, lastver=lastver)
 
-        await self.cellinfo.set('synapse:version', s_version.version)
+        self.cellinfo.set('synapse:version', s_version.version)
 
-        node = await self.hive.open(('cellvers',))
-        self.cellvers = await node.dict(nexs=True)
+        self.nexsvers = self.cellinfo.get('nexus:version', (0, 0))
+        self.nexspatches = ()
+
+        await self._bumpCellVers('cell:storage', (
+            (2, self._storCellAuthMigration),
+        ), nexs=False)
 
         self.auth = await self._initCellAuth()
 
@@ -1233,8 +1260,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         if auth_passwd is not None:
             user = await self.auth.getUserByName('root')
 
-            if not await user.tryPasswd(auth_passwd, nexs=False):
-                await user.setPasswd(auth_passwd, nexs=False)
+            if not await user.tryPasswd(auth_passwd, nexs=False, enforce_policy=False):
+                await user.setPasswd(auth_passwd, nexs=False, enforce_policy=False)
 
         self.boss = await s_boss.Boss.anit()
         self.onfini(self.boss)
@@ -1244,11 +1271,6 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             'cell': self
         }
 
-        # a tuple of (vers, func) tuples
-        # it is expected that this is set by
-        # initServiceStorage
-        self.cellupdaters = ()
-
         self.permdefs = None
         self.permlook = None
 
@@ -1262,17 +1284,13 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         # initialize network backend infrastructure
         await self._initAhaRegistry()
 
-        # initialize network daemons (but do not listen yet)
-        # to allow registration of callbacks and shared objects
-        # within phase 2/4.
-        await self._initCellHttp()
-        await self._initCellDmon()
-
         # phase 2 - service storage
         await self.initServiceStorage()
         # phase 3 - nexus subsystem
         await self.initNexusSubsystem()
 
+        await self.configNexsVers()
+
         # We can now do nexus-safe operations
         await self._initInauguralConfig()
 
@@ -1281,6 +1299,128 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         # phase 5 - service networking
         await self.initServiceNetwork()
 
+    async def _storCellHiveMigration(self):
+        logger.warning(f'migrating Cell ({self.getCellType()}) info out of hive')
+
+        async with await self.hive.open(('cellvers',)) as versnode:
+            versdict = await versnode.dict()
+            for key, valu in versdict.items():
+                self.cellvers.set(key, valu)
+
+        async with await self.hive.open(('cellinfo',)) as infonode:
+            infodict = await infonode.dict()
+            for key, valu in infodict.items():
+                self.cellinfo.set(key, valu)
+
+        logger.warning(f'...Cell ({self.getCellType()}) info migration complete!')
+
+    async def _storCellAuthMigration(self):
+        if self.conf.get('auth:ctor') is not None:
+            return
+
+        logger.warning(f'migrating Cell ({self.getCellType()}) auth out of hive')
+
+        authkv = self.slab.getSafeKeyVal('auth')
+
+        async with await self.hive.open(('auth',)) as rootnode:
+
+            rolekv = authkv.getSubKeyVal('role:info:')
+            rolenamekv = authkv.getSubKeyVal('role:name:')
+
+            async with await rootnode.open(('roles',)) as roles:
+                for iden, node in roles:
+                    roledict = await node.dict()
+                    roleinfo = roledict.pack()
+
+                    roleinfo['iden'] = iden
+                    roleinfo['name'] = node.valu
+                    roleinfo['authgates'] = {}
+                    roleinfo.setdefault('admin', False)
+                    roleinfo.setdefault('rules', ())
+
+                    rolekv.set(iden, roleinfo)
+                    rolenamekv.set(node.valu, iden)
+
+            userkv = authkv.getSubKeyVal('user:info:')
+            usernamekv = authkv.getSubKeyVal('user:name:')
+
+            async with await rootnode.open(('users',)) as users:
+                for iden, node in users:
+                    userdict = await node.dict()
+                    userinfo = userdict.pack()
+
+                    userinfo['iden'] = iden
+                    userinfo['name'] = node.valu
+                    userinfo['authgates'] = {}
+                    userinfo.setdefault('admin', False)
+                    userinfo.setdefault('rules', ())
+                    userinfo.setdefault('locked', False)
+                    userinfo.setdefault('passwd', None)
+                    userinfo.setdefault('archived', False)
+
+                    realroles = []
+                    for userrole in userinfo.get('roles', ()):
+                        if rolekv.get(userrole) is None:
+                            mesg = f'Unknown role {userrole} on user {iden} during migration, ignoring.'
+                            logger.warning(mesg)
+                            continue
+
+                        realroles.append(userrole)
+
+                    userinfo['roles'] = tuple(realroles)
+
+                    userkv.set(iden, userinfo)
+                    usernamekv.set(node.valu, iden)
+
+                    varskv = authkv.getSubKeyVal(f'user:{iden}:vars:')
+                    async with await node.open(('vars',)) as varnodes:
+                        for name, varnode in varnodes:
+                            varskv.set(name, varnode.valu)
+
+                    profkv = authkv.getSubKeyVal(f'user:{iden}:profile:')
+                    async with await node.open(('profile',)) as profnodes:
+                        for name, profnode in profnodes:
+                            profkv.set(name, profnode.valu)
+
+            gatekv = authkv.getSubKeyVal('gate:info:')
+            async with await rootnode.open(('authgates',)) as authgates:
+                for gateiden, node in authgates:
+                    gateinfo = {
+                        'iden': gateiden,
+                        'type': node.valu
+                    }
+                    gatekv.set(gateiden, gateinfo)
+
+                    async with await node.open(('users',)) as usernodes:
+                        for useriden, usernode in usernodes:
+                            if (user := userkv.get(useriden)) is None:
+                                mesg = f'Unknown user {useriden} on gate {gateiden} during migration, ignoring.'
+                                logger.warning(mesg)
+                                continue
+
+                            userinfo = await usernode.dict()
+                            userdict = userinfo.pack()
+                            authkv.set(f'gate:{gateiden}:user:{useriden}', userdict)
+
+                            user['authgates'][gateiden] = userdict
+                            userkv.set(useriden, user)
+
+                    async with await node.open(('roles',)) as rolenodes:
+                        for roleiden, rolenode in rolenodes:
+                            if (role := rolekv.get(roleiden)) is None:
+                                mesg = f'Unknown role {roleiden} on gate {gateiden} during migration, ignoring.'
+                                logger.warning(mesg)
+                                continue
+
+                            roleinfo = await rolenode.dict()
+                            roledict = roleinfo.pack()
+                            authkv.set(f'gate:{gateiden}:role:{roleiden}', roledict)
+
+                            role['authgates'][gateiden] = roledict
+                            rolekv.set(roleiden, role)
+
+        logger.warning(f'...Cell ({self.getCellType()}) auth migration complete!')
+
     def getPermDef(self, perm):
         perm = tuple(perm)
         if self.permlook is None:
@@ -1412,10 +1552,52 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         # ( and do so using _bumpCellVers )
         pass
 
+    async def setNexsVers(self, vers):
+        if self.nexsvers < NEXUS_VERSION:
+            await self._push('nexs:vers:set', NEXUS_VERSION)
+
+    @s_nexus.Pusher.onPush('nexs:vers:set')
+    async def _setNexsVers(self, vers):
+        if vers > self.nexsvers:
+            self.cellvers.set('nexus:version', vers)
+            self.nexsvers = vers
+            await self.configNexsVers()
+
+    async def configNexsVers(self):
+        for meth, orig in self.nexspatches:
+            setattr(self, meth, orig)
+
+        if self.nexsvers == NEXUS_VERSION:
+            return
+
+        patches = []
+        if self.nexsvers < (2, 177):
+            patches.extend([
+                ('popUserVarValu', self._popUserVarValuV0),
+                ('setUserVarValu', self._setUserVarValuV0),
+                ('popUserProfInfo', self._popUserProfInfoV0),
+                ('setUserProfInfo', self._setUserProfInfoV0),
+            ])
+
+        self.nexspatches = []
+        for meth, repl in patches:
+            self.nexspatches.append((meth, getattr(self, meth)))
+            setattr(self, meth, repl)
+
+    async def setCellVers(self, name, vers, nexs=True):
+        if nexs:
+            await self._push('cell:vers:set', name, vers)
+        else:
+            await self._setCellVers(name, vers)
+
+    @s_nexus.Pusher.onPush('cell:vers:set')
+    async def _setCellVers(self, name, vers):
+        self.cellvers.set(name, vers)
+
     async def _bumpCellVers(self, name, updates, nexs=True):
 
         if self.inaugural:
-            await self.cellvers.set(name, updates[-1][0], nexs=nexs)
+            await self.setCellVers(name, updates[-1][0], nexs=nexs)
             return
 
         curv = self.cellvers.get(name, 0)
@@ -1427,7 +1609,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
             await callback()
 
-            await self.cellvers.set(name, vers, nexs=nexs)
+            await self.setCellVers(name, vers, nexs=nexs)
 
             curv = vers
 
@@ -1436,10 +1618,10 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
     async def _runFreeSpaceLoop(self):
 
-        nexsroot = self.getCellNexsRoot()
-
         while not self.isfini:
 
+            nexsroot = self.getCellNexsRoot()
+
             self._checkspace.clear()
 
             disk = shutil.disk_usage(self.dirn)
@@ -1486,37 +1668,61 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
             await self.waitfini(self.SYSCTL_CHECK_FREQ)
 
-    def _getAhaAdmin(self):
-        name = self.conf.get('aha:admin')
-        if name is not None:
-            return name
-
     async def _initAhaRegistry(self):
 
-        ahaurl = self.conf.get('aha:registry')
-        if ahaurl is not None:
+        ahaurls = self.conf.get('aha:registry')
+        if ahaurls is not None:
+
+            await s_telepath.addAhaUrl(ahaurls)
+            if self.ahaclient is not None:
+                await self.ahaclient.fini()
+
+            async def onlink(proxy):
+                ahauser = self.conf.get('aha:user', 'root')
+                newurls = await proxy.getAhaUrls(user=ahauser)
+                oldurls = self.conf.get('aha:registry')
+                if isinstance(oldurls, str):
+                    oldurls = (oldurls,)
+                elif isinstance(oldurls, list):
+                    oldurls = tuple(oldurls)
+                if newurls and newurls != oldurls:
+                    if oldurls[0].startswith('tcp://'):
+                        s_common.deprecated('aha:registry: tcp:// client values.')
+                        logger.warning('tcp:// based aha:registry options are deprecated and will be removed in Synapse v3.0.0')
+                        return
 
-            info = await s_telepath.addAhaUrl(ahaurl)
-            if self.ahaclient is None:
-                self.ahaclient = await s_telepath.Client.anit(info.get('url'))
-                self.ahaclient._fini_atexit = True
-                self.onfini(self.ahaclient)
+                    self.modCellConf({'aha:registry': newurls})
+                    self.ahaclient.setBootUrls(newurls)
 
-            async def finiaha():
-                await s_telepath.delAhaUrl(ahaurl)
+            self.ahaclient = await s_telepath.Client.anit(ahaurls, onlink=onlink)
+            self.onfini(self.ahaclient)
 
-            self.onfini(finiaha)
+            async def fini():
+                await s_telepath.delAhaUrl(ahaurls)
 
+            self.ahaclient.onfini(fini)
+
+        ahaadmin = self.conf.get('aha:admin')
         ahauser = self.conf.get('aha:user')
-        ahanetw = self.conf.get('aha:network')
 
-        ahaadmin = self._getAhaAdmin()
         if ahaadmin is not None:
             await self._addAdminUser(ahaadmin)
 
         if ahauser is not None:
             await self._addAdminUser(ahauser)
 
+    def _getDmonListen(self):
+
+        lisn = self.conf.get('dmon:listen', s_common.novalu)
+        if lisn is not s_common.novalu:
+            return lisn
+
+        ahaname = self.conf.get('aha:name')
+        ahanetw = self.conf.get('aha:network')
+        if ahaname is not None and ahanetw is not None:
+            hostname = f'{ahaname}.{ahanetw}'
+            return f'ssl://0.0.0.0:0?hostname={hostname}&ca={ahanetw}'
+
     async def _addAdminUser(self, username):
         # add the user in a pre-nexus compatible way
         user = await self.auth.getUserByName(username)
@@ -1532,6 +1738,9 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         if user.isLocked():
             await user.setLocked(False, logged=False)
 
+    async def initServiceEarly(self):
+        pass
+
     async def initServiceStorage(self):
         pass
 
@@ -1544,7 +1753,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             if self.minfree is not None:
                 self.schedCoro(self._runFreeSpaceLoop())
 
-    async def initServiceNetwork(self):
+    async def _bindDmonListen(self):
+
+        # functionalized so downstream code can bind early.
+        if self.sockaddr is not None:
+            return
 
         # start a unix local socket daemon listener
         sockpath = os.path.join(self.dirn, 'sock')
@@ -1552,24 +1765,25 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         try:
             await self.dmon.listen(sockurl)
-        except asyncio.CancelledError:  # pragma: no cover  TODO:  remove once >= py 3.8 only
-            raise
         except OSError as e:
             logger.error(f'Failed to listen on unix socket at: [{sockpath}][{e}]')
             logger.error('LOCAL UNIX SOCKET WILL BE UNAVAILABLE')
         except Exception:  # pragma: no cover
             logging.exception('Unknown dmon listen error.')
-            raise
 
-        self.sockaddr = None
-
-        turl = self.conf.get('dmon:listen')
+        turl = self._getDmonListen()
         if turl is not None:
-            self.sockaddr = await self.dmon.listen(turl)
             logger.info(f'dmon listening: {turl}')
+            self.sockaddr = await self.dmon.listen(turl)
+
+    async def initServiceNetwork(self):
+
+        await self._bindDmonListen()
 
         await self._initAhaService()
 
+        self.netready.set()
+
         port = self.conf.get('https:port')
         if port is not None:
             await self.addHttpsPort(port)
@@ -1628,27 +1842,34 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         if ahaname is None:
             return
 
-        ahalead = self.conf.get('aha:leader')
         ahanetw = self.conf.get('aha:network')
+        if ahanetw is None:
+            return
 
         ahainfo = await self.getAhaInfo()
         if ahainfo is None:
             return
 
+        ahalead = self.conf.get('aha:leader')
+
         self.ahasvcname = f'{ahaname}.{ahanetw}'
 
         async def _runAhaRegLoop():
 
             while not self.isfini:
+
                 try:
                     proxy = await self.ahaclient.proxy()
+
                     info = await self.getAhaInfo()
                     await proxy.addAhaSvc(ahaname, info, network=ahanetw)
                     if self.isactive and ahalead is not None:
                         await proxy.addAhaSvc(ahalead, info, network=ahanetw)
+
                 except Exception as e:
                     logger.exception(f'Error registering service {self.ahasvcname} with AHA: {e}')
                     await self.waitfini(1)
+
                 else:
                     await proxy.waitfini()
 
@@ -1729,6 +1950,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
     async def setNexsIndx(self, indx):
         return await self.nexsroot.setindex(indx)
 
+    def getMyUrl(self, user='root'):
+        host = self.conf.req('aha:name')
+        network = self.conf.req('aha:network')
+        return f'aha://{host}.{network}'
+
     async def promote(self, graceful=False):
         '''
         Transform this cell from a passive follower to
@@ -1739,48 +1965,39 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             mesg = 'promote() called on non-mirror'
             raise s_exc.BadConfValu(mesg=mesg)
 
-        ahaname = self.conf.get('aha:name')
-        logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful} ahaname={ahaname}')
+        _dispname = f' ahaname={self.conf.get("aha:name")}' if self.conf.get('aha:name') else ''
+        logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful}{_dispname}.')
 
         if graceful:
 
-            if ahaname is None: # pragma: no cover
-                mesg = 'Cannot gracefully promote without aha:name configured.'
-                raise s_exc.BadArg(mesg=mesg)
-
-            ahanetw = self.conf.get('aha:network')
-            if ahanetw is None: # pragma: no cover
-                mesg = 'Cannot gracefully promote without aha:network configured.'
-                raise s_exc.BadArg(mesg=mesg)
+            myurl = self.getMyUrl()
 
-            myurl = f'aha://{ahaname}.{ahanetw}'
-            logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff to ahaname={ahaname}')
+            logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff{_dispname}.')
             async with await s_telepath.openurl(mirurl) as lead:
-                logger.debug(f'PROMOTION: Requesting leadership handoff to ahaname={ahaname}')
                 await lead.handoff(myurl)
-                logger.warning(f'PROMOTION: Completed leadership handoff to ahaname={ahaname}')
+                logger.warning(f'PROMOTION: Completed leadership handoff to {myurl}{_dispname}')
                 return
 
-        logger.debug(f'PROMOTION: Clearing mirror configuration for ahaname={ahaname}')
+        logger.debug(f'PROMOTION: Clearing mirror configuration{_dispname}.')
         self.modCellConf({'mirror': None})
 
-        logger.debug(f'PROMOTION: Promoting the nexus root for ahaname={ahaname}')
+        logger.debug(f'PROMOTION: Promoting the nexus root{_dispname}.')
         await self.nexsroot.promote()
 
-        logger.debug(f'PROMOTION: Setting the cell as active ahaname={ahaname}')
+        logger.debug(f'PROMOTION: Setting the cell as active{_dispname}.')
         await self.setCellActive(True)
 
-        logger.warning(f'PROMOTION: Finished leadership promotion ahaname={ahaname}')
+        logger.warning(f'PROMOTION: Finished leadership promotion{_dispname}.')
 
     async def handoff(self, turl, timeout=30):
         '''
         Hand off leadership to a mirror in a transactional fashion.
         '''
-        ahaname = self.conf.get("aha:name")
-        logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)} from ahaname={ahaname}')
+        _dispname = f' ahaname={self.conf.get("aha:name")}' if self.conf.get('aha:name') else ''
+        logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)}{_dispname}.')
         async with await s_telepath.openurl(turl) as cell:
 
-            logger.debug(f'HANDOFF: Connected to {s_urlhelp.sanitizeUrl(turl)} from ahaname={ahaname}')
+            logger.debug(f'HANDOFF: Connected to {s_urlhelp.sanitizeUrl(turl)}{_dispname}.')
 
             if self.iden != await cell.getCellIden(): # pragma: no cover
                 mesg = 'Mirror handoff remote cell iden does not match!'
@@ -1790,33 +2007,34 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 mesg = 'Cannot handoff mirror leadership to myself!'
                 raise s_exc.BadArg(mesg=mesg)
 
-            logger.debug(f'HANDOFF: Obtaining nexus applylock ahaname={ahaname}')
+            logger.debug(f'HANDOFF: Obtaining nexus lock{_dispname}.')
 
-            async with self.nexsroot.applylock:
+            async with self.nexslock:
 
-                logger.debug(f'HANDOFF: Obtained nexus applylock ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Obtained nexus lock{_dispname}.')
                 indx = await self.getNexsIndx()
 
-                logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}, ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}{_dispname}.')
                 if not await cell.waitNexsOffs(indx - 1, timeout=timeout): # pragma: no cover
                     mndx = await cell.getNexsIndx()
                     mesg = f'Remote mirror did not catch up in time: {mndx}/{indx}.'
                     raise s_exc.NotReady(mesg=mesg)
 
-                logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion{_dispname}.')
                 await cell.promote()
 
-                logger.debug(f'HANDOFF: Setting the service as inactive ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Setting the service as inactive{_dispname}.')
                 await self.setCellActive(False)
 
-                logger.debug(f'HANDOFF: Configuring service to use the new leader as its mirror ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Configuring service to sync from new leader{_dispname}.')
                 self.modCellConf({'mirror': turl})
 
-                logger.debug(f'HANDOFF: Restarting the nexus ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Restarting the nexus{_dispname}.')
                 await self.nexsroot.startup()
 
-            logger.debug(f'HANDOFF: Released nexus applylock ahaname={ahaname}')
-        logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)} ahaname={self.conf.get("aha:name")}')
+            logger.debug(f'HANDOFF: Released nexus lock{_dispname}.')
+
+        logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)}{_dispname}.')
 
     async def reqAhaProxy(self, timeout=None):
         if self.ahaclient is None:
@@ -1849,7 +2067,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             await proxy.fini()
             return
 
-        ahanetw = self.conf.get('aha:network')
+        ahanetw = self.conf.req('aha:network')
         try:
             await proxy.addAhaSvc(ahalead, ahainfo, network=ahanetw)
         except asyncio.CancelledError:  # pragma: no cover
@@ -1959,6 +2177,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             self.onfini(self.activebase)
             self._fireActiveCoros()
             await self._execCellUpdates()
+            await self.setNexsVers(NEXUS_VERSION)
             await self.initServiceActive()
         else:
             await self._killActiveCoros()
@@ -2115,7 +2334,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         try:
 
-            async with self.nexsroot.applylock:
+            async with self.nexslock:
 
                 logger.debug('Syncing LMDB Slabs')
 
@@ -2361,7 +2580,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 self.backupstreaming = False
 
     async def isUserAllowed(self, iden, perm, gateiden=None, default=False):
-        user = self.auth.user(iden)  # type: s_hiveauth.HiveUser
+        user = self.auth.user(iden)  # type: s_auth.User
         if user is None:
             return False
 
@@ -2386,36 +2605,59 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
     async def getUserProfile(self, iden):
         user = await self.auth.reqUser(iden)
-        return user.profile.pack()
+        return dict(user.profile.items())
 
-    async def getUserProfInfo(self, iden, name):
+    async def getUserProfInfo(self, iden, name, default=None):
         user = await self.auth.reqUser(iden)
-        return user.profile.get(name)
+        return user.profile.get(name, defv=default)
+
+    async def _setUserProfInfoV0(self, iden, name, valu):
+        path = ('auth', 'users', iden, 'profile', name)
+        return await self.hive._push('hive:set', path, valu)
 
     async def setUserProfInfo(self, iden, name, valu):
         user = await self.auth.reqUser(iden)
-        return await user.profile.set(name, valu)
+        return await user.setProfileValu(name, valu)
+
+    async def _popUserProfInfoV0(self, iden, name, default=None):
+        path = ('auth', 'users', iden, 'profile', name)
+        return await self.hive._push('hive:pop', path)
 
     async def popUserProfInfo(self, iden, name, default=None):
         user = await self.auth.reqUser(iden)
-        return await user.profile.pop(name, default=default)
+        return await user.popProfileValu(name, default=default)
 
     async def iterUserVars(self, iden):
         user = await self.auth.reqUser(iden)
         for item in user.vars.items():
             yield item
+            await asyncio.sleep(0)
 
-    async def getUserVarValu(self, iden, name):
+    async def iterUserProfInfo(self, iden):
         user = await self.auth.reqUser(iden)
-        return user.vars.get(name)
+        for item in user.profile.items():
+            yield item
+            await asyncio.sleep(0)
+
+    async def getUserVarValu(self, iden, name, default=None):
+        user = await self.auth.reqUser(iden)
+        return user.vars.get(name, defv=default)
+
+    async def _setUserVarValuV0(self, iden, name, valu):
+        path = ('auth', 'users', iden, 'vars', name)
+        return await self.hive._push('hive:set', path, valu)
 
     async def setUserVarValu(self, iden, name, valu):
         user = await self.auth.reqUser(iden)
-        return await user.vars.set(name, valu)
+        return await user.setVarValu(name, valu)
+
+    async def _popUserVarValuV0(self, iden, name, default=None):
+        path = ('auth', 'users', iden, 'vars', name)
+        return await self.hive._push('hive:pop', path)
 
     async def popUserVarValu(self, iden, name, default=None):
         user = await self.auth.reqUser(iden)
-        return await user.vars.pop(name, default=default)
+        return await user.popVarValu(name, default=default)
 
     async def addUserRule(self, iden, rule, indx=None, gateiden=None):
         user = await self.auth.reqUser(iden)
@@ -2807,7 +3049,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             sslctx = self.initSslCtx(certpath, pkeypath)
 
         kwargs = {
-            'xheaders': self.conf.reqConfValu('https:parse:proxy:remoteip')
+            'xheaders': self.conf.req('https:parse:proxy:remoteip')
         }
         serv = self.wapp.listen(port, address=addr, ssl_options=sslctx, **kwargs)
         self.httpds.append(serv)
@@ -2958,9 +3200,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
     async def _initCellDmon(self):
 
-        ahainfo = {
-            'name': self.ahasvcname
-        }
+        ahainfo = {'name': self.ahasvcname}
 
         self.dmon = await s_daemon.Daemon.anit(ahainfo=ahainfo)
         self.dmon.share('*', self)
@@ -2969,11 +3209,17 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
     async def _initCellHive(self):
         db = self.slab.initdb('hive')
-        hive = await s_hive.SlabHive.anit(self.slab, db=db, nexsroot=self.getCellNexsRoot())
+        hive = await s_hive.SlabHive.anit(self.slab, db=db, nexsroot=self.getCellNexsRoot(), cell=self)
         self.onfini(hive)
 
         return hive
 
+    async def _initSlabFile(self, path, readonly=False):
+        slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE, readonly=readonly)
+        slab.addResizeCallback(self.checkFreeSpace)
+        self.onfini(slab)
+        return slab
+
     async def _initCellSlab(self, readonly=False):
 
         s_common.gendir(self.dirn, 'slabs')
@@ -2985,44 +3231,31 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             _slab.initdb('hive')
             await _slab.fini()
 
-        self.slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE, readonly=readonly)
-        self.slab.addResizeCallback(self.checkFreeSpace)
-
-        self.onfini(self.slab.fini)
+        self.slab = await self._initSlabFile(path)
 
     async def _initCellAuth(self):
 
+        # Add callbacks
+        self.on('user:del', self._onUserDelEvnt)
+
         authctor = self.conf.get('auth:ctor')
         if authctor is not None:
             s_common.deprecated('auth:ctor cell config option', curv='2.157.0')
             ctor = s_dyndeps.getDynLocal(authctor)
-            auth = await ctor(self)
-        else:
-            auth = await self._initCellHiveAuth()
-
-        # Add callbacks
-        self.on('user:del', self._onUserDelEvnt)
-
-        return auth
-
-    def getCellNexsRoot(self):
-        # the "cell scope" nexusroot only exists if we are *not* embedded
-        # (aka we dont have a self.cellparent)
-        if self.cellparent is None:
-            return self.nexsroot
-
-    async def _initCellHiveAuth(self):
+            return await ctor(self)
 
         maxusers = self.conf.get('max:users')
+        policy = self.conf.get('auth:passwd:policy')
 
         seed = s_common.guid((self.iden, 'hive', 'auth'))
 
-        node = await self.hive.open(('auth',))
-        auth = await s_hiveauth.Auth.anit(
-            node,
+        auth = await s_auth.Auth.anit(
+            self.slab,
+            'auth',
             seed=seed,
             nexsroot=self.getCellNexsRoot(),
-            maxusers=maxusers
+            maxusers=maxusers,
+            policy=policy
         )
 
         auth.link(self.dist)
@@ -3035,6 +3268,12 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         self.onfini(auth.fini)
         return auth
 
+    def getCellNexsRoot(self):
+        # the "cell scope" nexusroot only exists if we are *not* embedded
+        # (aka we dont have a self.cellparent)
+        if self.cellparent is None:
+            return self.nexsroot
+
     async def _initInauguralConfig(self):
         if self.inaugural:
             icfg = self.conf.get('inaugural')
@@ -3044,7 +3283,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                     name = rnfo.get('name')
                     logger.debug(f'Adding inaugural role {name}')
                     iden = s_common.guid((self.iden, 'auth', 'role', name))
-                    role = await self.auth.addRole(name, iden)  # type: s_hiveauth.HiveRole
+                    role = await self.auth.addRole(name, iden)  # type: s_auth.Role
 
                     for rule in rnfo.get('rules', ()):
                         await role.addRule(rule)
@@ -3054,7 +3293,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                     email = unfo.get('email')
                     iden = s_common.guid((self.iden, 'auth', 'user', name))
                     logger.debug(f'Adding inaugural user {name}')
-                    user = await self.auth.addUser(name, email=email, iden=iden)  # type: s_hiveauth.HiveUser
+                    user = await self.auth.addUser(name, email=email, iden=iden)  # type: s_auth.User
 
                     if unfo.get('admin'):
                         await user.setAdmin(True)
@@ -3221,7 +3460,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         Args:
             link (s_link.Link): The link object.
-            user (s_hive.HiveUser): The heavy user object.
+            user (s_auth.User): The heavy user object.
             path (str): The path requested.
 
         Notes:
@@ -3245,7 +3484,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         '''
         extra = {**kwargs}
         sess = s_scope.get('sess')  # type: s_daemon.Sess
-        user = s_scope.get('user')  # type: s_hiveauth.HiveUser
+        user = s_scope.get('user')  # type: s_auth.User
         if user:
             extra['user'] = user.iden
             extra['username'] = user.name
@@ -3402,13 +3641,24 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         return pars
 
     async def _initCellBoot(self):
+        # NOTE: best hook point for custom provisioning
 
-        pnfo = await self._bootCellProv()
+        isok, pnfo = await self._bootCellProv()
 
         # check this before we setup loadTeleCell()
         if not self._mustBootMirror():
             return
 
+        if not isok:
+            # The way that we get to this requires the following states to be true:
+            # 1. self.dirn/cell.guid file is NOT present in the service directory.
+            # 2. mirror config is present.
+            # 3. aha:provision config is not set OR the aha:provision guid matches the self.dirn/prov.done file.
+            mesg = 'Service has been configured to boot from an upstream mirror, but has entered into an invalid ' \
+                   'state. This may have been caused by manipulation of the service storage or an error during a ' \
+                   f'backup / restore operation. {pnfo.get("mesg")}'
+            raise s_exc.FatalErr(mesg=mesg)
+
         async with s_telepath.loadTeleCell(self.dirn):
             await self._bootCellMirror(pnfo)
 
@@ -3540,7 +3790,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         provurl = self.conf.get('aha:provision')
         if provurl is None:
-            return
+            return False, {'mesg': 'No aha:provision configuration has been provided to allow the service to '
+                                   'bootstrap via AHA.'}
 
         doneiden = None
 
@@ -3553,7 +3804,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         providen = urlinfo.get('path').strip('/')
 
         if doneiden == providen:
-            return
+            return False, {'mesg': f'The aha:provision URL guid matches the service prov.done guid, '
+                                   f'aha:provision={provurl}'}
 
         logger.info(f'Provisioning {self.getCellType()} from AHA service.')
 
@@ -3613,7 +3865,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         logger.info(f'Done provisioning {self.getCellType()} AHA service.')
 
-        return provconf, providen
+        return True, {'conf': provconf, 'iden': providen}
 
     async def _bootProvConf(self, provconf):
         '''
@@ -3695,23 +3947,14 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             await self.nexsroot.enNexsLog()
             await self.sync()
 
-    async def _bootCellMirror(self, pnfo):
-        # this function must assume almost nothing is initialized
-        # but that's ok since it will only run rarely.
-        # It assumes it has a tuple of (provisioning configuration, provisioning iden) available
-        murl = self.conf.reqConfValu('mirror')
-        provconf, providen = pnfo
-
-        logger.warning(f'Bootstrap mirror from: {murl} (this could take a while!)')
+    async def _initCloneCell(self, proxy):
 
         tarpath = s_common.genpath(self.dirn, 'tmp', 'bootstrap.tgz')
-
         try:
 
-            async with await s_telepath.openurl(murl) as cell:
-                await cell.readyToMirror()
+                await proxy.readyToMirror()
                 with s_common.genfile(tarpath) as fd:
-                    async for byts in cell.iterNewBackupArchive(remove=True):
+                    async for byts in proxy.iterNewBackupArchive(remove=True):
                         fd.write(byts)
 
                 with tarfile.open(tarpath) as tgz:
@@ -3726,6 +3969,18 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             if os.path.isfile(tarpath):
                 os.unlink(tarpath)
 
+    async def _bootCellMirror(self, pnfo):
+        # this function must assume almost nothing is initialized
+        # but that's ok since it will only run rarely.
+        # It assumes it has a tuple of (provisioning configuration, provisioning iden) available
+        murl = self.conf.req('mirror')
+        provconf, providen = pnfo.get('conf'), pnfo.get('iden')
+
+        logger.warning(f'Bootstrap mirror from: {murl} (this could take a while!)')
+
+        async with await s_telepath.openurl(murl) as proxy:
+            await self._initCloneCell(proxy)
+
         # Remove aha:provision from cell.yaml if it exists and the iden differs.
         mnfo = s_common.yamlload(self.dirn, 'cell.yaml')
         if mnfo:
@@ -3821,15 +4076,12 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         try:
 
-            if 'dmon:listen' not in cell.conf:
-                await cell.dmon.listen(opts.telepath)
-                logger.info(f'...{cell.getCellType()} API (telepath): {opts.telepath}')
-            else:
-                lisn = cell.conf.get('dmon:listen')
-                if lisn is None:
-                    lisn = cell.getLocalUrl()
+            turl = cell._getDmonListen()
+            if turl is None:
+                turl = opts.telepath
+                await cell.dmon.listen(turl)
 
-                logger.info(f'...{cell.getCellType()} API (telepath): {lisn}')
+            logger.info(f'...{cell.getCellType()} API (telepath): {turl}')
 
             if 'https:port' not in cell.conf:
                 await cell.addHttpsPort(opts.https)
@@ -3997,6 +4249,12 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
     async def _onLoadHiveTree(self, tree, path, trim):
         return await self.hive.loadHiveTree(tree, path=path, trim=trim)
 
+    async def iterSlabData(self, name, prefix=''):
+        slabkv = self.slab.getSafeKeyVal(name, prefix=prefix, create=False)
+        for key, valu in slabkv.items():
+            yield key, valu
+            await asyncio.sleep(0)
+
     @s_nexus.Pusher.onPushAuto('sync')
     async def sync(self):
         '''
@@ -4065,6 +4323,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 'type': self.getCellType(),
                 'iden': self.getCellIden(),
                 'active': self.isactive,
+                'started': self.startms,
                 'ready': self.nexsroot.ready.is_set(),
                 'commit': self.COMMIT,
                 'version': self.VERSION,