synapse 2.197.0__py311-none-any.whl → 2.199.0__py311-none-any.whl

synapse/lib/auth.py

@@ -375,11 +375,32 @@ class Auth(s_nexus.Pusher):
 
             await self.fire('cell:beholder', **behold)
 
-    @s_nexus.Pusher.onPushAuto('user:info')
     async def setUserInfo(self, iden, name, valu, gateiden=None, logged=True, mesg=None):
 
         user = await self.reqUser(iden)
 
+        if name == 'locked' and not valu and user.isArchived():
+            raise s_exc.BadArg(mesg='Cannot unlock archived user.', user=iden, username=user.name)
+
+        if name in ('locked', 'archived') and not valu:
+            self.checkUserLimit()
+
+        await self._push('user:info', iden, name, valu, gateiden=gateiden, logged=logged, mesg=mesg)
+
+    @s_nexus.Pusher.onPush('user:info')
+    async def _setUserInfo(self, iden, name, valu, gateiden=None, logged=True, mesg=None):
+        user = await self.reqUser(iden)
+
+        if self.nexsroot and self.nexsroot.cell.nexsvers >= (2, 198):
+            # If the nexus version is less than 2.197 then the leader hasn't been upgraded yet and
+            # we don't want to get into a schism because we're bouncing edits and the leader is
+            # applying them.
+            if name == 'locked' and not valu and user.isArchived():
+                return
+
+        if name in ('locked', 'archived') and not valu:
+            self.checkUserLimit()
+
         if gateiden is not None:
             info = user.genGateInfo(gateiden)
             info[name] = s_msgpack.deepcopy(valu)
@@ -392,9 +413,6 @@ class Auth(s_nexus.Pusher):
             user.info[name] = s_msgpack.deepcopy(valu)
             self.userdefs.set(iden, user.info)
 
-        if name in ('locked', 'archived') and not valu:
-            self.checkUserLimit()
-
         if mesg is None:
             mesg = {
                 'iden': iden,
@@ -720,6 +738,16 @@ class Auth(s_nexus.Pusher):
         self.roleidenbyname.delete(role.name)
         await self.feedBeholder('role:del', {'iden': iden})
 
+    def clearAuthCache(self):
+        '''
+        Clear all auth caches.
+        '''
+        self.userbyidencache.clear()
+        self.useridenbynamecache.clear()
+        self.rolebyidencache.clear()
+        self.roleidenbynamecache.clear()
+        self.authgates.clear()
+
 class AuthGate():
     '''
     The storage object for object specific rules for users/roles.
@@ -975,7 +1003,7 @@ class User(Ruler):
         if nexs:
             return await self.auth.setUserInfo(self.iden, name, valu, gateiden=gateiden, mesg=mesg)
         else:
-            return await self.auth._hndlsetUserInfo(self.iden, name, valu, gateiden=gateiden, logged=nexs, mesg=mesg)
+            return await self.auth._setUserInfo(self.iden, name, valu, gateiden=gateiden, logged=nexs, mesg=mesg)
 
     async def setName(self, name):
         return await self.auth.setUserName(self.iden, name)
@@ -1289,7 +1317,7 @@ class User(Ruler):
         if nexs:
             await self.auth.setUserInfo(self.iden, 'roles', roles, mesg=mesg)
         else:
-            await self.auth._hndlsetUserInfo(self.iden, 'roles', roles, logged=nexs, mesg=mesg)
+            await self.auth._setUserInfo(self.iden, 'roles', roles, logged=nexs, mesg=mesg)
 
     def isLocked(self):
         return self.info.get('locked')
@@ -1332,7 +1360,7 @@ class User(Ruler):
         if logged:
             await self.auth.setUserInfo(self.iden, 'admin', admin, gateiden=gateiden)
         else:
-            await self.auth._hndlsetUserInfo(self.iden, 'admin', admin, gateiden=gateiden, logged=logged)
+            await self.auth._setUserInfo(self.iden, 'admin', admin, gateiden=gateiden, logged=logged)
 
     async def setLocked(self, locked, logged=True):
         if not isinstance(locked, bool):
@@ -1352,9 +1380,9 @@ class User(Ruler):
                 await self.auth.setUserInfo(self.iden, 'policy:attempts', 0)
 
         else:
-            await self.auth._hndlsetUserInfo(self.iden, 'locked', locked, logged=logged)
+            await self.auth._setUserInfo(self.iden, 'locked', locked, logged=logged)
             if resetAttempts:
-                await self.auth._hndlsetUserInfo(self.iden, 'policy:attempts', 0)
+                await self.auth._setUserInfo(self.iden, 'policy:attempts', 0)
 
     async def setArchived(self, archived):
         if not isinstance(archived, bool):
@@ -1541,7 +1569,7 @@ class User(Ruler):
             if nexs:
                 await self.auth.setUserInfo(self.iden, 'policy:previous', previous[:prevvalu])
             else:
-                await self.auth._hndlsetUserInfo(self.iden, 'policy:previous', previous[:prevvalu], logged=nexs)
+                await self.auth._setUserInfo(self.iden, 'policy:previous', previous[:prevvalu], logged=nexs)
 
     async def setPasswd(self, passwd, nexs=True, enforce_policy=True):
         # Prevent empty string or non-string values
@@ -1559,4 +1587,4 @@ class User(Ruler):
         if nexs:
             await self.auth.setUserInfo(self.iden, 'passwd', shadow)
         else:
-            await self.auth._hndlsetUserInfo(self.iden, 'passwd', shadow, logged=nexs)
+            await self.auth._setUserInfo(self.iden, 'passwd', shadow, logged=nexs)

synapse/lib/cell.py

@@ -62,7 +62,7 @@ import synapse.tools.backup as s_t_backup
 
 logger = logging.getLogger(__name__)
 
-NEXUS_VERSION = (2, 177)
+NEXUS_VERSION = (2, 198)
 
 SLAB_MAP_SIZE = 128 * s_const.mebibyte
 SSLCTX_CACHE_SIZE = 64
@@ -1625,16 +1625,34 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         pass
 
     async def setNexsVers(self, vers):
-        if self.nexsvers < NEXUS_VERSION:
-            await self._push('nexs:vers:set', NEXUS_VERSION)
+        if self.nexsvers < vers:
+            await self._push('nexs:vers:set', vers)
 
     @s_nexus.Pusher.onPush('nexs:vers:set')
     async def _setNexsVers(self, vers):
         if vers > self.nexsvers:
-            self.cellvers.set('nexus:version', vers)
+            await self._migrNexsVers(vers)
+            self.cellinfo.set('nexus:version', vers)
             self.nexsvers = vers
             await self.configNexsVers()
 
+    async def _migrNexsVers(self, newvers):
+        if self.nexsvers < (2, 198) and newvers >= (2, 198) and self.conf.get('auth:ctor') is None:
+            # This "migration" will lock all archived users. Once the nexus version is bumped to
+            # >=2.198, then the bottom-half nexus handler for user:info (Auth._setUserInfo()) will
+            # begin rejecting unlock requests for archived users.
+
+            authkv = self.slab.getSafeKeyVal('auth')
+            userkv = authkv.getSubKeyVal('user:info:')
+
+            for iden, info in userkv.items():
+                if info.get('archived') and not info.get('locked'):
+                    info['locked'] = True
+                    userkv.set(iden, info)
+
+            # Clear the auth caches so the changes get picked up by the already running auth subsystem
+            self.auth.clearAuthCache()
+
     async def configNexsVers(self):
         for meth, orig in self.nexspatches:
             setattr(self, meth, orig)

synapse/lib/hive.py

@@ -424,7 +424,7 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
         return node.valu
 
     async def getTeleApi(self, link, mesg, path):
-
+        s_common.deprecated('Hive.getTeleApi', curv='2.198.0', eolv='2.199.0')
         auth = await self.getHiveAuth()
 
         if not self.conf.get('auth:en'):
@@ -759,6 +759,7 @@ def iterpath(path):
         yield path[:i + 1]
 
 async def openurl(url, **opts):
+    s_common.deprecated('synapse.lib.hive.openurl()', curv='2.198.0', eolv='2.199.0')
     prox = await s_telepath.openurl(url, **opts)
     return await TeleHive.anit(prox)
 

synapse/lib/hiveauth.py

@@ -1,3 +1,9 @@
+# pragma: no cover
+
+###
+### THIS WHOLE MODULE IS DEPRECATED AND EXPECTED TO BE REMOVED O/A v2.198.0
+###
+
 import logging
 import dataclasses
 
@@ -30,12 +36,13 @@ reqValidRules = s_config.getJsValidator({
 
 def getShadow(passwd):  # pragma: no cover
     '''This API is deprecated.'''
-    s_common.deprecated('hiveauth.getShadow()', curv='2.110.0')
+    s_common.deprecated('hiveauth.getShadow()', curv='2.110.0', eolv='2.199.0')
     salt = s_common.guid()
     hashed = s_common.guid((salt, passwd))
     return (salt, hashed)
 
 def textFromRule(rule):
+    s_common.deprecated('hiveauth.textFromRule()', curv='2.198.0', eolv='2.199.0') # pragma: no cover
     text = '.'.join(rule[1])
     if not rule[0]:
         text = '!' + text
@@ -43,6 +50,7 @@ def textFromRule(rule):
 
 @dataclasses.dataclass(slots=True)
 class _allowedReason:
+    s_common.deprecated('hiveauth._allowedReason()', curv='2.198.0', eolv='2.199.0')
     value: Union[bool | None]
     default: bool = False
     isadmin: bool = False
@@ -135,6 +143,7 @@ class Auth(s_nexus.Pusher):
         Args:
             node (HiveNode): The root of the persistent storage for auth
         '''
+        s_common.deprecated('Auth.__anit__()', curv='2.198.0', eolv='2.199.0')
         # Derive an iden from the parent
         iden = 'auth:' + ':'.join(node.full)
         await s_nexus.Pusher.__anit__(self, iden, nexsroot=nexsroot)

synapse/lib/jsonstor.py

@@ -46,7 +46,7 @@ class JsonStor(s_base.Base):
             self.dirty.pop(buid, None)
             await asyncio.sleep(0)
 
-    def _incRefObj(self, buid, valu=1):
+    async def _incRefObj(self, buid, valu=1):
 
         refs = 0
 
@@ -62,6 +62,7 @@ class JsonStor(s_base.Base):
         # remove the meta entries
         for lkey, byts in self.slab.scanByPref(buid, db=self.metadb):
             self.slab.pop(lkey, db=self.metadb)
+            await asyncio.sleep(0)
 
         # remove the item data
         self.slab.pop(buid, db=self.itemdb)
@@ -88,7 +89,7 @@ class JsonStor(s_base.Base):
 
         oldb = self.slab.replace(pkey, buid, db=self.pathdb)
         if oldb is not None:
-            self._incRefObj(oldb, -1)
+            await self._incRefObj(oldb, -1)
 
         self.slab.put(buid + b'refs', s_msgpack.en(1), db=self.metadb)
 
@@ -124,7 +125,7 @@ class JsonStor(s_base.Base):
         pkey = self._pathToPkey(path)
         buid = self.slab.pop(pkey, db=self.pathdb)
         if buid is not None:
-            self._incRefObj(buid, valu=-1)
+            await self._incRefObj(buid, valu=-1)
 
     async def setPathLink(self, srcpath, dstpath):
         '''
@@ -140,9 +141,9 @@ class JsonStor(s_base.Base):
 
         oldb = self.slab.pop(srcpkey, db=self.pathdb)
         if oldb is not None:
-            self._incRefObj(oldb, valu=-1)
+            await self._incRefObj(oldb, valu=-1)
 
-        self._incRefObj(buid, valu=1)
+        await self._incRefObj(buid, valu=1)
         self.slab.put(srcpkey, buid, db=self.pathdb)
 
     async def getPathObjProp(self, path, prop):

synapse/lib/layer.py

@@ -3543,9 +3543,8 @@ class Layer(s_nexus.Pusher):
         if self.nodeDelHook is not None:
             self.nodeDelHook()
 
-        self._wipeNodeData(buid)
-        # TODO edits to become async so we can sleep(0) on large deletes?
-        self._delNodeEdges(buid)
+        await self._wipeNodeData(buid)
+        await self._delNodeEdges(buid)
 
         self.buidcache.pop(buid, None)
 
@@ -3957,13 +3956,14 @@ class Layer(s_nexus.Pusher):
         for _, lval in self.layrslab.scanByDups(verb.encode(), db=self.byverb):
             yield (s_common.ehex(lval[:32]), verb, s_common.ehex(lval[32:]))
 
-    def _delNodeEdges(self, buid):
+    async def _delNodeEdges(self, buid):
         for lkey, n2buid in self.layrslab.scanByPref(buid, db=self.edgesn1):
             venc = lkey[32:]
             self.layrslab.delete(venc, buid + n2buid, db=self.byverb)
             self.layrslab.delete(lkey, n2buid, db=self.edgesn1)
             self.layrslab.delete(n2buid + venc, buid, db=self.edgesn2)
             self.layrslab.delete(buid + n2buid, venc, db=self.edgesn1n2)
+            await asyncio.sleep(0)
 
     def getStorIndx(self, stortype, valu):
 
@@ -4473,7 +4473,7 @@ class Layer(s_nexus.Pusher):
 
             await self.waitfini(1)
 
-    def _wipeNodeData(self, buid):
+    async def _wipeNodeData(self, buid):
         '''
         Remove all node data for a buid
         '''
@@ -4482,6 +4482,7 @@ class Layer(s_nexus.Pusher):
             buid = lkey[:32]
             self.dataslab.delete(lkey, db=self.nodedata)
             self.dataslab.delete(abrv, buid, db=self.dataname)
+            await asyncio.sleep(0)
 
     async def getModelVers(self):
         return self.layrinfo.get('model:version', (-1, -1, -1))

synapse/lib/node.py

@@ -63,18 +63,24 @@ class Node:
     def __repr__(self):
         return f'Node{{{self.pack()}}}'
 
-    async def addEdge(self, verb, n2iden):
+    async def addEdge(self, verb, n2iden, extra=None):
         if self.form.isrunt:
             mesg = f'Edges cannot be used with runt nodes: {self.form.full}'
-            raise s_exc.IsRuntForm(mesg=mesg, form=self.form.full)
+            exc = s_exc.IsRuntForm(mesg=mesg, form=self.form.full)
+            if extra is not None:
+                exc = extra(exc)
+            raise exc
 
         async with self.snap.getNodeEditor(self) as editor:
             return await editor.addEdge(verb, n2iden)
 
-    async def delEdge(self, verb, n2iden):
+    async def delEdge(self, verb, n2iden, extra=None):
         if self.form.isrunt:
             mesg = f'Edges cannot be used with runt nodes: {self.form.full}'
-            raise s_exc.IsRuntForm(mesg=mesg, form=self.form.full)
+            exc = s_exc.IsRuntForm(mesg=mesg, form=self.form.full)
+            if extra is not None:
+                exc = extra(exc)
+            raise exc
 
         async with self.snap.getNodeEditor(self) as editor:
             return await editor.delEdge(verb, n2iden)

synapse/lib/parser.py

@@ -6,6 +6,7 @@ import lark  # type: ignore
 import regex  # type: ignore
 
 import synapse.exc as s_exc
+import synapse.common as s_common
 
 import synapse.lib.ast as s_ast
 import synapse.lib.coro as s_coro
@@ -72,6 +73,7 @@ terminalEnglishMap = {
     'LSQB': '[',
     'MCASEBARE': 'case multi-value',
     'MODSET': '+= or -=',
+    'MODSETMULTI': '++= or --=',
     'NONQUOTEWORD': 'unquoted value',
     'NOT': 'not',
     'NULL': 'null',
@@ -92,8 +94,8 @@ terminalEnglishMap = {
     'TRY': 'try',
     'TRIPLEQUOTEDSTRING': 'triple-quoted string',
     'TRYSET': '?=',
-    'TRYSETPLUS': '?+=',
-    'TRYSETMINUS': '?-=',
+    'TRYMODSET': '?+= or ?-=',
+    'TRYMODSETMULTI': '?++= or ?--=',
     'UNIVNAME': 'universal property',
     'UNSET': 'unset',
     'EXPRUNIVNAME': 'universal property',
@@ -158,7 +160,7 @@ class AstConverter(lark.Transformer):
 
         # Keep the original text for error printing and weird subquery argv parsing
         self.text = text
-        self.texthash = hashlib.md5(text.encode(errors='surrogatepass'), usedforsecurity=False).hexdigest()
+        self.texthash = s_common.queryhash(text)
 
     def metaToAstInfo(self, meta, isterm=False):
         if isinstance(meta, lark.tree.Meta) and meta.empty:
@@ -248,9 +250,14 @@ class AstConverter(lark.Transformer):
     @lark.v_args(meta=True)
     def embedquery(self, meta, kids):
         assert len(kids) == 1
-        astinfo = self.metaToAstInfo(meta)
-        text = kids[0].getAstText()
-        return s_ast.EmbedQuery(kids[0].astinfo, text, kids=kids)
+        astinfo = AstInfo(self.text,
+            meta.start_pos + 2, meta.end_pos - 1,
+            meta.line, meta.end_line,
+            meta.column, meta.end_column, False)
+
+        kids[0].astinfo = astinfo
+
+        return s_ast.EmbedQuery(astinfo, kids[0].getAstText(), kids=kids)
 
     @lark.v_args(meta=True)
     def funccall(self, meta, kids):
@@ -489,18 +496,28 @@ class Parser:
         Convert lark exception to synapse BadSyntax exception
         '''
         mesg = regex.split('[\n]', str(e))[0]
-        at = len(self.text)
-        line = None
-        column = None
+        soff = eoff = len(self.text)
+        sline = eline = None
+        scol = ecol = None
         token = None
         if isinstance(e, lark.exceptions.UnexpectedToken):
             expected = sorted(set(terminalEnglishMap[t] for t in e.expected))
-            at = e.pos_in_stream
-            line = e.line
-            column = e.column
             token = e.token.value
-            valu = terminalEnglishMap.get(e.token.type, e.token.value)
-            mesg = f"Unexpected token '{valu}' at line {line}, column {column}," \
+            soff = e.pos_in_stream
+            eoff = soff + len(token)
+
+            lines = token.splitlines()
+            sline = e.line
+            eline = sline + len(lines) - 1
+
+            scol = e.column
+            if len(lines) > 1:
+                ecol = len(lines[-1])
+            else:
+                ecol = scol + len(token)
+
+            valu = terminalEnglishMap.get(e.token.type, token)
+            mesg = f"Unexpected token '{valu}' at line {sline}, column {scol}," \
                    f' expecting one of: {", ".join(expected)}'
 
         elif isinstance(e, lark.exceptions.VisitError):
@@ -514,16 +531,23 @@ class Parser:
         elif isinstance(e, lark.exceptions.UnexpectedCharacters):  # pragma: no cover
             expected = sorted(set(terminalEnglishMap[t] for t in e.allowed))
             mesg += f'.  Expecting one of: {", ".join(expected)}'
-            at = e.pos_in_stream
-            line = e.line
-            column = e.column
+            soff = eoff = e.pos_in_stream
+            sline = eline = e.line
+            scol = ecol = e.column
         elif isinstance(e, lark.exceptions.UnexpectedEOF):  # pragma: no cover
             expected = sorted(set(terminalEnglishMap[t] for t in e.expected))
             mesg += ' ' + ', '.join(expected)
-            line = e.line
-            column = e.column
-
-        return s_exc.BadSyntax(at=at, text=self.text, mesg=mesg, line=line, column=column, token=token)
+            sline = eline = e.line
+            scol = ecol = e.column
+
+        highlight = {
+            'hash': s_common.queryhash(self.text),
+            'lines': (sline, eline),
+            'columns': (scol, ecol),
+            'offsets': (soff, eoff),
+        }
+        return s_exc.BadSyntax(at=soff, text=self.text, mesg=mesg, line=sline,
+                               column=scol, token=token, highlight=highlight)
 
     def eval(self):
         try:
@@ -661,6 +685,7 @@ ruleClassMap = {
     'editpropdel': lambda astinfo, kids: s_ast.EditPropDel(astinfo, kids[1:]),
     'editpropset': s_ast.EditPropSet,
     'editcondpropset': s_ast.EditCondPropSet,
+    'editpropsetmulti': s_ast.EditPropSetMulti,
     'edittagadd': s_ast.EditTagAdd,
     'edittagdel': lambda astinfo, kids: s_ast.EditTagDel(astinfo, kids[1:]),
     'edittagpropset': s_ast.EditTagPropSet,

synapse/lib/schemas.py

@@ -880,6 +880,19 @@ _reqValidPkgdefSchema = {
                     'type': 'string',
                     'pattern': s_grammar.re_scmd
                 },
+                'endpoints': {
+                    'type': 'array',
+                    'items': {
+                        'type': 'object',
+                        'properties': {
+                            'path': {'type': 'string'},
+                            'host': {'type': 'string'},
+                            'desc': {'type': 'string'},
+                        },
+                        'required': ['path'],
+                        'additionalProperties': False
+                    }
+                },
                 'cmdargs': {
                     'type': ['array', 'null'],
                     'items': {'$ref': '#/definitions/cmdarg'},