synapse 2.196.0__py311-none-any.whl → 2.198.0__py311-none-any.whl

synapse/lib/cell.py

@@ -62,7 +62,7 @@ import synapse.tools.backup as s_t_backup
 
 logger = logging.getLogger(__name__)
 
-NEXUS_VERSION = (2, 177)
+NEXUS_VERSION = (2, 198)
 
 SLAB_MAP_SIZE = 128 * s_const.mebibyte
 SSLCTX_CACHE_SIZE = 64
@@ -1625,16 +1625,34 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         pass
 
     async def setNexsVers(self, vers):
-        if self.nexsvers < NEXUS_VERSION:
-            await self._push('nexs:vers:set', NEXUS_VERSION)
+        if self.nexsvers < vers:
+            await self._push('nexs:vers:set', vers)
 
     @s_nexus.Pusher.onPush('nexs:vers:set')
     async def _setNexsVers(self, vers):
         if vers > self.nexsvers:
-            self.cellvers.set('nexus:version', vers)
+            await self._migrNexsVers(vers)
+            self.cellinfo.set('nexus:version', vers)
             self.nexsvers = vers
             await self.configNexsVers()
 
+    async def _migrNexsVers(self, newvers):
+        if self.nexsvers < (2, 198) and newvers >= (2, 198) and self.conf.get('auth:ctor') is None:
+            # This "migration" will lock all archived users. Once the nexus version is bumped to
+            # >=2.198, then the bottom-half nexus handler for user:info (Auth._setUserInfo()) will
+            # begin rejecting unlock requests for archived users.
+
+            authkv = self.slab.getSafeKeyVal('auth')
+            userkv = authkv.getSubKeyVal('user:info:')
+
+            for iden, info in userkv.items():
+                if info.get('archived') and not info.get('locked'):
+                    info['locked'] = True
+                    userkv.set(iden, info)
+
+            # Clear the auth caches so the changes get picked up by the already running auth subsystem
+            self.auth.clearAuthCache()
+
     async def configNexsVers(self):
         for meth, orig in self.nexspatches:
             setattr(self, meth, orig)
@@ -4478,7 +4496,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         for task in self.boss.ps():
 
-            item = task.pack()
+            item = task.packv2()
             item['service'] = self.ahasvcname
 
             yield item
@@ -4500,7 +4518,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         task = self.boss.get(iden)
         if task is not None:
-            item = task.pack()
+            item = task.packv2()
             item['service'] = self.ahasvcname
             return item
 

synapse/lib/config.py

@@ -32,16 +32,16 @@ def localSchemaRefHandler(uri):
     try:
         parts = urllib.parse.urlparse(uri)
     except ValueError:
-        raise s_exc.BadUrl(f'Malformed URI: {uri}.') from None
+        raise s_exc.BadUrl(mesg=f'Malformed URI: {uri}.') from None
 
     filename = s_data.path('jsonschemas', parts.hostname, *parts.path.split('/'))
 
     # Check for path traversal. Unlikely, but still check
     if not filename.startswith(s_data.path('jsonschemas', parts.hostname)):
-        raise s_exc.BadArg(f'Path traversal in schema URL: {uri}.')
+        raise s_exc.BadArg(mesg=f'Path traversal in schema URL: {uri}.')
 
     if not os.path.exists(filename) or not os.path.isfile(filename):
-        raise s_exc.NoSuchFile(f'Local JSON schema not found for {uri}.')
+        raise s_exc.NoSuchFile(mesg=f'Local JSON schema not found for {uri}.')
 
     with open(filename, 'r') as fp:
         return json.load(fp)

synapse/lib/hive.py

@@ -424,7 +424,7 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
         return node.valu
 
     async def getTeleApi(self, link, mesg, path):
-
+        s_common.deprecated('Hive.getTeleApi', curv='2.198.0', eolv='2.199.0')
         auth = await self.getHiveAuth()
 
         if not self.conf.get('auth:en'):
@@ -759,6 +759,7 @@ def iterpath(path):
         yield path[:i + 1]
 
 async def openurl(url, **opts):
+    s_common.deprecated('synapse.lib.hive.openurl()', curv='2.198.0', eolv='2.199.0')
     prox = await s_telepath.openurl(url, **opts)
     return await TeleHive.anit(prox)
 

synapse/lib/hiveauth.py

@@ -1,3 +1,9 @@
+# pragma: no cover
+
+###
+### THIS WHOLE MODULE IS DEPRECATED AND EXPECTED TO BE REMOVED O/A v2.198.0
+###
+
 import logging
 import dataclasses
 
@@ -30,12 +36,13 @@ reqValidRules = s_config.getJsValidator({
 
 def getShadow(passwd):  # pragma: no cover
     '''This API is deprecated.'''
-    s_common.deprecated('hiveauth.getShadow()', curv='2.110.0')
+    s_common.deprecated('hiveauth.getShadow()', curv='2.110.0', eolv='2.199.0')
     salt = s_common.guid()
     hashed = s_common.guid((salt, passwd))
     return (salt, hashed)
 
 def textFromRule(rule):
+    s_common.deprecated('hiveauth.textFromRule()', curv='2.198.0', eolv='2.199.0') # pragma: no cover
     text = '.'.join(rule[1])
     if not rule[0]:
         text = '!' + text
@@ -43,6 +50,7 @@ def textFromRule(rule):
 
 @dataclasses.dataclass(slots=True)
 class _allowedReason:
+    s_common.deprecated('hiveauth._allowedReason()', curv='2.198.0', eolv='2.199.0')
     value: Union[bool | None]
     default: bool = False
     isadmin: bool = False
@@ -135,6 +143,7 @@ class Auth(s_nexus.Pusher):
         Args:
             node (HiveNode): The root of the persistent storage for auth
         '''
+        s_common.deprecated('Auth.__anit__()', curv='2.198.0', eolv='2.199.0')
         # Derive an iden from the parent
         iden = 'auth:' + ':'.join(node.full)
         await s_nexus.Pusher.__anit__(self, iden, nexsroot=nexsroot)

synapse/lib/jsonstor.py

@@ -46,7 +46,7 @@ class JsonStor(s_base.Base):
             self.dirty.pop(buid, None)
             await asyncio.sleep(0)
 
-    def _incRefObj(self, buid, valu=1):
+    async def _incRefObj(self, buid, valu=1):
 
         refs = 0
 
@@ -62,6 +62,7 @@ class JsonStor(s_base.Base):
         # remove the meta entries
         for lkey, byts in self.slab.scanByPref(buid, db=self.metadb):
             self.slab.pop(lkey, db=self.metadb)
+            await asyncio.sleep(0)
 
         # remove the item data
         self.slab.pop(buid, db=self.itemdb)
@@ -88,7 +89,7 @@ class JsonStor(s_base.Base):
 
         oldb = self.slab.replace(pkey, buid, db=self.pathdb)
         if oldb is not None:
-            self._incRefObj(oldb, -1)
+            await self._incRefObj(oldb, -1)
 
         self.slab.put(buid + b'refs', s_msgpack.en(1), db=self.metadb)
 
@@ -124,7 +125,7 @@ class JsonStor(s_base.Base):
         pkey = self._pathToPkey(path)
         buid = self.slab.pop(pkey, db=self.pathdb)
         if buid is not None:
-            self._incRefObj(buid, valu=-1)
+            await self._incRefObj(buid, valu=-1)
 
     async def setPathLink(self, srcpath, dstpath):
         '''
@@ -140,9 +141,9 @@ class JsonStor(s_base.Base):
 
         oldb = self.slab.pop(srcpkey, db=self.pathdb)
         if oldb is not None:
-            self._incRefObj(oldb, valu=-1)
+            await self._incRefObj(oldb, valu=-1)
 
-        self._incRefObj(buid, valu=1)
+        await self._incRefObj(buid, valu=1)
         self.slab.put(srcpkey, buid, db=self.pathdb)
 
     async def getPathObjProp(self, path, prop):

synapse/lib/layer.py

@@ -3543,9 +3543,8 @@ class Layer(s_nexus.Pusher):
         if self.nodeDelHook is not None:
             self.nodeDelHook()
 
-        self._wipeNodeData(buid)
-        # TODO edits to become async so we can sleep(0) on large deletes?
-        self._delNodeEdges(buid)
+        await self._wipeNodeData(buid)
+        await self._delNodeEdges(buid)
 
         self.buidcache.pop(buid, None)
 
@@ -3957,13 +3956,14 @@ class Layer(s_nexus.Pusher):
         for _, lval in self.layrslab.scanByDups(verb.encode(), db=self.byverb):
             yield (s_common.ehex(lval[:32]), verb, s_common.ehex(lval[32:]))
 
-    def _delNodeEdges(self, buid):
+    async def _delNodeEdges(self, buid):
         for lkey, n2buid in self.layrslab.scanByPref(buid, db=self.edgesn1):
             venc = lkey[32:]
             self.layrslab.delete(venc, buid + n2buid, db=self.byverb)
             self.layrslab.delete(lkey, n2buid, db=self.edgesn1)
             self.layrslab.delete(n2buid + venc, buid, db=self.edgesn2)
             self.layrslab.delete(buid + n2buid, venc, db=self.edgesn1n2)
+            await asyncio.sleep(0)
 
     def getStorIndx(self, stortype, valu):
 
@@ -4473,7 +4473,7 @@ class Layer(s_nexus.Pusher):
 
             await self.waitfini(1)
 
-    def _wipeNodeData(self, buid):
+    async def _wipeNodeData(self, buid):
         '''
         Remove all node data for a buid
         '''
@@ -4482,6 +4482,7 @@ class Layer(s_nexus.Pusher):
             buid = lkey[:32]
             self.dataslab.delete(lkey, db=self.nodedata)
             self.dataslab.delete(abrv, buid, db=self.dataname)
+            await asyncio.sleep(0)
 
     async def getModelVers(self):
         return self.layrinfo.get('model:version', (-1, -1, -1))

synapse/lib/multislabseqn.py

@@ -150,7 +150,7 @@ class MultiSlabSeqn(s_base.Base):
                     firstidx = firstitem[0]  # might not match the separately stored first index due to culling
 
                     if firstidx < fnstartidx:
-                        raise s_exc.BadCoreStore('Multislab:  filename inconsistent with contents')
+                        raise s_exc.BadCoreStore(mesg='Multislab:  filename inconsistent with contents')
 
                     lastidx = seqn.index() - 1
 
@@ -161,7 +161,7 @@ class MultiSlabSeqn(s_base.Base):
             self.firstindx = lowindx
 
         if self.firstindx > self.indx:
-            raise s_exc.BadCoreStore('Invalid firstindx value')
+            raise s_exc.BadCoreStore(mesg='Invalid firstindx value')
 
         await self._initTailSlab(fnstartidx)
 

synapse/lib/node.py

@@ -63,18 +63,24 @@ class Node:
     def __repr__(self):
         return f'Node{{{self.pack()}}}'
 
-    async def addEdge(self, verb, n2iden):
+    async def addEdge(self, verb, n2iden, extra=None):
         if self.form.isrunt:
             mesg = f'Edges cannot be used with runt nodes: {self.form.full}'
-            raise s_exc.IsRuntForm(mesg=mesg, form=self.form.full)
+            exc = s_exc.IsRuntForm(mesg=mesg, form=self.form.full)
+            if extra is not None:
+                exc = extra(exc)
+            raise exc
 
         async with self.snap.getNodeEditor(self) as editor:
             return await editor.addEdge(verb, n2iden)
 
-    async def delEdge(self, verb, n2iden):
+    async def delEdge(self, verb, n2iden, extra=None):
         if self.form.isrunt:
             mesg = f'Edges cannot be used with runt nodes: {self.form.full}'
-            raise s_exc.IsRuntForm(mesg=mesg, form=self.form.full)
+            exc = s_exc.IsRuntForm(mesg=mesg, form=self.form.full)
+            if extra is not None:
+                exc = extra(exc)
+            raise exc
 
         async with self.snap.getNodeEditor(self) as editor:
             return await editor.delEdge(verb, n2iden)

synapse/lib/parser.py

@@ -6,6 +6,7 @@ import lark  # type: ignore
 import regex  # type: ignore
 
 import synapse.exc as s_exc
+import synapse.common as s_common
 
 import synapse.lib.ast as s_ast
 import synapse.lib.coro as s_coro
@@ -72,6 +73,7 @@ terminalEnglishMap = {
     'LSQB': '[',
     'MCASEBARE': 'case multi-value',
     'MODSET': '+= or -=',
+    'MODSETMULTI': '++= or --=',
     'NONQUOTEWORD': 'unquoted value',
     'NOT': 'not',
     'NULL': 'null',
@@ -92,8 +94,8 @@ terminalEnglishMap = {
     'TRY': 'try',
     'TRIPLEQUOTEDSTRING': 'triple-quoted string',
     'TRYSET': '?=',
-    'TRYSETPLUS': '?+=',
-    'TRYSETMINUS': '?-=',
+    'TRYMODSET': '?+= or ?-=',
+    'TRYMODSETMULTI': '?++= or ?--=',
     'UNIVNAME': 'universal property',
     'UNSET': 'unset',
     'EXPRUNIVNAME': 'universal property',
@@ -158,7 +160,7 @@ class AstConverter(lark.Transformer):
 
         # Keep the original text for error printing and weird subquery argv parsing
         self.text = text
-        self.texthash = hashlib.md5(text.encode(errors='surrogatepass'), usedforsecurity=False).hexdigest()
+        self.texthash = s_common.queryhash(text)
 
     def metaToAstInfo(self, meta, isterm=False):
         if isinstance(meta, lark.tree.Meta) and meta.empty:
@@ -248,9 +250,14 @@ class AstConverter(lark.Transformer):
     @lark.v_args(meta=True)
     def embedquery(self, meta, kids):
         assert len(kids) == 1
-        astinfo = self.metaToAstInfo(meta)
-        text = kids[0].getAstText()
-        return s_ast.EmbedQuery(kids[0].astinfo, text, kids=kids)
+        astinfo = AstInfo(self.text,
+            meta.start_pos + 2, meta.end_pos - 1,
+            meta.line, meta.end_line,
+            meta.column, meta.end_column, False)
+
+        kids[0].astinfo = astinfo
+
+        return s_ast.EmbedQuery(astinfo, kids[0].getAstText(), kids=kids)
 
     @lark.v_args(meta=True)
     def funccall(self, meta, kids):
@@ -489,18 +496,28 @@ class Parser:
         Convert lark exception to synapse BadSyntax exception
         '''
         mesg = regex.split('[\n]', str(e))[0]
-        at = len(self.text)
-        line = None
-        column = None
+        soff = eoff = len(self.text)
+        sline = eline = None
+        scol = ecol = None
         token = None
         if isinstance(e, lark.exceptions.UnexpectedToken):
             expected = sorted(set(terminalEnglishMap[t] for t in e.expected))
-            at = e.pos_in_stream
-            line = e.line
-            column = e.column
             token = e.token.value
-            valu = terminalEnglishMap.get(e.token.type, e.token.value)
-            mesg = f"Unexpected token '{valu}' at line {line}, column {column}," \
+            soff = e.pos_in_stream
+            eoff = soff + len(token)
+
+            lines = token.splitlines()
+            sline = e.line
+            eline = sline + len(lines) - 1
+
+            scol = e.column
+            if len(lines) > 1:
+                ecol = len(lines[-1])
+            else:
+                ecol = scol + len(token)
+
+            valu = terminalEnglishMap.get(e.token.type, token)
+            mesg = f"Unexpected token '{valu}' at line {sline}, column {scol}," \
                    f' expecting one of: {", ".join(expected)}'
 
         elif isinstance(e, lark.exceptions.VisitError):
@@ -514,16 +531,23 @@ class Parser:
         elif isinstance(e, lark.exceptions.UnexpectedCharacters):  # pragma: no cover
             expected = sorted(set(terminalEnglishMap[t] for t in e.allowed))
             mesg += f'.  Expecting one of: {", ".join(expected)}'
-            at = e.pos_in_stream
-            line = e.line
-            column = e.column
+            soff = eoff = e.pos_in_stream
+            sline = eline = e.line
+            scol = ecol = e.column
         elif isinstance(e, lark.exceptions.UnexpectedEOF):  # pragma: no cover
             expected = sorted(set(terminalEnglishMap[t] for t in e.expected))
             mesg += ' ' + ', '.join(expected)
-            line = e.line
-            column = e.column
-
-        return s_exc.BadSyntax(at=at, text=self.text, mesg=mesg, line=line, column=column, token=token)
+            sline = eline = e.line
+            scol = ecol = e.column
+
+        highlight = {
+            'hash': s_common.queryhash(self.text),
+            'lines': (sline, eline),
+            'columns': (scol, ecol),
+            'offsets': (soff, eoff),
+        }
+        return s_exc.BadSyntax(at=soff, text=self.text, mesg=mesg, line=sline,
+                               column=scol, token=token, highlight=highlight)
 
     def eval(self):
         try:
@@ -661,6 +685,7 @@ ruleClassMap = {
     'editpropdel': lambda astinfo, kids: s_ast.EditPropDel(astinfo, kids[1:]),
     'editpropset': s_ast.EditPropSet,
     'editcondpropset': s_ast.EditCondPropSet,
+    'editpropsetmulti': s_ast.EditPropSetMulti,
     'edittagadd': s_ast.EditTagAdd,
     'edittagdel': lambda astinfo, kids: s_ast.EditTagDel(astinfo, kids[1:]),
     'edittagpropset': s_ast.EditTagPropSet,