synapse 2.170.0__tar.gz → 2.173.1__tar.gz

{synapse-2.170.0/synapse.egg-info → synapse-2.173.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse
-Version: 2.170.0
+Version: 2.173.1
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License: Apache License 2.0

{synapse-2.170.0 → synapse-2.173.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.170.0'
+version = '2.173.1'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]

{synapse-2.170.0 → synapse-2.173.1}/synapse/common.py

@@ -1183,6 +1183,26 @@ def httpcodereason(code):
     except ValueError:
         return f'Unknown HTTP status code {code}'
 
+def trimText(text: str, n: int = 256, placeholder: str = '...') -> str:
+    '''
+    Trim a text string larger than n characters and add a placeholder at the end.
+
+    Args:
+        text: String to trim.
+        n: Number of characters to allow.
+        placeholder: Placeholder text.
+
+    Returns:
+        The original string or the trimmed string.
+    '''
+    if len(text) <= n:
+        return text
+    plen = len(placeholder)
+    mlen = n - plen
+    assert plen > 0
+    assert n > plen
+    return f'{text[:mlen]}{placeholder}'
+
 # TODO:  Switch back to using asyncio.wait_for when we are using py 3.12+
 # This is a workaround for a race where asyncio.wait_for can end up
 # ignoring cancellation https://github.com/python/cpython/issues/86296

{synapse-2.170.0 → synapse-2.173.1}/synapse/cortex.py

@@ -56,6 +56,7 @@ import synapse.lib.stormwhois as s_stormwhois  # NOQA
 import synapse.lib.stormtypes as s_stormtypes
 
 import synapse.lib.stormlib.aha as s_stormlib_aha  # NOQA
+import synapse.lib.stormlib.env as s_stormlib_env  # NOQA
 import synapse.lib.stormlib.gen as s_stormlib_gen  # NOQA
 import synapse.lib.stormlib.gis as s_stormlib_gis  # NOQA
 import synapse.lib.stormlib.hex as s_stormlib_hex  # NOQA
@@ -4548,7 +4549,11 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                 continue
 
             view.layers = [lyr for lyr in view.layers if lyr.iden != layriden]
-            await view.info.set('layers', [lyr.iden for lyr in view.layers])
+            layridens = [lyr.iden for lyr in view.layers]
+            await view.info.set('layers', layridens)
+
+            mesg = {'iden': view.iden, 'layers': layridens}
+            await self.feedBeholder('view:setlayers', mesg, gates=[view.iden, layridens[0]])
 
             if view.parent.iden == viewiden:
                 if newparent is None:
@@ -4558,6 +4563,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                     view.parent = newview
                     await view.info.set('parent', newparent)
 
+                mesg = {'iden': view.iden, 'name': 'parent', 'valu': newparent}
+                await self.feedBeholder('view:set', mesg, gates=[view.iden, layridens[0]])
+
         if not layrinuse and (layr := self.layers.get(layriden)) is not None:
             del self.layers[layriden]
 
@@ -4682,6 +4690,13 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         return self.layers.get(iden)
 
+    def reqLayer(self, iden=None):
+        layr = self.getLayer(iden=iden)
+        if layr is None:
+            mesg = f'No layer found with iden: {iden}'
+            raise s_exc.NoSuchLayer(mesg=mesg, iden=iden)
+        return layr
+
     def listLayers(self):
         return self.layers.values()
 
@@ -5038,10 +5053,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                 await self.setNexsIndx(maxindx)
 
     async def saveLayerNodeEdits(self, layriden, edits, meta):
-        layr = self.getLayer(layriden)
-        if layr is None:
-            mesg = f'No layer found with iden: {layriden}'
-            raise s_exc.NoSuchLayer(mesg=mesg, iden=layriden)
+        layr = self.reqLayer(layriden)
         return await layr.saveNodeEdits(edits, meta)
 
     async def cloneLayer(self, iden, ldef=None):
@@ -6905,6 +6917,83 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         self.slab.delete(name.encode(), db=self.vaultsbynamedb)
         self.slab.delete(bidn, db=self.vaultsdb)
 
+    def _propAllowedReason(self, user, perms, gateiden=None, default=None):
+        '''
+        Similar to allowed, but always prefer the default value specified by the caller.
+        Default values are still pulled from permdefs if there is a match there; but still prefer caller default.
+        This results in a ternary response that can be used to know if a rule had a positive/negative or no match.
+        The matching reason metadata is also returned.
+        '''
+        if default is None:
+            permdef = self.getPermDef(perms)
+            if permdef:
+                default = permdef.get('default', default)
+
+        return user.getAllowedReason(perms, gateiden=gateiden, default=default)
+
+    def confirmPropSet(self, user, prop, layriden):
+        meta0 = self._propAllowedReason(user, prop.setperms[0], gateiden=layriden)
+
+        if meta0.isadmin:
+            return
+
+        allowed0 = meta0.value
+
+        meta1 = self._propAllowedReason(user, prop.setperms[1], gateiden=layriden)
+        allowed1 = meta1.value
+
+        if allowed0:
+            if allowed1:
+                return
+            elif allowed1 is False:
+                # This is a allow-with-precedence case.
+                # Inspect meta to determine if the rule a0 is more specific than rule a1
+                if len(meta0.rule) >= len(meta1.rule):
+                    return
+                user.raisePermDeny(prop.setperms[0], gateiden=layriden)
+            return
+
+        if allowed1:
+            if allowed0 is None:
+                return
+            # allowed0 here is False. This is a deny-with-precedence case.
+            # Inspect meta to determine if the rule a1 is more specific than rule a0
+            if len(meta1.rule) > len(meta0.rule):
+                return
+
+        user.raisePermDeny(prop.setperms[0], gateiden=layriden)
+
+    def confirmPropDel(self, user, prop, layriden):
+        meta0 = self._propAllowedReason(user, prop.delperms[0], gateiden=layriden)
+
+        if meta0.isadmin:
+            return
+
+        allowed0 = meta0.value
+        meta1 = self._propAllowedReason(user, prop.delperms[1], gateiden=layriden)
+        allowed1 = meta1.value
+
+        if allowed0:
+            if allowed1:
+                return
+            elif allowed1 is False:
+                # This is a allow-with-precedence case.
+                # Inspect meta to determine if the rule a0 is more specific than rule a1
+                if len(meta0.rule) >= len(meta1.rule):
+                    return
+                user.raisePermDeny(prop.delperms[0], gateiden=layriden)
+            return
+
+        if allowed1:
+            if allowed0 is None:
+                return
+            # allowed0 here is False. This is a deny-with-precedence case.
+            # Inspect meta to determine if the rule a1 is more specific than rule a0
+            if len(meta1.rule) > len(meta0.rule):
+                return
+
+        user.raisePermDeny(prop.delperms[0], gateiden=layriden)
+
 @contextlib.asynccontextmanager
 async def getTempCortex(mods=None):
     '''
@@ -6922,7 +7011,10 @@ async def getTempCortex(mods=None):
     '''
     with s_common.getTempDir() as dirn:
         logger.debug(f'Creating temporary cortex as {dirn}')
-        async with await Cortex.anit(dirn) as core:
+        conf = {
+            'health:sysctl:checks': False,
+        }
+        async with await Cortex.anit(dirn, conf=conf) as core:
             if mods:
                 for mod in mods:
                     await core.loadCoreModule(mod)

{synapse-2.170.0 → synapse-2.173.1}/synapse/lib/agenda.py

@@ -572,7 +572,8 @@ class Agenda(s_base.Base):
         self._next_indx += 1
 
         if iden in self.appts:
-            raise s_exc.DupIden()
+            mesg = f'Cron job already exists with iden: {iden}'
+            raise s_exc.DupIden(iden=iden, mesg=mesg)
 
         if not query:
             raise ValueError('"query" key of cdef parameter is not present or empty')
@@ -622,20 +623,22 @@ class Agenda(s_base.Base):
         if appt is not None:
             return appt
 
-        mesg = f'No cron job with id: {iden}'
+        mesg = f'No cron job with iden {iden}'
         raise s_exc.NoSuchIden(iden=iden, mesg=mesg)
 
     async def enable(self, iden):
         appt = self.appts.get(iden)
         if appt is None:
-            raise s_exc.NoSuchIden()
+            mesg = f'No cron job with iden: {iden}'
+            raise s_exc.NoSuchIden(iden=iden, mesg=mesg)
 
         await self.mod(iden, appt.query)
 
     async def disable(self, iden):
         appt = self.appts.get(iden)
         if appt is None:
-            raise s_exc.NoSuchIden()
+            mesg = f'No cron job with iden: {iden}'
+            raise s_exc.NoSuchIden(iden=iden, mesg=mesg)
 
         appt.enabled = False
         await appt.save()
@@ -646,7 +649,8 @@ class Agenda(s_base.Base):
         '''
         appt = self.appts.get(iden)
         if appt is None:
-            raise s_exc.NoSuchIden()
+            mesg = f'No cron job with iden: {iden}'
+            raise s_exc.NoSuchIden(iden=iden, mesg=mesg)
 
         if not query:
             raise ValueError('empty query')
@@ -664,7 +668,8 @@ class Agenda(s_base.Base):
         '''
         appt = self.appts.get(croniden)
         if appt is None:
-            raise s_exc.NoSuchIden()
+            mesg = f'No cron job with iden: {croniden}'
+            raise s_exc.NoSuchIden(iden=croniden, mesg=mesg)
 
         appt.view = viewiden
 
@@ -676,7 +681,8 @@ class Agenda(s_base.Base):
         '''
         appt = self.appts.get(iden)
         if appt is None:
-            raise s_exc.NoSuchIden()
+            mesg = f'No cron job with iden: {iden}'
+            raise s_exc.NoSuchIden(iden=iden, mesg=mesg)
 
         try:
             heappos = self.apptheap.index(appt)

{synapse-2.170.0 → synapse-2.173.1}/synapse/lib/ast.py

@@ -733,8 +733,9 @@ class SubQuery(Oper):
                 retn.append(valunode.ndef[1])
 
                 if len(retn) > limit:
-                    mesg = f'Subquery used as a value yielded too many (>{limit}) nodes'
-                    raise self.addExcInfo(s_exc.BadTypeValu(mesg=mesg))
+                    query = self.kids[0].text
+                    mesg = f'Subquery used as a value yielded too many (>{limit}) nodes. {s_common.trimText(query)}'
+                    raise self.addExcInfo(s_exc.BadTypeValu(mesg=mesg, text=query))
 
         return retn
 
@@ -987,12 +988,12 @@ class ForLoop(Oper):
                         try:
                             numitems = len(item)
                         except TypeError:
-                            mesg = f'Number of items to unpack does not match the number of variables: {repr(item)[:256]}'
+                            mesg = f'Number of items to unpack does not match the number of variables: {s_common.trimText(repr(item))}'
                             exc = s_exc.StormVarListError(mesg=mesg, names=name)
                             raise self.kids[1].addExcInfo(exc)
 
                         if len(name) != numitems:
-                            mesg = f'Number of items to unpack does not match the number of variables: {repr(item)[:256]}'
+                            mesg = f'Number of items to unpack does not match the number of variables: {s_common.trimText(repr(item))}'
                             exc = s_exc.StormVarListError(mesg=mesg, names=name, numitems=numitems)
                             raise self.kids[1].addExcInfo(exc)
 
@@ -1053,12 +1054,12 @@ class ForLoop(Oper):
                         try:
                             numitems = len(item)
                         except TypeError:
-                            mesg = f'Number of items to unpack does not match the number of variables: {repr(item)[:256]}'
+                            mesg = f'Number of items to unpack does not match the number of variables: {s_common.trimText(repr(item))}'
                             exc = s_exc.StormVarListError(mesg=mesg, names=name)
                             raise self.kids[1].addExcInfo(exc)
 
                         if len(name) != numitems:
-                            mesg = f'Number of items to unpack does not match the number of variables: {repr(item)[:256]}'
+                            mesg = f'Number of items to unpack does not match the number of variables: {s_common.trimText(repr(item))}'
                             exc = s_exc.StormVarListError(mesg=mesg, names=name, numitems=numitems)
                             raise self.kids[1].addExcInfo(exc)
 
@@ -1306,7 +1307,7 @@ class VarListSetOper(Oper):
             item = [i async for i in s_stormtypes.toiter(item)]
 
             if len(item) < len(names):
-                mesg = f'Attempting to assign more items than we have variables to assign to: {repr(item)[:256]}'
+                mesg = f'Attempting to assign more items than we have variables to assign to: {s_common.trimText(repr(item))}'
                 exc = s_exc.StormVarListError(mesg=mesg, names=names, numitems=len(item))
                 raise self.kids[0].addExcInfo(exc)
 
@@ -1322,7 +1323,7 @@ class VarListSetOper(Oper):
             item = [i async for i in s_stormtypes.toiter(item)]
 
             if len(item) < len(names):
-                mesg = f'Attempting to assign more items than we have variables to assign to: {repr(item)[:256]}'
+                mesg = f'Attempting to assign more items than we have variables to assign to: {s_common.trimText(repr(item))}'
                 exc = s_exc.StormVarListError(mesg=mesg, names=names, numitems=len(item))
                 raise self.kids[0].addExcInfo(exc)
 

{synapse-2.170.0 → synapse-2.173.1}/synapse/lib/cache.py

@@ -66,7 +66,7 @@ class FixedCache:
 
     def get(self, key):
         if self.iscorocall:
-            raise s_exc.BadArg('cache was initialized with coroutine.  Must use aget')
+            raise s_exc.BadArg(mesg='cache was initialized with coroutine.  Must use aget')
 
         valu = self.cache.get(key, s_common.novalu)
         if valu is not s_common.novalu:
@@ -81,7 +81,7 @@ class FixedCache:
 
     async def aget(self, key):
         if not self.iscorocall:
-            raise s_exc.BadOperArg('cache was initialized with non coroutine.  Must use get')
+            raise s_exc.BadOperArg(mesg='cache was initialized with non coroutine.  Must use get')
 
         valu = self.cache.get(key, s_common.novalu)
         if valu is not s_common.novalu:

{synapse-2.170.0 → synapse-2.173.1}/synapse/lib/cell.py

@@ -55,7 +55,6 @@ import synapse.lib.lmdbslab as s_lmdbslab
 import synapse.lib.thisplat as s_thisplat
 
 import synapse.lib.crypto.passwd as s_passwd
-import synapse.lib.platforms.linux as s_linux
 
 import synapse.tools.backup as s_t_backup
 
@@ -1465,7 +1464,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
     async def _runSysctlLoop(self):
         while not self.isfini:
             fixvals = []
-            sysctls = s_linux.getSysctls()
+            sysctls = s_thisplat.getSysctls()
 
             for name, valu in self.SYSCTL_VALS.items():
                 if (sysval := sysctls.get(name)) != valu:
@@ -1484,7 +1483,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             extra = await self.getLogExtra(sysctls=fixvals)
             logger.warning(mesg, extra=extra)
 
-            await asyncio.sleep(self.SYSCTL_CHECK_FREQ)
+            await self.waitfini(self.SYSCTL_CHECK_FREQ)
 
     def _getAhaAdmin(self):
         name = self.conf.get('aha:admin')
@@ -4143,6 +4142,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         async with await s_spooled.Set.anit(dirn=self.dirn, cell=self) as sset:
             yield sset
 
+    @contextlib.asynccontextmanager
+    async def getSpooledDict(self):
+        async with await s_spooled.Dict.anit(dirn=self.dirn, cell=self) as sdict:
+            yield sdict
+
     async def addSignalHandlers(self):
         await s_base.Base.addSignalHandlers(self)
 

{synapse-2.170.0 → synapse-2.173.1}/synapse/lib/coro.py

@@ -39,6 +39,18 @@ async def agen(item):
     for x in item:
         yield x
 
+async def pause(genr, iterations=10):
+    idx = 0
+
+    async for out in agen(genr):
+        yield out
+        idx += 1
+
+        if idx % iterations == 0:
+            await asyncio.sleep(0)
+
+    return
+
 def executor(func, *args, **kwargs):
     '''
     Execute a non-coroutine function in the ioloop executor pool.

{synapse-2.170.0 → synapse-2.173.1}/synapse/lib/hiveauth.py

@@ -1,7 +1,7 @@
 import logging
 import dataclasses
 
-from typing import Union
+from typing import Optional, Union
 
 import synapse.exc as s_exc
 import synapse.common as s_common
@@ -941,16 +941,38 @@ class HiveUser(HiveRuler):
         if not self.allowed(perm):
             await self.addRule((True, perm), indx=0)
 
-    def allowed(self, perm, default=None, gateiden=None):
+    def allowed(self,
+                perm: tuple[str, ...],
+                default: Optional[str] = None,
+                gateiden: Optional[str] = None,
+                deepdeny: bool = False) -> Union[bool, None]:
+        '''
+        Check if a user is allowed a given permission.
+
+        Args:
+            perm: The permission tuple to check.
+            default: The default rule value if there is no match.
+            gateiden: The gate iden to check against.
+            deepdeny: If True, give precedence for checking deny rules which are more specific than the requested
+                      permission.
+
+        Notes:
+            The use of the deepdeny argument is intended for checking a less-specific part of a permissions tree, in
+            order to know about possible short circuit options. Using it to check a more specific part may have
+            unintended results.
+
+        Returns:
+            The allowed value of the permission.
+        '''
         perm = tuple(perm)
-        return self.permcache.get((perm, default, gateiden))
+        return self.permcache.get((perm, default, gateiden, deepdeny))
 
     def _allowed(self, pkey):
         '''
         NOTE: This must remain in sync with any changes to _getAllowedReason()!
         '''
 
-        perm, default, gateiden = pkey
+        perm, default, gateiden, deepdeny = pkey
 
         if self.info.get('locked'):
             return False
@@ -958,6 +980,9 @@ class HiveUser(HiveRuler):
         if self.info.get('admin'):
             return True
 
+        if deepdeny and self._hasDeepDeny(perm, gateiden):
+            return False
+
         # 1. check authgate user rules
         if gateiden is not None:
 
@@ -1055,6 +1080,58 @@ class HiveUser(HiveRuler):
 
         return _allowedReason(default, default=True)
 
+    def _hasDeepDeny(self, perm, gateiden):
+
+        permlen = len(perm)
+
+        # 1. check authgate user rules
+        if gateiden is not None:
+
+            info = self.authgates.get(gateiden)
+            if info is not None:
+
+                if info.get('admin'):
+                    return False
+
+                for allow, path in info.get('rules', ()):
+                    if allow:
+                        continue
+                    if path[:permlen] == perm and len(path) > permlen:
+                        return True
+
+        # 2. check user rules
+        for allow, path in self.info.get('rules', ()):
+            if allow:
+                continue
+
+            if path[:permlen] == perm and len(path) > permlen:
+                return True
+
+        # 3. check authgate role rules
+        if gateiden is not None:
+
+            for role in self.getRoles():
+
+                info = role.authgates.get(gateiden)
+                if info is None:
+                    continue
+
+                for allow, path in info.get('rules', ()):
+                    if allow:
+                        continue
+                    if path[:permlen] == perm and len(path) > permlen:
+                        return True
+
+        # 4. check role rules
+        for role in self.getRoles():
+            for allow, path in role.info.get('rules', ()):
+                if allow:
+                    continue
+                if path[:permlen] == perm and len(path) > permlen:
+                    return True
+
+        return False
+
     def clearAuthCache(self):
         self.permcache.clear()
         self.allowedcache.clear()