synapse 2.185.0__tar.gz → 2.187.0__tar.gz

{synapse-2.185.0/synapse.egg-info → synapse-2.187.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse
-Version: 2.185.0
+Version: 2.187.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License: Apache License 2.0

{synapse-2.185.0 → synapse-2.187.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.185.0'
+version = '2.187.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]
@@ -84,7 +84,6 @@ include-package-data = true
 
 [tool.setuptools.package-data]
 'synapse' = [
-    'assets/**',
     'data/*.mpk',
     'data/certs/**',
     'data/attack-flow/**',

{synapse-2.185.0 → synapse-2.187.0}/synapse/cortex.py

@@ -1938,12 +1938,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                 return
 
     async def coreQueuePuts(self, name, items):
-        await self._push('queue:puts', name, items)
+        return await self._push('queue:puts', name, items)
 
     @s_nexus.Pusher.onPush('queue:puts', passitem=True)
     async def _coreQueuePuts(self, name, items, nexsitem):
         nexsoff, nexsmesg = nexsitem
-        await self.multiqueue.puts(name, items, reqid=nexsoff)
+        return await self.multiqueue.puts(name, items, reqid=nexsoff)
 
     @s_nexus.Pusher.onPushAuto('queue:cull')
     async def coreQueueCull(self, name, offs):
@@ -2056,9 +2056,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                     continue
 
                 if not regex.fullmatch(regx[i], parts[i]):
-                    return False
+                    mesg = f'Tag part ({parts[i]}) of tag ({tagname}) does not match the tag model regex: [{regx[i]}]'
+                    return (False, mesg)
 
-        return True
+        return (True, None)
 
     async def getTagPrune(self, tagname):
         return self.tagprune.get(tagname)

{synapse-2.185.0 → synapse-2.187.0}/synapse/exc.py

@@ -141,6 +141,8 @@ class BadTag(SynErr): pass
 class BadTime(SynErr): pass
 class BadUrl(SynErr): pass
 
+class TypeMismatch(SynErr): pass
+
 class CantDelCmd(SynErr): pass
 class CantDelNode(SynErr): pass
 class CantDelForm(SynErr): pass

{synapse-2.185.0 → synapse-2.187.0}/synapse/lib/ast.py

@@ -4579,7 +4579,7 @@ class EditTagAdd(Edit):
         else:
             oper_offset = 0
 
-        excignore = (s_exc.BadTypeValu, s_exc.BadTag) if oper_offset == 1 else ()
+        excignore = (s_exc.BadTypeValu,) if oper_offset == 1 else ()
 
         hasval = len(self.kids) > 2 + oper_offset
 

{synapse-2.185.0 → synapse-2.187.0}/synapse/lib/cell.py

@@ -35,6 +35,7 @@ import synapse.lib.boss as s_boss
 import synapse.lib.coro as s_coro
 import synapse.lib.hive as s_hive
 import synapse.lib.link as s_link
+import synapse.lib.task as s_task
 import synapse.lib.cache as s_cache
 import synapse.lib.const as s_const
 import synapse.lib.drive as s_drive
@@ -206,6 +207,14 @@ class CellApi(s_base.Base):
     async def initCellApi(self):
         pass
 
+    @adminapi(log=True)
+    async def freeze(self, timeout=30):
+        return await self.cell.freeze(timeout=timeout)
+
+    @adminapi(log=True)
+    async def resume(self):
+        return await self.cell.resume()
+
     async def allowed(self, perm, default=None):
         '''
         Check if the user has the requested permission.
@@ -1102,6 +1111,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         self.auth = None
         self.cellparent = parent
         self.sessions = {}
+        self.paused = False
         self.isactive = False
         self.activebase = None
         self.inaugural = False
@@ -1767,8 +1777,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         return await self.drive.addItemInfo(info, path=path, reldir=reldir)
 
-    async def getDriveInfo(self, iden):
-        return self.drive.getItemInfo(iden)
+    async def getDriveInfo(self, iden, typename=None):
+        return self.drive.getItemInfo(iden, typename=typename)
+
+    def reqDriveInfo(self, iden, typename=None):
+        return self.drive.reqItemInfo(iden, typename=typename)
 
     async def getDrivePath(self, path, reldir=s_drive.rootdir):
         '''
@@ -4461,6 +4474,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 'run': await self.getCellRunId(),
                 'type': self.getCellType(),
                 'iden': self.getCellIden(),
+                'paused': self.paused,
                 'active': self.isactive,
                 'started': self.startms,
                 'ready': self.nexsroot.ready.is_set(),
@@ -4932,3 +4946,52 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         key = tuple(sorted(opts.items()))
         return self._sslctx_cache.get(key)
+
+    async def freeze(self, timeout=30):
+
+        if self.paused:
+            mesg = 'The service is already frozen.'
+            raise s_exc.BadState(mesg=mesg)
+
+        logger.warning(f'Freezing service for volume snapshot.')
+
+        logger.warning('...acquiring nexus lock to prevent edits.')
+
+        try:
+            await asyncio.wait_for(self.nexslock.acquire(), timeout=timeout)
+
+        except asyncio.TimeoutError:
+            logger.warning('...nexus lock acquire timed out!')
+            logger.warning('Aborting freeze and resuming normal operation.')
+
+            mesg = 'Nexus lock acquire timed out.'
+            raise s_exc.TimeOut(mesg=mesg)
+
+        self.paused = True
+
+        try:
+
+            logger.warning('...committing pending transactions.')
+            await self.slab.syncLoopOnce()
+
+            logger.warning('...flushing dirty buffers to disk.')
+            await s_task.executor(os.sync)
+
+            logger.warning('...done!')
+
+        except Exception:
+            self.paused = False
+            self.nexslock.release()
+            logger.exception('Failed to freeze. Resuming normal operation.')
+            raise
+
+    async def resume(self):
+
+        if not self.paused:
+            mesg = 'The service is not frozen.'
+            raise s_exc.BadState(mesg=mesg)
+
+        logger.warning('Resuming normal operations from a freeze.')
+
+        self.paused = False
+        self.nexslock.release()

{synapse-2.185.0 → synapse-2.187.0}/synapse/lib/drive.py

@@ -22,6 +22,7 @@ LKEY_INFO = b'\x02' # <bidn> = <info>
 LKEY_DATA = b'\x03' # <bidn> <vers> = <data>
 LKEY_VERS = b'\x04' # <bidn> <vers> = <versinfo>
 LKEY_INFO_BYTYPE = b'\x05' # <type> 00 <bidn> = 01
+LKEY_TYPE_VERS = b'\x06' # <type> = <uint64>
 
 rootdir = '00000000000000000000000000000000'
 
@@ -60,24 +61,36 @@ class Drive(s_base.Base):
 
         return [reqValidName(p.strip().lower()) for p in path]
 
-    def getItemInfo(self, iden):
-        return self._getItemInfo(s_common.uhex(iden))
+    def _reqInfoType(self, info, typename):
+        infotype = info.get('type')
+        if infotype != typename:
+            mesg = f'Drive item has the wrong type. Expected: {typename} got {infotype}.'
+            raise s_exc.TypeMismatch(mesg=mesg, expected=typename, got=infotype)
+
+    def getItemInfo(self, iden, typename=None):
+        info = self._getItemInfo(s_common.uhex(iden))
+        if typename is not None:
+            self._reqInfoType(info, typename)
+        return info
 
     def _getItemInfo(self, bidn):
         byts = self.slab.get(LKEY_INFO + bidn, db=self.dbname)
         if byts is not None:
             return s_msgpack.un(byts)
 
-    def reqItemInfo(self, iden):
-        return self._reqItemInfo(s_common.uhex(iden))
+    def reqItemInfo(self, iden, typename=None):
+        return self._reqItemInfo(s_common.uhex(iden), typename=typename)
 
-    def _reqItemInfo(self, bidn):
+    def _reqItemInfo(self, bidn, typename=None):
         info = self._getItemInfo(bidn)
-        if info is not None:
-            return info
+        if info is None:
+            mesg = f'No drive item with ID {s_common.ehex(bidn)}.'
+            raise s_exc.NoSuchIden(mesg=mesg)
 
-        mesg = f'No drive item with ID {s_common.ehex(bidn)}.'
-        raise s_exc.NoSuchIden(mesg=mesg)
+        if typename is not None:
+            self._reqInfoType(info, typename)
+
+        return info
 
     async def setItemPath(self, iden, path):
         '''
@@ -494,10 +507,27 @@ class Drive(s_base.Base):
         if byts is not None:
             return s_msgpack.un(byts)
 
-    async def setTypeSchema(self, typename, schema, callback=None):
+    def getTypeSchemaVersion(self, typename):
+        verskey = LKEY_TYPE_VERS + typename.encode()
+        byts = self.slab.get(verskey, db=self.dbname)
+        if byts is not None:
+            return s_msgpack.un(byts)
+
+    async def setTypeSchema(self, typename, schema, callback=None, vers=None):
 
         reqValidName(typename)
 
+        if vers is not None:
+            vers = int(vers)
+            curv = self.getTypeSchemaVersion(typename)
+            if curv is not None:
+                if vers == curv:
+                    return False
+
+                if vers < curv:
+                    mesg = f'Cannot downgrade drive schema version for type {typename}.'
+                    raise s_exc.BadVersion(mesg=mesg)
+
         vtor = s_config.getJsValidator(schema)
 
         self.validators[typename] = vtor
@@ -506,6 +536,10 @@ class Drive(s_base.Base):
 
         self.slab.put(lkey, s_msgpack.en(schema), db=self.dbname)
 
+        if vers is not None:
+            verskey = LKEY_TYPE_VERS + typename.encode()
+            self.slab.put(verskey, s_msgpack.en(vers), db=self.dbname)
+
         if callback is not None:
             async for info in self.getItemsByType(typename):
                 bidn = s_common.uhex(info.get('iden'))
@@ -516,6 +550,7 @@ class Drive(s_base.Base):
                     vtor(data)
                     self.slab.put(LKEY_DATA + bidn + versindx, s_msgpack.en(data), db=self.dbname)
                     await asyncio.sleep(0)
+        return True
 
     async def getItemsByType(self, typename):
         tkey = typename.encode() + b'\x00'

{synapse-2.185.0 → synapse-2.187.0}/synapse/lib/hive.py

@@ -351,7 +351,7 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
                         await self.cell.auth._hndlsetUserProfileValu(iden, name, valu)
 
             elif full[0] == 'cellvers':
-                self.cell.setCellVers(full[-1], valu, nexs=False)
+                await self.cell.setCellVers(full[-1], valu, nexs=False)
 
         node = await self._getHiveNode(full)