synapse 2.166.0__tar.gz → 2.168.0__tar.gz

{synapse-2.166.0/synapse.egg-info → synapse-2.168.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse
-Version: 2.166.0
+Version: 2.168.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License: Apache License 2.0
@@ -36,7 +36,7 @@ Requires-Dist: lark==1.1.9
 Requires-Dist: Pygments<2.18.0,>=2.7.4
 Requires-Dist: packaging<24.0,>=20.0
 Requires-Dist: fastjsonschema<2.20.0,>=2.18.0
-Requires-Dist: stix2-validator<3.2.0,>=3.0.0
+Requires-Dist: stix2-validator<4.0.0,>=3.2.0
 Requires-Dist: vcrpy<5.2.0,>=4.3.1
 Requires-Dist: base58<2.2.0,>=2.1.0
 Requires-Dist: python-bitcoinlib<0.13.0,>=0.11.0
@@ -46,7 +46,7 @@ Requires-Dist: scalecodec<1.3.0,>=1.0.2
 Requires-Dist: cbor2<5.7.0,>=5.4.1
 Requires-Dist: bech32==1.2.0
 Requires-Dist: oauthlib<4.0.0,>=3.2.1
-Requires-Dist: idna<3.7,>=3.6
+Requires-Dist: idna<3.8,>=3.6
 Requires-Dist: python-dateutil<3.0,>=2.8
 Requires-Dist: pytz<2024.1,>=2023.3
 Requires-Dist: beautifulsoup4[html5lib]<5.0,>=4.11.1

{synapse-2.166.0 → synapse-2.168.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.166.0'
+version = '2.168.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]
@@ -40,7 +40,7 @@ dependencies = [
     'Pygments>=2.7.4,<2.18.0',
     'packaging>=20.0,<24.0',
     'fastjsonschema>=2.18.0,<2.20.0',
-    'stix2-validator>=3.0.0,<3.2.0',
+    'stix2-validator>=3.2.0,<4.0.0',
     'vcrpy>=4.3.1,<5.2.0',
     'base58>=2.1.0,<2.2.0',
     'python-bitcoinlib>=0.11.0,<0.13.0',
@@ -50,7 +50,7 @@ dependencies = [
     'cbor2>=5.4.1,<5.7.0',
     'bech32==1.2.0',
     'oauthlib>=3.2.1,<4.0.0',
-    'idna>=3.6,<3.7',
+    'idna>=3.6,<3.8',
     'python-dateutil>=2.8,<3.0',
     'pytz>=2023.3,<2024.1',
     'beautifulsoup4[html5lib]>=4.11.1,<5.0',

{synapse-2.166.0 → synapse-2.168.0}/synapse/axon.py

@@ -1509,7 +1509,6 @@ class Axon(s_cell.Cell):
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
-
             try:
                 data = aiohttp.FormData()
                 data._is_multipart = True
@@ -1523,7 +1522,9 @@ class Axon(s_cell.Cell):
 
                     sha256 = field.get('sha256')
                     if sha256:
-                        valu = self.get(s_common.uhex(sha256))
+                        sha256b = s_common.uhex(sha256)
+                        await self._reqHas(sha256b)
+                        valu = self.get(sha256b)
                     else:
                         valu = field.get('value')
                         if not isinstance(valu, (bytes, str)):
@@ -1547,9 +1548,6 @@ class Axon(s_cell.Cell):
                     }
                     return info
 
-            except asyncio.CancelledError:  # pramga: no cover
-                raise
-
             except Exception as e:
                 logger.exception(f'Error POSTing files to [{s_urlhelp.sanitizeUrl(url)}]')
                 err = s_common.err(e)
@@ -1586,9 +1584,8 @@ class Axon(s_cell.Cell):
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
-
             try:
-
+                await self._reqHas(sha256)
                 async with sess.request(method, url, headers=headers, params=params,
                                         data=self.get(sha256), ssl=ssl) as resp:
 
@@ -1601,9 +1598,6 @@ class Axon(s_cell.Cell):
                     }
                     return info
 
-            except asyncio.CancelledError:  # pramga: no cover
-                raise
-
             except Exception as e:
                 logger.exception(f'Error streaming [{sha256}] to [{s_urlhelp.sanitizeUrl(url)}]')
                 err = s_common.err(e)

{synapse-2.166.0 → synapse-2.168.0}/synapse/cortex.py

@@ -881,7 +881,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         self.maxnodes = self.conf.get('max:nodes')
         self.nodecount = 0
+
         self.migration = False
+        self._migration_lock = asyncio.Lock()
 
         self.stormmods = {}     # name: mdef
         self.stormpkgs = {}     # name: pkgdef
@@ -1016,6 +1018,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             (2, self._storLayrFeedDefaults),
         ), nexs=False)
 
+    async def _viewNomergeToProtected(self):
+        for view in self.views.values():
+            nomerge = view.info.get('nomerge', False)
+            await view.setViewInfo('protected', nomerge)
+            await view.setViewInfo('nomerge', None)
+
     async def _storUpdateMacros(self):
         for name, node in await self.hive.open(('cortex', 'storm', 'macros')):
 
@@ -1050,6 +1058,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
     def getStormMacro(self, name, user=None):
 
+        if not name:
+            raise s_exc.BadArg(mesg=f'Macro names must be at least 1 character long')
+
         if len(name) > 491:
             raise s_exc.BadArg(mesg='Macro names may only be up to 491 chars.')
 
@@ -1140,6 +1151,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             raise s_exc.BadArg(mesg=f'Duplicate macro name: {name}')
 
         self.slab.put(name.encode(), s_msgpack.en(mdef), db=self.macrodb)
+        await self.feedBeholder('storm:macro:add', {'macro': mdef})
         return mdef
 
     async def delStormMacro(self, name, user=None):
@@ -1151,9 +1163,15 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
     @s_nexus.Pusher.onPush('storm:macro:del')
     async def _delStormMacro(self, name):
+        if not name:
+            raise s_exc.BadArg(mesg=f'Macro names must be at least 1 character long')
+
         byts = self.slab.pop(name.encode(), db=self.macrodb)
+
         if byts is not None:
-            return s_msgpack.un(byts)
+            macro = s_msgpack.un(byts)
+            await self.feedBeholder('storm:macro:del', {'name': name, 'iden': macro.get('iden')})
+            return macro
 
     async def modStormMacro(self, name, info, user=None):
         if user is not None:
@@ -1183,6 +1201,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         else:
             self.slab.put(name.encode(), s_msgpack.en(mdef), db=self.macrodb)
 
+        await self.feedBeholder('storm:macro:mod', {'macro': mdef, 'info': info})
         return mdef
 
     async def setStormMacroPerm(self, name, scope, iden, level, user=None):
@@ -1201,6 +1220,14 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         reqValidStormMacro(mdef)
 
         self.slab.put(name.encode(), s_msgpack.en(mdef), db=self.macrodb)
+
+        info = {
+            'scope': scope,
+            'iden': iden,
+            'level': level
+        }
+
+        await self.feedBeholder('storm:macro:set:perm', {'macro': mdef, 'info': info})
         return mdef
 
     async def getStormMacros(self, user=None):
@@ -1400,6 +1427,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         await self._bumpCellVers('cortex:defaults', (
             (1, self._addAllLayrRead),
+            (2, self._viewNomergeToProtected),
         ))
 
     async def _addAllLayrRead(self):
@@ -1454,12 +1482,21 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         await self.stormdmons.start()
         await self.agenda.clearRunningStatus()
 
-        for view in self.views.values():
-            await view.initTrigTask()
-            await view.initMergeTask()
+        async def _runMigrations():
+            # Run migrations when this cortex becomes active. This is to prevent
+            # migrations getting skipped in a zero-downtime upgrade path
+            # (upgrade mirror, promote mirror).
+            await self._checkLayerModels()
 
-        for layer in self.layers.values():
-            await layer.initLayerActive()
+            # Once migrations are complete, start the view and layer tasks.
+            for view in self.views.values():
+                await view.initTrigTask()
+                await view.initMergeTask()
+
+            for layer in self.layers.values():
+                await layer.initLayerActive()
+
+        self.runActiveTask(_runMigrations())
 
         await self.initStormPool()
 
@@ -3571,6 +3608,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                         break
 
                     yield ioff, layr.iden, SYNC_NODEEDITS, item, meta
+                    await asyncio.sleep(0)
 
             if layr.isdeleted:
                 yield layr.deloffs, layr.iden, SYNC_LAYR_DEL, (), {}
@@ -4337,7 +4375,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         return ret
 
     async def _checkLayerModels(self):
-        with self.enterMigrationMode():
+        async with self.enterMigrationMode():
             mrev = s_modelrev.ModelRev(self)
             await mrev.revCoreLayers()
 
@@ -4458,6 +4496,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         if view is None:
             raise s_exc.NoSuchView(mesg=f'No such view {iden=}', iden=iden)
 
+        if view.info.get('protected'):
+            mesg = f'Cannot delete view ({iden}) that has protected set.'
+            raise s_exc.CantDelView(mesg=mesg)
+
         return await self._push('view:del', iden)
 
     @s_nexus.Pusher.onPush('view:del')
@@ -6068,11 +6110,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         appt = await self.agenda.get(iden)
         await appt.edits(edits)
 
-    @contextlib.contextmanager
-    def enterMigrationMode(self):
-        self.migration = True
-        yield
-        self.migration = False
+    @contextlib.asynccontextmanager
+    async def enterMigrationMode(self):
+        async with self._migration_lock:
+            self.migration = True
+            yield
+            self.migration = False
 
     async def iterFormRows(self, layriden, form, stortype=None, startvalu=None):
         '''

{synapse-2.166.0 → synapse-2.168.0}/synapse/exc.py

@@ -145,6 +145,7 @@ class CantDelForm(SynErr): pass
 class CantDelProp(SynErr): pass
 class CantDelType(SynErr): pass
 class CantDelUniv(SynErr): pass
+class CantDelView(SynErr): pass
 class CantMergeView(SynErr): pass
 class CantRevLayer(SynErr): pass
 class CliFini(SynErr):

{synapse-2.166.0 → synapse-2.168.0}/synapse/lib/aha.py

@@ -782,6 +782,8 @@ class AhaCell(s_cell.Cell):
 
         svcname, svcnetw, svcfull = self._nameAndNetwork(name, network)
 
+        logger.info(f'Deleting service [{svcfull}].', extra=await self.getLogExtra(name=svcname, netw=svcnetw))
+
         full = ('aha', 'svcfull', svcfull)
         path = ('aha', 'services', svcnetw, svcname)
 
@@ -805,7 +807,8 @@ class AhaCell(s_cell.Cell):
     async def _setAhaSvcDown(self, name, linkiden, network=None):
         svcname, svcnetw, svcfull = self._nameAndNetwork(name, network)
         path = ('aha', 'services', svcnetw, svcname)
-        await self.jsonstor.cmpDelPathObjProp(path, 'svcinfo/online', linkiden)
+        if await self.jsonstor.cmpDelPathObjProp(path, 'svcinfo/online', linkiden):
+            await self.jsonstor.setPathObjProp(path, 'svcinfo/ready', False)
 
         # Check if we have any links which may need to be removed
         current_sessions = {s_common.guid(iden): sess for iden, sess in self.dmon.sessions.items()}

{synapse-2.166.0 → synapse-2.168.0}/synapse/lib/base.py

@@ -143,6 +143,7 @@ class Base:
         self._fini_atexit = False
         self._active_tasks = None       # the set of free running tasks associated with me
         self._context_managers = None   # the set of context managers i must fini
+        self._syn_signal_tasks = None   # initialized as a Set when addSignalHandlers is called.
 
     async def postAnit(self):
         '''
@@ -559,14 +560,20 @@ class Base:
         '''
         Register SIGTERM/SIGINT signal handlers with the ioloop to fini this object.
         '''
+        if self._syn_signal_tasks is None:
+            self._syn_signal_tasks = set()
 
         def sigterm():
-            print('Caught SIGTERM, shutting down.')
-            asyncio.create_task(self.fini())
+            logger.warning('Caught SIGTERM, shutting down.')
+            task = asyncio.create_task(self.fini())
+            self._syn_signal_tasks.add(task)
+            task.add_done_callback(self._syn_signal_tasks.discard)
 
         def sigint():
-            print('Caught SIGINT, shutting down.')
-            asyncio.create_task(self.fini())
+            logger.warning('Caught SIGINT, shutting down.')
+            task = asyncio.create_task(self.fini())
+            self._syn_signal_tasks.add(task)
+            task.add_done_callback(self._syn_signal_tasks.discard)
 
         loop = asyncio.get_running_loop()
         loop.add_signal_handler(signal.SIGINT, sigint)

{synapse-2.166.0 → synapse-2.168.0}/synapse/lib/cell.py

@@ -668,26 +668,32 @@ class CellApi(s_base.Base):
 
     @adminapi()
     async def listHiveKey(self, path=None):
+        s_common.deprecated('CellApi.listHiveKey', curv='2.167.0')
         return await self.cell.listHiveKey(path=path)
 
     @adminapi(log=True)
     async def getHiveKeys(self, path):
+        s_common.deprecated('CellApi.getHiveKeys', curv='2.167.0')
         return await self.cell.getHiveKeys(path)
 
     @adminapi(log=True)
     async def getHiveKey(self, path):
+        s_common.deprecated('CellApi.getHiveKey', curv='2.167.0')
         return await self.cell.getHiveKey(path)
 
     @adminapi(log=True)
     async def setHiveKey(self, path, valu):
+        s_common.deprecated('CellApi.setHiveKey', curv='2.167.0')
         return await self.cell.setHiveKey(path, valu)
 
     @adminapi(log=True)
     async def popHiveKey(self, path):
+        s_common.deprecated('CellApi.popHiveKey', curv='2.167.0')
         return await self.cell.popHiveKey(path)
 
     @adminapi(log=True)
     async def saveHiveTree(self, path=()):
+        s_common.deprecated('CellApi.saveHiveTree', curv='2.167.0')
         return await self.cell.saveHiveTree(path=path)
 
     @adminapi()
@@ -1419,7 +1425,6 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         if ahaurl is not None:
 
             info = await s_telepath.addAhaUrl(ahaurl)
-            self.ahaclient = info.get('client')
             if self.ahaclient is None:
                 self.ahaclient = await s_telepath.Client.anit(info.get('url'))
                 self.ahaclient._fini_atexit = True
@@ -2895,6 +2900,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         if isnew:
             path = os.path.join(self.dirn, 'hiveboot.yaml')
             if os.path.isfile(path):
+                s_common.deprdate('Initial hive config from hiveboot.yaml', '2024-05-05')
                 logger.debug(f'Loading cell hive from {path}')
                 tree = s_common.yamlload(path)
                 if tree is not None:
@@ -4079,7 +4085,10 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         await s_base.Base.addSignalHandlers(self)
 
         def sighup():
-            asyncio.create_task(self.reload())
+            logger.info('Caught SIGHUP, running reloadable subsystems.')
+            task = asyncio.create_task(self.reload())
+            self._syn_signal_tasks.add(task)
+            task.add_done_callback(self._syn_signal_tasks.discard)
 
         loop = asyncio.get_running_loop()
         loop.add_signal_handler(signal.SIGHUP, sighup)

{synapse-2.166.0 → synapse-2.168.0}/synapse/lib/hive.py

@@ -473,12 +473,15 @@ class HiveApi(s_base.Base):
         self.onfini(self._onHapiFini)
 
     async def loadHiveTree(self, tree, path=(), trim=False):
+        s_common.deprecated('HiveApi.loadHiveTree', curv='2.167.0')
         return await self.hive.loadHiveTree(tree, path=path, trim=trim)
 
     async def saveHiveTree(self, path=()):
+        s_common.deprecated('HiveApi.saveHiveTree', curv='2.167.0')
         return await self.hive.saveHiveTree(path=path)
 
     async def treeAndSync(self, path, iden):
+        s_common.deprecated('HiveApi.treeAndSync', curv='2.167.0')
 
         node = await self.hive.open(path)
 
@@ -510,16 +513,22 @@ class HiveApi(s_base.Base):
         return
 
     async def setAndSync(self, path, valu, iden, nexs=False):
+        s_common.deprecated('HiveApi.setAndSync', curv='2.167.0')
+
         valu = await self.hive.set(path, valu, nexs=nexs)
         await self.msgq.put(('hive:sync', {'iden': iden}))
         return valu
 
     async def addAndSync(self, path, valu, iden):
+        s_common.deprecated('HiveApi.addAndSync', curv='2.167.0')
+
         valu = await self.hive.add(path, valu)
         await self.msgq.put(('hive:sync', {'iden': iden}))
         return valu
 
     async def popAndSync(self, path, iden, nexs=False):
+        s_common.deprecated('HiveApi.popAndSync', curv='2.167.0')
+
         valu = await self.hive.pop(path, nexs=nexs)
         await self.msgq.put(('hive:sync', {'iden': iden}))
         return valu
@@ -531,9 +540,11 @@ class HiveApi(s_base.Base):
         self.msgq.put_nowait(mesg)
 
     async def get(self, full):
+        s_common.deprecated('HiveApi.get', curv='2.167.0')
         return await self.hive.get(full)
 
     async def edits(self):
+        s_common.deprecated('HiveApi.edits', curv='2.167.0')
 
         while not self.isfini:
 

{synapse-2.166.0 → synapse-2.168.0}/synapse/lib/layer.py

@@ -165,11 +165,13 @@ class LayerApi(s_cell.CellApi):
         await self._reqUserAllowed(self.liftperm)
         async for item in self.layr.syncNodeEdits(offs, wait=wait):
             yield item
+            await asyncio.sleep(0)
 
     async def syncNodeEdits2(self, offs, wait=True):
         await self._reqUserAllowed(self.liftperm)
         async for item in self.layr.syncNodeEdits2(offs, wait=wait):
             yield item
+            await asyncio.sleep(0)
 
     async def getEditIndx(self):
         '''

{synapse-2.166.0 → synapse-2.168.0}/synapse/lib/modelrev.py

@@ -768,6 +768,12 @@ class ModelRev:
         Returns:
             None
         '''
+        if opts is None:
+            opts = {}
+
+        # Migrations only run on leaders
+        opts['mirror'] = False
+
         async def _runStorm():
             async for mesgtype, mesginfo in self.core.storm(text, opts=opts):
                 if mesgtype == 'print':

{synapse-2.166.0 → synapse-2.168.0}/synapse/lib/modules.py

@@ -28,4 +28,5 @@ coremods = (
     'synapse.models.biz.BizModule',
     'synapse.models.belief.BeliefModule',
     'synapse.models.science.ScienceModule',
+    'synapse.models.planning.PlanModule',
 )

{synapse-2.166.0 → synapse-2.168.0}/synapse/lib/node.py

@@ -472,10 +472,12 @@ class Node:
                                    form=self.form.full, tag=tag)
 
         pref = name + '.'
+        exists = self.tags.get(name, s_common.novalu) is not s_common.novalu
 
         todel = [(len(t), t) for t in self.tags.keys() if t.startswith(pref)]
 
-        if len(path) > 1:
+        # only prune when we're actually deleting a tag
+        if len(path) > 1 and exists:
 
             parent = '.'.join(path[:-1])
 
@@ -511,7 +513,7 @@ class Node:
             edits.append((s_layer.EDIT_TAG_DEL, (subtag, None), ()))
 
         edits.extend(self._getTagPropDel(name))
-        if self.getTag(name, defval=s_common.novalu) is not s_common.novalu:
+        if exists:
             edits.append((s_layer.EDIT_TAG_DEL, (name, None), ()))
 
         return edits

{synapse-2.166.0 → synapse-2.168.0}/synapse/lib/schemas.py

@@ -172,7 +172,7 @@ reqValidView = s_config.getJsValidator({
         'parent': {'type': ['string', 'null'], 'pattern': s_config.re_iden},
         'creator': {'type': 'string', 'pattern': s_config.re_iden},
         'created': {'type': 'integer', 'minimum': 0},
-        'nomerge': {'type': 'boolean'},
+        'protected': {'type': 'boolean', 'default': False},
         'merging': {'type': 'boolean'},
         'layers': {
             'type': 'array',