synapse 2.165.0__tar.gz → 2.167.0__tar.gz

{synapse-2.165.0/synapse.egg-info → synapse-2.167.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse
-Version: 2.165.0
+Version: 2.167.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License: Apache License 2.0
@@ -36,7 +36,7 @@ Requires-Dist: lark==1.1.9
 Requires-Dist: Pygments<2.18.0,>=2.7.4
 Requires-Dist: packaging<24.0,>=20.0
 Requires-Dist: fastjsonschema<2.20.0,>=2.18.0
-Requires-Dist: stix2-validator<4.0.0,>=3.0.0
+Requires-Dist: stix2-validator<4.0.0,>=3.2.0
 Requires-Dist: vcrpy<5.2.0,>=4.3.1
 Requires-Dist: base58<2.2.0,>=2.1.0
 Requires-Dist: python-bitcoinlib<0.13.0,>=0.11.0
@@ -46,7 +46,7 @@ Requires-Dist: scalecodec<1.3.0,>=1.0.2
 Requires-Dist: cbor2<5.7.0,>=5.4.1
 Requires-Dist: bech32==1.2.0
 Requires-Dist: oauthlib<4.0.0,>=3.2.1
-Requires-Dist: idna<3.7,>=3.6
+Requires-Dist: idna<3.8,>=3.6
 Requires-Dist: python-dateutil<3.0,>=2.8
 Requires-Dist: pytz<2024.1,>=2023.3
 Requires-Dist: beautifulsoup4[html5lib]<5.0,>=4.11.1

{synapse-2.165.0 → synapse-2.167.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.165.0'
+version = '2.167.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]
@@ -40,7 +40,7 @@ dependencies = [
     'Pygments>=2.7.4,<2.18.0',
     'packaging>=20.0,<24.0',
     'fastjsonschema>=2.18.0,<2.20.0',
-    'stix2-validator>=3.0.0,<4.0.0',
+    'stix2-validator>=3.2.0,<4.0.0',
     'vcrpy>=4.3.1,<5.2.0',
     'base58>=2.1.0,<2.2.0',
     'python-bitcoinlib>=0.11.0,<0.13.0',
@@ -50,7 +50,7 @@ dependencies = [
     'cbor2>=5.4.1,<5.7.0',
     'bech32==1.2.0',
     'oauthlib>=3.2.1,<4.0.0',
-    'idna>=3.6,<3.7',
+    'idna>=3.6,<3.8',
     'python-dateutil>=2.8,<3.0',
     'pytz>=2023.3,<2024.1',
     'beautifulsoup4[html5lib]>=4.11.1,<5.0',

{synapse-2.165.0 → synapse-2.167.0}/synapse/axon.py

@@ -1509,7 +1509,6 @@ class Axon(s_cell.Cell):
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
-
             try:
                 data = aiohttp.FormData()
                 data._is_multipart = True
@@ -1523,7 +1522,9 @@ class Axon(s_cell.Cell):
 
                     sha256 = field.get('sha256')
                     if sha256:
-                        valu = self.get(s_common.uhex(sha256))
+                        sha256b = s_common.uhex(sha256)
+                        await self._reqHas(sha256b)
+                        valu = self.get(sha256b)
                     else:
                         valu = field.get('value')
                         if not isinstance(valu, (bytes, str)):
@@ -1547,9 +1548,6 @@ class Axon(s_cell.Cell):
                     }
                     return info
 
-            except asyncio.CancelledError:  # pramga: no cover
-                raise
-
             except Exception as e:
                 logger.exception(f'Error POSTing files to [{s_urlhelp.sanitizeUrl(url)}]')
                 err = s_common.err(e)
@@ -1586,9 +1584,8 @@ class Axon(s_cell.Cell):
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
-
             try:
-
+                await self._reqHas(sha256)
                 async with sess.request(method, url, headers=headers, params=params,
                                         data=self.get(sha256), ssl=ssl) as resp:
 
@@ -1601,9 +1598,6 @@ class Axon(s_cell.Cell):
                     }
                     return info
 
-            except asyncio.CancelledError:  # pramga: no cover
-                raise
-
             except Exception as e:
                 logger.exception(f'Error streaming [{sha256}] to [{s_urlhelp.sanitizeUrl(url)}]')
                 err = s_common.err(e)

{synapse-2.165.0 → synapse-2.167.0}/synapse/cmds/cortex.py

@@ -212,7 +212,7 @@ class StormCmd(s_cli.Cmd):
                    * nodeedits (default),
                    * count (just counts of nodeedits), or
                    * none (no such messages emitted).
-        --show-prov:  Show provenance messages.
+        --show-prov:  Deprecated. This no longer does anything.
         --raw: Print the nodes in their raw format. This overrides --hide-tags and --hide-props.
         --debug: Display cmd debug information along with nodes in raw format. This overrides other display arguments.
         --path: Get path information about returned nodes.
@@ -256,7 +256,6 @@ class StormCmd(s_cli.Cmd):
             'err': self._onErr,
             'node:edits': self._onNodeEdits,
             'node:edits:count': self._onNodeEditsCount,
-            'prov:new': self._onProvNew,  # TODO remove in 3.0.0
         }
         self._indented = False
 
@@ -281,10 +280,6 @@ class StormCmd(s_cli.Cmd):
         s_cli.Cmd.printf(self, '.' * count, addnl=False, color=NODEEDIT_COLOR)
         self._indented = True
 
-    def _onProvNew(self, mesg, opts):
-        if opts.get('show-prov'):
-            self.printf(repr(mesg), color=PROVNEW_COLOR)
-
     def _printNodeProp(self, name, valu):
         self.printf(f'        {name} = {valu}')
 

{synapse-2.165.0 → synapse-2.167.0}/synapse/common.py

@@ -973,6 +973,12 @@ def unjsonsafe_nodeedits(nodeedits):
 
     return retn
 
+def reprauthrule(rule):
+    text = '.'.join(rule[1])
+    if not rule[0]:
+        text = '!' + text
+    return text
+
 def reqJsonSafeStrict(item):
     '''
     Require the item to be safe to serialize to JSON without type coercion issues.

{synapse-2.165.0 → synapse-2.167.0}/synapse/cortex.py

@@ -53,7 +53,6 @@ import synapse.lib.crypto.rsa as s_rsa
 import synapse.lib.stormhttp as s_stormhttp  # NOQA
 import synapse.lib.stormwhois as s_stormwhois  # NOQA
 
-import synapse.lib.provenance as s_provenance
 import synapse.lib.stormtypes as s_stormtypes
 
 import synapse.lib.stormlib.aha as s_stormlib_aha  # NOQA
@@ -74,6 +73,7 @@ import synapse.lib.stormlib.smtp as s_stormlib_smtp  # NOQA
 import synapse.lib.stormlib.stix as s_stormlib_stix  # NOQA
 import synapse.lib.stormlib.yaml as s_stormlib_yaml  # NOQA
 import synapse.lib.stormlib.basex as s_stormlib_basex  # NOQA
+import synapse.lib.stormlib.cache as s_stormlib_cache  # NOQA
 import synapse.lib.stormlib.graph as s_stormlib_graph  # NOQA
 import synapse.lib.stormlib.iters as s_stormlib_iters  # NOQA
 import synapse.lib.stormlib.macro as s_stormlib_macro
@@ -89,6 +89,7 @@ import synapse.lib.stormlib.random as s_stormlib_random  # NOQA
 import synapse.lib.stormlib.scrape as s_stormlib_scrape   # NOQA
 import synapse.lib.stormlib.infosec as s_stormlib_infosec  # NOQA
 import synapse.lib.stormlib.project as s_stormlib_project  # NOQA
+import synapse.lib.stormlib.spooled as s_stormlib_spooled  # NOQA
 import synapse.lib.stormlib.version as s_stormlib_version  # NOQA
 import synapse.lib.stormlib.easyperm as s_stormlib_easyperm  # NOQA
 import synapse.lib.stormlib.ethereum as s_stormlib_ethereum  # NOQA
@@ -286,10 +287,8 @@ class CoreApi(s_cell.CellApi):
         s_common.deprecated('CoreApi.addNode')
         async with await self.cell.snap(user=self.user) as snap:
             self.user.confirm(('node', 'add', form), gateiden=snap.wlyr.iden)
-            with s_provenance.claim('coreapi', meth='node:add', user=snap.user.iden):
-
-                node = await snap.addNode(form, valu, props=props)
-                return node.pack()
+            node = await snap.addNode(form, valu, props=props)
+            return node.pack()
 
     async def addNodes(self, nodes):
         '''
@@ -317,16 +316,14 @@ class CoreApi(s_cell.CellApi):
             done[formname] = True
 
         async with await self.cell.snap(user=self.user) as snap:
-            with s_provenance.claim('coreapi', meth='node:add', user=snap.user.iden):
 
-                snap.strict = False
-
-                async for node in snap.addNodes(nodes):
+            snap.strict = False
+            async for node in snap.addNodes(nodes):
 
-                    if node is not None:
-                        node = node.pack()
+                if node is not None:
+                    node = node.pack()
 
-                    yield node
+                yield node
 
     async def getFeedFuncs(self):
         '''
@@ -361,9 +358,8 @@ class CoreApi(s_cell.CellApi):
                                            })
 
         async with await self.cell.snap(user=self.user, view=view) as snap:
-            with s_provenance.claim('feed:data', name=name, user=snap.user.iden):
-                snap.strict = False
-                await snap.addFeedData(name, items)
+            snap.strict = False
+            await snap.addFeedData(name, items)
 
     async def count(self, text, opts=None):
         '''
@@ -1002,7 +998,16 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             'multiqueue': self.multiqueue,
         })
 
-        await self.auth.addAuthGate('cortex', 'cortex')
+        # TODO - Remove this in 3.0.0
+        ag = await self.auth.addAuthGate('cortex', 'cortex')
+        for (useriden, user) in ag.gateusers.items():
+            mesg = f'User {useriden} ({user.name}) has a rule on the "cortex" authgate. This authgate is not used ' \
+                   f'for permission checks and will be removed in Synapse v3.0.0.'
+            logger.warning(mesg, extra=await self.getLogExtra(user=useriden, username=user.name))
+        for (roleiden, role) in ag.gateroles.items():
+            mesg = f'Role {roleiden} ({role.name}) has a rule on the "cortex" authgate. This authgate is not used ' \
+                   f'for permission checks and will be removed in Synapse v3.0.0.'
+            logger.warning(mesg, extra=await self.getLogExtra(role=roleiden, rolename=role.name))
 
         self._initVaults()
 
@@ -1011,6 +1016,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             (2, self._storLayrFeedDefaults),
         ), nexs=False)
 
+    async def _viewNomergeToProtected(self):
+        for view in self.views.values():
+            nomerge = view.info.get('nomerge', False)
+            await view.setViewInfo('protected', nomerge)
+            await view.setViewInfo('nomerge', None)
+
     async def _storUpdateMacros(self):
         for name, node in await self.hive.open(('cortex', 'storm', 'macros')):
 
@@ -1045,6 +1056,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
     def getStormMacro(self, name, user=None):
 
+        if not name:
+            raise s_exc.BadArg(mesg=f'Macro names must be at least 1 character long')
+
         if len(name) > 491:
             raise s_exc.BadArg(mesg='Macro names may only be up to 491 chars.')
 
@@ -1135,6 +1149,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             raise s_exc.BadArg(mesg=f'Duplicate macro name: {name}')
 
         self.slab.put(name.encode(), s_msgpack.en(mdef), db=self.macrodb)
+        await self.feedBeholder('storm:macro:add', {'macro': mdef})
         return mdef
 
     async def delStormMacro(self, name, user=None):
@@ -1146,9 +1161,15 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
     @s_nexus.Pusher.onPush('storm:macro:del')
     async def _delStormMacro(self, name):
+        if not name:
+            raise s_exc.BadArg(mesg=f'Macro names must be at least 1 character long')
+
         byts = self.slab.pop(name.encode(), db=self.macrodb)
+
         if byts is not None:
-            return s_msgpack.un(byts)
+            macro = s_msgpack.un(byts)
+            await self.feedBeholder('storm:macro:del', {'name': name, 'iden': macro.get('iden')})
+            return macro
 
     async def modStormMacro(self, name, info, user=None):
         if user is not None:
@@ -1178,6 +1199,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         else:
             self.slab.put(name.encode(), s_msgpack.en(mdef), db=self.macrodb)
 
+        await self.feedBeholder('storm:macro:mod', {'macro': mdef, 'info': info})
         return mdef
 
     async def setStormMacroPerm(self, name, scope, iden, level, user=None):
@@ -1196,6 +1218,14 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         reqValidStormMacro(mdef)
 
         self.slab.put(name.encode(), s_msgpack.en(mdef), db=self.macrodb)
+
+        info = {
+            'scope': scope,
+            'iden': iden,
+            'level': level
+        }
+
+        await self.feedBeholder('storm:macro:set:perm', {'macro': mdef, 'info': info})
         return mdef
 
     async def getStormMacros(self, user=None):
@@ -1395,6 +1425,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         await self._bumpCellVers('cortex:defaults', (
             (1, self._addAllLayrRead),
+            (2, self._viewNomergeToProtected),
         ))
 
     async def _addAllLayrRead(self):
@@ -1447,6 +1478,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     async def initServiceActive(self):
 
         await self.stormdmons.start()
+        await self.agenda.clearRunningStatus()
 
         for view in self.views.values():
             await view.initTrigTask()
@@ -1472,19 +1504,28 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
     async def initStormPool(self):
 
-        byts = self.slab.get(b'storm:pool', db='cell:conf')
-        if byts is None:
-            return
+        try:
+
+            byts = self.slab.get(b'storm:pool', db='cell:conf')
+            if byts is None:
+                return
 
-        url, opts = s_msgpack.un(byts)
+            url, opts = s_msgpack.un(byts)
 
-        self.stormpoolurl = url
-        self.stormpoolopts = opts
+            self.stormpoolurl = url
+            self.stormpoolopts = opts
 
-        self.stormpool = await s_telepath.open(url)
+            async def onlink(proxy, urlinfo):
+                _url = s_urlhelp.sanitizeUrl(s_telepath.zipurl(urlinfo))
+                logger.debug(f'Stormpool client connected to {_url}')
 
-        # make this one a fini weakref vs the fini() handler
-        self.onfini(self.stormpool)
+            self.stormpool = await s_telepath.open(url, onlink=onlink)
+
+            # make this one a fini weakref vs the fini() handler
+            self.onfini(self.stormpool)
+
+        except Exception as e:  # pragma: no cover
+            logger.exception(f'Error starting stormpool, it will not be available: {e}')
 
     async def finiStormPool(self):
 
@@ -3478,14 +3519,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         buid = s_common.uhex(iden)
         async with await self.snap(user=user) as snap:
 
-            with s_provenance.claim('coreapi', meth='tag:add', user=snap.user.iden):
-
-                node = await snap.getNodeByBuid(buid)
-                if node is None:
-                    raise s_exc.NoSuchIden(iden=iden)
+            node = await snap.getNodeByBuid(buid)
+            if node is None:
+                raise s_exc.NoSuchIden(iden=iden)
 
-                await node.addTag(tag, valu=valu)
-                return node.pack()
+            await node.addTag(tag, valu=valu)
+            return node.pack()
 
     async def addNode(self, user, form, valu, props=None):
 
@@ -3505,14 +3544,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         async with await self.snap(user=user) as snap:
 
-            with s_provenance.claim('coreapi', meth='tag:del', user=snap.user.iden):
-
-                node = await snap.getNodeByBuid(buid)
-                if node is None:
-                    raise s_exc.NoSuchIden(iden=iden)
+            node = await snap.getNodeByBuid(buid)
+            if node is None:
+                raise s_exc.NoSuchIden(iden=iden)
 
-                await node.delTag(tag)
-                return node.pack()
+            await node.delTag(tag)
+            return node.pack()
 
     async def _onCoreFini(self):
         '''
@@ -4130,7 +4167,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     @s_nexus.Pusher.onPushAuto('http:api:mod')
     async def modHttpExtApi(self, iden, name, valu):
         # Created, Creator, Updated are not mutable
-        if name in ('name', 'desc', 'runas', 'methods', 'authenticated', 'perms', 'readonly', 'vars'):
+        if name in ('name', 'desc', 'runas', 'methods', 'authenticated', 'pool', 'perms', 'readonly', 'vars'):
             # Schema takes care of these values
             pass
         elif name == 'owner':
@@ -4447,6 +4484,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         if view is None:
             raise s_exc.NoSuchView(mesg=f'No such view {iden=}', iden=iden)
 
+        if view.info.get('protected'):
+            mesg = f'Cannot delete view ({iden}) that has protected set.'
+            raise s_exc.CantDelView(mesg=mesg)
+
         return await self._push('view:del', iden)
 
     @s_nexus.Pusher.onPush('view:del')
@@ -5280,12 +5321,14 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         return mirropts
 
     async def _getMirrorProxy(self):
-        if isinstance(self.stormpool, s_telepath.Pool) and self.stormpool.size() == 0:
-            mesg = 'Storm query mirror pool is empty, running locally instead.'
-            logger.warning(mesg)
+
+        if self.stormpool is None:  # pragma: no cover
+            return None
+
+        if self.stormpool.size() == 0:
+            logger.warning('Storm query mirror pool is empty, running query locally.')
             return None
 
-        proxy = None
         try:
             timeout = self.stormpoolopts.get('timeout:connection')
             proxy = await self.stormpool.proxy(timeout=timeout)
@@ -5293,10 +5336,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             if proxyname is not None and proxyname == self.ahasvcname:
                 # we are part of the pool and were selected. Convert to local use.
                 return None
+
+            return proxy
+
         except (TimeoutError, s_exc.IsFini):
-            mesg = 'Unable to get proxy for query mirror, running locally instead.'
-            logger.warning(mesg)
-        return proxy
+            logger.warning('Timeout waiting for pool mirror, running query locally.')
+            return None
 
     async def storm(self, text, opts=None):
 
@@ -5767,16 +5812,15 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
     async def _initCoreMods(self):
 
-        with s_provenance.claim('init', meth='_initCoreMods'):
-            for ctor, modu in list(self.modules.items()):
+        for ctor, modu in list(self.modules.items()):
 
-                try:
-                    await s_coro.ornot(modu.initCoreModule)
-                except asyncio.CancelledError:  # pragma: no cover  TODO:  remove once >= py 3.8 only
-                    raise
-                except Exception:
-                    logger.exception(f'module initCoreModule failed: {ctor}')
-                    self.modules.pop(ctor, None)
+            try:
+                await s_coro.ornot(modu.initCoreModule)
+            except asyncio.CancelledError:  # pragma: no cover  TODO:  remove once >= py 3.8 only
+                raise
+            except Exception:
+                logger.exception(f'module initCoreModule failed: {ctor}')
+                self.modules.pop(ctor, None)
 
     def _loadCoreModule(self, ctor, conf=None):
 
@@ -6033,6 +6077,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         elif name == 'doc':
             appt.doc = str(valu)
 
+        elif name == 'pool':
+            appt.pool = bool(valu)
+
         else:
             mesg = f'editCronJob name {name} is not supported for editing.'
             raise s_exc.BadArg(mesg=mesg)

{synapse-2.165.0 → synapse-2.167.0}/synapse/datamodel.py

@@ -873,10 +873,42 @@ class Model:
         for ifname in form.type.info.get('interfaces', ()):
             self._addFormIface(form, ifname)
 
+        self._checkFormDisplay(form)
+
         self.formprefixcache.clear()
 
         return form
 
+    def _checkFormDisplay(self, form):
+
+        formtype = self.types.get(form.full)
+
+        display = formtype.info.get('display')
+        if display is None:
+            return
+
+        for column in display.get('columns', ()):
+            coltype = column.get('type')
+            colopts = column.get('opts')
+
+            if coltype == 'prop':
+                curf = form
+                propname = colopts.get('name')
+                parts = propname.split('::')
+
+                for partname in parts:
+                    prop = curf.prop(partname)
+                    if prop is None:
+                        mesg = (f'Form {form.name} defines prop column {propname}'
+                               f' but {curf.full} has no property named {partname}.')
+                        raise s_exc.BadFormDef(mesg=mesg)
+
+                    curf = self.form(prop.type.name)
+
+            else:
+                mesg = f'Form {form.name} defines column with invalid type ({coltype}).'
+                raise s_exc.BadFormDef(mesg=mesg)
+
     def delForm(self, formname):
 
         form = self.forms.get(formname)

{synapse-2.165.0 → synapse-2.167.0}/synapse/exc.py

@@ -145,6 +145,7 @@ class CantDelForm(SynErr): pass
 class CantDelProp(SynErr): pass
 class CantDelType(SynErr): pass
 class CantDelUniv(SynErr): pass
+class CantDelView(SynErr): pass
 class CantMergeView(SynErr): pass
 class CantRevLayer(SynErr): pass
 class CliFini(SynErr):