synapse 3.5.0__tar.gz → 3.7.0__tar.gz

{synapse-3.5.0/synapse.egg-info → synapse-3.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 3.5.0
+Version: 3.7.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0
@@ -20,18 +20,18 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: cryptography<51.0.0,>=46.0.7
 Requires-Dist: msgpack<1.3.0,>=1.2.1
-Requires-Dist: xxhash<3.8.0,>=1.4.4
+Requires-Dist: xxhash<4.1.0,>=1.4.4
 Requires-Dist: lmdb<1.8.0,>=1.7.0
 Requires-Dist: tornado<7.0.0,>=6.2.0
 Requires-Dist: regex>=2022.9.11
 Requires-Dist: PyYAML<6.1.0,>=5.4
 Requires-Dist: aiohttp<3.15.0,>=3.14.1
-Requires-Dist: aiohttp-socks<0.12.0,>=0.11.0
+Requires-Dist: aiohttp-socks<0.13.0,>=0.12.0
 Requires-Dist: aiosmtplib<6.0.0,>=5.1.2
 Requires-Dist: prompt_toolkit<3.1.0,>=3.0.29
 Requires-Dist: lark==1.3.1
 Requires-Dist: mistletoe<2.0.0,>=1.6.0
-Requires-Dist: Pygments<2.21.0,>=2.7.4
+Requires-Dist: Pygments<2.22.0,>=2.7.4
 Requires-Dist: packaging<27.0,>=20.0
 Requires-Dist: fastjsonschema<2.22.0,>=2.20.0
 Requires-Dist: stix2-validator<3.3.0,>=3.2.0

{synapse-3.5.0 → synapse-3.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '3.5.0'
+version = '3.7.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]
@@ -24,18 +24,18 @@ classifiers = [
 dependencies = [
     'cryptography>=46.0.7,<51.0.0',
     'msgpack>=1.2.1,<1.3.0',
-    'xxhash>=1.4.4,<3.8.0',
+    'xxhash>=1.4.4,<4.1.0',
     'lmdb>=1.7.0,<1.8.0',
     'tornado>=6.2.0,<7.0.0',
     'regex>=2022.9.11',
     'PyYAML>=5.4,<6.1.0',
     'aiohttp>=3.14.1,<3.15.0',
-    'aiohttp-socks>=0.11.0,<0.12.0',
+    'aiohttp-socks>=0.12.0,<0.13.0',
     'aiosmtplib>=5.1.2,<6.0.0',
     'prompt_toolkit>=3.0.29,<3.1.0',
     'lark==1.3.1',
     'mistletoe>=1.6.0,<2.0.0',
-    'Pygments>=2.7.4,<2.21.0',
+    'Pygments>=2.7.4,<2.22.0',
     'packaging>=20.0,<27.0',
     'fastjsonschema>=2.20.0,<2.22.0',
     'stix2-validator>=3.2.0,<3.3.0',

{synapse-3.5.0 → synapse-3.7.0}/synapse/axon.py

@@ -1504,6 +1504,9 @@ class Axon(s_cell.Cell):
             if first:
                 first = False
                 delt = boff - offs
+                if delt == 0:
+                    continue
+
                 yield byts[-delt:]
                 continue
 

{synapse-3.5.0 → synapse-3.7.0}/synapse/cortex.py

@@ -459,9 +459,13 @@ class CoreApi(s_cell.CellApi):
         self.user.confirm(('pkg', 'add'))
         return await self.cell.addStormPkg(pkgdef, verify=verify)
 
-    async def delStormPkg(self, iden):
+    async def delStormPkg(self, iden, *, uninstall=False, keep=()):
         self.user.confirm(('pkg', 'del'))
-        return await self.cell.delStormPkg(iden)
+        return await self.cell.delStormPkg(iden, uninstall=uninstall, keep=keep)
+
+    @s_cell.adminapi()
+    async def getStormPkgStatus(self, name):
+        return self.cell.getStormPkgStatus(name)
 
     @s_cell.adminapi()
     async def getStormPkgs(self):
@@ -755,6 +759,11 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
         self.stormpkgvars = {}   # type: Dict[str, s_lmdbslab.SafeKeyVal]
         self.stormpkgstate = {}  # type: Dict[str, s_lmdbslab.SafeKeyVal]
 
+        # name: running onload / uninstall task for a package. Only populated while
+        # the cortex is active.
+        self.stormpkgonload = {}
+        self.stormpkguninsttask = {}
+
         # storm services, keyed by the cell type name of the service. a service
         # is unique by cell type for a deployment, so no separate iden is needed.
         self.svcs = {}
@@ -873,6 +882,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
         self.tagmeta = self.cortexdata.getSubKeyVal('tagmeta:')
         self.cmddefs = self.cortexdata.getSubKeyVal('storm:cmds:')
         self.pkgdefs = self.cortexdata.getSubKeyVal('storm:packages:')
+        self.pkguninst = self.cortexdata.getSubKeyVal('storm:pkg:uninstall:')
         self.svcdefs = self.cortexdata.getSubKeyVal('storm:services:')
         self.quedefs = self.cortexdata.getSubKeyVal('storm:queues:')
 
@@ -1218,7 +1228,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             {'perm': ('pkg', 'add'), 'gate': 'cortex',
              'desc': 'Controls access to adding storm packages.'},
             {'perm': ('pkg', 'del'), 'gate': 'cortex',
-             'desc': 'Controls access to deleting storm packages.'},
+             'desc': 'Controls access to deleting storm packages, including uninstalling them and removing their vaults.'},
 
             {'perm': ('storm',), 'gate': 'cortex',
              'desc': 'Controls all Storm permissions.'},
@@ -1333,7 +1343,14 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             await view.initMergeTask()
 
         for pkgdef in list(self.stormpkgs.values()):
-            self._runStormPkgOnload(pkgdef)
+            pkgname = pkgdef.get('name')
+
+            # a package which is being uninstalled resumes its uninstall, and one
+            # whose uninstall failed stays quiet until it is retried or deleted.
+            if (status := self.pkguninst.get(pkgname)) is None:
+                self._runStormPkgOnload(pkgdef)
+            elif status.get('status') == 'uninstalling':
+                self._initStormPkgUninstall(pkgname)
 
         # a passive cortex does not register storm service packages, so reconcile
         # them now that we are the leader. the service may have changed the
@@ -2189,18 +2206,38 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             self._reqStormPkgSigned(pkgdef)
 
         await self._normStormPkg(pkgdef)
+
+        name = pkgdef.get('name')
+        if self._isStormPkgUninstalling(name):
+            mesg = f'Storm package {name} is currently being uninstalled.'
+            raise s_exc.BadArg(mesg=mesg, name=name)
+
         return await self._push('pkg:add', pkgdef)
 
     @s_nexus.Pusher.onPush('pkg:add')
     async def _addStormPkg(self, pkgdef):
         name = pkgdef.get('name')
 
+        # a readonly cortex reads the status through to the leader's current value, which
+        # may be ahead of the event being replayed, so only a writer consults it.
+        reinstall = False
+        if not self.readonly:
+
+            # an add which raced a leader side check must not disturb a running uninstall
+            if self._isStormPkgUninstalling(name):
+                return
+
+            # adding a package whose uninstall failed abandons the uninstall and
+            # reinstalls it, even if the definition is unchanged.
+            if self.pkguninst.pop(name, None) is not None:
+                reinstall = True
+
         if not self.readonly:
             # pkgdefs holds the last-loaded def, so bounce an identical re-add
             # and drop the old def on a changed re-add.
             olddef = self.pkgdefs.get(name, None)
             if olddef is not None:
-                if s_hashitem.hashitem(pkgdef) != s_hashitem.hashitem(olddef):
+                if reinstall or s_hashitem.hashitem(pkgdef) != s_hashitem.hashitem(olddef):
                     self._dropStormPkg(olddef)
                 else:
                     return
@@ -2229,21 +2266,86 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             if self._reqAddableVaultType(vtdef):
                 await self._addVaultType(vtdef)
 
+        gates, perms = self._getStormPkgBeholdPerms(pkgdef)
+        await self.feedBeholder('pkg:add', pkgdef, gates=gates, perms=perms)
+
+    def _getStormPkgBeholdPerms(self, pkgdef):
         gates = []
         perms = []
         pkgperms = pkgdef.get('perms')
         if pkgperms:
             gates = [p['gate'] for p in pkgperms if p.get('gate') is not None]
             perms = [p['perm'] for p in pkgperms if p.get('perm') is not None]
-        await self.feedBeholder('pkg:add', pkgdef, gates=gates, perms=perms)
+        return gates, perms
+
+    def getStormPkgStatus(self, name):
+        '''
+        Get the uninstall status of a Storm package.
 
-    async def delStormPkg(self, name):
+        Args:
+            name (str): The name of the package.
+
+        Returns:
+            (dict|None): A dict with the `status` ( "uninstalling" or "failed" ), the `keep` items,
+            the `time` the status last changed and, for a failed uninstall, the failure `mesg`. None
+            if the package is not being uninstalled.
+        '''
+        return self.pkguninst.get(name)
+
+    def _isStormPkgUninstalling(self, name):
+        status = self.pkguninst.get(name)
+        return status is not None and status.get('status') == 'uninstalling'
+
+    def _reqStormPkgKeep(self, pkgdef, keep):
+        '''
+        Validate the items to keep when uninstalling a package.
+        '''
+        valid = {kdef['name'] for kdef in pkgdef.get('uninstall', {}).get('keep', ())}
+        if pkgdef.get('vaults'):
+            valid.add('vaults')
+
+        if isinstance(keep, str) or not isinstance(keep, (list, tuple)):
+            mesg = 'Uninstall keep items must be a list of strings.'
+            raise s_exc.BadArg(mesg=mesg)
+
+        for item in keep:
+            if not isinstance(item, str) or item not in valid:
+                mesg = f'Invalid uninstall keep item: {item}'
+                raise s_exc.BadArg(mesg=mesg, valid=sorted(valid))
+
+        return list(dict.fromkeys(keep))
+
+    async def delStormPkg(self, name, *, uninstall=False, keep=()):
+        '''
+        Delete a Storm package.
+
+        Args:
+            name (str): The name of the package.
+            uninstall (bool): Run the package uninstall routine, which removes the package
+                configuration, state, queues and vaults as well as the package itself. The
+                uninstall runs in the background and survives a restart.
+            keep (list): Items to keep when uninstalling. These must be declared by the
+                package, or be `vaults` for a package which declares vault types.
+        '''
         pkgdef = self.pkgdefs.get(name)
         if pkgdef is None:
             mesg = f'No storm package: {name}.'
             raise s_exc.NoSuchPkg(mesg=mesg)
 
-        return await self._push('pkg:del', name)
+        if not uninstall:
+            if keep:
+                mesg = 'Uninstall keep items require an uninstall.'
+                raise s_exc.BadArg(mesg=mesg)
+
+            return await self._push('pkg:del', name)
+
+        if self._isStormPkgUninstalling(name):
+            mesg = f'Storm package {name} is already being uninstalled.'
+            raise s_exc.BadArg(mesg=mesg, name=name)
+
+        keep = self._reqStormPkgKeep(pkgdef, keep)
+
+        return await self._push('pkg:uninstall', name, keep, s_common.now())
 
     @s_nexus.Pusher.onPush('pkg:del')
     async def _delStormPkg(self, name):
@@ -2257,20 +2359,179 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
         if pkgdef is None:
             return
 
+        # a deleted package must not leave work running, and deleting is how an
+        # operator abandons an uninstall.
+        self._cancelStormPkgOnload(name)
+        if (task := self.stormpkguninsttask.pop(name, None)) is not None:
+            task.cancel()
+
+        if not self.readonly:
+            self.pkguninst.pop(name, None)
+
+        await self._dropStormPkgDef(name, pkgdef)
+
+    async def _dropStormPkgDef(self, name, pkgdef):
+
         if not self.readonly:
             self.pkgdefs.pop(name, None)
         self._dropStormPkg(pkgdef)
 
         self._clearPermDefs()
 
-        gates = []
-        perms = []
-        pkgperms = pkgdef.get('perms')
-        if pkgperms:
-            gates = [p['gate'] for p in pkgperms if p.get('gate') is not None]
-            perms = [p['perm'] for p in pkgperms if p.get('perm') is not None]
+        gates, perms = self._getStormPkgBeholdPerms(pkgdef)
         await self.feedBeholder('pkg:del', {'name': name}, gates=gates, perms=perms)
 
+    @s_nexus.Pusher.onPush('pkg:uninstall')
+    async def _uninstStormPkg(self, name, keep, time):
+
+        pkgdef = self.stormpkgs.get(name)
+        if pkgdef is None:
+            return
+
+        if not self.readonly:
+
+            # a replay must not restart or re-announce the uninstall
+            if self._isStormPkgUninstalling(name):
+                return
+
+            self.pkguninst.set(name, {'status': 'uninstalling', 'keep': keep, 'time': time})
+
+        gates, perms = self._getStormPkgBeholdPerms(pkgdef)
+        await self.feedBeholder('pkg:uninstall:init', {'name': name, 'keep': keep}, gates=gates, perms=perms)
+
+        if self.isactive:
+            self._initStormPkgUninstall(name)
+
+    def _initStormPkgUninstall(self, name):
+        '''
+        Start the background uninstall task for a package.
+        '''
+        # a finished task may not have been reaped yet, so a retry must not be mistaken for a duplicate
+        if (task := self.stormpkguninsttask.get(name)) is not None and not task.done():
+            return
+
+        async def _uninstall():
+
+            async with await self.boss.promote(f'storm:pkg:uninstall:{name}', self.auth.rootuser,
+                                               info={'name': name}, background=True, protected=True):
+
+                if self.safemode:
+                    await self.fire('core:pkg:uninstall:skipped', pkg=name, reason='safemode')
+                    return
+
+                if (task := self._cancelStormPkgOnload(name)) is not None:
+                    await asyncio.gather(task, return_exceptions=True)
+
+                pkgdef = self.stormpkgs.get(name)
+                status = self.pkguninst.get(name)
+
+                # the package was deleted or re-added while we were waiting
+                if pkgdef is None or status is None or status.get('status') != 'uninstalling':
+                    return
+
+                keep = status.get('keep')
+                logextra = self.getLogExtra(pkg=name, vers=pkgdef.get('version'))
+
+                await self.fire('core:pkg:uninstall:start', pkg=name, keep=keep)
+
+                failed = None
+
+                if (udef := pkgdef.get('uninstall')) is not None:
+
+                    try:
+                        opts = copy.deepcopy(udef.get('queryopts', {}))
+                        opts.setdefault('vars', {})['keep'] = list(keep)
+
+                        async for mesg in self.storm(udef['query'], opts=opts):
+                            match mesg[0]:
+                                case 'print':
+                                    logger.info(f'{name} uninstall output: {mesg[1].get("mesg")}', extra=logextra)
+                                case 'warn':
+                                    logger.warning(f'{name} uninstall output: {mesg[1].get("mesg")}', extra=logextra)
+                                case 'err':
+                                    logger.error(f'{name} uninstall output: {mesg[1]}', extra=logextra)
+                                    failed = f'{mesg[1][0]}: {mesg[1][1].get("mesg")}'
+                            await asyncio.sleep(0)
+
+                    except Exception as exc:
+                        logger.warning(f'Uninstall query failed for package: {name}', exc_info=exc, extra=logextra)
+                        failed = f'{exc.__class__.__name__}: {exc}'
+
+                if failed is not None:
+                    await self._push('pkg:uninstall:fail', name, failed, s_common.now())
+                    await self.fire('core:pkg:uninstall:failed', pkg=name, mesg=failed)
+                    return
+
+                await self._push('pkg:uninstall:fini', name, keep)
+                await self.fire('core:pkg:uninstall:complete', pkg=name)
+
+        task = self.runActiveTask(_uninstall())
+
+        def _fini(task):
+            if self.stormpkguninsttask.get(name) is task:
+                self.stormpkguninsttask.pop(name, None)
+
+        self.stormpkguninsttask[name] = task
+        task.add_done_callback(_fini)
+
+    @s_nexus.Pusher.onPush('pkg:uninstall:fail')
+    async def _failStormPkgUninstall(self, name, mesg, time):
+
+        pkgdef = self.stormpkgs.get(name)
+        if pkgdef is None:
+            return
+
+        if not self.readonly:
+
+            # a replay, or a failure which raced a delete or re-add, is not a failure of the current uninstall
+            status = self.pkguninst.get(name)
+            if status is None or status.get('status') != 'uninstalling':
+                return
+
+            status.update({'status': 'failed', 'mesg': mesg, 'time': time})
+            self.pkguninst.set(name, status)
+
+        gates, perms = self._getStormPkgBeholdPerms(pkgdef)
+        await self.feedBeholder('pkg:uninstall:fail', {'name': name, 'mesg': mesg}, gates=gates, perms=perms)
+
+    @s_nexus.Pusher.onPush('pkg:uninstall:fini')
+    async def _finiStormPkgUninstall(self, name, keep):
+        '''
+        Remove everything the cortex owns for a package, then the package.
+        '''
+        # source the def from the in-memory registry for the same reason _delStormPkg does
+        pkgdef = self.stormpkgs.get(name)
+        if pkgdef is None:
+            return
+
+        if not self.readonly:
+
+            status = self.pkguninst.get(name)
+            if status is None or status.get('status') != 'uninstalling':
+                return
+
+            if 'vaults' not in keep:
+                for vtype in pkgdef.get('vaults', {}):
+                    for vault in [v for v in self._getVaults() if v.get('type') == vtype]:
+                        await self._hndldelVault(vault.get('iden'))
+                    await self._delVaultType(vtype)
+
+            for qinfo in [q for q in self.stormpkgqueue.list() if q.get('pkgname') == name]:
+                await self._delStormPkgQueue(name, qinfo.get('name'))
+
+            await self._getStormPkgVarKV(name).truncate()
+            await self._getStormPkgStateKV(name).truncate()
+
+            self.stormpkgvars.pop(name, None)
+            self.stormpkgstate.pop(name, None)
+
+            self.pkguninst.pop(name, None)
+
+        await self._dropStormPkgDef(name, pkgdef)
+
+        gates, perms = self._getStormPkgBeholdPerms(pkgdef)
+        await self.feedBeholder('pkg:uninstall:fini', {'name': name, 'keep': keep}, gates=gates, perms=perms)
+
     def getStormPkgSvc(self, name):
         '''
         Return the cell type name of the storm service which delivered a package.
@@ -2338,6 +2599,9 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             name = pkgdef.get('name', '')
             logger.exception(f'Error loading pkg: {name}, {str(e)}')
 
+    def getPkgProvides(self):
+        return {name: self.VERSION for name in self.PKG_PROVIDES}
+
     async def _getStormDepPkg(self, pkgname):
         '''
         Resolve a dependency/conflict name to a currently loaded package def.
@@ -2525,6 +2789,22 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
         # Validate package def (post normalization)
         s_schemas.reqValidPkgdef(pkgdef)
 
+        if (uninstall := pkgdef.get('uninstall')) is not None:
+
+            if validstorm:
+                await self.getStormQuery(uninstall.get('query'))
+
+            keepnames = set()
+            for kdef in uninstall.get('keep', ()):
+                kname = kdef.get('name')
+
+                # vaults are kept by the cortex, not the package uninstall query
+                if kname == 'vaults' or kname in keepnames:
+                    mesg = f'Uninstall keep name is reserved or duplicated: {kname}'
+                    raise s_exc.BadPkgDef(mesg=mesg, name=kname)
+
+                keepnames.add(kname)
+
         for configvar in pkgdef.get('configvars', ()):
             self._reqStormPkgVarType(pkgname, configvar.get('type'))
 
@@ -2691,7 +2971,28 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
 
                 await self.fire('core:pkg:onload:complete', pkg=name, storvers=curvers)
 
-            self.runActiveTask(_onload())
+            # a re-add must not leave the previous onload running alongside the new one
+            self._cancelStormPkgOnload(name)
+
+            task = self.runActiveTask(_onload())
+
+            def _fini(task):
+                if self.stormpkgonload.get(name) is task:
+                    self.stormpkgonload.pop(name, None)
+
+            self.stormpkgonload[name] = task
+            task.add_done_callback(_fini)
+
+    def _cancelStormPkgOnload(self, name):
+        '''
+        Cancel the running onload/inits task for a package, if there is one.
+
+        Returns:
+            (asyncio.Task|None): The cancelled task, which the caller may await.
+        '''
+        if (task := self.stormpkgonload.pop(name, None)) is not None:
+            task.cancel()
+        return task
 
     # N.B. This function is intentionally not async in order to prevent possible user race conditions for code
     # executing outside of the nexus lock.
@@ -7373,11 +7674,11 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
 
     async def replaceVaultSecrets(self, iden, valu):
         '''
-        Replace the entire vault config.
+        Replace the entire vault secrets.
 
         Args:
             iden (str): The iden of the vault to edit.
-            valu (str): New secrets object to store on the vault.
+            valu (dict): New secrets object to store on the vault.
 
         Raises:
             synapse.exc.BadArg: `valu` is not a dictionary.
@@ -7426,7 +7727,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
 
         Raises: None
 
-        Yields: tuples of vault info: (<iden>, <name>, <type>, <scope>).
+        Yields: vault definition dictionaries.
         '''
         for vault in self._getVaults():
             yield vault

{synapse-3.5.0 → synapse-3.7.0}/synapse/data/skills/storm/SKILL.md

@@ -30,6 +30,16 @@ The `storm_validate` MCP tool validates Storm syntax without executing the query
 1. Call `storm_validate` with the query text.
 2. If `valid` is `false`, read `err` / `mesg` (which include line/column info), fix the query, and re-validate before proceeding.
 
+### Linting Storm (Required)
+
+The `storm_lint` MCP tool lints a query against the Cortex's own data model (including extended forms and properties) without executing it. Lint every query before running it, and fix or consider each finding.
+
+- Arguments: `query`, plus optional `select` / `ignore` / `enable` lists of rule codes or code prefixes.
+- Returns `{"findings": [{"rule", "severity", "line", "column", "mesg"}, ...]}`; an empty list means the query is clean.
+- A syntax error is reported as a single `E001` finding with its line and column, so a clean lint also implies valid syntax.
+- Only `storm` mode queries are supported; `lookup` mode text usually returns an `E001` syntax error.
+- Other findings report code which does not run as intended on Synapse 3.x (`R`), pipeline usage which may not do what was intended (`P`), and Storm which could be written more simply (`S`). Suppress a deliberate finding with a trailing `// noqa` or `// noqa: <code>` comment on that line; `E001` cannot be suppressed.
+
 ### Running Storm Queries
 
 The `storm` MCP tool runs a Storm query against the Cortex and returns a **page** of result messages: `{"messages": [(<type>, <info>), ...], "cursor": <str-or-null>}`. Each message is a `(<type>, <info>)` tuple as yielded by the Storm runtime (e.g. `node`, `print`, `warn`, `err`, `fini`).
@@ -653,6 +663,7 @@ inet:fqdn { -> inet:dns:a [ +#reviewed ] } // still has inet:fqdn nodes in the p
 init { ... }                                  // runs once before any nodes are allowed through the pipeline
 empty { ... }                                 // runs if pipeline is empty
 fini { ... }                                  // runs once after all nodes have passed through the pipeline
+exec { ... }                                  // runs once, discarding the nodes it yields; inbound nodes pass through
 ```
 
 ### Built-in Commands
@@ -661,7 +672,6 @@ fini { ... }                                  // runs once after all nodes have
 limit 10                                      // limit output count
 count                                         // count nodes
 uniq                                          // deduplicate
-spin                                          // consume without output
 max :prop                                     // keep max by property
 min :prop                                     // keep min by property
 tee { query1 } { query2 }                   // branch pipeline
@@ -974,6 +984,7 @@ $func("positional", key=1)
 - Double-underscore `__` prefix for module-internal variables and functions: they cannot be accessed from outside the module that defines them (e.g. `function __getJson()`)
 - Module variables at top level: `$setup = $lib.import(...)`, `$srcnode = (null)`
 - Format strings with backticks for interpolation: `` `text {$var}` ``
+- Avoid `spin`: run a query whose nodes are not needed in an `exec { ... }` block, or isolate it in a function, and a query that returns a value uses `return()`, or `fini { return() }` when more than one node can flow
 - Parenthesize values and conditions so Storm evaluates them as typed expressions rather than bare strings or edit syntax: `(true)` / `(false)` / `(null)` (not `$lib.true` / `$lib.false` / `$lib.null`), numbers `(42)`, collections `([])` / `({})`, and comparisons like `if ($a = null)`
 - Use `$lib.raise(ErrName, "message")` for errors, `$lib.exit("message")` for fatal exits
 - Use `return()` with empty parens to return `(null)`
@@ -985,7 +996,7 @@ $func("positional", key=1)
 
 1. **Isolate the failing query** -- extract the minimal Storm snippet that reproduces the error.
 
-2. **Check the error location** -- the `storm_validate` result (and `BadSyntax`) provides `line` and `column`. Look at that exact position in the query.
+2. **Lint and check the error location** -- run `storm_lint` on the query; its findings (including an `E001` syntax error) and the `storm_validate` result (and `BadSyntax`) provide `line` and `column`. Look at that exact position in the query.
 
 3. **Build up incrementally and check the output at each step** -- for complex queries, construct the query one pipeline stage at a time, running each stage with the `storm` tool and confirming it yields the expected nodes/output before adding the next operation. This pins down which stage changes the results. The focus here is verifying the expected output at each step; you can also `storm_validate` each stage for syntax, or do both:
    ```storm
@@ -1010,6 +1021,7 @@ $func("positional", key=1)
 | Tool / File | Purpose |
 |-------------|---------|
 | `storm_validate` MCP tool | Validate Storm syntax without executing a query |
+| `storm_lint` MCP tool | Lint a query against the Cortex data model (syntax errors as `E001`, plus style/performance findings) |
 | `storm` / `call_storm` MCP tools | Run Storm queries against the Cortex (page of result messages / return a value) |
 | `view_list` / `view_get` MCP tools | List views (for the `view` storm opt) and read the user's default view |
 | `view_fork` / `view_del` / `view_merge` MCP tools | Fork, delete, and merge views (use fork+del to safely test ingest) |

{synapse-3.5.0 → synapse-3.7.0}/synapse/data/v2modelmap.yaml

@@ -666,6 +666,18 @@ changes:
       src:
         became: :caller:phone
         goals: [renamed]
+  tel:txtmesg:
+    props:
+      from:
+        became: :actor:phone
+        goals: [renamed]
+      svctype:
+        goals: [model-refinement]
+        reason: The message service type is now recorded with the `:type` taxonomy (for example `sms`, `mms`, or `rcs`).
+      to:
+        became: :to:phone
+        goals: [renamed]
+        reason: The inherited `inet:service:message:to` property is the destination account, role, or channel.
   transport:cargo:
     props:
       loaded:

{synapse-3.5.0 → synapse-3.7.0}/synapse/datamodel.py

@@ -279,7 +279,7 @@ class Prop:
             'form': self.form.name,
             'relname': self.name,
             'base': self.name.split(':')[-1],
-            'computed': int(self.info.get('computed', False)),
+            'computed': bool(self.info.get('computed', False)),
             'extmodel': self.isext,
         }
 

{synapse-3.5.0 → synapse-3.7.0}/synapse/lib/autodoc.py

@@ -1752,22 +1752,35 @@ async def processStormModulesMd(md, pkgname, modules):
         md.addHead(mname, lvl=2, anchor=anchor)
 
         for apidef in apidefs:
-
-            apiname = apidef['name']
-            apidesc = apidef['desc']
-            apitype = apidef['type']
-
-            callsig = genCallsig(apitype)
-            md.addHead(f'{apiname}{callsig}', lvl=3)
-            if depr := apidef.get('deprecated'):
-                md.addLines(*genDeprecationWarningMd(apiname, depr))
-            md.addLines(*prepareMdLines(apidesc))
-            md.addLines(*getArgLinesMd(apitype))
-            md.addLines(*getReturnLinesMd(apitype))
+            _renderApiDefMd(md, apidef, 3)
 
     if not hasapi:
         md.addLines('This package does not export any Storm APIs.', '')
 
+def _renderApiDefMd(md, apidef, lvl):
+    '''
+    Render one module apidef: a function, or a class followed by each of its methods one level down.
+    '''
+    apiname = apidef['name']
+    apidesc = apidef['desc']
+    apitype = apidef['type']
+
+    callsig = genCallsig(apitype)
+    md.addHead(f'{apiname}{callsig}', lvl=lvl)
+    if depr := apidef.get('deprecated'):
+        md.addLines(*genDeprecationWarningMd(apiname, depr))
+
+    md.addLines(*prepareMdLines(apidesc))
+    md.addLines(*getArgLinesMd(apitype))
+
+    # a class returns nothing itself: its methods each document their own returns
+    if apitype.get('type') != 'class':
+        md.addLines(*getReturnLinesMd(apitype))
+        return
+
+    for methdef in apitype['methods']:
+        _renderApiDefMd(md, methdef, lvl + 1)
+
 def _renderDepsTableMd(rows):
     lines = ['| Name | Version | Optional | Description |', '|---|---|---|---|']
     for name, vers, optional, desc in rows: