synapse 2.221.0__tar.gz → 2.223.0__tar.gz

{synapse-2.221.0/synapse.egg-info → synapse-2.223.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 2.221.0
+Version: 2.223.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0

{synapse-2.221.0 → synapse-2.223.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.221.0'
+version = '2.223.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]

{synapse-2.221.0 → synapse-2.223.0}/synapse/cortex.py

@@ -61,6 +61,7 @@ import synapse.lib.stormlib.gen as s_stormlib_gen  # NOQA
 import synapse.lib.stormlib.gis as s_stormlib_gis  # NOQA
 import synapse.lib.stormlib.hex as s_stormlib_hex  # NOQA
 import synapse.lib.stormlib.log as s_stormlib_log  # NOQA
+import synapse.lib.stormlib.pkg as s_stormlib_pkg  # NOQA
 import synapse.lib.stormlib.xml as s_stormlib_xml  # NOQA
 import synapse.lib.stormlib.auth as s_stormlib_auth  # NOQA
 import synapse.lib.stormlib.cell as s_stormlib_cell  # NOQA
@@ -72,6 +73,7 @@ import synapse.lib.stormlib.mime as s_stormlib_mime  # NOQA
 import synapse.lib.stormlib.pack as s_stormlib_pack  # NOQA
 import synapse.lib.stormlib.smtp as s_stormlib_smtp  # NOQA
 import synapse.lib.stormlib.stix as s_stormlib_stix  # NOQA
+import synapse.lib.stormlib.task as s_stormlib_task  # NOQA
 import synapse.lib.stormlib.yaml as s_stormlib_yaml  # NOQA
 import synapse.lib.stormlib.basex as s_stormlib_basex  # NOQA
 import synapse.lib.stormlib.cache as s_stormlib_cache  # NOQA
@@ -624,7 +626,8 @@ class CoreApi(s_cell.CellApi):
         Returns:
             AsyncIterator[Tuple(buid, valu)]
         '''
-        self.user.confirm(('layer', 'lift', layriden))
+        if not self.user.allowed(('layer', 'lift', layriden)):
+            self.user.confirm(('layer', 'read', layriden))
         async for item in self.cell.iterFormRows(layriden, form, stortype=stortype, startvalu=startvalu):
             yield item
 
@@ -642,7 +645,8 @@ class CoreApi(s_cell.CellApi):
         Returns:
             AsyncIterator[Tuple(buid, valu)]
         '''
-        self.user.confirm(('layer', 'lift', layriden))
+        if not self.user.allowed(('layer', 'lift', layriden)):
+            self.user.confirm(('layer', 'read', layriden))
         async for item in self.cell.iterPropRows(layriden, form, prop, stortype=stortype, startvalu=startvalu):
             yield item
 
@@ -659,7 +663,8 @@ class CoreApi(s_cell.CellApi):
         Returns:
             AsyncIterator[Tuple(buid, valu)]
         '''
-        self.user.confirm(('layer', 'lift', layriden))
+        if not self.user.allowed(('layer', 'lift', layriden)):
+            self.user.confirm(('layer', 'read', layriden))
         async for item in self.cell.iterUnivRows(layriden, prop, stortype=stortype, startvalu=startvalu):
             yield item
 
@@ -680,7 +685,8 @@ class CoreApi(s_cell.CellApi):
             This yields (buid, (tagvalu, form)) instead of just buid, valu in order to allow resuming an interrupted
             call by feeding the last value retrieved into starttupl
         '''
-        self.user.confirm(('layer', 'lift', layriden))
+        if not self.user.allowed(('layer', 'lift', layriden)):
+            self.user.confirm(('layer', 'read', layriden))
         async for item in self.cell.iterTagRows(layriden, tag, form=form, starttupl=starttupl):
             yield item
 
@@ -699,7 +705,8 @@ class CoreApi(s_cell.CellApi):
         Returns:
             AsyncIterator[Tuple(buid, valu)]
         '''
-        self.user.confirm(('layer', 'lift', layriden))
+        if not self.user.allowed(('layer', 'lift', layriden)):
+            self.user.confirm(('layer', 'read', layriden))
         async for item in self.cell.iterTagPropRows(layriden, tag, prop, form=form, stortype=stortype,
                                                     startvalu=startvalu):
             yield item
@@ -1366,6 +1373,30 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
     def _initCorePerms(self):
         self._cortex_permdefs.extend((
+            {'perm': ('axon', 'upload'), 'gate': 'cortex',
+             'desc': 'Controls the ability to upload a file to the Axon.'},
+            {'perm': ('axon', 'get'), 'gate': 'cortex',
+             'desc': 'Controls the ability to retrieve a file from the Axon.'},
+            {'perm': ('axon', 'has'), 'gate': 'cortex',
+             'desc': 'Controls the ability to check if the Axon contains a file.'},
+            {'perm': ('axon', 'del'), 'gate': 'cortex',
+             'desc': 'Controls the ability to remove a file from the Axon.'},
+
+            {'perm': ('layer', 'add'), 'gate': 'cortex',
+             'desc': 'Controls the ability to add Layers to the cortex.'},
+            {'perm': ('layer', 'del'), 'gate': 'cortex',
+             'desc': 'Controls the ability to remove Layers from the cortex.'},
+            {'perm': ('layer', 'read'), 'gate': 'layer',
+             'desc': 'Controls the ability to read/lift from a Layer.'},
+            {'perm': ('layer', 'read', '<iden>'), 'gate': 'cortex',
+             'desc': 'Controls the ability to read/lift from a specific Layer.'},
+            {'perm': ('layer', 'set', '<name>'), 'gate': 'layer',
+             'desc': 'Controls the ability to configure properties of a Layer.'},
+            {'perm': ('layer', 'write'), 'gate': 'layer',
+             'desc': 'Controls the ability to write to a Layer.'},
+            {'perm': ('layer', 'write', '<iden>'), 'gate': 'cortex',
+             'desc': 'Controls the ability to write to a specific Layer.'},
+
             {'perm': ('model', 'form', 'add'), 'gate': 'cortex',
              'desc': 'Controls access to adding extended model forms.'},
             {'perm': ('model', 'form', 'add', '<form>'), 'gate': 'cortex',
@@ -1519,15 +1550,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             {'perm': ('view', 'set', '<setting>'), 'gate': 'view',
              'desc': 'Controls access to change view settings.',
              'ex': 'view.set.name'},
-
-            {'perm': ('axon', 'upload'), 'gate': 'cortex',
-             'desc': 'Controls the ability to upload a file to the Axon.'},
-            {'perm': ('axon', 'get'), 'gate': 'cortex',
-             'desc': 'Controls the ability to retrieve a file from the Axon.'},
-            {'perm': ('axon', 'has'), 'gate': 'cortex',
-             'desc': 'Controls the ability to check if the Axon contains a file.'},
-            {'perm': ('axon', 'del'), 'gate': 'cortex',
-             'desc': 'Controls the ability to remove a file from the Axon.'},
         ))
         for pdef in self._cortex_permdefs:
             s_schemas.reqValidPermDef(pdef)
@@ -2242,6 +2264,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         self.onfini(slab.fini)
 
         self.multiqueue = await slab.getMultiQueue('cortex:queue', nexsroot=self.nexsroot)
+        self.stormpkgqueue = await slab.getMultiQueue('storm:pkg:queue', nexsroot=self.nexsroot)
 
     async def _initStormGraphs(self):
         path = os.path.join(self.dirn, 'slabs', 'graphs.lmdb')
@@ -3042,8 +3065,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                         if not ok:
                             break
 
-                        curvers = vers
-                        await self.setStormPkgVar(name, varname, vers)
+                        curvers = max(vers, await self.getStormPkgVar(name, varname, default=-1))
+                        await self.setStormPkgVar(name, varname, curvers)
                         logger.info(f'{name} finished init vers={vers}: {vname}', extra=logextra)
 
                 if onload is not None:
@@ -3328,6 +3351,91 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         for item in pkgvars.items():
             yield item
 
+    async def addStormPkgQueue(self, pkgname, name):
+        guid = s_common.guid((pkgname, name))
+        if self.stormpkgqueue.exists(guid):
+            mesg = f'Queue named {name} already exists for package {pkgname}!'
+            raise s_exc.DupName(mesg=mesg)
+
+        info = {
+            'iden': guid,
+            'name': name,
+            'pkgname': pkgname,
+            'created': s_common.now()
+        }
+
+        await self._push('storm:pkg:queue:add', pkgname, name, info)
+
+    @s_nexus.Pusher.onPush('storm:pkg:queue:add')
+    async def _addStormPkgQueue(self, pkgname, name, info):
+        guid = s_common.guid((pkgname, name))
+        if self.stormpkgqueue.exists(guid):
+            return
+        await self.stormpkgqueue.add(guid, info)
+
+    async def listStormPkgQueues(self, pkgname=None):
+        for pkginfo in self.stormpkgqueue.list():
+            if pkgname is None or pkginfo['meta'].get('pkgname') == pkgname:
+                yield pkginfo
+
+    async def getStormPkgQueue(self, pkgname, name):
+        guid = s_common.guid((pkgname, name))
+        return self.stormpkgqueue.status(guid)
+
+    async def delStormPkgQueue(self, pkgname, name):
+        guid = s_common.guid((pkgname, name))
+        if not self.stormpkgqueue.exists(guid):
+            mesg = f'No queue named {name} exists for package {pkgname}!'
+            raise s_exc.NoSuchName(mesg=mesg)
+
+        await self._push('storm:pkg:queue:del', pkgname, name)
+
+    @s_nexus.Pusher.onPush('storm:pkg:queue:del')
+    async def _delStormPkgQueue(self, pkgname, name):
+        guid = s_common.guid((pkgname, name))
+        if not self.stormpkgqueue.exists(guid):
+            return
+        await self.stormpkgqueue.rem(guid)
+
+    async def stormPkgQueueGet(self, pkgname, name, offs=0, wait=False):
+        guid = s_common.guid((pkgname, name))
+        async for item in self.stormpkgqueue.gets(guid, offs, cull=False, wait=wait):
+            return item
+
+    async def stormPkgQueueGets(self, pkgname, name, offs=0, wait=False, size=None):
+        count = 0
+        guid = s_common.guid((pkgname, name))
+        async for item in self.stormpkgqueue.gets(guid, offs, cull=False, wait=wait):
+
+            yield item
+
+            count += 1
+            if size is not None and count >= size:
+                return
+
+    async def stormPkgQueuePuts(self, pkgname, name, items):
+        return await self._push('storm:pkg:queue:puts', pkgname, name, items)
+
+    @s_nexus.Pusher.onPush('storm:pkg:queue:puts', passitem=True)
+    async def _stormPkgQueuePuts(self, pkgname, name, items, nexsitem):
+        nexsoff, nexsmesg = nexsitem
+        guid = s_common.guid((pkgname, name))
+        return await self.stormpkgqueue.puts(guid, items, reqid=nexsoff)
+
+    @s_nexus.Pusher.onPushAuto('storm:pkg:queue:cull')
+    async def stormPkgQueueCull(self, pkgname, name, offs):
+        guid = s_common.guid((pkgname, name))
+        await self.stormpkgqueue.cull(guid, offs)
+
+    @s_nexus.Pusher.onPushAuto('storm:pkg:queue:pop')
+    async def stormPkgQueuePop(self, pkgname, name, offs):
+        guid = s_common.guid((pkgname, name))
+        return await self.stormpkgqueue.pop(guid, offs)
+
+    async def stormPkgQueueSize(self, pkgname, name):
+        guid = s_common.guid((pkgname, name))
+        return self.stormpkgqueue.size(guid)
+
     async def _cortexHealth(self, health):
         health.update('cortex', 'nominal')
 
@@ -4548,34 +4656,25 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         self.addStormCmd(s_stormlib_cortex.StormPoolGetCmd)
         self.addStormCmd(s_stormlib_cortex.StormPoolSetCmd)
 
-        for cdef in s_stormsvc.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_storm.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_aha.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_gen.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_auth.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_macro.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_model.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_cortex.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_vault.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_index.stormcmds:
+        cmdmods = [
+            s_storm,
+            s_stormsvc,
+            s_stormlib_aha,
+            s_stormlib_auth,
+            s_stormlib_cortex,
+            s_stormlib_gen,
+            s_stormlib_index,
+            s_stormlib_macro,
+            s_stormlib_model,
+            s_stormlib_pkg,
+            s_stormlib_vault,
+        ]
+
+        for cmod in cmdmods:
+            for cdef in cmod.stormcmds:
+                await self._trySetStormCmd(cdef.get('name'), cdef)
+
+        for cdef in s_stormlib_task.stormcmds:
             await self._trySetStormCmd(cdef.get('name'), cdef)
 
     async def _initPureStormCmds(self):

{synapse-2.221.0 → synapse-2.223.0}/synapse/cryotank.py

@@ -39,7 +39,7 @@ class CryoTank(s_base.Base):
     A CryoTank implements a stream of structured data.
     '''
     async def __anit__(self, dirn, iden, conf=None):
-
+        s_common.deprecated('synapse.cryotank.CryoTank', curv='2.223.0')
         await s_base.Base.__anit__(self)
 
         if conf is None:

{synapse-2.221.0 → synapse-2.223.0}/synapse/data/lark/storm.lark

@@ -238,10 +238,10 @@ liftreverse: _REVERSE "(" (liftformtag | liftpropby | liftprop | liftbyarray | l
 
 _DEREF.3:  /\*(?=\$)/
 
-liftformtag: ((PROPS | UNIVNAME | WILDPROPS) | _DEREF _varvalu) tagname [_cmpr _valu]
-liftpropby: ((PROPS | EMBEDPROPS | UNIVNAME) | _DEREF _varvalu) _cmpr _valu
-liftprop: ((PROPS | UNIVNAME | WILDPROPS) | _DEREF _varvalu)
-liftbyarray: ((PROPS | EMBEDPROPS | UNIVNAME) | _DEREF _varvalu) "*[" _safe_cmpr _valu "]"
+liftformtag: (PROPS | UNIVNAME | WILDPROPS | derefprops) tagname [_cmpr _valu]
+liftpropby: (PROPS | EMBEDPROPS | UNIVNAME | derefprops) _cmpr _valu
+liftprop: (PROPS | UNIVNAME | WILDPROPS | derefprops)
+liftbyarray: (PROPS | EMBEDPROPS | UNIVNAME | derefprops) "*[" _safe_cmpr _valu "]"
 lifttagtag: (_HASH | _HASHSPACE) tagname [_cmpr _valu]
 liftbytag: (tagname | tagnamewithspace) [_cmpr _valu]
 liftbytagprop: (tagprop | tagpropwithspace) [_cmpr _valu]
@@ -249,7 +249,7 @@ liftbyformtagprop: formtagprop [_cmpr _valu]
 
 tagprop: tagname _COLONNOSPACE (BASEPROP | _varvalu)
 tagpropwithspace: tagnamewithspace _COLONNOSPACE (BASEPROP | _varvalu) -> tagprop
-formtagprop: ((PROPS | UNIVNAME | WILDPROPS) | _DEREF _varvalu) tagname _COLONNOSPACE (BASEPROP | _varvalu)
+formtagprop: (PROPS | UNIVNAME | WILDPROPS | derefprops) tagname _COLONNOSPACE (BASEPROP | _varvalu)
 _COLONNOSPACE.2: /(?<!\s):/
 
 _funcarg: (VARTOKN [EQNOSPACE _valu])
@@ -484,7 +484,7 @@ EMBEDPROPS.2: /[a-z_][a-z0-9_]*(:[a-z0-9_]+)+((\:\:|\:|\.)[a-z0-9_]+)*(?![:.a-z0
 UNIVNAME.2: /(?<=^|[\s\|\{\(\[+=-])\.[a-z_][a-z0-9_]*([:.][a-z0-9_]+)*/
 univprop:  UNIVNAME | "." _varvalu
 // A full property or a universal property
-formname: PROPS | _DEREF _varvalu
+formname: PROPS | derefprops
 // A relative property
 relprop: RELNAME | _COLONDOLLAR _varvaluatom
 
@@ -494,6 +494,9 @@ RELNAME: /(?<!\w):\.?[a-z_][a-z0-9_]*(?:(\:\:|\:|\.)[a-z_][a-z0-9_]*)*/
 // Similar to PROPS but does not require a colon
 BASEPROP: /[a-z_][a-z0-9_]*(?:(\:\:|\:|\.)[a-z_][a-z0-9_]*)*/
 
+// Derefed variable for prop names
+derefprops: _DEREF _varvalu
+
 // The entry point for a $(...) expression.  The initial dollar sign is now optional
 _dollarexprs: "$"? dollaroper
 

{synapse-2.221.0 → synapse-2.223.0}/synapse/lib/ast.py

@@ -2,7 +2,6 @@ import types
 import asyncio
 import decimal
 import fnmatch
-import hashlib
 import logging
 import binascii
 import itertools
@@ -1833,7 +1832,7 @@ class LiftProp(LiftOper):
 
         assert len(self.kids) == 1
 
-        name = await tostr(await self.kids[0].compute(runt, path))
+        name = await self.kids[0].compute(runt, path)
 
         prop = runt.model.props.get(name)
         if prop is not None:
@@ -4029,6 +4028,13 @@ class FormName(Value):
     async def compute(self, runt, path):
         return await self.kids[0].compute(runt, path)
 
+class DerefProps(Value):
+    async def compute(self, runt, path):
+        valu = await toprim(await self.kids[0].compute(runt, path))
+        if (exc := await s_stormtypes.typeerr(valu, str)) is not None:
+            raise self.kids[0].addExcInfo(exc)
+        return valu
+
 class RelProp(PropName):
     pass
 
@@ -4521,9 +4527,6 @@ class N1Walk(Oper):
 
     def buildfilter(self, runt, destforms, cmpr):
 
-        if not isinstance(destforms, (tuple, list)):
-            destforms = (destforms,)
-
         if '*' in destforms:
             if cmpr is not None:
                 mesg = 'Wild card walk operations do not support comparison.'
@@ -4608,6 +4611,11 @@ class N1Walk(Oper):
                 dest = await self.kids[1].compute(runt, path)
                 dest = await s_stormtypes.toprim(dest)
 
+                if isinstance(dest, (tuple, list)):
+                    dest = [await s_stormtypes.tostr(form) for form in dest]
+                else:
+                    dest = (await s_stormtypes.tostr(dest),)
+
                 destfilt = self.buildfilter(runt, dest, cmpr)
 
             for verb in verbs:

{synapse-2.221.0 → synapse-2.223.0}/synapse/lib/layer.py

@@ -119,6 +119,7 @@ class LayerApi(s_cell.CellApi):
 
         self.layr = layr
         self.liftperm = ('layer', 'lift', self.layr.iden)
+        self.readperm = ('layer', 'read', self.layr.iden)
         self.writeperm = ('layer', 'write', self.layr.iden)
 
     async def iterLayerNodeEdits(self):
@@ -126,7 +127,8 @@ class LayerApi(s_cell.CellApi):
         Scan the full layer and yield artificial nodeedit sets.
         '''
 
-        await self._reqUserAllowed(self.liftperm)
+        if not self.allowed(self.liftperm):
+            await self._reqUserAllowed(self.readperm)
         async for item in self.layr.iterLayerNodeEdits():
             yield item
             await asyncio.sleep(0)
@@ -165,13 +167,15 @@ class LayerApi(s_cell.CellApi):
 
         Once caught up with storage, yield them in realtime.
         '''
-        await self._reqUserAllowed(self.liftperm)
+        if not self.allowed(self.liftperm):
+            await self._reqUserAllowed(self.readperm)
         async for item in self.layr.syncNodeEdits(offs, wait=wait, reverse=reverse):
             yield item
             await asyncio.sleep(0)
 
     async def syncNodeEdits2(self, offs, wait=True):
-        await self._reqUserAllowed(self.liftperm)
+        if not self.allowed(self.liftperm):
+            await self._reqUserAllowed(self.readperm)
         async for item in self.layr.syncNodeEdits2(offs, wait=wait):
             yield item
             await asyncio.sleep(0)
@@ -180,18 +184,21 @@ class LayerApi(s_cell.CellApi):
         '''
         Returns what will be the *next* nodeedit log index.
         '''
-        await self._reqUserAllowed(self.liftperm)
+        if not self.allowed(self.liftperm):
+            await self._reqUserAllowed(self.readperm)
         return await self.layr.getEditIndx()
 
     async def getEditSize(self):
         '''
         Return the total number of (edits, meta) pairs in the layer changelog.
         '''
-        await self._reqUserAllowed(self.liftperm)
+        if not self.allowed(self.liftperm):
+            await self._reqUserAllowed(self.readperm)
         return await self.layr.getEditSize()
 
     async def getIden(self):
-        await self._reqUserAllowed(self.liftperm)
+        if not self.allowed(self.liftperm):
+            await self._reqUserAllowed(self.readperm)
         return self.layr.iden
 
 BUID_CACHE_SIZE = 10000
@@ -3350,7 +3357,7 @@ class Layer(s_nexus.Pusher):
         nodeedits = [(buid, newp_formname, [set_edit])]
 
         _, changes = await self.saveNodeEdits(nodeedits, meta)
-        return bool(changes[0][2])
+        return any(c[2] for c in changes)
 
     async def delStorNode(self, buid, meta):
         '''
@@ -3434,7 +3441,7 @@ class Layer(s_nexus.Pusher):
             if changed: # pragma: no cover
                 return changed
 
-            return bool(changes[0][2])
+            return any(c[2] for c in changes)
 
         for n2iden, edges in n2edges.items():
             edits = [(EDIT_EDGE_DEL, edge, ()) for edge in edges]
@@ -3455,7 +3462,7 @@ class Layer(s_nexus.Pusher):
         nodeedits = [(buid, oldp_formname, [del_edit])]
 
         _, changes = await self.saveNodeEdits(nodeedits, meta)
-        return bool(changes[0][2])
+        return any(c[2] for c in changes)
 
     async def delNodeData(self, buid, meta, name=None):
         '''
@@ -3480,7 +3487,7 @@ class Layer(s_nexus.Pusher):
         nodeedits = [(buid, sode.get('form'), edits)]
 
         _, changes = await self.saveNodeEdits(nodeedits, meta)
-        return bool(changes[0][2])
+        return any(c[2] for c in changes)
 
     async def delEdge(self, n1buid, verb, n2buid, meta):
         sode = self._getStorNode(n1buid)
@@ -3497,7 +3504,7 @@ class Layer(s_nexus.Pusher):
         nodeedits = [(n1buid, sode.get('form'), edits)]
 
         _, changes = await self.saveNodeEdits(nodeedits, meta)
-        return bool(changes[0][2])
+        return any(c[2] for c in changes)
 
     async def storNodeEdits(self, nodeedits, meta):
 

{synapse-2.221.0 → synapse-2.223.0}/synapse/lib/nexus.py

@@ -572,7 +572,7 @@ class NexsRoot(s_base.Base):
 
                     offs, args = item
                     if offs != self.nexslog.index():
-                        logger.error('Local Nexus offset is out of sync from remote cell! Aborting mirror sync')
+                        logger.error(f'Local Nexus offset is out of sync from remote cell! Aborting mirror sync. Local offs={self.nexslog.index()}, Remote {offs=}')
                         await self.fini()
                         return
 

{synapse-2.221.0 → synapse-2.223.0}/synapse/lib/parser.py

@@ -669,6 +669,7 @@ ruleClassMap = {
     'condsetoper': s_ast.CondSetOper,
     'condtrysetoper': lambda astinfo, kids: s_ast.CondSetOper(astinfo, kids, errok=True),
     'condsubq': s_ast.SubqCond,
+    'derefprops': s_ast.DerefProps,
     'dollarexpr': s_ast.DollarExpr,
     'edgeaddn1': s_ast.EditEdgeAdd,
     'edgedeln1': s_ast.EditEdgeDel,

{synapse-2.221.0 → synapse-2.223.0}/synapse/lib/rstorm.py

@@ -344,6 +344,7 @@ class StormRst(s_base.Base):
             'storm-cortex': self._handleStormCortex,
             'storm-envvar': self._handleStormEnvVar,
             'storm-expect': self._handleStormExpect,
+            'storm-python-path': self._handlePythonPath,
             'storm-multiline': self._handleStormMultiline,
             'storm-mock-http': self._handleStormMockHttp,
             'storm-vcr-opts': self._handleStormVcrOpts,
@@ -374,6 +375,22 @@ class StormRst(s_base.Base):
             raise s_exc.NoSuchName(mesg=f'The {directive} directive is not supported', directive=directive)
         return handler
 
+    async def _handlePythonPath(self, text):
+        '''
+        Add the text to sys.path.
+
+        Args:
+            text: The path to add.
+        '''
+        if not os.path.isdir(text):
+            raise s_exc.NoSuchDir(mesg=f'The path {text} is not a directory', path=text)
+        if text not in sys.path:
+            logger.debug(f'Inserting {text} into sys.path')
+            sys.path.insert(0, text)
+            def onfini():
+                sys.path.remove(text)
+            self.onfini(onfini)
+
     async def _handleStorm(self, text):
         '''
         Run a Storm command and generate text from the output.
@@ -618,7 +635,8 @@ class StormRst(s_base.Base):
         query = query.replace('--fail-ok', '')
         query = query.strip()
 
-        env = self.context.get('shell-env')
+        env = dict(os.environ)
+        env.update(self.context.get('shell-env', {}))
 
         stderr = None
         if opts.include_stderr:

{synapse-2.221.0 → synapse-2.223.0}/synapse/lib/schemas.py

@@ -816,6 +816,10 @@ _reqValidPkgdefSchema = {
                     'items': {'type': 'array',
                         'items': {'type': 'string'}},
                 },
+                'asroot:ondeny:import': {
+                    'type': 'string',
+                    'enum': ['allow', 'warn', 'deny'],
+                },
             },
             'additionalProperties': True,
             'required': ['name', 'storm']

{synapse-2.221.0 → synapse-2.223.0}/synapse/lib/snap.py

@@ -1305,6 +1305,11 @@ class Snap(s_base.Base):
             mesg = 'The snapshot is in read-only mode.'
             raise s_exc.IsReadOnly(mesg=mesg)
 
+        useriden = meta.get('user')
+        if useriden is None:
+            mesg = 'meta is missing user key. Cannot process edits.'
+            raise s_exc.BadArg(mesg=mesg, name='user')
+
         wlyr = self.wlyr
         nodes = []
         callbacks = []
@@ -1344,12 +1349,12 @@ class Snap(s_base.Base):
                 if etyp == s_layer.EDIT_NODE_ADD:
                     node.bylayer['ndef'] = wlyr.iden
                     callbacks.append((node.form.wasAdded, (node,)))
-                    callbacks.append((self.view.runNodeAdd, (node,)))
+                    callbacks.append((self.view.runNodeAdd, (node, useriden)))
                     continue
 
                 if etyp == s_layer.EDIT_NODE_DEL:
                     callbacks.append((node.form.wasDeleted, (node,)))
-                    callbacks.append((self.view.runNodeDel, (node,)))
+                    callbacks.append((self.view.runNodeDel, (node, useriden)))
                     continue
 
                 if etyp == s_layer.EDIT_PROP_SET:
@@ -1365,7 +1370,7 @@ class Snap(s_base.Base):
                     node.bylayer['props'][name] = wlyr.iden
 
                     callbacks.append((prop.wasSet, (node, oldv)))
-                    callbacks.append((self.view.runPropSet, (node, prop, oldv)))
+                    callbacks.append((self.view.runPropSet, (node, prop, oldv, useriden)))
                     continue
 
                 if etyp == s_layer.EDIT_PROP_DEL:
@@ -1381,7 +1386,7 @@ class Snap(s_base.Base):
                     node.bylayer['props'].pop(name, None)
 
                     callbacks.append((prop.wasDel, (node, oldv)))
-                    callbacks.append((self.view.runPropSet, (node, prop, oldv)))
+                    callbacks.append((self.view.runPropSet, (node, prop, oldv, useriden)))
                     continue
 
                 if etyp == s_layer.EDIT_TAG_SET:
@@ -1391,7 +1396,7 @@ class Snap(s_base.Base):
                     node.tags[tag] = valu
                     node.bylayer['tags'][tag] = wlyr.iden
 
-                    callbacks.append((self.view.runTagAdd, (node, tag, valu)))
+                    callbacks.append((self.view.runTagAdd, (node, tag, valu, useriden,)))
                     continue
 
                 if etyp == s_layer.EDIT_TAG_DEL:
@@ -1401,7 +1406,7 @@ class Snap(s_base.Base):
                     node.tags.pop(tag, None)
                     node.bylayer['tags'].pop(tag, None)
 
-                    callbacks.append((self.view.runTagDel, (node, tag, oldv)))
+                    callbacks.append((self.view.runTagDel, (node, tag, oldv, useriden)))
                     continue
 
                 if etyp == s_layer.EDIT_TAGPROP_SET:
@@ -1436,14 +1441,15 @@ class Snap(s_base.Base):
                 if etyp == s_layer.EDIT_EDGE_ADD:
                     verb, n2iden = parms
                     n2 = await self.getNodeByBuid(s_common.uhex(n2iden))
-                    callbacks.append((self.view.runEdgeAdd, (node, verb, n2)))
+                    callbacks.append((self.view.runEdgeAdd, (node, verb, n2, useriden)))
 
                 if etyp == s_layer.EDIT_EDGE_DEL:
                     verb, n2iden = parms
                     n2 = await self.getNodeByBuid(s_common.uhex(n2iden))
-                    callbacks.append((self.view.runEdgeDel, (node, verb, n2)))
+                    callbacks.append((self.view.runEdgeDel, (node, verb, n2, useriden)))
 
-        [await func(*args) for (func, args) in callbacks]
+        for func, args in callbacks:
+            await func(*args)
 
         if actualedits:
             await self.fire('node:edits', edits=actualedits)