synapse 2.221.0__tar.gz → 2.222.0__tar.gz

{synapse-2.221.0/synapse.egg-info → synapse-2.222.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 2.221.0
+Version: 2.222.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0

{synapse-2.221.0 → synapse-2.222.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.221.0'
+version = '2.222.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]

{synapse-2.221.0 → synapse-2.222.0}/synapse/cortex.py

@@ -624,7 +624,8 @@ class CoreApi(s_cell.CellApi):
         Returns:
             AsyncIterator[Tuple(buid, valu)]
         '''
-        self.user.confirm(('layer', 'lift', layriden))
+        if not self.user.allowed(('layer', 'lift', layriden)):
+            self.user.confirm(('layer', 'read', layriden))
         async for item in self.cell.iterFormRows(layriden, form, stortype=stortype, startvalu=startvalu):
             yield item
 
@@ -642,7 +643,8 @@ class CoreApi(s_cell.CellApi):
         Returns:
             AsyncIterator[Tuple(buid, valu)]
         '''
-        self.user.confirm(('layer', 'lift', layriden))
+        if not self.user.allowed(('layer', 'lift', layriden)):
+            self.user.confirm(('layer', 'read', layriden))
         async for item in self.cell.iterPropRows(layriden, form, prop, stortype=stortype, startvalu=startvalu):
             yield item
 
@@ -659,7 +661,8 @@ class CoreApi(s_cell.CellApi):
         Returns:
             AsyncIterator[Tuple(buid, valu)]
         '''
-        self.user.confirm(('layer', 'lift', layriden))
+        if not self.user.allowed(('layer', 'lift', layriden)):
+            self.user.confirm(('layer', 'read', layriden))
         async for item in self.cell.iterUnivRows(layriden, prop, stortype=stortype, startvalu=startvalu):
             yield item
 
@@ -680,7 +683,8 @@ class CoreApi(s_cell.CellApi):
             This yields (buid, (tagvalu, form)) instead of just buid, valu in order to allow resuming an interrupted
             call by feeding the last value retrieved into starttupl
         '''
-        self.user.confirm(('layer', 'lift', layriden))
+        if not self.user.allowed(('layer', 'lift', layriden)):
+            self.user.confirm(('layer', 'read', layriden))
         async for item in self.cell.iterTagRows(layriden, tag, form=form, starttupl=starttupl):
             yield item
 
@@ -699,7 +703,8 @@ class CoreApi(s_cell.CellApi):
         Returns:
             AsyncIterator[Tuple(buid, valu)]
         '''
-        self.user.confirm(('layer', 'lift', layriden))
+        if not self.user.allowed(('layer', 'lift', layriden)):
+            self.user.confirm(('layer', 'read', layriden))
         async for item in self.cell.iterTagPropRows(layriden, tag, prop, form=form, stortype=stortype,
                                                     startvalu=startvalu):
             yield item
@@ -1366,6 +1371,30 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
     def _initCorePerms(self):
         self._cortex_permdefs.extend((
+            {'perm': ('axon', 'upload'), 'gate': 'cortex',
+             'desc': 'Controls the ability to upload a file to the Axon.'},
+            {'perm': ('axon', 'get'), 'gate': 'cortex',
+             'desc': 'Controls the ability to retrieve a file from the Axon.'},
+            {'perm': ('axon', 'has'), 'gate': 'cortex',
+             'desc': 'Controls the ability to check if the Axon contains a file.'},
+            {'perm': ('axon', 'del'), 'gate': 'cortex',
+             'desc': 'Controls the ability to remove a file from the Axon.'},
+
+            {'perm': ('layer', 'add'), 'gate': 'cortex',
+             'desc': 'Controls the ability to add Layers to the cortex.'},
+            {'perm': ('layer', 'del'), 'gate': 'cortex',
+             'desc': 'Controls the ability to remove Layers from the cortex.'},
+            {'perm': ('layer', 'read'), 'gate': 'layer',
+             'desc': 'Controls the ability to read/lift from a Layer.'},
+            {'perm': ('layer', 'read', '<iden>'), 'gate': 'cortex',
+             'desc': 'Controls the ability to read/lift from a specific Layer.'},
+            {'perm': ('layer', 'set', '<name>'), 'gate': 'layer',
+             'desc': 'Controls the ability to configure properties of a Layer.'},
+            {'perm': ('layer', 'write'), 'gate': 'layer',
+             'desc': 'Controls the ability to write to a Layer.'},
+            {'perm': ('layer', 'write', '<iden>'), 'gate': 'cortex',
+             'desc': 'Controls the ability to write to a specific Layer.'},
+
             {'perm': ('model', 'form', 'add'), 'gate': 'cortex',
              'desc': 'Controls access to adding extended model forms.'},
             {'perm': ('model', 'form', 'add', '<form>'), 'gate': 'cortex',
@@ -1519,15 +1548,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             {'perm': ('view', 'set', '<setting>'), 'gate': 'view',
              'desc': 'Controls access to change view settings.',
              'ex': 'view.set.name'},
-
-            {'perm': ('axon', 'upload'), 'gate': 'cortex',
-             'desc': 'Controls the ability to upload a file to the Axon.'},
-            {'perm': ('axon', 'get'), 'gate': 'cortex',
-             'desc': 'Controls the ability to retrieve a file from the Axon.'},
-            {'perm': ('axon', 'has'), 'gate': 'cortex',
-             'desc': 'Controls the ability to check if the Axon contains a file.'},
-            {'perm': ('axon', 'del'), 'gate': 'cortex',
-             'desc': 'Controls the ability to remove a file from the Axon.'},
         ))
         for pdef in self._cortex_permdefs:
             s_schemas.reqValidPermDef(pdef)

{synapse-2.221.0 → synapse-2.222.0}/synapse/data/lark/storm.lark

@@ -238,10 +238,10 @@ liftreverse: _REVERSE "(" (liftformtag | liftpropby | liftprop | liftbyarray | l
 
 _DEREF.3:  /\*(?=\$)/
 
-liftformtag: ((PROPS | UNIVNAME | WILDPROPS) | _DEREF _varvalu) tagname [_cmpr _valu]
-liftpropby: ((PROPS | EMBEDPROPS | UNIVNAME) | _DEREF _varvalu) _cmpr _valu
-liftprop: ((PROPS | UNIVNAME | WILDPROPS) | _DEREF _varvalu)
-liftbyarray: ((PROPS | EMBEDPROPS | UNIVNAME) | _DEREF _varvalu) "*[" _safe_cmpr _valu "]"
+liftformtag: (PROPS | UNIVNAME | WILDPROPS | derefprops) tagname [_cmpr _valu]
+liftpropby: (PROPS | EMBEDPROPS | UNIVNAME | derefprops) _cmpr _valu
+liftprop: (PROPS | UNIVNAME | WILDPROPS | derefprops)
+liftbyarray: (PROPS | EMBEDPROPS | UNIVNAME | derefprops) "*[" _safe_cmpr _valu "]"
 lifttagtag: (_HASH | _HASHSPACE) tagname [_cmpr _valu]
 liftbytag: (tagname | tagnamewithspace) [_cmpr _valu]
 liftbytagprop: (tagprop | tagpropwithspace) [_cmpr _valu]
@@ -249,7 +249,7 @@ liftbyformtagprop: formtagprop [_cmpr _valu]
 
 tagprop: tagname _COLONNOSPACE (BASEPROP | _varvalu)
 tagpropwithspace: tagnamewithspace _COLONNOSPACE (BASEPROP | _varvalu) -> tagprop
-formtagprop: ((PROPS | UNIVNAME | WILDPROPS) | _DEREF _varvalu) tagname _COLONNOSPACE (BASEPROP | _varvalu)
+formtagprop: (PROPS | UNIVNAME | WILDPROPS | derefprops) tagname _COLONNOSPACE (BASEPROP | _varvalu)
 _COLONNOSPACE.2: /(?<!\s):/
 
 _funcarg: (VARTOKN [EQNOSPACE _valu])
@@ -484,7 +484,7 @@ EMBEDPROPS.2: /[a-z_][a-z0-9_]*(:[a-z0-9_]+)+((\:\:|\:|\.)[a-z0-9_]+)*(?![:.a-z0
 UNIVNAME.2: /(?<=^|[\s\|\{\(\[+=-])\.[a-z_][a-z0-9_]*([:.][a-z0-9_]+)*/
 univprop:  UNIVNAME | "." _varvalu
 // A full property or a universal property
-formname: PROPS | _DEREF _varvalu
+formname: PROPS | derefprops
 // A relative property
 relprop: RELNAME | _COLONDOLLAR _varvaluatom
 
@@ -494,6 +494,9 @@ RELNAME: /(?<!\w):\.?[a-z_][a-z0-9_]*(?:(\:\:|\:|\.)[a-z_][a-z0-9_]*)*/
 // Similar to PROPS but does not require a colon
 BASEPROP: /[a-z_][a-z0-9_]*(?:(\:\:|\:|\.)[a-z_][a-z0-9_]*)*/
 
+// Derefed variable for prop names
+derefprops: _DEREF _varvalu
+
 // The entry point for a $(...) expression.  The initial dollar sign is now optional
 _dollarexprs: "$"? dollaroper
 

{synapse-2.221.0 → synapse-2.222.0}/synapse/lib/ast.py

@@ -2,7 +2,6 @@ import types
 import asyncio
 import decimal
 import fnmatch
-import hashlib
 import logging
 import binascii
 import itertools
@@ -1833,7 +1832,7 @@ class LiftProp(LiftOper):
 
         assert len(self.kids) == 1
 
-        name = await tostr(await self.kids[0].compute(runt, path))
+        name = await self.kids[0].compute(runt, path)
 
         prop = runt.model.props.get(name)
         if prop is not None:
@@ -4029,6 +4028,13 @@ class FormName(Value):
     async def compute(self, runt, path):
         return await self.kids[0].compute(runt, path)
 
+class DerefProps(Value):
+    async def compute(self, runt, path):
+        valu = await toprim(await self.kids[0].compute(runt, path))
+        if (exc := await s_stormtypes.typeerr(valu, str)) is not None:
+            raise self.kids[0].addExcInfo(exc)
+        return valu
+
 class RelProp(PropName):
     pass
 

{synapse-2.221.0 → synapse-2.222.0}/synapse/lib/layer.py

@@ -119,6 +119,7 @@ class LayerApi(s_cell.CellApi):
 
         self.layr = layr
         self.liftperm = ('layer', 'lift', self.layr.iden)
+        self.readperm = ('layer', 'read', self.layr.iden)
         self.writeperm = ('layer', 'write', self.layr.iden)
 
     async def iterLayerNodeEdits(self):
@@ -126,7 +127,8 @@ class LayerApi(s_cell.CellApi):
         Scan the full layer and yield artificial nodeedit sets.
         '''
 
-        await self._reqUserAllowed(self.liftperm)
+        if not self.allowed(self.liftperm):
+            await self._reqUserAllowed(self.readperm)
         async for item in self.layr.iterLayerNodeEdits():
             yield item
             await asyncio.sleep(0)
@@ -165,13 +167,15 @@ class LayerApi(s_cell.CellApi):
 
         Once caught up with storage, yield them in realtime.
         '''
-        await self._reqUserAllowed(self.liftperm)
+        if not self.allowed(self.liftperm):
+            await self._reqUserAllowed(self.readperm)
         async for item in self.layr.syncNodeEdits(offs, wait=wait, reverse=reverse):
             yield item
             await asyncio.sleep(0)
 
     async def syncNodeEdits2(self, offs, wait=True):
-        await self._reqUserAllowed(self.liftperm)
+        if not self.allowed(self.liftperm):
+            await self._reqUserAllowed(self.readperm)
         async for item in self.layr.syncNodeEdits2(offs, wait=wait):
             yield item
             await asyncio.sleep(0)
@@ -180,18 +184,21 @@ class LayerApi(s_cell.CellApi):
         '''
         Returns what will be the *next* nodeedit log index.
         '''
-        await self._reqUserAllowed(self.liftperm)
+        if not self.allowed(self.liftperm):
+            await self._reqUserAllowed(self.readperm)
         return await self.layr.getEditIndx()
 
     async def getEditSize(self):
         '''
         Return the total number of (edits, meta) pairs in the layer changelog.
         '''
-        await self._reqUserAllowed(self.liftperm)
+        if not self.allowed(self.liftperm):
+            await self._reqUserAllowed(self.readperm)
         return await self.layr.getEditSize()
 
     async def getIden(self):
-        await self._reqUserAllowed(self.liftperm)
+        if not self.allowed(self.liftperm):
+            await self._reqUserAllowed(self.readperm)
         return self.layr.iden
 
 BUID_CACHE_SIZE = 10000
@@ -3350,7 +3357,7 @@ class Layer(s_nexus.Pusher):
         nodeedits = [(buid, newp_formname, [set_edit])]
 
         _, changes = await self.saveNodeEdits(nodeedits, meta)
-        return bool(changes[0][2])
+        return any(c[2] for c in changes)
 
     async def delStorNode(self, buid, meta):
         '''
@@ -3434,7 +3441,7 @@ class Layer(s_nexus.Pusher):
             if changed: # pragma: no cover
                 return changed
 
-            return bool(changes[0][2])
+            return any(c[2] for c in changes)
 
         for n2iden, edges in n2edges.items():
             edits = [(EDIT_EDGE_DEL, edge, ()) for edge in edges]
@@ -3455,7 +3462,7 @@ class Layer(s_nexus.Pusher):
         nodeedits = [(buid, oldp_formname, [del_edit])]
 
         _, changes = await self.saveNodeEdits(nodeedits, meta)
-        return bool(changes[0][2])
+        return any(c[2] for c in changes)
 
     async def delNodeData(self, buid, meta, name=None):
         '''
@@ -3480,7 +3487,7 @@ class Layer(s_nexus.Pusher):
         nodeedits = [(buid, sode.get('form'), edits)]
 
         _, changes = await self.saveNodeEdits(nodeedits, meta)
-        return bool(changes[0][2])
+        return any(c[2] for c in changes)
 
     async def delEdge(self, n1buid, verb, n2buid, meta):
         sode = self._getStorNode(n1buid)
@@ -3497,7 +3504,7 @@ class Layer(s_nexus.Pusher):
         nodeedits = [(n1buid, sode.get('form'), edits)]
 
         _, changes = await self.saveNodeEdits(nodeedits, meta)
-        return bool(changes[0][2])
+        return any(c[2] for c in changes)
 
     async def storNodeEdits(self, nodeedits, meta):
 

{synapse-2.221.0 → synapse-2.222.0}/synapse/lib/parser.py

@@ -669,6 +669,7 @@ ruleClassMap = {
     'condsetoper': s_ast.CondSetOper,
     'condtrysetoper': lambda astinfo, kids: s_ast.CondSetOper(astinfo, kids, errok=True),
     'condsubq': s_ast.SubqCond,
+    'derefprops': s_ast.DerefProps,
     'dollarexpr': s_ast.DollarExpr,
     'edgeaddn1': s_ast.EditEdgeAdd,
     'edgedeln1': s_ast.EditEdgeDel,

{synapse-2.221.0 → synapse-2.222.0}/synapse/lib/rstorm.py

@@ -344,6 +344,7 @@ class StormRst(s_base.Base):
             'storm-cortex': self._handleStormCortex,
             'storm-envvar': self._handleStormEnvVar,
             'storm-expect': self._handleStormExpect,
+            'storm-python-path': self._handlePythonPath,
             'storm-multiline': self._handleStormMultiline,
             'storm-mock-http': self._handleStormMockHttp,
             'storm-vcr-opts': self._handleStormVcrOpts,
@@ -374,6 +375,22 @@ class StormRst(s_base.Base):
             raise s_exc.NoSuchName(mesg=f'The {directive} directive is not supported', directive=directive)
         return handler
 
+    async def _handlePythonPath(self, text):
+        '''
+        Add the text to sys.path.
+
+        Args:
+            text: The path to add.
+        '''
+        if not os.path.isdir(text):
+            raise s_exc.NoSuchDir(mesg=f'The path {text} is not a directory', path=text)
+        if text not in sys.path:
+            logger.debug(f'Inserting {text} into sys.path')
+            sys.path.insert(0, text)
+            def onfini():
+                sys.path.remove(text)
+            self.onfini(onfini)
+
     async def _handleStorm(self, text):
         '''
         Run a Storm command and generate text from the output.
@@ -618,7 +635,8 @@ class StormRst(s_base.Base):
         query = query.replace('--fail-ok', '')
         query = query.strip()
 
-        env = self.context.get('shell-env')
+        env = dict(os.environ)
+        env.update(self.context.get('shell-env', {}))
 
         stderr = None
         if opts.include_stderr:

{synapse-2.221.0 → synapse-2.222.0}/synapse/lib/schemas.py

@@ -816,6 +816,10 @@ _reqValidPkgdefSchema = {
                     'items': {'type': 'array',
                         'items': {'type': 'string'}},
                 },
+                'asroot:ondeny:import': {
+                    'type': 'string',
+                    'enum': ['allow', 'warn', 'deny'],
+                },
             },
             'additionalProperties': True,
             'required': ['name', 'storm']

{synapse-2.221.0 → synapse-2.222.0}/synapse/lib/stormlib/cortex.py

@@ -1029,7 +1029,7 @@ class HttpReq(s_stormtypes.StormType):
             if not isinstance(body, bytes):
                 body = await s_stormtypes.toprim(body)
                 body = s_json.dumps(body)
-                headers['Content-Type'] = 'application/json; charset=utf8"'
+                headers['Content-Type'] = 'application/json; charset=utf8'
                 headers['Content-Length'] = len(body)
 
         await self._methSendCode(code)

{synapse-2.221.0 → synapse-2.222.0}/synapse/lib/stormtypes.py

@@ -1595,8 +1595,16 @@ class LibBase(Lib):
 
             if not asroot:
                 permtext = ' or '.join(('.'.join(p) for p in rootperms))
-                mesg = f'Module ({name}) requires permission: {permtext}'
-                raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
+
+                match mdef.get('asroot:ondeny:import', 'deny'):
+                    case 'allow':
+                        pass
+                    case 'warn':
+                        mesg = f'Module ({name}) permissions will not be elevated. Missing permission: {permtext}.'
+                        await self.runt.warnonce(mesg, log=False)
+                    case _:
+                        mesg = f'Module ({name}) requires permission: {permtext}'
+                        raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
 
         else:
             perm = ('storm', 'asroot', 'mod') + tuple(name.split('.'))
@@ -7837,7 +7845,8 @@ class Layer(Prim):
 
         layr = self.runt.snap.core.reqLayer(self.valu.get('iden'))
 
-        self.runt.confirm(('layer', 'edits', 'read'), gateiden=layr.iden)
+        if not self.runt.allowed(('layer', 'edits', 'read'), gateiden=layr.iden):
+            self.runt.confirm(('layer', 'read'), gateiden=layr.iden)
 
         if reverse:
             wait = False
@@ -10439,5 +10448,10 @@ async def totype(valu, basetypes=False) -> str:
 async def typeerr(name, reqt):
     if not isinstance(name, reqt):
         styp = await totype(name, basetypes=True)
-        mesg = f"Expected value of type '{reqt}', got '{styp}' with value {name}."
+
+        reqtname = str(reqt)
+        if (clsname := getattr(reqt, '__name__')):
+            reqtname = clsname
+
+        mesg = f"Expected value of type '{reqtname}', got '{styp}' with value {name}."
         return s_exc.StormRuntimeError(mesg=mesg, name=name, type=styp)

{synapse-2.221.0 → synapse-2.222.0}/synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 221, 0)
+version = (2, 222, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = '6ae7cd60524df7dc861bb91fe25309999eba6d70'
+commit = '1ab0342cfb8272e9f9a629f1755b0e33b78b78b3'

{synapse-2.221.0 → synapse-2.222.0}/synapse/models/orgs.py

@@ -729,6 +729,9 @@ class OuModule(s_module.CoreModule):
 
                     ('ext:id', ('str', {'strip': True}), {
                         'doc': 'An external identifier for the technique.'}),
+
+                    ('parent', ('ou:technique', {}), {
+                        'doc': 'The parent technique for the technique.'}),
                 )),
                 ('ou:technique:taxonomy', {}, ()),
                 ('ou:orgtype', {}, ()),

{synapse-2.221.0 → synapse-2.222.0}/synapse/tests/test_lib_grammar.py

@@ -1267,7 +1267,7 @@ _ParseResults = [
     'Query: [EditTagPropDel: [TagProp: [TagName: [Const: baz, Const: faz], Const: lol]]]',
     'Query: [EditTagPropSet: [TagProp: [TagName: [Const: baz, Const: faz], Const: lol], Const: =, Const: 20]]',
     'Query: [LiftTagProp: [TagProp: [TagName: [Const: tag], Const: somegeoloctypebecauseihatelife], Const: near=, List: [VarValue: [Const: lat], VarValue: [Const: long]]]]',
-    'Query: [LiftPropBy: [VarValue: [Const: foo], Const: near=, Const: 20]]',
+    'Query: [LiftPropBy: [DerefProps: [VarValue: [Const: foo]], Const: near=, Const: 20]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, VarDeref: [VarDeref: [VarDeref: [VarDeref: [VarDeref: [VarValue: [Const: foo], Const: woot], Const: var], VarValue: [Const: bar]], Const: mar], VarValue: [Const: car]]]]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, VarDeref: [VarDeref: [VarValue: [Const: foo], VarValue: [Const: space key]], Const: subkey]]]',
     'Query: [ForLoop: [Const: iterkey, VarDeref: [VarDeref: [VarValue: [Const: foo], VarValue: [Const: bar key]], VarValue: [Const: biz key]], SubQuery: [Query: [LiftPropBy: [Const: inet:ipv4, Const: =, VarDeref: [VarDeref: [VarDeref: [VarValue: [Const: foo], VarValue: [Const: bar key]], VarValue: [Const: biz key]], VarValue: [Const: iterkey]]]]]]]',
@@ -1337,11 +1337,11 @@ _ParseResults = [
     'Query: [VarEvalOper: [FuncCall: [VarDeref: [VarValue: [Const: lib], Const: print], CallArgs: [DollarExpr: [FuncCall: [VarDeref: [DollarExpr: [RelPropValue: [Const: prop]], Const: upper], CallArgs: [], CallKwargs: []]]], CallKwargs: []]]]',
     'Query: [VarEvalOper: [FuncCall: [VarDeref: [VarValue: [Const: lib], Const: print], CallArgs: [VarDeref: [DollarExpr: [ExprDict: [Const: unicode, Const: 1]], DollarExpr: [RelPropValue: [Const: prop]]]], CallKwargs: []]]]',
     'Query: [VarEvalOper: [FuncCall: [VarDeref: [VarValue: [Const: lib], Const: print], CallArgs: [DollarExpr: [ExprNode: [VarDeref: [DollarExpr: [ExprDict: [Const: unicode, Const: 1]], DollarExpr: [RelPropValue: [Const: prop]]], Const: +, DollarExpr: [Const: 2]]]], CallKwargs: []]]]',
-    'Query: [LiftFormTag: [VarValue: [Const: form], TagName: [VarValue: [Const: tag]]]]',
-    'Query: [LiftFormTagProp: [FormTagProp: [VarValue: [Const: form], TagName: [VarValue: [Const: tag]], VarValue: [Const: prop]]]]',
+    'Query: [LiftFormTag: [DerefProps: [VarValue: [Const: form]], TagName: [VarValue: [Const: tag]]]]',
+    'Query: [LiftFormTagProp: [FormTagProp: [DerefProps: [VarValue: [Const: form]], TagName: [VarValue: [Const: tag]], VarValue: [Const: prop]]]]',
     'Query: [LiftProp: [Const: inet:ipv4]]',
     'Query: [LiftPropBy: [Const: inet:ipv4, Const: =, Const: 1.2.3.4]]',
-    'Query: [LiftPropBy: [VarValue: [Const: form], Const: =, VarValue: [Const: valu]]]',
+    'Query: [LiftPropBy: [DerefProps: [VarValue: [Const: form]], Const: =, VarValue: [Const: valu]]]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [Const: inet:dns*], isjoin=False]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [Const: inet:dns:*], isjoin=False]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [List: [Const: meta:source, Const: inet:dns:a]], isjoin=False]',

{synapse-2.221.0 → synapse-2.222.0}/synapse/tests/test_lib_rstorm.py

@@ -1,4 +1,5 @@
 import os
+import sys
 
 import vcr
 
@@ -226,10 +227,10 @@ Bye!
 shell_input00 = '''
 Shell with environment variable.
 
-.. shell-env:: SYN_LOG_LEVEL=DEBUG SYN_FOO=BAR
-.. shell:: python3 -c "import os; print('LEVEL', os.environ.get('SYN_LOG_LEVEL')); print('FOO', os.environ.get('SYN_FOO'))"
+.. shell-env:: SYN_HEHE_HAHA=BEEP SYN_FOO=BAR
+.. shell:: python3 -c "import os; print('HEHE_HAHA', os.environ.get('SYN_HEHE_HAHA')); print('FOO', os.environ.get('SYN_FOO'))"
 .. shell-env::
-.. shell:: python3 -c "import os; print('LEVEL', os.environ.get('SYN_LOG_LEVEL')); print('FOO', os.environ.get('SYN_FOO'))"
+.. shell:: python3 -c "import os; print('HEHE_HAHA', os.environ.get('SYN_HEHE_HAHA')); print('FOO', os.environ.get('SYN_FOO'))"
 '''
 
 shell_output00 = '''
@@ -237,17 +238,17 @@ Shell with environment variable.
 
 ::
 
-  python3 -c "import os; print('LEVEL', os.environ.get('SYN_LOG_LEVEL')); print('FOO', os.environ.get('SYN_FOO'))"
+  python3 -c "import os; print('HEHE_HAHA', os.environ.get('SYN_HEHE_HAHA')); print('FOO', os.environ.get('SYN_FOO'))"
 
-  LEVEL DEBUG
+  HEHE_HAHA BEEP
   FOO BAR
 
 
 ::
 
-  python3 -c "import os; print('LEVEL', os.environ.get('SYN_LOG_LEVEL')); print('FOO', os.environ.get('SYN_FOO'))"
+  python3 -c "import os; print('HEHE_HAHA', os.environ.get('SYN_HEHE_HAHA')); print('FOO', os.environ.get('SYN_FOO'))"
 
-  LEVEL None
+  HEHE_HAHA None
   FOO None
 
 
@@ -776,3 +777,50 @@ class RStormLibTest(s_test.SynTest):
                 fd.write(fix_input_for_cli(fail02).encode())
             with self.raises(s_exc.StormRuntimeError):
                 await get_rst_text(path)
+
+    async def test_rstorm_python_path(self):
+        content = '''#comment
+import synapse.lib.cell as s_cell
+import synapse.lib.stormsvc as s_stormsvc
+
+class SomeApi(s_stormsvc.StormSvc, s_cell.CellApi):
+    _storm_svc_name = 'someservice'
+    _storm_svc_vers = '0.1.0',
+    _storm_svc_pkgs = ()
+
+class SomeService(s_cell.Cell):
+    cellapi = SomeApi
+'''
+        with self.getTestDir() as dirn:
+            with s_common.genfile(dirn, 'somefile.py') as fd:
+                fd.write(content.encode())
+            pythonpath_rst_in = f'''
+.. storm-cortex:: default
+.. storm-python-path:: {dirn}
+.. storm-svc:: somefile.SomeService fooservice {{"https:port": 0}}
+.. storm:: service.list
+
+hello world
+            '''
+            rst_path = s_common.genpath(dirn, 'test.rst')
+
+            with s_common.genfile(rst_path) as fd:
+                fd.write(pythonpath_rst_in.encode())
+
+            pythonpath_rst_out = await get_rst_text(rst_path)
+            self.notin('storm-python-path', pythonpath_rst_out)
+            self.isin('(fooservice) (someservice @ 0.1.0)', pythonpath_rst_out)
+            self.isin('hello world', pythonpath_rst_out)
+            # Fini handler cleaned up the path manipulation
+            self.notin(dirn, sys.path)
+
+            # Sad path
+            pythonpath_rst_in = f'''
+.. storm-python-path:: {dirn}/{s_common.guid()}
+hello world
+            '''
+            rst_path = s_common.genpath(dirn, 'test.rst')
+            with s_common.genfile(rst_path) as fd:
+                fd.write(pythonpath_rst_in.encode())
+            with self.raises(s_exc.NoSuchDir):
+                await get_rst_text(rst_path)

{synapse-2.221.0 → synapse-2.222.0}/synapse/tests/test_lib_storm.py

@@ -1,6 +1,5 @@
 import copy
 import asyncio
-import datetime
 import itertools
 import urllib.parse as u_parse
 import unittest.mock as mock
@@ -17,6 +16,7 @@ import synapse.lib.storm as s_storm
 import synapse.lib.httpapi as s_httpapi
 import synapse.lib.msgpack as s_msgpack
 import synapse.lib.version as s_version
+import synapse.lib.stormtypes as s_stormtypes
 
 import synapse.tests.utils as s_t_utils
 from synapse.tests.utils import alist
@@ -4874,6 +4874,79 @@ class StormTest(s_t_utils.SynTest):
             self.len(1, links)
             self.eq({'type': 'runtime'}, links[0][1])
 
+    async def test_storm_derefprops(self):
+        async with self.getTestCore() as core:
+            await core.addTagProp('score', ('int', {}), {})
+
+            mesg = "Expected value of type 'str', got '"
+
+            # editnodeadd
+            msgs = await core.stormlist('$form = inet:fqdn [ *$form=foobar.com ]')
+            self.stormHasNoWarnErr(msgs)
+
+            invals = [10, None, False, [], {}]
+
+            for inval in invals:
+                opts = {'vars': {'form': inval}}
+                with self.raises(s_exc.StormRuntimeError) as exc:
+                    await core.nodes('[ *$form=valu ]', opts=opts)
+                self.true(exc.exception.get('mesg').startswith(mesg))
+
+            # liftprop
+            msgs = await core.stormlist('$form = inet:fqdn *$form')
+            self.stormHasNoWarnErr(msgs)
+
+            for inval in invals:
+                opts = {'vars': {'form': inval}}
+                with self.raises(s_exc.StormRuntimeError) as exc:
+                    await core.nodes('*$form', opts=opts)
+                self.true(exc.exception.get('mesg').startswith(mesg))
+
+            # liftpropby
+            msgs = await core.stormlist('$form = inet:fqdn *$form=foobar.com')
+            self.stormHasNoWarnErr(msgs)
+
+            for inval in invals:
+                opts = {'vars': {'form': inval}}
+                with self.raises(s_exc.StormRuntimeError) as exc:
+                    await core.nodes('*$form=newp', opts=opts)
+                self.true(exc.exception.get('mesg').startswith(mesg))
+
+            # liftformtag
+            msgs = await core.stormlist('$form = inet:fqdn *$form#foo')
+            self.stormHasNoWarnErr(msgs)
+
+            for inval in invals:
+                opts = {'vars': {'form': inval}}
+                with self.raises(s_exc.StormRuntimeError) as exc:
+                    await core.nodes('*$form#newp', opts=opts)
+                self.true(exc.exception.get('mesg').startswith(mesg))
+
+            # liftbyarray
+            msgs = await core.stormlist('$form = test:arrayform *$form*[=(10)]')
+            self.stormHasNoWarnErr(msgs)
+
+            for inval in invals:
+                opts = {'vars': {'form': inval}}
+                with self.raises(s_exc.StormRuntimeError) as exc:
+                    await core.nodes('*$form*[="newp"]', opts=opts)
+                self.true(exc.exception.get('mesg').startswith(mesg))
+
+            # formtagprop
+            msgs = await core.stormlist('$form = inet:fqdn *$form#foo:score')
+            self.stormHasNoWarnErr(msgs)
+
+            for inval in invals:
+                opts = {'vars': {'form': inval}}
+                with self.raises(s_exc.StormRuntimeError) as exc:
+                    await core.nodes('*$form#newp:score', opts=opts)
+                self.true(exc.exception.get('mesg').startswith(mesg))
+
+            # Check Storm Str types
+            name = s_stormtypes.Str('inet:fqdn')
+            msgs = await core.stormlist('*$form', opts={'vars': {'form': name}})
+            self.stormHasNoWarnErr(msgs)
+
     async def test_storm_nested_root(self):
         async with self.getTestCore() as core:
             self.eq(20, await core.callStorm('''