synapse 2.152.0__py311-none-any.whl → 2.154.0__py311-none-any.whl

synapse/axon.py

@@ -677,27 +677,29 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
         async for item in self.cell.iterMpkFile(sha256):
             yield item
 
-    async def readlines(self, sha256):
+    async def readlines(self, sha256, errors='ignore'):
         '''
         Yield lines from a multi-line text file in the axon.
 
         Args:
             sha256 (bytes): The sha256 hash of the file.
+            errors (str): Specify how encoding errors should handled.
 
         Yields:
             str: Lines of text
         '''
         await self._reqUserAllowed(('axon', 'get'))
-        async for item in self.cell.readlines(sha256):
+        async for item in self.cell.readlines(sha256, errors=errors):
             yield item
 
-    async def csvrows(self, sha256, dialect='excel', **fmtparams):
+    async def csvrows(self, sha256, dialect='excel', errors='ignore', **fmtparams):
         '''
         Yield CSV rows from a CSV file.
 
         Args:
             sha256 (bytes): The sha256 hash of the file.
             dialect (str): The CSV dialect to use.
+            errors (str): Specify how encoding errors should handled.
             **fmtparams: The CSV dialect format parameters.
 
         Notes:
@@ -719,21 +721,22 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
             list: Decoded CSV rows.
         '''
         await self._reqUserAllowed(('axon', 'get'))
-        async for item in self.cell.csvrows(sha256, dialect, **fmtparams):
+        async for item in self.cell.csvrows(sha256, dialect, errors=errors, **fmtparams):
             yield item
 
-    async def jsonlines(self, sha256):
+    async def jsonlines(self, sha256, errors='ignore'):
         '''
         Yield JSON objects from JSONL (JSON lines) file.
 
         Args:
             sha256 (bytes): The sha256 hash of the file.
+            errors (str): Specify how encoding errors should handled.
 
         Yields:
             object: Decoded JSON objects.
         '''
         await self._reqUserAllowed(('axon', 'get'))
-        async for item in self.cell.jsonlines(sha256):
+        async for item in self.cell.jsonlines(sha256, errors=errors):
             yield item
 
 
@@ -1350,7 +1353,7 @@ class Axon(s_cell.Cell):
         finally:
             link.txfini()
 
-    async def readlines(self, sha256):
+    async def readlines(self, sha256, errors='ignore'):
 
         sha256 = s_common.uhex(sha256)
         await self._reqHas(sha256)
@@ -1360,7 +1363,7 @@ class Axon(s_cell.Cell):
         feedtask = None
 
         try:
-            todo = s_common.todo(_spawn_readlines, sock00)
+            todo = s_common.todo(_spawn_readlines, sock00, errors=errors)
             async with await s_base.Base.anit() as scope:
 
                 scope.schedCoro(s_coro.spawn(todo, log_conf=await self._getSpawnLogConf()))
@@ -1384,7 +1387,7 @@ class Axon(s_cell.Cell):
             if feedtask is not None:
                 await feedtask
 
-    async def csvrows(self, sha256, dialect='excel', **fmtparams):
+    async def csvrows(self, sha256, dialect='excel', errors='ignore', **fmtparams):
         await self._reqHas(sha256)
         if dialect not in csv.list_dialects():
             raise s_exc.BadArg(mesg=f'Invalid CSV dialect, use one of {csv.list_dialects()}')
@@ -1394,7 +1397,7 @@ class Axon(s_cell.Cell):
         feedtask = None
 
         try:
-            todo = s_common.todo(_spawn_readrows, sock00, dialect, fmtparams)
+            todo = s_common.todo(_spawn_readrows, sock00, dialect, fmtparams, errors=errors)
             async with await s_base.Base.anit() as scope:
 
                 scope.schedCoro(s_coro.spawn(todo, log_conf=await self._getSpawnLogConf()))
@@ -1418,8 +1421,8 @@ class Axon(s_cell.Cell):
             if feedtask is not None:
                 await feedtask
 
-    async def jsonlines(self, sha256):
-        async for line in self.readlines(sha256):
+    async def jsonlines(self, sha256, errors='ignore'):
+        async for line in self.readlines(sha256, errors=errors):
             line = line.strip()
             if not line:
                 continue
@@ -1745,9 +1748,9 @@ class Axon(s_cell.Cell):
                     'err': err,
                 }
 
-def _spawn_readlines(sock): # pragma: no cover
+def _spawn_readlines(sock, errors='ignore'): # pragma: no cover
     try:
-        with sock.makefile('r') as fd:
+        with sock.makefile('r', errors=errors) as fd:
 
             try:
 
@@ -1763,11 +1766,11 @@ def _spawn_readlines(sock): # pragma: no cover
         mesg = s_common.retnexc(e)
         sock.sendall(s_msgpack.en(mesg))
 
-def _spawn_readrows(sock, dialect, fmtparams): # pragma: no cover
+def _spawn_readrows(sock, dialect, fmtparams, errors='ignore'): # pragma: no cover
     try:
 
         # Assume utf8 encoding and ignore errors.
-        with sock.makefile('r', errors='ignore') as fd:
+        with sock.makefile('r', errors=errors) as fd:
 
             try:
 

synapse/cortex.py

@@ -36,6 +36,7 @@ import synapse.lib.grammar as s_grammar
 import synapse.lib.httpapi as s_httpapi
 import synapse.lib.msgpack as s_msgpack
 import synapse.lib.modules as s_modules
+import synapse.lib.schemas as s_schemas
 import synapse.lib.spooled as s_spooled
 import synapse.lib.version as s_version
 import synapse.lib.urlhelp as s_urlhelp
@@ -74,11 +75,13 @@ import synapse.lib.stormlib.iters as s_stormlib_iters  # NOQA
 import synapse.lib.stormlib.macro as s_stormlib_macro
 import synapse.lib.stormlib.model as s_stormlib_model
 import synapse.lib.stormlib.oauth as s_stormlib_oauth  # NOQA
+import synapse.lib.stormlib.stats as s_stormlib_stats  # NOQA
 import synapse.lib.stormlib.storm as s_stormlib_storm  # NOQA
 import synapse.lib.stormlib.backup as s_stormlib_backup  # NOQA
-import synapse.lib.stormlib.hashes as s_stormlib_hashes # NOQA
-import synapse.lib.stormlib.random as s_stormlib_random # NOQA
-import synapse.lib.stormlib.scrape as s_stormlib_scrape  # NOQA
+import synapse.lib.stormlib.cortex as s_stormlib_cortex  # NOQA
+import synapse.lib.stormlib.hashes as s_stormlib_hashes  # NOQA
+import synapse.lib.stormlib.random as s_stormlib_random  # NOQA
+import synapse.lib.stormlib.scrape as s_stormlib_scrape   # NOQA
 import synapse.lib.stormlib.infosec as s_stormlib_infosec  # NOQA
 import synapse.lib.stormlib.project as s_stormlib_project  # NOQA
 import synapse.lib.stormlib.version as s_stormlib_version  # NOQA
@@ -574,7 +577,7 @@ class CoreApi(s_cell.CellApi):
 
         view = self.cell.getView(viewiden, user=self.user)
         if view is None:
-            raise s_exc.NoSuchView(iden=viewiden)
+            raise s_exc.NoSuchView(mesg=f'No such view iden={viewiden}', iden=viewiden)
 
         wlyr = view.layers[0]
         parts = name.split('.')
@@ -1045,6 +1048,10 @@ class CoreApi(s_cell.CellApi):
         async for item in self.cell.watchAllUserNotifs(offs=offs):
             yield item
 
+    @s_cell.adminapi()
+    async def getHttpExtApiByPath(self, path):
+        return await self.cell.getHttpExtApiByPath(path)
+
 class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     '''
     A Cortex implements the synapse hypergraph.
@@ -1163,6 +1170,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         # NOTE: we may not make *any* nexus actions in this method
         self.macrodb = self.slab.initdb('storm:macros')
+        self.httpextapidb = self.slab.initdb('http:ext:apis')
 
         if self.inaugural:
             await self.cellinfo.set('cortex:version', s_version.version)
@@ -1239,6 +1247,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         self.stormiface_scrape = self.conf.get('storm:interface:scrape')
 
         self._initCortexHttpApi()
+        self._exthttpapis = {}  # iden -> adef; relies on cpython ordered dictionary behavior.
+        self._exthttpapiorder = b'exthttpapiorder'
+        self._exthttpapicache = s_cache.FixedCache(self._getHttpExtApiByPath, size=1000)
+        self._initCortexExtHttpApi()
 
         self.model = s_datamodel.Model()
 
@@ -1259,10 +1271,14 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         await self._initOAuthManager()
 
+        stormdmonhive = await self.hive.open(('cortex', 'storm', 'dmons'))
+        self.stormdmonhive = await stormdmonhive.dict()
         self.stormdmons = await s_storm.DmonManager.anit(self)
         self.onfini(self.stormdmons)
+
         self.agenda = await s_agenda.Agenda.anit(self)
         self.onfini(self.agenda)
+
         await self._initStormGraphs()
 
         self.trigson = self.conf.get('trigger:enable')
@@ -2221,10 +2237,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
     async def _initStormDmons(self):
 
-        node = await self.hive.open(('cortex', 'storm', 'dmons'))
-
-        self.stormdmonhive = await node.dict()
-
         for iden, ddef in self.stormdmonhive.items():
             try:
                 await self.runStormDmon(iden, ddef)
@@ -3291,7 +3303,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         view = self.views.get(iden)
         if view is None:
-            raise s_exc.NoSuchView(iden=iden)
+            raise s_exc.NoSuchView(mesg=f'No such view {iden=}', iden=iden)
 
         async with await s_queue.Window.anit(maxsize=10000) as wind:
 
@@ -3848,11 +3860,11 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         async with await s_base.Base.anit() as base:
 
-            def addlayr(layr, newlayer=False):
+            def addlayr(layr, newlayer=False, startoffs=topoffs):
                 '''
                 A new layer joins the live stream
                 '''
-                genr = genrfunc(layr, topoffs, newlayer=newlayer)
+                genr = genrfunc(layr, startoffs, newlayer=newlayer)
                 layrgenrs[layr.iden] = genr
                 task = base.schedCoro(genr.__anext__())
                 task.iden = layr.iden
@@ -3873,6 +3885,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
             # First, catch up to what was the current offset when we started, guaranteeing order
 
+            logger.debug(f'_syncNodeEdits() running catch-up sync to offs={topoffs}')
+
             genrs = [genrfunc(layr, offsdict.get(layr.iden, 0), endoff=topoffs) for layr in self.layers.values()]
             async for item in s_common.merggenr(genrs, lambda x, y: x[0] < y[0]):
                 yield item
@@ -3884,6 +3898,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
             # After we've caught up, read on genrs from all the layers simultaneously
 
+            logger.debug('_syncNodeEdits() entering into live sync')
+
+            lastoffs = {}
+
             todo.clear()
 
             for layr in self.layers.values():
@@ -3922,6 +3940,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
                         yield result
 
+                        lastoffs[layriden] = result[0]
+
                         # Re-add a task to wait on the next iteration of the generator
                         genr = layrgenrs[layriden]
                         task = base.schedCoro(genr.__anext__())
@@ -3929,9 +3949,21 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                         todo.add(task)
 
                     except StopAsyncIteration:
+
                         # Help out the garbage collector
                         del layrgenrs[layriden]
 
+                        layr = self.getLayer(iden=layriden)
+                        if layr is None or not layr.logedits:
+                            logger.debug(f'_syncNodeEdits() removed {layriden=} from sync')
+                            continue
+
+                        startoffs = lastoffs[layriden] + 1 if layriden in lastoffs else topoffs
+                        logger.debug(f'_syncNodeEdits() restarting {layriden=} live sync from offs={startoffs}')
+                        addlayr(layr, startoffs=startoffs)
+
+                        await self.waitfini(1)
+
     async def spliceHistory(self, user):
         '''
         Yield splices backwards from the end of the nodeedit log.
@@ -4152,6 +4184,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         self.addStormCmd(s_storm.SpliceListCmd)
         self.addStormCmd(s_storm.SpliceUndoCmd)
         self.addStormCmd(s_stormlib_macro.MacroExecCmd)
+        self.addStormCmd(s_stormlib_stats.StatsCountByCmd)
 
         for cdef in s_stormsvc.stormcmds:
             await self._trySetStormCmd(cdef.get('name'), cdef)
@@ -4171,6 +4204,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         for cdef in s_stormlib_model.stormcmds:
             await self._trySetStormCmd(cdef.get('name'), cdef)
 
+        for cdef in s_stormlib_cortex.stormcmds:
+            await self._trySetStormCmd(cdef.get('name'), cdef)
+
     async def _initPureStormCmds(self):
         oldcmds = []
         for name, cdef in self.cmdhive.items():
@@ -4257,6 +4293,158 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         self.addHttpApi('/api/v1/axon/files/by/sha256/([0-9a-fA-F]{64}$)', CortexAxonHttpBySha256V1, {'cell': self})
         self.addHttpApi('/api/v1/axon/files/by/sha256/(.*)', CortexAxonHttpBySha256InvalidV1, {'cell': self})
 
+        self.addHttpApi('/api/ext/(.*)', s_httpapi.ExtApiHandler, {'cell': self})
+
+    def _initCortexExtHttpApi(self):
+        self._exthttpapis.clear()
+        self._exthttpapicache.clear()
+
+        byts = self.slab.get(self._exthttpapiorder, self.httpextapidb)
+        if byts is None:
+            return
+
+        order = s_msgpack.un(byts)
+
+        for iden in order:
+            byts = self.slab.get(s_common.uhex(iden), self.httpextapidb)
+            if byts is None:  # pragma: no cover
+                logger.error(f'Missing HTTP API definition for iden={iden}')
+                continue
+            adef = s_msgpack.un(byts)
+            self._exthttpapis[adef.get('iden')] = adef
+
+    async def addHttpExtApi(self, adef):
+        path = adef.get('path')
+        try:
+            _ = regex.compile(path)
+        except Exception as e:
+            mesg = f'Invalid path for Extended HTTP API - cannot compile regular expression for [{path}] : {e}'
+            raise s_exc.BadArg(mesg=mesg) from None
+        adef['iden'] = s_common.guid()
+        adef['created'] = s_common.now()
+        adef['updated'] = adef['created']
+        adef = s_schemas.reqValidHttpExtAPIConf(adef)
+        return await self._push('http:api:add', adef)
+
+    @s_nexus.Pusher.onPush('http:api:add')
+    async def _addHttpExtApi(self, adef):
+        iden = adef.get('iden')
+        self.slab.put(s_common.uhex(iden), s_msgpack.en(adef), db=self.httpextapidb)
+
+        order = self.slab.get(self._exthttpapiorder, db=self.httpextapidb)
+        if order is None:
+            self.slab.put(self._exthttpapiorder, s_msgpack.en([iden]), db=self.httpextapidb)
+        else:
+            order = s_msgpack.un(order)  # type: tuple
+            if iden not in order:
+                # Replay safety
+                order = order + (iden, )  # New handlers go to the end of the list of handlers
+                self.slab.put(self._exthttpapiorder, s_msgpack.en(order), db=self.httpextapidb)
+
+        # Re-initialize the HTTP API list from the index order
+        self._initCortexExtHttpApi()
+        return adef
+
+    @s_nexus.Pusher.onPushAuto('http:api:del')
+    async def delHttpExtApi(self, iden):
+        if s_common.isguid(iden) is False:
+            raise s_exc.BadArg(mesg=f'Must provide an iden. Got {iden}')
+
+        self.slab.pop(s_common.uhex(iden), db=self.httpextapidb)
+
+        byts = self.slab.get(self._exthttpapiorder, self.httpextapidb)
+        order = list(s_msgpack.un(byts))
+        if iden in order:
+            order.remove(iden)
+            self.slab.put(self._exthttpapiorder, s_msgpack.en(order), db=self.httpextapidb)
+
+        self._initCortexExtHttpApi()
+
+        return
+
+    @s_nexus.Pusher.onPushAuto('http:api:mod')
+    async def modHttpExtApi(self, iden, name, valu):
+        # Created, Creator, Updated are not mutable
+        if name in ('name', 'desc', 'runas', 'methods', 'authenticated', 'perms', 'readonly', 'vars'):
+            # Schema takes care of these values
+            pass
+        elif name == 'owner':
+            _obj = await self.getUserDef(valu, packroles=False)
+            if _obj is None:
+                raise s_exc.NoSuchUser(mesg=f'Cannot set owner={valu} on extended httpapi, it does not exist.')
+        elif name == 'view':
+            _obj = self.getView(valu)
+            if _obj is None:
+                raise s_exc.NoSuchView(mesg=f'Cannot set view={valu} on extended httpapi, it does not exist.')
+        elif name == 'path':
+            try:
+                _ = regex.compile(valu)
+            except Exception as e:
+                mesg = f'Invalid path for Extended HTTP API - cannot compile regular expression for [{valu}] : {e}'
+                raise s_exc.BadArg(mesg=mesg) from None
+        else:
+            raise s_exc.BadArg(mesg=f'Cannot set {name=} on extended httpapi')
+
+        byts = self.slab.get(s_common.uhex(iden), db=self.httpextapidb)
+        if byts is None:
+            raise s_exc.NoSuchIden(mesg=f'No http api for {iden=}', iden=iden)
+
+        adef = s_msgpack.un(byts)
+        adef[name] = valu
+        adef['updated'] = s_common.now()
+        adef = s_schemas.reqValidHttpExtAPIConf(adef)
+        self.slab.put(s_common.uhex(iden), s_msgpack.en(adef), db=self.httpextapidb)
+
+        self._initCortexExtHttpApi()
+
+        return adef
+
+    @s_nexus.Pusher.onPushAuto('http:api:indx')
+    async def setHttpApiIndx(self, iden, indx):
+        if indx < 0:
+            raise s_exc.BadArg(mesg=f'indx must be greater than or equal to 0; got {indx}')
+        byts = self.slab.get(self._exthttpapiorder, db=self.httpextapidb)
+        if byts is None:
+            raise s_exc.SynErr(mesg='No Extended HTTP handlers registered. Cannot set order.')
+
+        order = list(s_msgpack.un(byts))
+
+        if iden not in order:
+            raise s_exc.NoSuchIden(mesg=f'Extended HTTP API is not set: {iden}')
+
+        if order.index(iden) == indx:
+            return indx
+        order.remove(iden)
+        # indx values > length of the list end up at the end of the list.
+        order.insert(indx, iden)
+        self.slab.put(self._exthttpapiorder, s_msgpack.en(order), db=self.httpextapidb)
+        self._initCortexExtHttpApi()
+        return order.index(iden)
+
+    async def getHttpExtApis(self):
+        return copy.deepcopy(list(self._exthttpapis.values()))
+
+    async def getHttpExtApi(self, iden):
+        adef = self._exthttpapis.get(iden)
+        if adef is None:
+            raise s_exc.NoSuchIden(mesg=f'No extended http api for {iden=}', iden=iden)
+        return copy.deepcopy(adef)
+
+    async def getHttpExtApiByPath(self, path):
+        iden, args = self._exthttpapicache.get(path)
+        adef = copy.deepcopy(self._exthttpapis.get(iden))
+        return adef, args
+
+    def _getHttpExtApiByPath(self, key):
+        # Cache callback.
+        # Returns (iden, args) or (None, ()) for caching.
+        for iden, adef in self._exthttpapis.items():
+            match = regex.fullmatch(adef.get('path'), key)
+            if match is None:
+                continue
+            return iden, match.groups()
+        return None, ()
+
     async def getCellApi(self, link, user, path):
 
         if not path:
@@ -4560,7 +4748,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     async def delView(self, iden):
         view = self.views.get(iden)
         if view is None:
-            raise s_exc.NoSuchView(iden=iden)
+            raise s_exc.NoSuchView(mesg=f'No such view {iden=}', iden=iden)
 
         return await self._push('view:del', iden)
 
@@ -4633,7 +4821,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         '''
         view = self.getView(iden)
         if view is None:
-            raise s_exc.NoSuchView(iden=iden)
+            raise s_exc.NoSuchView(mesg=f'No such view {iden=}', iden=iden)
 
         await view.setLayers(layers)
         self._calcViewsByLayer()
@@ -5432,7 +5620,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         view = self.views.get(viewiden)
         if view is None:
-            raise s_exc.NoSuchView(iden=viewiden)
+            raise s_exc.NoSuchView(mesg=f'No such view iden={viewiden}', iden=viewiden)
 
         user.confirm(('view', 'read'), gateiden=viewiden)
 
@@ -5750,7 +5938,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         view = self.getView(viewiden)
         if view is None:
-            raise s_exc.NoSuchView(iden=viewiden)
+            raise s_exc.NoSuchView(mesg=f'No such view iden={viewiden}', iden=viewiden)
 
         async with await self.snap(view=view) as snap:
             snap.strict = False

synapse/exc.py

@@ -319,5 +319,3 @@ class FatalErr(SynErr):
     pass
 
 class LmdbLock(SynErr): pass
-
-proxy_admin_mesg = 'Specifying a proxy to the HTTP library requires admin privileges.'

synapse/lib/ast.py

@@ -215,19 +215,18 @@ class Query(AstNode):
         if genr is None:
             genr = runt.getInput()
 
-        genr = self.run(runt, genr)
+        async with contextlib.aclosing(self.run(runt, genr)) as agen:
+            async for node, path in agen:
 
-        async for node, path in genr:
-
-            runt.tick()
+                runt.tick()
 
-            yield node, path
+                yield node, path
 
-            count += 1
+                count += 1
 
-            limit = runt.getOpt('limit')
-            if limit is not None and count >= limit:
-                break
+                limit = runt.getOpt('limit')
+                if limit is not None and count >= limit:
+                    break
 
 class Lookup(Query):
     '''
@@ -594,10 +593,15 @@ class SubQuery(Oper):
         async for item in self.kids[0].run(runt, genr):
             yield item
 
-    async def _compute(self, runt, limit):
+    async def _compute(self, runt, path, limit):
+
         retn = []
 
-        async with runt.getSubRuntime(self.kids[0]) as runt:
+        opts = {}
+        if path is not None:
+            opts['vars'] = path.vars.copy()
+
+        async with runt.getSubRuntime(self.kids[0], opts=opts) as runt:
             async for valunode, valupath in runt.execute():
 
                 retn.append(valunode.ndef[1])
@@ -616,7 +620,7 @@ class SubQuery(Oper):
         Its value is the primary property of the node yielded, or the returned value.
         '''
         try:
-            retn = await self._compute(runt, 1)
+            retn = await self._compute(runt, path, 1)
 
         except s_stormctrl.StormReturn as e:
             # a subquery assignment with a return; just use the returned value
@@ -632,7 +636,7 @@ class SubQuery(Oper):
         Use subquery as an array.
         '''
         try:
-            return await self._compute(runt, 128)
+            return await self._compute(runt, path, 128)
         except s_stormctrl.StormReturn as e:
             # a subquery assignment with a return; just use the returned value
             return e.item
@@ -2710,6 +2714,10 @@ class TagValuCond(Cond):
         if isinstance(lnode, VarValue) or not lnode.isconst:
             async def cond(node, path):
                 name = await lnode.compute(runt, path)
+                if '*' in name:
+                    mesg = f'Wildcard tag names may not be used in conjunction with tag value comparison: {name}'
+                    raise self.addExcInfo(s_exc.StormRuntimeError(mesg=mesg, name=name))
+
                 valu = await rnode.compute(runt, path)
                 return cmprctor(valu)(node.tags.get(name))
 
@@ -2796,6 +2804,10 @@ class TagPropCond(Cond):
             tag = await self.kids[0].compute(runt, path)
             name = await self.kids[1].compute(runt, path)
 
+            if '*' in tag:
+                mesg = f'Wildcard tag names may not be used in conjunction with tagprop value comparison: {tag}'
+                raise self.addExcInfo(s_exc.StormRuntimeError(mesg=mesg, name=tag))
+
             prop = runt.model.getTagProp(name)
             if prop is None:
                 mesg = f'No such tag property: {name}'
@@ -3019,7 +3031,8 @@ class VarDeref(Value):
         name = await self.kids[1].compute(runt, path)
 
         valu = s_stormtypes.fromprim(base, path=path)
-        return await valu.deref(name)
+        with s_scope.enter({'runt': runt}):
+            return await valu.deref(name)
 
 class FuncCall(Value):
 
@@ -3097,7 +3110,7 @@ async def expr_prefix(x, y):
     return x.startswith(y)
 
 async def expr_re(x, y):
-    if regex.search(await tostr(y), await tostr(x)):
+    if regex.search(await tostr(y), await tostr(x), flags=regex.I):
         return True
     return False
 
@@ -3186,8 +3199,11 @@ class ExprAndNode(Value):
 class TagName(Value):
 
     def prepare(self):
-        self.isconst = not self.kids or (len(self.kids) == 1 and isinstance(self.kids[0], Const))
-        self.constval = self.kids[0].value() if self.isconst and self.kids else None
+        self.isconst = not self.kids or all(isinstance(k, Const) for k in self.kids)
+        if self.isconst and self.kids:
+            self.constval = '.'.join([k.value() for k in self.kids])
+        else:
+            self.constval = None
 
     async def compute(self, runt, path):
 
@@ -3593,8 +3609,9 @@ class EditNodeAdd(Edit):
             async for node, path in genr:
                 yield node, path
 
-        async for item in s_base.schedGenr(feedfunc()):
-            yield item
+        async with contextlib.aclosing(s_base.schedGenr(feedfunc())) as agen:
+            async for item in agen:
+                yield item
 
 class EditPropSet(Edit):
 
@@ -4383,11 +4400,13 @@ class Function(AstNode):
                 subr.funcscope = True
                 try:
                     if self.hasemit:
-                        async for item in await subr.emitter():
-                            yield item
+                        async with contextlib.aclosing(await subr.emitter()) as agen:
+                            async for item in agen:
+                                yield item
                     else:
-                        async for node, path in subr.execute():
-                            yield node, path
+                        async with contextlib.aclosing(subr.execute()) as agen:
+                            async for node, path in agen:
+                                yield node, path
                 except s_stormctrl.StormStop:
                     return
 

synapse/lib/autodoc.py

@@ -173,7 +173,7 @@ def getArgLines(rtype):
             for obj in atyp:
                 assert isinstance(obj, str)
             tdata = ', '.join([f'``{obj}``' for obj in atyp])
-            rline = f'The input type may one one of the following: {tdata}.'
+            rline = f'The input type may be one of the following: {tdata}.'
             line = f'    {name}: {desc} {rline}'
         elif isinstance(atyp, dict):
             logger.warning('Fully declarative input types are not yet supported.')
@@ -287,7 +287,7 @@ def getReturnLines(rtype, known_types=None, types_prefix=None, suffix=None, isst
     elif isinstance(rtype, dict):
         returns = rtype.get('returns')
         assert returns is not None, f'Invalid returns for {rtype}'
-        name = returns.get('name', 'Returns')
+        name = returns.get('name', 'Returns').title()
 
         desc = returns.get('desc')
         rettype = returns.get('type')