synapse 2.218.0__py311-none-any.whl → 2.219.0__py311-none-any.whl

synapse/cortex.py

@@ -897,6 +897,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         self.stormmods = {}     # name: mdef
         self.stormpkgs = {}     # name: pkgdef
         self.stormvars = None   # type: s_lmdbslab.SafeKeyVal
+        self.stormpkgvars = {}  # type: Dict[str, s_lmdbslab.SafeKeyVal]
 
         self.svcsbyiden = {}
         self.svcsbyname = {}
@@ -2883,11 +2884,23 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         mods = pkgdef.get('modules', ())
         cmds = pkgdef.get('commands', ())
         onload = pkgdef.get('onload')
+        inits = pkgdef.get('inits')
         svciden = pkgdef.get('svciden')
 
         if onload is not None and validstorm:
             await self.getStormQuery(onload)
 
+        if inits is not None:
+            lastver = None
+            for initdef in inits.get('versions'):
+                curver = initdef.get('version')
+                if lastver is not None and not curver > lastver:
+                    raise s_exc.BadPkgDef(mesg='Init versions must be monotonically increasing.', version=curver)
+                lastver = curver
+
+                if validstorm:
+                    await self.getStormQuery(initdef.get('query'))
+
         for mdef in mods:
             mdef.setdefault('modconf', {})
             if svciden:
@@ -2968,28 +2981,88 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
     def _runStormPkgOnload(self, pkgdef):
         name = pkgdef.get('name')
+        inits = pkgdef.get('inits')
         onload = pkgdef.get('onload')
+        pkgvers = pkgdef.get('version')
 
-        if onload is not None and self.isactive:
+        if (onload is not None or inits is not None) and self.isactive:
             async def _onload():
                 if self.safemode:
                     await self.fire('core:pkg:onload:skipped', pkg=name, reason='safemode')
                     return
 
                 await self.fire('core:pkg:onload:start', pkg=name)
-                try:
-                    async for mesg in self.storm(onload):
-                        if mesg[0] == 'print':
-                            logger.info(f'{name} onload output: {mesg[1].get("mesg")}')
-                        if mesg[0] == 'warn':
-                            logger.warning(f'{name} onload output: {mesg[1].get("mesg")}')
-                        if mesg[0] == 'err':
-                            logger.error(f'{name} onload output: {mesg[1]}')
-                        await asyncio.sleep(0)
-                except asyncio.CancelledError:  # pragma: no cover
-                    raise
-                except Exception as exc:  # pragma: no cover
-                    logger.warning(f'onload failed for package: {name}', exc_info=exc)
+
+                logextra = await self.getLogExtra(pkg=name, vers=pkgvers)
+
+                if inits is not None:
+                    varname = inits['key']
+                    curvers = await self.getStormPkgVar(name, varname, default=-1)
+                    inaugural = curvers == -1
+
+                    for initdef in inits['versions']:
+
+                        vers = initdef['version']
+                        vname = initdef['name']
+
+                        if vers <= curvers:
+                            continue
+
+                        if inaugural and not initdef.get('inaugural'):
+                            await self.setStormPkgVar(name, varname, vers)
+                            continue
+
+                        logextra['synapse']['initvers'] = vers
+
+                        logger.info(f'{name} starting init vers={vers}: {vname}', extra=logextra)
+
+                        ok = True
+
+                        try:
+                            async for mesg in self.storm(initdef['query']):
+                                match mesg[0]:
+                                    case 'print':
+                                        msg = f'{name} init vers={vers} output: {mesg[1].get("mesg")}'
+                                        logger.info(msg, extra=logextra)
+                                    case 'warn':
+                                        msg = f'{name} init vers={vers} output: {mesg[1].get("mesg")}'
+                                        logger.warning(msg, extra=logextra)
+                                    case 'err':
+                                        msg = f'{name} init vers={vers} output: {mesg[1]}'
+                                        logger.error(msg, extra=logextra)
+                                        ok = False
+                                await asyncio.sleep(0)
+                        except asyncio.CancelledError:  # pragma: no cover
+                            raise
+                        except Exception as exc:  # pragma: no cover
+                            msg = f'{name} init failed for vers={vers}: {vname}'
+                            logger.warning(msg, exc_info=exc, extra=logextra)
+                            ok = False
+
+                        if not ok:
+                            break
+
+                        curvers = vers
+                        await self.setStormPkgVar(name, varname, vers)
+                        logger.info(f'{name} finished init vers={vers}: {vname}', extra=logextra)
+
+                if onload is not None:
+                    try:
+                        async for mesg in self.storm(onload):
+                            if mesg[0] == 'print':
+                                logger.info(f'{name} onload output: {mesg[1].get("mesg")}', extra=logextra)
+                            if mesg[0] == 'warn':
+                                logger.warning(f'{name} onload output: {mesg[1].get("mesg")}', extra=logextra)
+                            if mesg[0] == 'err':
+                                logger.error(f'{name} onload output: {mesg[1]}', extra=logextra)
+                            await asyncio.sleep(0)
+                    except asyncio.CancelledError:  # pragma: no cover
+                        raise
+                    except Exception as exc:  # pragma: no cover
+                        logger.warning(f'onload failed for package: {name}', exc_info=exc, extra=logextra)
+
+                    logger.info(f'{name} finished onload', extra=logextra)
+
                 await self.fire('core:pkg:onload:complete', pkg=name)
 
             self.runActiveTask(_onload())
@@ -3229,6 +3302,32 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         for item in self.stormvars.items():
             yield item
 
+    # Storm package vars APIs
+
+    def _getStormPkgVarKV(self, name):
+        if (pkgvars := self.stormpkgvars.get(name)) is None:
+            self.stormpkgvars[name] = pkgvars = self.cortexdata.getSubKeyVal(f'stormpkg:vars:{name}:')
+        return pkgvars
+
+    async def getStormPkgVar(self, name, key, default=None):
+        pkgvars = self._getStormPkgVarKV(name)
+        return pkgvars.get(key, defv=default)
+
+    @s_nexus.Pusher.onPushAuto('storm:pkg:var:pop')
+    async def popStormPkgVar(self, name, key, default=None):
+        pkgvars = self._getStormPkgVarKV(name)
+        return pkgvars.pop(key, defv=default)
+
+    @s_nexus.Pusher.onPushAuto('storm:pkg:var:set')
+    async def setStormPkgVar(self, name, key, valu):
+        pkgvars = self._getStormPkgVarKV(name)
+        return pkgvars.set(key, valu)
+
+    async def iterStormPkgVars(self, name):
+        pkgvars = self._getStormPkgVarKV(name)
+        for item in pkgvars.items():
+            yield item
+
     async def _cortexHealth(self, health):
         health.update('cortex', 'nominal')
 
@@ -5500,7 +5599,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         # TODO Remove the defaults in 3.0.0
         csize = pdef.get('chunk:size', s_const.layer_pdef_csize)
         qsize = pdef.get('queue:size', s_const.layer_pdef_qsize)
-        soffs = pdef.get('offs', -1)
+        soffs = max(pdef.get('offs', 0), 0)
 
         async with await s_base.Base.anit() as base:
 
@@ -5509,8 +5608,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             async def fill():
 
                 try:
-                    filloffs = await self.getStormVar(gvar, soffs)
-                    async for item in layr0.syncNodeEdits(filloffs + 1, wait=True):
+                    if (filloffs := await self.getStormVar(gvar)) is not None:
+                        filloffs += 1
+                    else:
+                        filloffs = soffs
+
+                    async for item in layr0.syncNodeEdits(filloffs, wait=True):
                         await queue.put(item)
                     await queue.close()
 

synapse/daemon.py

@@ -213,7 +213,8 @@ async def t2call(link, meth, args, kwargs):
         return
 
     except (asyncio.CancelledError, Exception) as e:
-        logger.exception(f'error during task: {meth.__name__} {e}')
+        if not isinstance(e, asyncio.CancelledError):
+            logger.exception(f'error during task: {meth.__name__} {e}')
         if not link.isfini:
             retn = s_common.retnexc(e)
             await link.tx(('t2:fini', {'retn': retn}))

synapse/lib/ast.py

@@ -1081,64 +1081,65 @@ class ForLoop(Oper):
             if valu is None:
                 valu = ()
 
-            async with contextlib.aclosing(s_coro.agen(valu)) as agen:
+            with s_scope.enter({'runt': runt}):
+                async with contextlib.aclosing(s_coro.agen(valu)) as agen:
 
-                try:
-                    agen, _ = await pullone(agen)
-                except TypeError:
-                    styp = await s_stormtypes.totype(valu, basetypes=True)
-                    mesg = f"'{styp}' object is not iterable: {s_common.trimText(repr(valu))}"
-                    raise self.kids[1].addExcInfo(s_exc.StormRuntimeError(mesg=mesg, type=styp)) from None
+                    try:
+                        agen, _ = await pullone(agen)
+                    except TypeError:
+                        styp = await s_stormtypes.totype(valu, basetypes=True)
+                        mesg = f"'{styp}' object is not iterable: {s_common.trimText(repr(valu))}"
+                        raise self.kids[1].addExcInfo(s_exc.StormRuntimeError(mesg=mesg, type=styp)) from None
 
-                async for item in agen:
+                    async for item in agen:
 
-                    if isinstance(name, (list, tuple)):
+                        if isinstance(name, (list, tuple)):
 
-                        try:
-                            numitems = len(item)
-                        except TypeError:
-                            mesg = f'Number of items to unpack does not match the number of variables: {s_common.trimText(repr(item))}'
-                            exc = s_exc.StormVarListError(mesg=mesg, names=name)
-                            raise self.kids[1].addExcInfo(exc)
+                            try:
+                                numitems = len(item)
+                            except TypeError:
+                                mesg = f'Number of items to unpack does not match the number of variables: {s_common.trimText(repr(item))}'
+                                exc = s_exc.StormVarListError(mesg=mesg, names=name)
+                                raise self.kids[1].addExcInfo(exc)
 
-                        if len(name) != numitems:
-                            mesg = f'Number of items to unpack does not match the number of variables: {s_common.trimText(repr(item))}'
-                            exc = s_exc.StormVarListError(mesg=mesg, names=name, numitems=numitems)
-                            raise self.kids[1].addExcInfo(exc)
+                            if len(name) != numitems:
+                                mesg = f'Number of items to unpack does not match the number of variables: {s_common.trimText(repr(item))}'
+                                exc = s_exc.StormVarListError(mesg=mesg, names=name, numitems=numitems)
+                                raise self.kids[1].addExcInfo(exc)
 
-                        if isinstance(item, s_stormtypes.Prim):
-                            item = await item.value()
+                            if isinstance(item, s_stormtypes.Prim):
+                                item = await item.value()
 
-                        for x, y in itertools.zip_longest(name, item):
-                            await path.setVar(x, y)
-                            await runt.setVar(x, y)
+                            for x, y in itertools.zip_longest(name, item):
+                                await path.setVar(x, y)
+                                await runt.setVar(x, y)
 
-                    else:
-                        # set both so inner subqueries have it in their runtime
-                        await path.setVar(name, item)
-                        await runt.setVar(name, item)
+                        else:
+                            # set both so inner subqueries have it in their runtime
+                            await path.setVar(name, item)
+                            await runt.setVar(name, item)
 
-                    try:
+                        try:
 
-                        # since it's possible to "multiply" the (node, path)
-                        # we must make a clone of the path to prevent yield-then-use.
-                        newg = s_common.agen((node, path.clone()))
-                        async for item in subq.inline(runt, newg):
-                            yield item
+                            # since it's possible to "multiply" the (node, path)
+                            # we must make a clone of the path to prevent yield-then-use.
+                            newg = s_common.agen((node, path.clone()))
+                            async for item in subq.inline(runt, newg):
+                                yield item
 
-                    except s_stormctrl.StormBreak as e:
-                        if (eitem := e.get('item')) is not None:
-                            yield eitem
-                        break
+                        except s_stormctrl.StormBreak as e:
+                            if (eitem := e.get('item')) is not None:
+                                yield eitem
+                            break
 
-                    except s_stormctrl.StormContinue as e:
-                        if (eitem := e.get('item')) is not None:
-                            yield eitem
-                        continue
+                        except s_stormctrl.StormContinue as e:
+                            if (eitem := e.get('item')) is not None:
+                                yield eitem
+                            continue
 
-                    finally:
-                        # for loops must yield per item they iterate over
-                        await asyncio.sleep(0)
+                        finally:
+                            # for loops must yield per item they iterate over
+                            await asyncio.sleep(0)
 
         # no nodes and a runt safe value should execute once
         if node is None and self.kids[1].isRuntSafe(runt):
@@ -1155,56 +1156,57 @@ class ForLoop(Oper):
             if valu is None:
                 valu = ()
 
-            async with contextlib.aclosing(s_coro.agen(valu)) as agen:
-                try:
-                    agen, _ = await pullone(agen)
-                except TypeError:
-                    styp = await s_stormtypes.totype(valu, basetypes=True)
-                    mesg = f"'{styp}' object is not iterable: {s_common.trimText(repr(valu))}"
-                    raise self.kids[1].addExcInfo(s_exc.StormRuntimeError(mesg=mesg, type=styp)) from None
+            with s_scope.enter({'runt': runt}):
+                async with contextlib.aclosing(s_coro.agen(valu)) as agen:
+                    try:
+                        agen, _ = await pullone(agen)
+                    except TypeError:
+                        styp = await s_stormtypes.totype(valu, basetypes=True)
+                        mesg = f"'{styp}' object is not iterable: {s_common.trimText(repr(valu))}"
+                        raise self.kids[1].addExcInfo(s_exc.StormRuntimeError(mesg=mesg, type=styp)) from None
 
-                async for item in agen:
+                    async for item in agen:
 
-                    if isinstance(name, (list, tuple)):
+                        if isinstance(name, (list, tuple)):
 
-                        try:
-                            numitems = len(item)
-                        except TypeError:
-                            mesg = f'Number of items to unpack does not match the number of variables: {s_common.trimText(repr(item))}'
-                            exc = s_exc.StormVarListError(mesg=mesg, names=name)
-                            raise self.kids[1].addExcInfo(exc)
+                            try:
+                                numitems = len(item)
+                            except TypeError:
+                                mesg = f'Number of items to unpack does not match the number of variables: {s_common.trimText(repr(item))}'
+                                exc = s_exc.StormVarListError(mesg=mesg, names=name)
+                                raise self.kids[1].addExcInfo(exc)
 
-                        if len(name) != numitems:
-                            mesg = f'Number of items to unpack does not match the number of variables: {s_common.trimText(repr(item))}'
-                            exc = s_exc.StormVarListError(mesg=mesg, names=name, numitems=numitems)
-                            raise self.kids[1].addExcInfo(exc)
+                            if len(name) != numitems:
+                                mesg = f'Number of items to unpack does not match the number of variables: {s_common.trimText(repr(item))}'
+                                exc = s_exc.StormVarListError(mesg=mesg, names=name, numitems=numitems)
+                                raise self.kids[1].addExcInfo(exc)
 
-                        if isinstance(item, s_stormtypes.Prim):
-                            item = await item.value()
+                            if isinstance(item, s_stormtypes.Prim):
+                                item = await item.value()
 
-                        for x, y in itertools.zip_longest(name, item):
-                            await runt.setVar(x, y)
+                            for x, y in itertools.zip_longest(name, item):
+                                await runt.setVar(x, y)
 
-                    else:
-                        await runt.setVar(name, item)
+                        else:
+                            await runt.setVar(name, item)
 
-                    try:
-                        async for jtem in subq.inline(runt, s_common.agen()):
-                            yield jtem
+                        try:
+                            async for jtem in subq.inline(runt, s_common.agen()):
+                                yield jtem
 
-                    except s_stormctrl.StormBreak as e:
-                        if (eitem := e.get('item')) is not None:
-                            yield eitem
-                        break
+                        except s_stormctrl.StormBreak as e:
+                            if (eitem := e.get('item')) is not None:
+                                yield eitem
+                            break
 
-                    except s_stormctrl.StormContinue as e:
-                        if (eitem := e.get('item')) is not None:
-                            yield eitem
-                        continue
+                        except s_stormctrl.StormContinue as e:
+                            if (eitem := e.get('item')) is not None:
+                                yield eitem
+                            continue
 
-                    finally:
-                        # for loops must yield per item they iterate over
-                        await asyncio.sleep(0)
+                        finally:
+                            # for loops must yield per item they iterate over
+                            await asyncio.sleep(0)
 
 class WhileLoop(Oper):
 

synapse/lib/layer.py

@@ -3338,6 +3338,33 @@ class Layer(s_nexus.Pusher):
             sode['nodedata'] = {name: s_msgpack.un(byts)}
             yield None, buid, sode
 
+    async def setStorNodeProp(self, buid, prop, valu, meta):
+        newp = self.core.model.reqProp(prop)
+
+        newp_valu = newp.type.norm(valu)[0]
+        newp_name = newp.name
+        newp_stortype = newp.type.stortype
+        newp_formname = newp.form.name
+
+        set_edit = (EDIT_PROP_SET, (newp_name, newp_valu, None, newp_stortype), ())
+        nodeedits = [(buid, newp_formname, [set_edit])]
+
+        _, changes = await self.saveNodeEdits(nodeedits, meta)
+        return bool(changes[0][2])
+
+    async def delStorNodeProp(self, buid, prop, meta):
+        pprop = self.core.model.reqProp(prop)
+
+        oldp_name = pprop.name
+        oldp_formname = pprop.form.name
+        oldp_stortype = pprop.type.stortype
+
+        del_edit = (EDIT_PROP_DEL, (oldp_name, None, oldp_stortype), ())
+        nodeedits = [(buid, oldp_formname, [del_edit])]
+
+        _, changes = await self.saveNodeEdits(nodeedits, meta)
+        return bool(changes[0][2])
+
     async def storNodeEdits(self, nodeedits, meta):
 
         saveoff, results = await self.saveNodeEdits(nodeedits, meta)

synapse/lib/schemas.py

@@ -734,6 +734,19 @@ _reqValidPkgdefSchema = {
         'desc': {'type': 'string'},
         'svciden': {'type': ['string', 'null'], 'pattern': s_config.re_iden},
         'onload': {'type': 'string'},
+        'inits': {
+            'type': 'object',
+            'properties': {
+                'key': {'type': 'string'},
+                'versions': {
+                    'type': 'array',
+                    'items': {'$ref': '#/definitions/initdef'},
+                    'minItems': 1,
+                },
+            },
+            'additionalProperties': True,
+            'required': ['key', 'versions'],
+        },
         'author': {
             'type': 'object',
             'properties': {
@@ -807,6 +820,18 @@ _reqValidPkgdefSchema = {
             'additionalProperties': True,
             'required': ['name', 'storm']
         },
+        'initdef': {
+            'type': 'object',
+            'properties': {
+                'desc': {'type': 'string'},
+                'inaugural': {'type': 'boolean', 'default': False},
+                'name': {'type': 'string'},
+                'query': {'type': 'string'},
+                'version': {'type': 'integer', 'minimum': 0},
+            },
+            'additionalProperties': False,
+            'required': ['name', 'query', 'version']
+        },
         'apidef': {
             'type': 'object',
             'properties': {