synapse 2.173.1__py311-none-any.whl → 2.175.0__py311-none-any.whl

synapse/axon.py

@@ -493,7 +493,7 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
 
     async def wants(self, sha256s):
         '''
-        Get a list of sha256 values the axon does not have from a input list.
+        Get a list of sha256 values the axon does not have from an input list.
 
         Args:
             sha256s (list): A list of sha256 values as bytes.

synapse/common.py

@@ -11,6 +11,7 @@ import types
 import base64
 import shutil
 import struct
+import typing
 import asyncio
 import decimal
 import fnmatch
@@ -379,7 +380,7 @@ def reqbytes(*paths):
     with reqfile(*paths) as fd:
         return fd.read()
 
-def genfile(*paths):
+def genfile(*paths) -> typing.BinaryIO:
     '''
     Create or open (for read/write) a file path join.
 
@@ -396,7 +397,7 @@ def genfile(*paths):
         to append.
 
     Returns:
-        io.BufferedRandom: A file-object which can be read/written too.
+        A file-object which can be read/written too.
     '''
     path = genpath(*paths)
     gendir(os.path.dirname(path))
@@ -535,13 +536,26 @@ def yamlload(*paths):
     with io.open(path, 'rb') as fd:
         return yamlloads(fd)
 
+def yamldump(obj, stream: typing.Optional[typing.BinaryIO] =None) -> bytes:
+    '''
+    Dump a object to yaml.
+
+    Args:
+        obj: The object to serialize.
+        stream: The optional stream to write the stream too.
+
+    Returns:
+        The raw yaml bytes if stream is not provided.
+    '''
+    return yaml.dump(obj, allow_unicode=True, default_flow_style=False,
+                     default_style='', explicit_start=True, explicit_end=True,
+                     encoding='utf8', stream=stream, Dumper=Dumper)
+
 def yamlsave(obj, *paths):
     path = genpath(*paths)
     with genfile(path) as fd:
         fd.truncate(0)
-        yaml.dump(obj, allow_unicode=True, default_flow_style=False,
-                  default_style='', explicit_start=True, explicit_end=True,
-                  encoding='utf8', stream=fd, Dumper=Dumper)
+        yamldump(obj, stream=fd)
 
 def yamlmod(obj, *paths):
     '''

synapse/cortex.py

@@ -42,6 +42,7 @@ import synapse.lib.schemas as s_schemas
 import synapse.lib.spooled as s_spooled
 import synapse.lib.version as s_version
 import synapse.lib.urlhelp as s_urlhelp
+import synapse.lib.hashitem as s_hashitem
 import synapse.lib.jsonstor as s_jsonstor
 import synapse.lib.modelrev as s_modelrev
 import synapse.lib.stormsvc as s_stormsvc
@@ -949,6 +950,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         # Initialize our storage and views
         await self._initCoreAxon()
+        await self._initJsonStor()
 
         await self._initCoreLayers()
         await self._initCoreViews()
@@ -1009,8 +1011,16 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         await self._bumpCellVers('cortex:storage', (
             (1, self._storUpdateMacros),
             (2, self._storLayrFeedDefaults),
+            (3, self._updateTriggerViewIdens),
         ), nexs=False)
 
+    async def _updateTriggerViewIdens(self):
+        for view in self.views.values():
+            for trigiden, trigger in await view.listTriggers():
+                if trigger.get('view') != view.iden:
+                    trigger.tdef['view'] = view.iden
+                    await view.trigdict.set(trigiden, trigger.tdef)
+
     async def _viewNomergeToProtected(self):
         for view in self.views.values():
             nomerge = view.info.get('nomerge', False)
@@ -1462,8 +1472,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     async def initServiceRuntime(self):
 
         # do any post-nexus initialization here...
-        await self._initJsonStor()
-
         if self.isactive:
             await self._checkNexsIndx()
 
@@ -2550,7 +2558,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         name = pkgdef.get('name')
         olddef = self.pkghive.get(name, None)
         if olddef is not None:
-            await self._dropStormPkg(olddef)
+            if s_hashitem.hashitem(pkgdef) != s_hashitem.hashitem(olddef):
+                await self._dropStormPkg(olddef)
+            else:
+                return
 
         await self.loadStormPkg(pkgdef)
         await self.pkghive.set(name, pkgdef)
@@ -2965,17 +2976,19 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         '''
         Delete storm packages associated with a service.
         '''
-        oldpkgs = []
-        for _, pdef in self.pkghive.items():
-            pkgiden = pdef.get('svciden')
-            if pkgiden and pkgiden == iden:
-                oldpkgs.append(pdef)
-
-        for pkg in oldpkgs:
+        for pkg in self.getStormSvcPkgs(iden):
             name = pkg.get('name')
             if name:
                 await self._delStormPkg(name)
 
+    def getStormSvcPkgs(self, iden):
+        pkgs = []
+        for _, pdef in self.pkghive.items():
+            pkgiden = pdef.get('svciden')
+            if pkgiden and pkgiden == iden:
+                pkgs.append(pdef)
+        return pkgs
+
     async def setStormSvcEvents(self, iden, edef):
         '''
         Set the event callbacks for a storm service. Extends the sdef dict.
@@ -6113,6 +6126,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         Args:
             iden (bytes):  The iden of the cron job to be deleted
         '''
+        await self._killCronTask(iden)
         try:
             await self.agenda.delete(iden)
         except s_exc.NoSuchIden:
@@ -6142,6 +6156,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         '''
         await self.agenda.enable(iden)
         await self.feedBeholder('cron:enable', {'iden': iden}, gates=[iden])
+        logger.info(f'Enabled cron job {iden}', extra=await self.getLogExtra(iden=iden, status='MODIFY'))
 
     @s_nexus.Pusher.onPushAuto('cron:disable')
     async def disableCronJob(self, iden):
@@ -6152,7 +6167,28 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             iden (bytes):  The iden of the cron job to be changed
         '''
         await self.agenda.disable(iden)
+        await self._killCronTask(iden)
         await self.feedBeholder('cron:disable', {'iden': iden}, gates=[iden])
+        logger.info(f'Disabled cron job {iden}', extra=await self.getLogExtra(iden=iden, status='MODIFY'))
+
+    async def killCronTask(self, iden):
+        if self.agenda.appts.get(iden) is None:
+            return False
+        return await self._push('cron:task:kill', iden)
+
+    @s_nexus.Pusher.onPush('cron:task:kill')
+    async def _killCronTask(self, iden):
+
+        appt = self.agenda.appts.get(iden)
+        if appt is None:
+            return False
+
+        task = appt.task
+        if task is None:
+            return False
+
+        self.schedCoro(task.kill())
+        return True
 
     async def listCronJobs(self):
         '''

synapse/daemon.py

@@ -178,7 +178,7 @@ async def t2call(link, meth, args, kwargs):
             if isinstance(e, asyncio.CancelledError):
                 logger.info('t2call task %s cancelled', meth.__name__)
             else:
-                logger.exception('error during task %s', meth.__name__)
+                logger.exception(f'error during task {meth.__name__} {e}')
 
             if isinstance(valu, types.AsyncGeneratorType):
                 await valu.aclose()
@@ -211,7 +211,7 @@ async def t2call(link, meth, args, kwargs):
         return
 
     except (asyncio.CancelledError, Exception) as e:
-        logger.exception('error during task: %s', meth.__name__)
+        logger.exception(f'error during task: {meth.__name__} {e}')
         if not link.isfini:
             retn = s_common.retnexc(e)
             await link.tx(('t2:fini', {'retn': retn}))
@@ -380,24 +380,23 @@ class Daemon(s_base.Base):
 
         link.schedCoro(rxloop())
 
-    async def _onLinkMesg(self, link, mesg):
+    async def _onLinkMesg(self, link: s_link.Link, mesg):
 
         try:
             func = self.mesgfuncs.get(mesg[0])
             if func is None:
-                logger.exception('Dmon.onLinkMesg Invalid: %.80r' % (mesg,))
+                logger.error(f'Dmon.onLinkMesg Invalid mesg: mesg={s_common.trimText(repr(mesg), n=80)} '
+                             f'link={link.getAddrInfo()}')
                 return
 
             await func(link, mesg)
 
-        except asyncio.CancelledError:  # pragma: no cover  # TODO: remove once >= py 3.8 only
-            raise
-
         except ConnectionResetError:
-            logger.debug('Dmon.onLinkMesg Handler: connection reset')
+            logger.debug(f'Dmon.onLinkMesg Handler: connection reset link={link.getAddrInfo()}')
 
         except Exception:
-            logger.exception('Dmon.onLinkMesg Handler: %.80r' % (mesg,))
+            logger.exception(f'Dmon.onLinkMesg Handler: mesg={s_common.trimText(repr(mesg), n=80)} '
+                             f'link={link.getAddrInfo()}')
 
     async def _onShareFini(self, link, mesg):
 
@@ -473,7 +472,7 @@ class Daemon(s_base.Base):
             reply[1]['sess'] = sess.iden
 
         except Exception as e:
-            logger.exception('tele:syn error')
+            logger.exception(f'tele:syn error: {e} link={link.getAddrInfo()}')
             reply[1]['retn'] = s_common.retnexc(e)
 
         await link.tx(reply)
@@ -501,7 +500,7 @@ class Daemon(s_base.Base):
         typename = valu.typename
         return ('task:fini', {'task': task, 'retn': retn, 'type': typename})
 
-    async def _onTaskV2Init(self, link, mesg):
+    async def _onTaskV2Init(self, link: s_link.Link, mesg):
 
         # t2:init is used by the pool sockets on the client
         name = mesg[1].get('name')
@@ -539,7 +538,7 @@ class Daemon(s_base.Base):
                 sess.onfini(sessitem)
 
         except (asyncio.CancelledError, Exception) as e:
-            logger.exception('on t2:init: %r' % (mesg,))
+            logger.exception(f'Error on t2:init: {s_common.trimText(repr(mesg), n=80)} link={link.getAddrInfo()}')
             if not link.isfini:
                 retn = s_common.retnexc(e)
                 await link.tx(('t2:fini', {'retn': retn}))

synapse/lib/agenda.py

@@ -274,6 +274,7 @@ class _Appt:
     def __init__(self, stor, iden, recur, indx, query, creator, recs, nexttime=None, view=None, created=None, pool=False):
         self.doc = ''
         self.name = ''
+        self.task = None
         self.stor = stor
         self.pool = pool
         self.iden = iden
@@ -801,7 +802,12 @@ class Agenda(s_base.Base):
 
         coro = self._runJob(user, appt)
         task = self.core.runActiveTask(coro)
+
         appt.task = await self.core.boss.promotetask(task, f'Cron {appt.iden}', user, info=info)
+        async def fini():
+            appt.task = None
+
+        appt.task.onfini(fini)
 
     async def _markfailed(self, appt, reason):
         now = self._getNowTick()

synapse/lib/ast.py

@@ -2097,6 +2097,10 @@ class PivotIn(PivotOper):
             async for pivo in runt.snap.nodesByPropArray(prop.full, '=', valu):
                 yield pivo, path.fork(pivo)
 
+        async for refsbuid in runt.snap.getNdefRefs(node.buid):
+            pivo = await runt.snap.getNodeByBuid(refsbuid)
+            yield pivo, path.fork(pivo)
+
 class N2WalkNPivo(PivotIn):
 
     async def run(self, runt, genr):
@@ -3017,11 +3021,11 @@ class ArrayCond(Cond):
 
     async def getCondEval(self, runt):
 
-        name = await self.kids[0].compute(runt, None)
         cmpr = await self.kids[1].compute(runt, None)
 
         async def cond(node, path):
 
+            name = await self.kids[0].compute(runt, None)
             prop = node.form.props.get(name)
             if prop is None:
                 raise self.kids[0].addExcInfo(s_exc.NoSuchProp.init(name))

synapse/lib/cell.py

@@ -1100,6 +1100,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         self.inaugural = False
         self.activecoros = {}
         self.sockaddr = None  # Default value...
+        self.https_listeners = []
         self.ahaclient = None
         self._checkspace = s_coro.Event()
         self._reloadfuncs = {}  # name -> func
@@ -2822,6 +2823,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
             self.addReloadableSystem('https:certs', reload)
 
+        self.https_listeners.append({'host': lhost, 'port': lport})
+
         return (lhost, lport)
 
     def initSslCtx(self, certpath, keypath):
@@ -2877,18 +2880,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         log_method(mesg, extra=extra)
 
-    async def _initCellHttp(self):
-
-        self.httpds = []
-        self.sessstor = s_lmdbslab.GuidStor(self.slab, 'http:sess')
-
-        async def fini():
-            [await s.fini() for s in self.sessions.values()]
-            for http in self.httpds:
-                http.stop()
-
-        self.onfini(fini)
-
+    async def _getCellHttpOpts(self):
         # Generate/Load a Cookie Secret
         secpath = os.path.join(self.dirn, 'cookie.secret')
         if not os.path.isfile(secpath):
@@ -2898,12 +2890,26 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         with s_common.getfile(secpath) as fd:
             secret = fd.read().decode('utf8')
 
-        opts = {
+        return {
             'cookie_secret': secret,
             'log_function': self._log_web_request,
             'websocket_ping_interval': 10
         }
 
+    async def _initCellHttp(self):
+
+        self.httpds = []
+        self.sessstor = s_lmdbslab.GuidStor(self.slab, 'http:sess')
+
+        async def fini():
+            [await s.fini() for s in self.sessions.values()]
+            for http in self.httpds:
+                http.stop()
+
+        self.onfini(fini)
+
+        opts = await self._getCellHttpOpts()
+
         self.wapp = t_web.Application(**opts)
         self._initCellHttpApis()
 
@@ -4070,6 +4076,9 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                     'name': self.conf.get('aha:name'),
                     'leader': self.conf.get('aha:leader'),
                     'network': self.conf.get('aha:network'),
+                },
+                'network': {
+                    'https': self.https_listeners,
                 }
             },
             'features': {

synapse/lib/jupyter.py

@@ -38,6 +38,7 @@ def getDocPath(fn, root=None):
     Raises:
         ValueError if the file does not exist or directory traversal attempted..
     '''
+    s_common.deprdate('synapse.lib.jupyter.getDocPath', '2024-08-26')
     cwd = pathlib.Path(os.getcwd())
     if root:
         cwd = pathlib.Path(root)
@@ -84,6 +85,7 @@ def getDocData(fp, root=None):
     Raises:
         ValueError if the file does not exist or directory traversal attempted..
     '''
+    s_common.deprdate('synapse.lib.jupyter.getDocData', '2024-08-26')
     fpath = getDocPath(fp, root)
     if fpath.endswith('.yaml'):
         return s_common.yamlload(fpath)
@@ -103,6 +105,7 @@ def getDocData(fp, root=None):
 @contextlib.asynccontextmanager
 async def genTempCoreProxy(mods=None):
     '''Get a temporary cortex proxy.'''
+    s_common.deprdate('synapse.lib.jupyter.genTempCoreProxy', '2024-08-26')
     with s_common.getTempDir() as dirn:
         async with await s_cortex.Cortex.anit(dirn) as core:
             if mods:
@@ -115,6 +118,8 @@ async def genTempCoreProxy(mods=None):
 
 @contextlib.asynccontextmanager
 async def genTempStormsvcProxy(cmdrcore, svcname, svcctor, conf=None):
+    s_common.deprdate('synapse.lib.jupyter.genTempStormsvcProxy', '2024-08-26')
+
     if conf is None:
         conf = {}
 
@@ -142,12 +147,14 @@ async def genTempStormsvcProxy(cmdrcore, svcname, svcctor, conf=None):
 
 async def getItemStorm(prox, outp=None):
     '''Get a Storm CLI instance with prepopulated locs'''
+    s_common.deprdate('synapse.lib.jupyter.getItemStorm', '2024-08-26')
     storm = await s_t_storm.StormCli.anit(prox, outp=outp)
     storm.echoline = True
     return storm
 
 async def getItemCmdr(prox, outp=None, locs=None):
     '''Get a Cmdr instance with prepopulated locs'''
+    s_common.deprdate('synapse.lib.jupyter.getItemCmdr', '2024-08-26')
     cmdr = await s_cmdr.getItemCmdr(prox, outp=outp)
     cmdr.echoline = True
     if locs:
@@ -159,6 +166,7 @@ def suppress_logging(suppress):
     '''
     Context manager to suppress specific loggers.
     '''
+    s_common.deprdate('synapse.lib.jupyter.suppress_logging', '2024-08-26')
     logs = {}
     if not suppress:
         yield None
@@ -197,6 +205,7 @@ class StormCore(s_base.Base):
         '''
         Context manager to suppress specific loggers.
         '''
+        s_common.deprdate('StormCore.suppress_logging', '2024-08-26')
         with suppress_logging(suppress):
             yield None
 
@@ -204,6 +213,7 @@ class StormCore(s_base.Base):
         '''
         Run a line of text directly via storm cli.
         '''
+        s_common.deprdate('StormCore.runCmdLine', '2024-08-26')
         await self.stormcli.runCmdLine(text, opts=opts)
 
     async def _runStorm(self, text, opts=None, cli=False, suppress_logging=False):
@@ -240,6 +250,7 @@ class StormCore(s_base.Base):
         Returns:
             list: A list of storm messages.
         '''
+        s_common.deprdate('StormCore.storm', '2024-08-26')
         mesgs = await self._runStorm(text, opts, cli, suppress_logging)
         if num is not None:
             nodes = [m for m in mesgs if m[0] == 'node']
@@ -265,12 +276,14 @@ class CmdrCore(s_base.Base):
         '''
         Add feed data to the cortex.
         '''
+        s_common.deprdate('CmdrCore.addFeedData', '2024-08-26')
         return await self.core.addFeedData(name, items, viewiden=viewiden)
 
     async def runCmdLine(self, text):
         '''
         Run a line of text directly via cmdr.
         '''
+        s_common.deprdate('CmdrCore.runCmdLine', '2024-08-26')
         await self.cmdr.runCmdLine(text)
 
     @contextlib.contextmanager
@@ -278,6 +291,7 @@ class CmdrCore(s_base.Base):
         '''
         Context manager to suppress specific loggers.
         '''
+        s_common.deprdate('CmdrCore.suppress_logging', '2024-08-26')
         with suppress_logging(suppress):
             yield None
 
@@ -318,6 +332,7 @@ class CmdrCore(s_base.Base):
         Returns:
             list: A list of storm messages.
         '''
+        s_common.deprdate('CmdrCore.storm', '2024-08-26')
         mesgs = await self._runStorm(text, opts, cmdr, suppress_logging)
         if num is not None:
             nodes = [m for m in mesgs if m[0] == 'node']
@@ -342,6 +357,7 @@ class CmdrCore(s_base.Base):
         Returns:
             list: A list of packed nodes.
         '''
+        s_common.deprdate('CmdrCore.eval', '2024-08-26')
         mesgs = await self._runStorm(text, opts, cmdr)
         for mesg in mesgs:
             if mesg[0] == 'err':  # pragma: no cover
@@ -375,6 +391,7 @@ async def getTempCoreProx(mods=None):
     Returns:
         s_telepath.Proxy
     '''
+    s_common.deprdate('synapse.lib.jupyter.getTempCoreProx', '2024-08-26')
     acm = genTempCoreProxy(mods)
     prox = await acm.__aenter__()
     # Use object.__setattr__ to hulk smash and avoid proxy getattr magick
@@ -399,6 +416,7 @@ async def getTempCoreStorm(mods=None, outp=None):
     Returns:
         StormCore: A StormCore instance.
     '''
+    s_common.deprdate('synapse.lib.jupyter.getTempCoreStorm', '2024-08-26')
     acm = genTempCoreProxy(mods)
     prox = await acm.__aenter__()
     stormcore = await StormCore.anit(prox, outp=outp)
@@ -419,6 +437,7 @@ async def getTempCoreCmdr(mods=None, outp=None):
     Returns:
         CmdrCore: A CmdrCore instance.
     '''
+    s_common.deprdate('synapse.lib.jupyter.getTempCoreCmdr', '2024-08-26')
     acm = genTempCoreProxy(mods)
     prox = await acm.__aenter__()
     cmdrcore = await CmdrCore.anit(prox, outp=outp)
@@ -441,6 +460,7 @@ async def getTempCoreCmdrStormsvc(svcname, svcctor, svcconf=None, outp=None):
     Returns:
         (CmdrCore, Proxy): A CmdrCore instance and proxy to the Storm service
     '''
+    s_common.deprdate('synapse.lib.jupyter.getTempCoreCmdrStormsvc', '2024-08-26')
     cmdrcore = await getTempCoreCmdr(outp=outp)
 
     acm = genTempStormsvcProxy(cmdrcore, svcname, svcctor, svcconf)
@@ -470,6 +490,7 @@ async def getTempCoreStormStormsvc(svcname, svcctor, svcconf=None, outp=None):
     Returns:
         (StormCore, Proxy): A StormCore instance and proxy to the Storm service
     '''
+    s_common.deprdate('synapse.lib.jupyter.getTempCoreStormStormsvc', '2024-08-26')
     stormcore = await getTempCoreStorm(outp=outp)
 
     acm = genTempStormsvcProxy(stormcore, svcname, svcctor, svcconf)