synapse 2.252.0__tar.gz → 2.253.0__tar.gz

{synapse-2.252.0/synapse.egg-info → synapse-2.253.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 2.252.0
+Version: 2.253.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0

{synapse-2.252.0 → synapse-2.253.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.252.0'
+version = '2.253.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]

{synapse-2.252.0 → synapse-2.253.0}/synapse/axon.py

@@ -48,6 +48,7 @@ class AxonHttpUploadV1(AxonHandlerMixin, s_httpapi.StreamHandler):
 
         if not await self.allowed(('axon', 'upload')):
             await self.finish()
+            return
 
         # max_body_size defaults to 100MB and requires a value
         self.request.connection.set_max_body_size(MAX_HTTP_UPLOAD_SIZE)

{synapse-2.252.0 → synapse-2.253.0}/synapse/cortex.py

@@ -1,5 +1,6 @@
 import os
 import copy
+import http
 import regex
 import asyncio
 import logging
@@ -175,9 +176,18 @@ async def wrap_liftgenr(iden, genr):
 
 class CortexAxonMixin:
 
-    async def prepare(self):
-        await self.cell.axready.wait()
-        await s_coro.ornot(super().prepare)
+    async def allowed(self, perm, default=False, gateiden=None):
+        # wait for the Axon only once the user has the permission
+        if not await super().allowed(perm, default=default, gateiden=gateiden):
+            return False
+
+        try:
+            await self.cell.getAxon()
+        except s_exc.TimeOut as e:
+            self.sendRestExc(e, status_code=http.HTTPStatus.SERVICE_UNAVAILABLE)
+            return False
+
+        return True
 
     def getAxon(self):
         return self.cell.axon
@@ -760,13 +770,13 @@ class CoreApi(s_cell.CellApi):
 
     async def getAxonUpload(self):
         self.user.confirm(('axon', 'upload'))
-        await self.cell.axready.wait()
+        await self.cell.getAxon()
         upload = await self.cell.axon.upload()
         return await s_axon.UpLoadProxy.anit(self.link, upload)
 
     async def getAxonBytes(self, sha256):
         self.user.confirm(('axon', 'get'))
-        await self.cell.axready.wait()
+        await self.cell.getAxon()
         async for byts in self.cell.axon.get(s_common.uhex(sha256)):
             yield byts
 
@@ -1044,6 +1054,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             (5, self._storCleanQueueAuthGates),
             (6, self._storCleanCronAuthGates),
             (7, self._storMigrDmonDdefView),
+            (8, self._storBackfillCreated),
         ), nexs=False)
 
         # Perform module loading
@@ -1225,6 +1236,26 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                 subkv.set(iden, ddef)
         logger.warning('...storm dmon ddef migration complete!')
 
+    async def _storBackfillCreated(self):
+        logger.warning('backfilling created on views, layers, cron jobs, and triggers')
+
+        def backfill(subkv):
+            for iden, info in subkv.items():
+                if info.get('created') is None:
+                    info['created'] = 0
+                    subkv.set(iden, info)
+
+        backfill(self.cortexdata.getSubKeyVal('layer:info:'))
+        backfill(self.cortexdata.getSubKeyVal('agenda:appt:'))
+
+        viewdefs = self.cortexdata.getSubKeyVal('view:info:')
+        backfill(viewdefs)
+
+        for iden in list(viewdefs.keys()):
+            backfill(self.cortexdata.getSubKeyVal(f'view:{iden}:trigger:'))
+
+        logger.warning('...created backfill complete!')
+
     async def _storUpdateMacros(self):
         for name, node in await self.hive.open(('cortex', 'storm', 'macros')):
 
@@ -6007,7 +6038,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     @s_nexus.Pusher.onPushAuto('storm:dmon:bump')
     async def bumpStormDmon(self, iden):
         ddef = self.stormdmondefs.get(iden)
-        if ddef is None:
+        if ddef is None or not ddef.get('enabled', True):
             return False
 
         if self.isactive:
@@ -6168,9 +6199,45 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     def getStormCmds(self):
         return list(self.stormcmds.items())
 
-    async def getAxon(self):
-        await self.axready.wait()
-        return self.axon.iden
+    async def getAxon(self, timeout=s_const.AXON_READY_TIMEOUT):
+        '''
+        Wait for the Axon to be ready and return its iden.
+
+        Args:
+            timeout (int): The maximum number of seconds to wait, or None to wait indefinitely.
+
+        Returns:
+            str: The iden of the Axon, or None if the Axon did not report one.
+
+        Raises:
+            s_exc.TimeOut: If the Axon is not ready within the timeout.
+        '''
+        if not self.axready.is_set():
+            try:
+                await s_common.wait_for(self.axready.wait(), timeout)
+            except asyncio.TimeoutError:
+                mesg = f'Timed out waiting {timeout} seconds for the Axon to be ready.'
+                raise s_exc.TimeOut(mesg=mesg, timeout=timeout) from None
+
+        if (cellinfo := self.axoninfo.get('cell')) is None:
+            return None
+
+        return cellinfo.get('iden')
+
+    async def getCellInfo(self):
+        '''
+        Return metadata specific for the Cortex.
+
+        Notes:
+            In addition to the base Cell information, the ``cell`` section
+            includes ``axon:ready``, which is True when the Axon is ready.
+
+        Returns:
+            Dict: A Dictionary of metadata.
+        '''
+        info = await super().getCellInfo()
+        info['cell']['axon:ready'] = self.axready.is_set()
+        return info
 
     def setFeedFunc(self, name, func):
         '''
@@ -6493,6 +6560,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                         yield pode
 
     async def exportStormToAxon(self, text, opts=None):
+        await self.getAxon()
         async with await self.axon.upload() as fd:
             async for pode in self.exportStorm(text, opts=opts):
                 await fd.write(s_msgpack.en(pode))
@@ -6513,6 +6581,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         # ensure that the user can make all node edits in the layer
         user.confirm(('node',), gateiden=view.layers[0].iden)
 
+        await self.getAxon()
+
         q = s_queue.Queue(maxsize=10000)
         feedexc = None
 

{synapse-2.252.0 → synapse-2.253.0}/synapse/lib/agenda.py

@@ -681,7 +681,10 @@ class Agenda(s_base.Base):
             mesg = f'No cron job with iden: {iden}'
             raise s_exc.NoSuchIden(iden=iden, mesg=mesg)
 
-        await self.mod(iden, {'query': appt.query})
+        await self.core.getStormQuery(appt.query)
+
+        appt.enabled = True
+        await appt.save()
 
     async def disable(self, iden):
         appt = self.appts.get(iden)
@@ -737,8 +740,6 @@ class Agenda(s_base.Base):
                     raise s_exc.BadConfValu(mesg=mesg)
                 heapq.heapify(self.apptheap)
 
-        appt.enabled = True  # in case it was disabled for a bad query
-
         await appt.save()
 
         if appt.nexttime and self.apptheap and self.apptheap[0] is appt:

{synapse-2.252.0 → synapse-2.253.0}/synapse/lib/aha.py

@@ -775,6 +775,7 @@ class AhaCell(s_cell.Cell):
             proxy = await self.nexsroot.client.proxy(timeout=timeout)
             async for item in proxy.callAhaPeerApi(iden, todo, timeout=timeout, skiprun=skiprun):
                 yield item
+            return
 
         queue = asyncio.Queue()
         async with await s_base.Base.anit() as base:
@@ -797,6 +798,7 @@ class AhaCell(s_cell.Cell):
             proxy = await self.nexsroot.client.proxy(timeout=timeout)
             async for item in proxy.callAhaPeerGenr(iden, todo, timeout=timeout, skiprun=skiprun):
                 yield item
+            return
 
         queue = asyncio.Queue()
         async with await s_base.Base.anit() as base:

{synapse-2.252.0 → synapse-2.253.0}/synapse/lib/cell.py

@@ -892,6 +892,15 @@ class CellApi(s_base.Base):
 
     @adminapi()
     async def getDiagInfo(self):
+        '''
+        Get diagnostic information about the Cell.
+
+        Returns:
+            dict: A dictionary with a ``slabs`` list containing a dict for each open LMDB slab. Each slab
+            dict includes the ``path``, map size settings, and a ``commitstats`` list of up to 1000
+            ``(starttime, xactopslen, delta)`` tuples for recent commits, oldest first. Times are in
+            milliseconds.
+        '''
         return {
             'slabs': await s_lmdbslab.Slab.getSlabStats(),
         }

{synapse-2.252.0 → synapse-2.253.0}/synapse/lib/const.py

@@ -52,5 +52,8 @@ assert layer_pdef_qsize <= layer_pdef_qsize_max
 MAX_LINE_SIZE = kibibyte * 64
 MAX_FIELD_SIZE = kibibyte * 64
 
+# Axon constants
+AXON_READY_TIMEOUT = 300  # seconds
+
 # Socket constants
 UNIX_SOCKET_PATH_MAX = 103

{synapse-2.252.0 → synapse-2.253.0}/synapse/lib/lmdbslab.py

@@ -204,6 +204,10 @@ class SafeKeyVal:
 
     def reqValidName(self, name):
 
+        if not isinstance(name, str):
+            mesg = f'SafeKeyVal key names must be strings, got {type(name).__name__}.'
+            raise s_exc.BadArg(mesg=mesg, name=s_common.trimText(repr(name)))
+
         _name = name.encode('utf-8')
 
         if self._prefix:
@@ -437,6 +441,7 @@ class HotCount(HotKeyVal):
         byts = name.encode()
         self.cache[byts] += valu
         self.dirty.add(byts)
+        self.slab.dirty = True
 
     def set(self, name: str, valu):
         byts = name.encode()
@@ -843,7 +848,7 @@ class Slab(s_base.Base):
                 'maxsize': slab.maxsize,
                 'growsize': slab.growsize,
                 'mapasync': True,
-
+                'commitstats': list(slab.commitstats),
             })
         return retn
 

{synapse-2.252.0 → synapse-2.253.0}/synapse/lib/storm.py

@@ -169,7 +169,7 @@ Modify an existing cron job's query or period.
 
 Notes:
     All times are interpreted as UTC.
-    Modifying a disabled cron job re-enables it.
+    Modifying a disabled cron job does not re-enable it. Use cron.enable to re-enable it.
 
     The --period argument uses the same syntax as cron.add:
         <periodicity>[/<value>...][@<time>]
@@ -1266,7 +1266,9 @@ class DmonManager(s_base.Base):
             return
         logger.debug('Starting Dmons')
         for dmon in dmons:
-            await dmon.run()
+            if dmon.enabled:
+                await dmon.run()
+
         self.enabled = True
         logger.debug('Started Dmons')
 
@@ -1307,6 +1309,7 @@ class StormDmon(s_base.Base):
         if self.task is not None:
             self.task.cancel()
         self.task = None
+        self.status = 'stopped'
         logger.debug(f'Stopped Dmon {self.iden}', extra=self.core.getLogExtra(iden=self.iden))
 
     async def run(self):
@@ -1317,7 +1320,8 @@ class StormDmon(s_base.Base):
 
     async def bump(self):
         await self.stop()
-        await self.run()
+        if self.enabled:
+            await self.run()
 
     def pack(self):
         retn = dict(self.ddef)
@@ -1692,8 +1696,8 @@ class Runtime(s_base.Base):
 
     async def popVar(self, name):
 
-        if self._isRootScope(name):
-            return self.root.popVar(name)
+        if name not in self.vars and self._isRootScope(name):
+            return await self.root.popVar(name)
 
         oldv = self.vars.pop(name, s_common.novalu)
         if isinstance(oldv, s_base.Base):

{synapse-2.252.0 → synapse-2.253.0}/synapse/lib/stormhttp.py

@@ -425,7 +425,7 @@ class LibHttp(s_stormtypes.Lib):
                     kwargs['proxy'] = proxy
 
                 if ssl_opts is not None:
-                    axonvers = self.runt.snap.core.axoninfo['synapse']['version']
+                    axonvers = await s_stormtypes.getAxonSynapseVersion(self.runt)
                     mesg = f'The ssl_opts argument requires an Axon Synapse version {s_stormtypes.AXON_MINVERS_SSLOPTS}, ' \
                            f'but the Axon is running {axonvers}'
                     s_version.reqVersion(axonvers, s_stormtypes.AXON_MINVERS_SSLOPTS, mesg=mesg)

{synapse-2.252.0 → synapse-2.253.0}/synapse/lib/stormlib/auth.py

@@ -2008,6 +2008,7 @@ class LibUsers(s_stormtypes.Lib):
 
     @s_stormtypes.stormfunc(readonly=True)
     async def _methUsersGet(self, iden=None):
+        iden = await s_stormtypes.tostr(iden, noneok=True)
         if iden is None:
             iden = self.runt.user.iden
 
@@ -2103,6 +2104,10 @@ class LibRoles(s_stormtypes.Lib):
 
     @s_stormtypes.stormfunc(readonly=True)
     async def _methRolesGet(self, iden):
+        iden = await s_stormtypes.tostr(iden, noneok=True)
+        if iden is None:
+            return None
+
         if self.runt.snap.core.hasRoleIden(iden):
             return Role(self.runt, iden)
 

{synapse-2.252.0 → synapse-2.253.0}/synapse/lib/stormtypes.py

@@ -1,6 +1,7 @@
 import bz2
 import copy
 import gzip
+import math
 import time
 import regex
 import types
@@ -108,6 +109,28 @@ async def resolveCoreProxyUrl(valu):
         case _:
             raise s_exc.BadArg(mesg='HTTP proxy argument must be a string or bool.')
 
+async def getAxonSynapseVersion(runt):
+    '''
+    Get the Synapse version of the Cortex's Axon.
+
+    Args:
+        runt (Runtime): The Storm runtime.
+
+    Returns:
+        tuple: The Synapse version of the Axon.
+
+    Raises:
+        s_exc.FeatureNotSupported: If the Axon version is unavailable.
+        s_exc.TimeOut: If the Axon is not ready within the default Axon timeout.
+    '''
+    await runt.snap.core.getAxon()
+
+    if (syninfo := runt.snap.core.axoninfo.get('synapse')) is None or (axonvers := syninfo.get('version')) is None:
+        mesg = 'Unable to determine the Synapse version of the Axon.'
+        raise s_exc.FeatureNotSupported(mesg=mesg)
+
+    return axonvers
+
 async def resolveAxonProxyArg(valu):
     '''
     Resolve a proxy value to the kwarg to set for an Axon HTTP call.
@@ -120,7 +143,7 @@ async def resolveAxonProxyArg(valu):
     '''
     runt = s_scope.get('runt')
 
-    axonvers = runt.snap.core.axoninfo['synapse']['version']
+    axonvers = await getAxonSynapseVersion(runt)
     if axonvers < AXON_MINVERS_PROXY:
         await runt.snap.warnonce(f'Axon version does not support proxy argument: {axonvers} < {AXON_MINVERS_PROXY}')
         return False, None
@@ -750,7 +773,7 @@ class LibDmon(Lib):
                       {'name': 'iden', 'type': 'str', 'desc': 'The GUID of the dmon to restart.'},
                   ),
                   'returns': {'type': 'boolean',
-                              'desc': 'True if the Dmon is restarted; False if the iden does not exist.'}}},
+                              'desc': 'True if the Dmon is restarted; False if the iden does not exist or the Dmon is disabled.'}}},
         {'name': 'stop', 'desc': 'Stop a Storm Dmon.',
          'type': {'type': 'function', '_funcname': '_libDmonStop',
                   'args': (
@@ -857,8 +880,7 @@ class LibDmon(Lib):
         viewiden = ddef['stormopts']['view']
         self.runt.confirm(('dmon', 'add'), gateiden=viewiden)
 
-        await self.runt.snap.core.bumpStormDmon(iden)
-        return True
+        return await self.runt.snap.core.bumpStormDmon(iden)
 
     async def _libDmonStop(self, iden):
         iden = await tostr(iden)
@@ -2556,7 +2578,7 @@ class LibAxon(Lib):
             kwargs['proxy'] = proxy
 
         if ssl_opts is not None:
-            axonvers = self.runt.snap.core.axoninfo['synapse']['version']
+            axonvers = await getAxonSynapseVersion(self.runt)
             mesg = f'The ssl_opts argument requires an Axon Synapse version {AXON_MINVERS_SSLOPTS}, ' \
                    f'but the Axon is running {axonvers}'
             s_version.reqVersion(axonvers, AXON_MINVERS_SSLOPTS, mesg=mesg)
@@ -2597,7 +2619,7 @@ class LibAxon(Lib):
             kwargs['proxy'] = proxy
 
         if ssl_opts is not None:
-            axonvers = self.runt.snap.core.axoninfo['synapse']['version']
+            axonvers = await getAxonSynapseVersion(self.runt)
             mesg = f'The ssl_opts argument requires an Axon Synapse version {AXON_MINVERS_SSLOPTS}, ' \
                    f'but the Axon is running {axonvers}'
             s_version.reqVersion(axonvers, AXON_MINVERS_SSLOPTS, mesg=mesg)
@@ -2706,6 +2728,8 @@ class LibAxon(Lib):
     async def metrics(self):
         if not self.runt.allowed(('axon', 'has')):
             self.runt.confirm(('storm', 'lib', 'axon', 'has'))
+
+        await self.runt.snap.core.getAxon()
         return await self.runt.snap.core.axon.metrics()
 
     async def upload(self, genr):
@@ -2792,7 +2816,12 @@ class LibAxon(Lib):
         '''
         Unpack bytes from a file in the Axon using struct.
         '''
-        if self.runt.snap.core.axoninfo.get('features', {}).get('unpack', 0) < 1:
+        if not self.runt.allowed(('axon', 'get')):
+            self.runt.confirm(('storm', 'lib', 'axon', 'get'))
+
+        await self.runt.snap.core.getAxon()
+
+        if (features := self.runt.snap.core.axoninfo.get('features')) is None or features.get('unpack', 0) < 1:
             mesg = 'The connected Axon does not support the the unpack API. Please update your Axon.'
             raise s_exc.FeatureNotSupported(mesg=mesg)
 
@@ -2800,10 +2829,6 @@ class LibAxon(Lib):
         fmt = await tostr(fmt)
         offs = await toint(offs)
 
-        if not self.runt.allowed(('axon', 'get')):
-            self.runt.confirm(('storm', 'lib', 'axon', 'get'))
-
-        await self.runt.snap.core.getAxon()
         return await self.runt.snap.core.axon.unpack(s_common.uhex(sha256), fmt, offs)
 
 @registry.registerLib
@@ -3446,8 +3471,17 @@ class LibTime(Lib):
                 break
 
     async def _fromunix(self, secs):
-        secs = float(secs)
-        return int(secs * 1000)
+
+        try:
+            valu = await tofloat(secs) * 1000
+        except s_exc.BadCast:
+            valu = None
+
+        if valu is None or not math.isfinite(valu):
+            mesg = f'Invalid unix epoch time: {s_common.trimText(await torepr(secs))}'
+            raise s_exc.BadArg(mesg=mesg)
+
+        return int(valu)
 
 @registry.registerLib
 class LibRegx(Lib):
@@ -8221,6 +8255,19 @@ class View(Prim):
                       {'name': 'name', 'type': 'str', 'desc': 'The name of the new View.', 'default': None},
                   ),
                   'returns': {'type': 'view', 'desc': 'The ``view`` object for the new View.', }}},
+        {'name': 'insertChildFork', 'desc': '''
+            Insert a new View between this View and all of its child Views.
+
+            Only admin permissions on this View are required, and every child View is
+            re-parented, including Views owned by other users. The new View receives a
+            copy of this View's quorum, so pending merge requests on the re-parented
+            child Views now target the new View. Child Views which are currently
+            merging are left under this View.''',
+         'type': {'type': 'function', '_funcname': '_methViewInsertChildFork',
+                  'args': (
+                      {'name': 'name', 'type': 'str', 'desc': 'The name of the new View.', 'default': None},
+                  ),
+                  'returns': {'type': 'view', 'desc': 'The ``view`` object for the new View.', }}},
         {'name': 'pack', 'desc': 'Get the View definition.',
          'type': {'type': 'function', '_funcname': '_methViewPack',
                   'returns': {'type': 'dict', 'desc': 'Dictionary containing the View definition.', }}},
@@ -8452,6 +8499,7 @@ class View(Prim):
 
             'fork': self._methViewFork,
             'insertParentFork': self._methViewInsertParentFork,
+            'insertChildFork': self._methViewInsertChildFork,
 
             'getMerges': self.getMerges,
             'delMergeVote': self.delMergeVote,
@@ -8742,6 +8790,22 @@ class View(Prim):
 
         return View(self.runt, newv, path=self.path)
 
+    async def _methViewInsertChildFork(self, name=None):
+        useriden = self.runt.user.iden
+        viewiden = self.valu.get('iden')
+
+        name = await tostr(name, noneok=True)
+
+        self.runt.reqAdmin(gateiden=viewiden)
+
+        self.runt.confirm(('view', 'add'))
+
+        view = self.runt.snap.core.reqView(viewiden)
+
+        newv = await view.insertChildFork(useriden, name=name)
+
+        return View(self.runt, newv, path=self.path)
+
     async def _methViewMerge(self, force=False):
         '''
         Merge a forked view back into its parent.
@@ -10595,6 +10659,25 @@ async def toint(valu, noneok=False):
         mesg = f'Failed to make an integer from {s_common.trimText(repr(valu))}.'
         raise s_exc.BadCast(mesg=mesg) from e
 
+async def tofloat(valu, noneok=False):
+
+    if noneok and valu is None:
+        return None
+
+    try:
+        return float(valu)
+    except OverflowError as e:
+        mesg = f'Failed to make a float from {s_common.trimText(repr(valu))}.'
+        raise s_exc.BadCast(mesg=mesg) from e
+    except (TypeError, ValueError):
+        pass
+
+    try:
+        return float(await tostr(valu))
+    except (s_exc.BadCast, ValueError) as e:
+        mesg = f'Failed to make a float from {s_common.trimText(repr(valu))}.'
+        raise s_exc.BadCast(mesg=mesg) from e
+
 async def toiter(valu, noneok=False):
 
     if noneok and valu is None:

{synapse-2.252.0 → synapse-2.253.0}/synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 252, 0)
+version = (2, 253, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = '21c2e2f9d60f9360a35a4a025628144bb0c9da1f'
+commit = '781db4966ec712dbffc9552eeba5d61a272662e9'