synapse 2.167.0__py311-none-any.whl → 2.169.0__py311-none-any.whl

synapse/cortex.py

@@ -573,14 +573,6 @@ class CoreApi(s_cell.CellApi):
     async def delStormDmon(self, iden):
         return await self.cell.delStormDmon(iden)
 
-    @s_cell.adminapi(log=True)
-    async def enableMigrationMode(self): # pragma: no cover
-        s_common.deprdate('CoreApi.enableMigrationMode', '2024-05-05')
-
-    @s_cell.adminapi(log=True)
-    async def disableMigrationMode(self): # pragma: no cover
-        s_common.deprdate('CoreApi.disableMigrationMode', '2024-05-05')
-
     @s_cell.adminapi()
     async def cloneLayer(self, iden, ldef=None):
 
@@ -881,7 +873,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         self.maxnodes = self.conf.get('max:nodes')
         self.nodecount = 0
+
         self.migration = False
+        self._migration_lock = asyncio.Lock()
 
         self.stormmods = {}     # name: mdef
         self.stormpkgs = {}     # name: pkgdef
@@ -1348,6 +1342,17 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
              'ex': 'node.prop.del.inet:ipv4.asn',
              'desc': 'Controls removing a specific property from a form of node in a layer.'},
 
+            {'perm': ('node', 'data', 'set'), 'gate': 'layer',
+             'desc': 'Permits a user to set node data in a given layer.'},
+            {'perm': ('node', 'data', 'set', '<key>'), 'gate': 'layer',
+              'ex': 'node.data.set.hehe',
+             'desc': 'Permits a user to set node data in a given layer for a specific key.'},
+            {'perm': ('node', 'data', 'pop'), 'gate': 'layer',
+             'desc': 'Permits a user to remove node data in a given layer.'},
+            {'perm': ('node', 'data', 'pop', '<key>'), 'gate': 'layer',
+             'ex': 'node.data.pop.hehe',
+             'desc': 'Permits a user to remove node data in a given layer for a specific key.'},
+
             {'perm': ('pkg', 'add'), 'gate': 'cortex',
              'desc': 'Controls access to adding storm packages.'},
             {'perm': ('pkg', 'del'), 'gate': 'cortex',
@@ -1480,12 +1485,21 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         await self.stormdmons.start()
         await self.agenda.clearRunningStatus()
 
-        for view in self.views.values():
-            await view.initTrigTask()
-            await view.initMergeTask()
+        async def _runMigrations():
+            # Run migrations when this cortex becomes active. This is to prevent
+            # migrations getting skipped in a zero-downtime upgrade path
+            # (upgrade mirror, promote mirror).
+            await self._checkLayerModels()
 
-        for layer in self.layers.values():
-            await layer.initLayerActive()
+            # Once migrations are complete, start the view and layer tasks.
+            for view in self.views.values():
+                await view.initTrigTask()
+                await view.initMergeTask()
+
+            for layer in self.layers.values():
+                await layer.initLayerActive()
+
+        self.runActiveTask(_runMigrations())
 
         await self.initStormPool()
 
@@ -3597,6 +3611,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                         break
 
                     yield ioff, layr.iden, SYNC_NODEEDITS, item, meta
+                    await asyncio.sleep(0)
 
             if layr.isdeleted:
                 yield layr.deloffs, layr.iden, SYNC_LAYR_DEL, (), {}
@@ -4363,7 +4378,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         return ret
 
     async def _checkLayerModels(self):
-        with self.enterMigrationMode():
+        async with self.enterMigrationMode():
             mrev = s_modelrev.ModelRev(self)
             await mrev.revCoreLayers()
 
@@ -6098,11 +6113,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         appt = await self.agenda.get(iden)
         await appt.edits(edits)
 
-    @contextlib.contextmanager
-    def enterMigrationMode(self):
-        self.migration = True
-        yield
-        self.migration = False
+    @contextlib.asynccontextmanager
+    async def enterMigrationMode(self):
+        async with self._migration_lock:
+            self.migration = True
+            yield
+            self.migration = False
 
     async def iterFormRows(self, layriden, form, stortype=None, startvalu=None):
         '''

synapse/lib/agenda.py

@@ -831,6 +831,9 @@ class Agenda(s_base.Base):
                 'view': appt.view,
                 'mirror': appt.pool,
                 'vars': {'auto': {'iden': appt.iden, 'type': 'cron'}},
+                '_loginfo': {
+                    'cron': appt.iden
+                }
             }
             opts = self.core._initStormOpts(opts)
 

synapse/lib/aha.py

@@ -807,7 +807,8 @@ class AhaCell(s_cell.Cell):
     async def _setAhaSvcDown(self, name, linkiden, network=None):
         svcname, svcnetw, svcfull = self._nameAndNetwork(name, network)
         path = ('aha', 'services', svcnetw, svcname)
-        await self.jsonstor.cmpDelPathObjProp(path, 'svcinfo/online', linkiden)
+        if await self.jsonstor.cmpDelPathObjProp(path, 'svcinfo/online', linkiden):
+            await self.jsonstor.setPathObjProp(path, 'svcinfo/ready', False)
 
         # Check if we have any links which may need to be removed
         current_sessions = {s_common.guid(iden): sess for iden, sess in self.dmon.sessions.items()}

synapse/lib/cell.py

@@ -1295,14 +1295,20 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
     async def _onBootOptimize(self):
 
+        bdir = s_common.genpath(self.dirn, 'backups')
         tdir = s_common.gendir(self.dirn, 'tmp')
         tdev = os.stat(tdir).st_dev
 
+        logger.warning('Collecting LMDB files for onboot optimization.')
+
         lmdbs = []
         for (root, dirs, files) in os.walk(self.dirn):
             for dirname in dirs:
                 filepath = os.path.join(root, dirname, 'data.mdb')
                 if os.path.isfile(filepath):
+                    if filepath.startswith(bdir):
+                        logger.debug(f'Skipping backup file {filepath}')
+                        continue
                     if os.stat(filepath).st_dev != tdev:
                         logger.warning(f'Unable to run onboot:optimize, {filepath} is not on the same volume as {tdir}')
                         return
@@ -2891,24 +2897,10 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         self.onfini(self.dmon.fini)
 
     async def _initCellHive(self):
-        isnew = not self.slab.dbexists('hive')
-
         db = self.slab.initdb('hive')
         hive = await s_hive.SlabHive.anit(self.slab, db=db, nexsroot=self.getCellNexsRoot())
         self.onfini(hive)
 
-        if isnew:
-            path = os.path.join(self.dirn, 'hiveboot.yaml')
-            if os.path.isfile(path):
-                s_common.deprdate('Initial hive config from hiveboot.yaml', '2024-05-05')
-                logger.debug(f'Loading cell hive from {path}')
-                tree = s_common.yamlload(path)
-                if tree is not None:
-                    # Pack and unpack the tree to avoid tuple/list issues
-                    # for in-memory structures.
-                    tree = s_common.tuplify(tree)
-                    await hive.loadHiveTree(tree)
-
         return hive
 
     async def _initCellSlab(self, readonly=False):

synapse/lib/httpapi.py

@@ -1354,6 +1354,9 @@ class ExtApiHandler(StormHandler):
             'user': useriden,
             'vars': varz,
             'view': adef.get('view'),
+            '_loginfo': {
+                'httpapi': iden
+            }
         }
 
         query = '\n'.join((self.storm_prefix, storm))

synapse/lib/layer.py

@@ -165,11 +165,13 @@ class LayerApi(s_cell.CellApi):
         await self._reqUserAllowed(self.liftperm)
         async for item in self.layr.syncNodeEdits(offs, wait=wait):
             yield item
+            await asyncio.sleep(0)
 
     async def syncNodeEdits2(self, offs, wait=True):
         await self._reqUserAllowed(self.liftperm)
         async for item in self.layr.syncNodeEdits2(offs, wait=wait):
             yield item
+            await asyncio.sleep(0)
 
     async def getEditIndx(self):
         '''

synapse/lib/modelrev.py

@@ -768,6 +768,12 @@ class ModelRev:
         Returns:
             None
         '''
+        if opts is None:
+            opts = {}
+
+        # Migrations only run on leaders
+        opts['mirror'] = False
+
         async def _runStorm():
             async for mesgtype, mesginfo in self.core.storm(text, opts=opts):
                 if mesgtype == 'print':

synapse/lib/modules.py

@@ -28,4 +28,5 @@ coremods = (
     'synapse.models.biz.BizModule',
     'synapse.models.belief.BeliefModule',
     'synapse.models.science.ScienceModule',
+    'synapse.models.planning.PlanModule',
 )

synapse/lib/snap.py

@@ -717,7 +717,9 @@ class Snap(s_base.Base):
 
         show_storage = False
 
-        self.core._logStormQuery(text, user, info={'mode': opts.get('mode', 'storm'), 'view': self.view.iden})
+        info = opts.get('_loginfo', {})
+        info.update({'mode': opts.get('mode', 'storm'), 'view': self.view.iden})
+        self.core._logStormQuery(text, user, info=info)
 
         # { form: ( embedprop, ... ) }
         embeds = opts.get('embeds')

synapse/lib/stormlib/aha.py

@@ -190,7 +190,7 @@ class AhaPool(s_stormtypes.StormType):
                        'desc': 'The name of the AHA service to add. It is easiest to use the relative name of a service, ending with "...".', },
                   ),
                   'returns': {'type': 'null', }}},
-        {'name': 'add', 'desc': '''Remove a service from the AHA pool.
+        {'name': 'del', 'desc': '''Remove a service from the AHA pool.
 
         Examples:
             Remove a service from a pool with its relative name::
@@ -203,7 +203,7 @@ class AhaPool(s_stormtypes.StormType):
                       {'name': 'svcname', 'type': 'str',
                        'desc': 'The name of the AHA service to remove. It is easiest to use the relative name of a service, ending with "...".', },
                   ),
-                  'returns': {'type': 'null', }}},
+                  'returns': {'type': ['null', 'str'], 'desc': 'The service removed from the pool or null if a service was not removed.'}}},
     )
     _storm_typename = 'aha:pool'
 
@@ -243,9 +243,20 @@ class AhaPool(s_stormtypes.StormType):
         proxy = await self.runt.snap.core.reqAhaProxy()
 
         poolname = self.poolinfo.get('name')
-        await proxy.delAhaPoolSvc(poolname, svcname)
+        newinfo = await proxy.delAhaPoolSvc(poolname, svcname)
+
+        tname = svcname
+        if tname.endswith('...'):
+            tname = tname[:-2]
+        deleted_service = None
+        deleted_services = [svc for svc in self.poolinfo.get('services').keys()
+                            if svc not in newinfo.get('services') and svc.startswith(tname)]
+        if deleted_services:
+            deleted_service = deleted_services[0]
+
+        self.poolinfo = newinfo
 
-        self.poolinfo = await proxy.getAhaPool(poolname)
+        return deleted_service
 
 stormcmds = (
     {
@@ -319,8 +330,12 @@ stormcmds = (
             $pool = $lib.aha.pool.get($cmdopts.poolname)
             if (not $pool) { $lib.exit(`No AHA service pool named: {$cmdopts.poolname}`) }
 
-            $pool.del($cmdopts.svcname)
-            $lib.print(`AHA service ({$cmdopts.svcname}) removed from service pool ({$pool.name})`)
+            $svc = $pool.del($cmdopts.svcname)
+            if $svc {
+                $lib.print(`AHA service ({$svc}) removed from service pool ({$pool.name})`)
+            } else {
+                $lib.print(`Did not remove ({$cmdopts.svcname}) from the service pool.`)
+            }
         ''',
     },
     {

synapse/lib/stormlib/easyperm.py

@@ -51,6 +51,14 @@ class LibEasyPerm(s_stormtypes.Lib):
                        'desc': 'Optional error message to present if user does not have required permission level.'},
                   ),
                   'returns': {'type': 'null'}}},
+        {'name': 'level.admin', 'desc': 'Constant for admin permission.',
+         'type': 'int', },
+        {'name': 'level.deny', 'desc': 'Constant for deny permission.',
+         'type': 'int', },
+        {'name': 'level.edit', 'desc': 'Constant for edit permission.',
+         'type': 'int', },
+        {'name': 'level.read', 'desc': 'Constant for read permission.',
+         'type': 'int', },
     )
     _storm_lib_path = ('auth', 'easyperm')
 

synapse/lib/stormlib/model.py

@@ -1,6 +1,7 @@
 import synapse.exc as s_exc
 import synapse.common as s_common
 
+import synapse.lib.node as s_node
 import synapse.lib.cache as s_cache
 import synapse.lib.stormtypes as s_stormtypes
 
@@ -692,3 +693,122 @@ class LibModelDeprecated(s_stormtypes.Lib):
         todo = s_common.todo('setDeprLock', name, locked)
         gatekeys = ((self.runt.user.iden, ('model', 'deprecated', 'lock'), None),)
         await self.runt.dyncall('cortex', todo, gatekeys=gatekeys)
+
+@s_stormtypes.registry.registerLib
+class LibModelMigration(s_stormtypes.Lib):
+    '''
+    A Storm library containing migration tools.
+    '''
+    _storm_locals = (
+        {'name': 'copyData', 'desc': 'Copy node data from the src node to the dst node.',
+         'type': {'type': 'function', '_funcname': '_methCopyData',
+                  'args': (
+                      {'name': 'src', 'type': 'node', 'desc': 'The node to copy data from.', },
+                      {'name': 'dst', 'type': 'node', 'desc': 'The node to copy data to.', },
+                      {'name': 'overwrite', 'type': 'boolean', 'default': False,
+                       'desc': 'Copy data even if the key exists on the destination node.', },
+                  ),
+                  'returns': {'type': 'null', }}},
+        {'name': 'copyEdges', 'desc': 'Copy edges from the src node to the dst node.',
+         'type': {'type': 'function', '_funcname': '_methCopyEdges',
+                  'args': (
+                      {'name': 'src', 'type': 'node', 'desc': 'The node to copy edges from.', },
+                      {'name': 'dst', 'type': 'node', 'desc': 'The node to copy edges to.', },
+                  ),
+                  'returns': {'type': 'null', }}},
+        {'name': 'copyTags', 'desc': 'Copy tags, tag timestamps, and tag props from the src node to the dst node.',
+         'type': {'type': 'function', '_funcname': '_methCopyTags',
+                  'args': (
+                      {'name': 'src', 'type': 'node', 'desc': 'The node to copy tags from.', },
+                      {'name': 'dst', 'type': 'node', 'desc': 'The node to copy tags to.', },
+                      {'name': 'overwrite', 'type': 'boolean', 'default': False,
+                       'desc': 'Copy tag property value even if the property exists on the destination node.', },
+                  ),
+                  'returns': {'type': 'null', }}},
+    )
+    _storm_lib_path = ('model', 'migration')
+
+    def getObjLocals(self):
+        return {
+            'copyData': self._methCopyData,
+            'copyEdges': self._methCopyEdges,
+            'copyTags': self._methCopyTags,
+        }
+
+    async def _methCopyData(self, src, dst, overwrite=False):
+
+        if not isinstance(src, s_node.Node):
+            raise s_exc.BadArg(mesg='$lib.model.migration.copyData() source argument must be a node.')
+        if not isinstance(dst, s_node.Node):
+            raise s_exc.BadArg(mesg='$lib.model.migration.copyData() dest argument must be a node.')
+
+        overwrite = await s_stormtypes.tobool(overwrite)
+
+        async with self.runt.snap.getEditor() as editor:
+
+            proto = editor.loadNode(dst)
+
+            async for name in src.iterDataKeys():
+                if overwrite or not await dst.hasData(name):
+                    self.runt.layerConfirm(('node', 'data', 'set', name))
+                    valu = await src.getData(name)
+                    await proto.setData(name, valu)
+
+    async def _methCopyEdges(self, src, dst):
+
+        if not isinstance(src, s_node.Node):
+            raise s_exc.BadArg(mesg='$lib.model.migration.copyEdges() source argument must be a node.')
+        if not isinstance(dst, s_node.Node):
+            raise s_exc.BadArg(mesg='$lib.model.migration.copyEdges() dest argument must be a node.')
+
+        snap = self.runt.snap
+
+        async with snap.getEditor() as editor:
+
+            proto = editor.loadNode(dst)
+            verbs = set()
+
+            async for (verb, n2iden) in src.iterEdgesN1():
+
+                if verb not in verbs:
+                    self.runt.layerConfirm(('node', 'edge', 'add', verb))
+                    verbs.add(verb)
+
+                if await snap.getNodeByBuid(s_common.uhex(n2iden)) is not None:
+                    await proto.addEdge(verb, n2iden)
+
+            dstiden = s_common.ehex(dst.buid)
+
+            async for (verb, n1iden) in src.iterEdgesN2():
+
+                if verb not in verbs:
+                    self.runt.layerConfirm(('node', 'edge', 'add', verb))
+                    verbs.add(verb)
+
+                n1proto = await editor.getNodeByBuid(s_common.uhex(n1iden))
+                if n1proto is not None:
+                    await n1proto.addEdge(verb, dstiden)
+
+    async def _methCopyTags(self, src, dst, overwrite=False):
+
+        if not isinstance(src, s_node.Node):
+            raise s_exc.BadArg(mesg='$lib.model.migration.copyTags() source argument must be a node.')
+        if not isinstance(dst, s_node.Node):
+            raise s_exc.BadArg(mesg='$lib.model.migration.copyTags() dest argument must be a node.')
+
+        overwrite = await s_stormtypes.tobool(overwrite)
+
+        snap = self.runt.snap
+
+        async with snap.getEditor() as editor:
+
+            proto = editor.loadNode(dst)
+
+            for name, valu in src.tags.items():
+                self.runt.layerConfirm(('node', 'tag', 'add', *name.split('.')))
+                await proto.addTag(name, valu=valu)
+
+            for tagname, tagprops in src.tagprops.items():
+                for propname, valu in tagprops.items():
+                    if overwrite or not dst.hasTagProp(tagname, propname):
+                        await proto.setTagProp(tagname, propname, valu) # use tag perms

synapse/lib/stormtypes.py

@@ -1604,8 +1604,12 @@ class LibBase(Lib):
                 continue
             vals.append(arg)
 
+        if len(vals) < 1:
+            mesg = '$lib.min() must have at least one argument or a list containing at least one value.'
+            raise s_exc.StormRuntimeError(mesg=mesg)
+
         ints = [await toint(x) for x in vals]
-        return min(*ints)
+        return min(ints)
 
     @stormfunc(readonly=True)
     async def _max(self, *args):
@@ -1618,8 +1622,12 @@ class LibBase(Lib):
                 continue
             vals.append(arg)
 
+        if len(vals) < 1:
+            mesg = '$lib.max() must have at least one argument or a list containing at least one value.'
+            raise s_exc.StormRuntimeError(mesg=mesg)
+
         ints = [await toint(x) for x in vals]
-        return max(*ints)
+        return max(ints)
 
     @staticmethod
     async def _get_mesg(mesg, **kwargs):
@@ -1707,6 +1715,13 @@ class LibDict(Lib):
     A Storm Library for interacting with dictionaries.
     '''
     _storm_locals = (
+        {'name': 'has', 'desc': 'Check a dictionary has a specific key.',
+         'type': {'type': 'function', '_funcname': '_has',
+                  'args': (
+                      {'name': 'valu', 'type': 'dict', 'desc': 'The dictionary being checked.'},
+                      {'name': 'name', 'type': 'str', 'desc': 'The key name to check.'},
+                  ),
+                  'returns': {'type': 'boolean', 'desc': 'True if the key is present, false if the key is not present.'}}},
         {'name': 'keys', 'desc': 'Retrieve a list of keys in the specified dictionary.',
          'type': {'type': 'function', '_funcname': '_keys',
                   'args': (

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 167, 0)
+version = (2, 169, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = '7b2983533ea27d69e547ac54d9aab953630137c2'
+commit = '41421efcf40bb7655af3f90cb7d6d335247fc726'

synapse/lib/view.py

@@ -55,6 +55,7 @@ class ViewApi(s_cell.CellApi):
         layr = self.view.layers[0]
         async for item in layr.syncNodeEdits2(offs, wait=wait):
             yield item
+            await asyncio.sleep(0)
 
     @s_cell.adminapi()
     async def saveNodeEdits(self, edits, meta):
@@ -867,20 +868,16 @@ class View(s_nexus.Pusher):  # type: ignore
 
             self.layers.append(layr)
 
-    async def eval(self, text, opts=None, log_info=None):
+    async def eval(self, text, opts=None):
         '''
         Evaluate a storm query and yield Nodes only.
         '''
         opts = self.core._initStormOpts(opts)
         user = self.core._userFromOpts(opts)
 
-        if log_info is None:
-            log_info = {}
-
-        log_info['mode'] = opts.get('mode', 'storm')
-        log_info['view'] = self.iden
-
-        self.core._logStormQuery(text, user, info=log_info)
+        info = opts.get('_loginfo', {})
+        info.update({'mode': opts.get('mode', 'storm'), 'view': self.iden})
+        self.core._logStormQuery(text, user, info=info)
 
         taskiden = opts.get('task')
         taskinfo = {'query': text, 'view': self.iden}
@@ -996,8 +993,9 @@ class View(s_nexus.Pusher):  # type: ignore
                                 count += 1
 
                         else:
-                            self.core._logStormQuery(text, user,
-                                                     info={'mode': opts.get('mode', 'storm'), 'view': self.iden})
+                            info = opts.get('_loginfo', {})
+                            info.update({'mode': opts.get('mode', 'storm'), 'view': self.iden})
+                            self.core._logStormQuery(text, user, info=info)
                             async for item in snap.storm(text, opts=opts, user=user):
                                 count += 1