synapse 2.168.0__tar.gz → 2.170.0__tar.gz

{synapse-2.168.0/synapse.egg-info → synapse-2.170.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse
-Version: 2.168.0
+Version: 2.170.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License: Apache License 2.0

{synapse-2.168.0 → synapse-2.170.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.168.0'
+version = '2.170.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]

{synapse-2.168.0 → synapse-2.170.0}/synapse/cortex.py

@@ -573,14 +573,6 @@ class CoreApi(s_cell.CellApi):
     async def delStormDmon(self, iden):
         return await self.cell.delStormDmon(iden)
 
-    @s_cell.adminapi(log=True)
-    async def enableMigrationMode(self): # pragma: no cover
-        s_common.deprdate('CoreApi.enableMigrationMode', '2024-05-05')
-
-    @s_cell.adminapi(log=True)
-    async def disableMigrationMode(self): # pragma: no cover
-        s_common.deprdate('CoreApi.disableMigrationMode', '2024-05-05')
-
     @s_cell.adminapi()
     async def cloneLayer(self, iden, ldef=None):
 
@@ -1350,6 +1342,17 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
              'ex': 'node.prop.del.inet:ipv4.asn',
              'desc': 'Controls removing a specific property from a form of node in a layer.'},
 
+            {'perm': ('node', 'data', 'set'), 'gate': 'layer',
+             'desc': 'Permits a user to set node data in a given layer.'},
+            {'perm': ('node', 'data', 'set', '<key>'), 'gate': 'layer',
+              'ex': 'node.data.set.hehe',
+             'desc': 'Permits a user to set node data in a given layer for a specific key.'},
+            {'perm': ('node', 'data', 'pop'), 'gate': 'layer',
+             'desc': 'Permits a user to remove node data in a given layer.'},
+            {'perm': ('node', 'data', 'pop', '<key>'), 'gate': 'layer',
+             'ex': 'node.data.pop.hehe',
+             'desc': 'Permits a user to remove node data in a given layer for a specific key.'},
+
             {'perm': ('pkg', 'add'), 'gate': 'cortex',
              'desc': 'Controls access to adding storm packages.'},
             {'perm': ('pkg', 'del'), 'gate': 'cortex',
@@ -3863,7 +3866,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                     with open(idenpath, 'w') as fd:
                         fd.write(jsoniden)
 
-            conf = {'cell:guid': jsoniden}
+            # Disable sysctl checks for embedded jsonstor server
+            conf = {'cell:guid': jsoniden, 'health:sysctl:checks': False}
             self.jsonstor = await s_jsonstor.JsonStorCell.anit(path, conf=conf, parent=self)
 
         self.onfini(self.jsonstor)
@@ -3940,7 +3944,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         turl = self.conf.get('axon')
         if turl is None:
             path = os.path.join(self.dirn, 'axon')
-            conf = {}
+            # Disable sysctl checks for embedded axon server
+            conf = {'health:sysctl:checks': False}
 
             proxyurl = self.conf.get('http:proxy')
             if proxyurl is not None:
@@ -4491,6 +4496,87 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         await self.feedBeholder('view:add', pack, gates=[iden])
         return pack
 
+    async def delViewWithLayer(self, iden):
+        '''
+        Delete a Cortex View and its write Layer if not in use by other View stacks.
+
+        Note:
+            Any children of the View will have their parent View updated to
+            the deleted View's parent (if present). The deleted View's write Layer
+            will also be removed from any child Views which contain it in their
+            Layer stack. If the Layer is used in Views which are not children of
+            the deleted View, the Layer will be preserved, otherwise it will be
+            deleted as well.
+        '''
+        view = self.views.get(iden)
+        if view is None:
+            raise s_exc.NoSuchView(mesg=f'No such view {iden=}', iden=iden)
+
+        if view.info.get('protected'):
+            mesg = f'Cannot delete view ({iden}) that has protected set.'
+            raise s_exc.CantDelView(mesg=mesg)
+
+        layriden = view.layers[0].iden
+        pareiden = None
+        if view.parent is not None:
+            pareiden = view.parent.iden
+
+        return await self._push('view:delwithlayer', iden, layriden, newparent=pareiden)
+
+    @s_nexus.Pusher.onPush('view:delwithlayer', passitem=True)
+    async def _delViewWithLayer(self, viewiden, layriden, nexsitem, newparent=None):
+
+        if viewiden == self.view.iden:
+            raise s_exc.SynErr(mesg='Cannot delete the main view')
+
+        if (view := self.views.get(viewiden)) is not None:
+
+            await self.hive.pop(('cortex', 'views', viewiden))
+            await view.delete()
+
+            self._calcViewsByLayer()
+            await self.feedBeholder('view:del', {'iden': viewiden}, gates=[viewiden])
+            await self.auth.delAuthGate(viewiden)
+
+        if newparent is not None:
+            newview = self.views.get(newparent)
+
+        layrinuse = False
+        for view in self.viewsbylayer[layriden]:
+            if not view.isForkOf(viewiden):
+                layrinuse = True
+                continue
+
+            view.layers = [lyr for lyr in view.layers if lyr.iden != layriden]
+            await view.info.set('layers', [lyr.iden for lyr in view.layers])
+
+            if view.parent.iden == viewiden:
+                if newparent is None:
+                    view.parent = None
+                    await view.info.pop('parent')
+                else:
+                    view.parent = newview
+                    await view.info.set('parent', newparent)
+
+        if not layrinuse and (layr := self.layers.get(layriden)) is not None:
+            del self.layers[layriden]
+
+            for pdef in layr.layrinfo.get('pushs', {}).values():
+                await self.delActiveCoro(pdef.get('iden'))
+
+            for pdef in layr.layrinfo.get('pulls', {}).values():
+                await self.delActiveCoro(pdef.get('iden'))
+
+            await self.feedBeholder('layer:del', {'iden': layriden}, gates=[layriden])
+            await self.auth.delAuthGate(layriden)
+            self.dynitems.pop(layriden)
+
+            await self.hive.pop(('cortex', 'layers', layriden))
+
+            await layr.delete()
+
+            layr.deloffs = nexsitem[0]
+
     async def delView(self, iden):
         view = self.views.get(iden)
         if view is None:
@@ -5153,12 +5239,15 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         return dmon
 
+    @s_cell.from_leader
     async def getStormDmon(self, iden):
         return self.stormdmons.getDmonDef(iden)
 
+    @s_cell.from_leader
     async def getStormDmons(self):
         return self.stormdmons.getDmonDefs()
 
+    @s_cell.from_leader
     async def getStormDmonLog(self, iden):
         return self.stormdmons.getDmonRunlog(iden)
 

{synapse-2.168.0 → synapse-2.170.0}/synapse/datamodel.py

@@ -330,11 +330,16 @@ class Form:
                 'prop': [],
                 'ndef': [],
                 'array': [],
+                'ndefarray': [],
             }
 
             for name, prop in self.props.items():
 
                 if isinstance(prop.type, s_types.Array):
+                    if isinstance(prop.type.arraytype, s_types.Ndef):
+                        self.refsout['ndefarray'].append(name)
+                        continue
+
                     typename = prop.type.arraytype.name
                     if self.modl.forms.get(typename) is not None:
                         self.refsout['array'].append((name, typename))

{synapse-2.168.0 → synapse-2.170.0}/synapse/lib/agenda.py

@@ -831,6 +831,9 @@ class Agenda(s_base.Base):
                 'view': appt.view,
                 'mirror': appt.pool,
                 'vars': {'auto': {'iden': appt.iden, 'type': 'cron'}},
+                '_loginfo': {
+                    'cron': appt.iden
+                }
             }
             opts = self.core._initStormOpts(opts)
 

{synapse-2.168.0 → synapse-2.170.0}/synapse/lib/ast.py

@@ -1943,6 +1943,12 @@ class PivotOut(PivotOper):
                 continue
 
             if isinstance(prop.type, s_types.Array):
+                if isinstance(prop.type.arraytype, s_types.Ndef):
+                    for item in valu:
+                        if (pivo := await runt.snap.getNodeByNdef(item)) is not None:
+                            yield pivo, path.fork(pivo)
+                    continue
+
                 typename = prop.type.opts.get('type')
                 if runt.model.forms.get(typename) is not None:
                     for item in valu:
@@ -2177,13 +2183,13 @@ class FormPivot(PivotOper):
 
             # plain old pivot...
             async def pgenr(node, strict=True):
-
-                valu = node.ndef[1]
-
                 if isarray:
-                    ngenr = runt.snap.nodesByPropArray(prop.full, '=', valu)
+                    if isinstance(prop.type.arraytype, s_types.Ndef):
+                        ngenr = runt.snap.nodesByPropArray(prop.full, '=', node.ndef)
+                    else:
+                        ngenr = runt.snap.nodesByPropArray(prop.full, '=', node.ndef[1])
                 else:
-                    ngenr = runt.snap.nodesByPropValu(prop.full, '=', valu)
+                    ngenr = runt.snap.nodesByPropValu(prop.full, '=', node.ndef[1])
 
                 # TODO cache/bypass normalization in loop!
                 async for pivo in ngenr:
@@ -2266,6 +2272,16 @@ class FormPivot(PivotOper):
                         if pivo is not None:
                             yield pivo
 
+                for refsname in refs.get('ndefarray'):
+
+                    found = True
+
+                    if (refsvalu := node.get(refsname)) is not None:
+                        for aval in refsvalu:
+                            if aval[0] == destform.name:
+                                if (pivo := await runt.snap.getNodeByNdef(aval)) is not None:
+                                    yield pivo
+
                 #########################################################################
                 # reverse "-> form" pivots (ie inet:fqdn -> inet:dns:a)
                 refs = destform.getRefsOut()
@@ -2303,6 +2319,14 @@ class FormPivot(PivotOper):
                     async for pivo in runt.snap.nodesByPropValu(refsprop.full, '=', node.ndef):
                         yield pivo
 
+                for refsname in refs.get('ndefarray'):
+
+                    found = True
+
+                    refsprop = destform.props.get(refsname)
+                    async for pivo in runt.snap.nodesByPropArray(refsprop.full, '=', node.ndef):
+                        yield pivo
+
                 if strict and not found:
                     mesg = f'No pivot found for {node.form.name} -> {destform.name}.'
                     raise self.addExcInfo(s_exc.NoSuchPivot(n1=node.form.name, n2=destform.name, mesg=mesg))
@@ -2389,6 +2413,12 @@ class PropPivotOut(PivotOper):
                 continue
 
             if prop.type.isarray:
+                if isinstance(prop.type.arraytype, s_types.Ndef):
+                    for item in valu:
+                        if (pivo := await runt.snap.getNodeByNdef(item)) is not None:
+                            yield pivo, path.fork(pivo)
+                    continue
+
                 fname = prop.type.arraytype.name
                 if runt.model.forms.get(fname) is None:
                     if not warned:
@@ -2442,6 +2472,14 @@ class PropPivot(PivotOper):
 
             # pivoting from an array prop to a non-array prop needs an extra loop
             if srcprop.type.isarray and not prop.type.isarray:
+                if isinstance(srcprop.type.arraytype, s_types.Ndef) and prop.isform:
+                    for aval in valu:
+                        if aval[0] != prop.form.name:
+                            continue
+
+                        if (pivo := await runt.snap.getNodeByNdef(aval)) is not None:
+                            yield pivo
+                    return
 
                 for arrayval in valu:
                     async for pivo in runt.snap.nodesByPropValu(prop.full, '=', arrayval):
@@ -2449,6 +2487,18 @@ class PropPivot(PivotOper):
 
                 return
 
+            if isinstance(srcprop.type, s_types.Ndef) and prop.isform:
+                if valu[0] != prop.form.name:
+                    return
+
+                pivo = await runt.snap.getNodeByNdef(valu)
+                if pivo is None:
+                    await runt.snap.warn(f'Missing node corresponding to ndef {valu}', log=False, ndef=valu)
+                    return
+                yield pivo
+
+                return
+
             if prop.type.isarray and not srcprop.type.isarray:
                 genr = runt.snap.nodesByPropArray(prop.full, '=', valu)
             else:
@@ -2463,7 +2513,6 @@ class PropPivot(PivotOper):
 
         if isinstance(name, list) or (prop := runt.model.props.get(name)) is None:
 
-            proplist = None
             if isinstance(name, list):
                 proplist = name
             else:
@@ -2485,7 +2534,7 @@ class PropPivot(PivotOper):
                     async for pivo in pgenr(node, srcprop, valu, strict=False):
                         yield pivo
 
-            return(listpivot)
+            return listpivot
 
         return self.pivogenr(runt, prop)
 
@@ -2771,15 +2820,24 @@ class TagCond(Cond):
         kid = self.kids[0]
 
         if not isinstance(kid, TagMatch):
-            # TODO:  we might hint based on variable value
             return []
 
-        if not kid.isconst or kid.hasglob():
+        if kid.hasglob():
             return []
 
-        return (
-            ('tag', {'name': await kid.compute(None, None)}),
-        )
+        if kid.isconst:
+            return (
+                ('tag', {'name': await kid.compute(None, None)}),
+            )
+
+        if kid.isRuntSafe(runt):
+            name = await kid.compute(runt, path)
+            if name and '*' not in name:
+                return (
+                    ('tag', {'name': name}),
+                )
+
+        return []
 
     async def getCondEval(self, runt):
 

{synapse-2.168.0 → synapse-2.170.0}/synapse/lib/cell.py

@@ -55,6 +55,7 @@ import synapse.lib.lmdbslab as s_lmdbslab
 import synapse.lib.thisplat as s_thisplat
 
 import synapse.lib.crypto.passwd as s_passwd
+import synapse.lib.platforms.linux as s_linux
 
 import synapse.tools.backup as s_t_backup
 
@@ -109,6 +110,25 @@ def adminapi(log=False):
 
     return decrfunc
 
+def from_leader(func):
+    '''
+    Decorator used to indicate that the decorated method must call up to the
+    leader to perform it's work.
+
+    This only works on Cell classes and subclasses. The decorated method name
+    MUST be the same as the telepath API name.
+    '''
+    @functools.wraps(func)
+    async def wrapper(self, *args, **kwargs):
+        if not self.isactive:
+            proxy = await self.nexsroot.client.proxy()
+            api = getattr(proxy, func.__name__)
+            return await api(*args, **kwargs)
+
+        return await func(self, *args, **kwargs)
+
+    return wrapper
+
 async def _doIterBackup(path, chunksize=1024):
     '''
     Create tarball and stream bytes.
@@ -924,6 +944,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             'minimum': 0,
             'maximum': 100,
         },
+        'health:sysctl:checks': {
+            'default': True,
+            'description': 'Enable sysctl parameter checks and warn if values are not optimal.',
+            'type': 'boolean',
+        },
         'aha:name': {
             'description': 'The name of the cell service in the aha service registry.',
             'type': 'string',
@@ -1049,6 +1074,12 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
     VERSION = s_version.version
     VERSTRING = s_version.verstring
 
+    SYSCTL_VALS = {
+        'vm.dirty_expire_centisecs': 20,
+        'vm.dirty_writeback_centisecs': 20,
+    }
+    SYSCTL_CHECK_FREQ = 60.0
+
     async def __anit__(self, dirn, conf=None, readonly=False, parent=None):
 
         # phase 1
@@ -1177,16 +1208,24 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         self.cellinfo = await node.dict()
         self.onfini(node)
 
-        node = await self.hive.open(('cellvers',))
-        self.cellvers = await node.dict(nexs=True)
+        # Check the cell version didn't regress
+        if (lastver := self.cellinfo.get('cell:version')) is not None and self.VERSION < lastver:
+            mesg = f'Cell version regression ({self.getCellType()}) is not allowed! Stored version: {lastver}, current version: {self.VERSION}.'
+            logger.error(mesg)
+            raise s_exc.BadVersion(mesg=mesg, currver=self.VERSION, lastver=lastver)
 
-        if self.inaugural:
-            await self.cellinfo.set('synapse:version', s_version.version)
+        await self.cellinfo.set('cell:version', self.VERSION)
+
+        # Check the synapse version didn't regress
+        if (lastver := self.cellinfo.get('synapse:version')) is not None and s_version.version < lastver:
+            mesg = f'Synapse version regression ({self.getCellType()}) is not allowed! Stored version: {lastver}, current version: {s_version.version}.'
+            logger.error(mesg)
+            raise s_exc.BadVersion(mesg=mesg, currver=s_version.version, lastver=lastver)
 
-        synvers = self.cellinfo.get('synapse:version')
+        await self.cellinfo.set('synapse:version', s_version.version)
 
-        if synvers is None or synvers < s_version.version:
-            await self.cellinfo.set('synapse:version', s_version.version)
+        node = await self.hive.open(('cellvers',))
+        self.cellvers = await node.dict(nexs=True)
 
         self.auth = await self._initCellAuth()
 
@@ -1217,6 +1256,9 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         self._health_funcs = []
         self.addHealthFunc(self._cellHealth)
 
+        if self.conf.get('health:sysctl:checks'):
+            self.schedCoro(self._runSysctlLoop())
+
         # initialize network backend infrastructure
         await self._initAhaRegistry()
 
@@ -1295,14 +1337,20 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
     async def _onBootOptimize(self):
 
+        bdir = s_common.genpath(self.dirn, 'backups')
         tdir = s_common.gendir(self.dirn, 'tmp')
         tdev = os.stat(tdir).st_dev
 
+        logger.warning('Collecting LMDB files for onboot optimization.')
+
         lmdbs = []
         for (root, dirs, files) in os.walk(self.dirn):
             for dirname in dirs:
                 filepath = os.path.join(root, dirname, 'data.mdb')
                 if os.path.isfile(filepath):
+                    if filepath.startswith(bdir):
+                        logger.debug(f'Skipping backup file {filepath}')
+                        continue
                     if os.stat(filepath).st_dev != tdev:
                         logger.warning(f'Unable to run onboot:optimize, {filepath} is not on the same volume as {tdir}')
                         return
@@ -1414,6 +1462,30 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
             await self._checkspace.timewait(timeout=self.FREE_SPACE_CHECK_FREQ)
 
+    async def _runSysctlLoop(self):
+        while not self.isfini:
+            fixvals = []
+            sysctls = s_linux.getSysctls()
+
+            for name, valu in self.SYSCTL_VALS.items():
+                if (sysval := sysctls.get(name)) != valu:
+                    fixvals.append({'name': name, 'expected': valu, 'actual': sysval})
+
+            if not fixvals:
+                # All sysctl parameters have been set to recommended values, no
+                # need to keep checking.
+                break
+
+            fixnames = [k['name'] for k in fixvals]
+            mesg = f'Sysctl values different than expected: {", ".join(fixnames)}. '
+            mesg += 'See https://synapse.docs.vertex.link/en/latest/synapse/devopsguide.html#performance-tuning '
+            mesg += 'for information about these sysctl parameters.'
+
+            extra = await self.getLogExtra(sysctls=fixvals)
+            logger.warning(mesg, extra=extra)
+
+            await asyncio.sleep(self.SYSCTL_CHECK_FREQ)
+
     def _getAhaAdmin(self):
         name = self.conf.get('aha:admin')
         if name is not None:
@@ -2891,24 +2963,10 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         self.onfini(self.dmon.fini)
 
     async def _initCellHive(self):
-        isnew = not self.slab.dbexists('hive')
-
         db = self.slab.initdb('hive')
         hive = await s_hive.SlabHive.anit(self.slab, db=db, nexsroot=self.getCellNexsRoot())
         self.onfini(hive)
 
-        if isnew:
-            path = os.path.join(self.dirn, 'hiveboot.yaml')
-            if os.path.isfile(path):
-                s_common.deprdate('Initial hive config from hiveboot.yaml', '2024-05-05')
-                logger.debug(f'Loading cell hive from {path}')
-                tree = s_common.yamlload(path)
-                if tree is not None:
-                    # Pack and unpack the tree to avoid tuple/list issues
-                    # for in-memory structures.
-                    tree = s_common.tuplify(tree)
-                    await hive.loadHiveTree(tree)
-
         return hive
 
     async def _initCellSlab(self, readonly=False):
@@ -3732,6 +3790,10 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         logconf = s_common.setlogging(logger, defval=opts.log_level,
                                       structlog=opts.structured_logging)
 
+        logger.info(f'Starting {cls.getCellType()} version {cls.VERSTRING}, Synapse version: {s_version.verstring}',
+                    extra={'synapse': {'svc_type': cls.getCellType(), 'svc_version': cls.VERSTRING,
+                                       'synapse_version': s_version.verstring}})
+
         await cls._initBootRestore(opts.dirn)
 
         try:

{synapse-2.168.0 → synapse-2.170.0}/synapse/lib/httpapi.py

@@ -1354,6 +1354,9 @@ class ExtApiHandler(StormHandler):
             'user': useriden,
             'vars': varz,
             'view': adef.get('view'),
+            '_loginfo': {
+                'httpapi': iden
+            }
         }
 
         query = '\n'.join((self.storm_prefix, storm))