synapse 2.215.0__py311-none-any.whl → 2.216.0__py311-none-any.whl

synapse/cortex.py

@@ -758,6 +758,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     confbase['mirror']['hidedocs'] = False  # type: ignore
     confbase['mirror']['hidecmdl'] = False  # type: ignore
 
+    confbase['safemode']['hidecmdl'] = False
+    confbase['safemode']['description'] = (
+        'Enable safe-mode which disables crons, triggers, dmons, storm '
+        'package onload handlers, view merge tasks, and storm pools.'
+    )
+
     confdefs = {
         'axon': {
             'description': 'A telepath URL for a remote axon.',
@@ -1572,9 +1578,11 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         if self.isactive:
             await self._checkLayerModels()
 
-        self.addActiveCoro(self.agenda.runloop)
+        if not self.safemode:
+            self.addActiveCoro(self.agenda.runloop)
 
         await self._initStormDmons()
+
         await self._initStormSvcs()
 
         # share ourself via the cell dmon as "cortex"
@@ -1621,6 +1629,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
     async def initStormPool(self):
 
+        if self.safemode:
+            return
+
         try:
 
             byts = self.slab.get(b'storm:pool', db='cell:conf')
@@ -2961,6 +2972,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         if onload is not None and self.isactive:
             async def _onload():
+                if self.safemode:
+                    await self.fire('core:pkg:onload:skipped', pkg=name, reason='safemode')
+                    return
+
                 await self.fire('core:pkg:onload:start', pkg=name)
                 try:
                     async for mesg in self.storm(onload):

synapse/lib/cell.py

@@ -930,6 +930,12 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
     confdefs = {}  # type: ignore  # This should be a JSONSchema properties list for an object.
     confbase = {
+        'safemode': {
+            'default': False,
+            'description': 'Boot the service in safe-mode.',
+            'type': 'boolean',
+            'hidecmdl': True,
+        },
         'cell:guid': {
             'description': 'An optional hard-coded GUID to store as the permanent GUID for the service.',
             'type': 'string',
@@ -1200,6 +1206,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             'tasks': 1,
         }
 
+        self.safemode = self.conf.req('safemode')
+        if self.safemode:
+            mesg = f'Booting {self.getCellType()} in safe-mode. Some functionality may be disabled.'
+            logger.warning(mesg)
+
         self.minfree = self.conf.get('limit:disk:free')
         if self.minfree is not None:
             self.minfree = self.minfree / 100
@@ -4816,6 +4827,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 'iden': self.getCellIden(),
                 'paused': self.paused,
                 'active': self.isactive,
+                'safemode': self.safemode,
                 'started': self.startms,
                 'ready': self.nexsroot.ready.is_set(),
                 'commit': self.COMMIT,

synapse/lib/cmd.py

@@ -3,15 +3,19 @@ import asyncio
 import argparse
 
 import synapse.exc as s_exc
-import synapse.common as s_common
 
 import synapse.lib.coro as s_coro
 import synapse.lib.output as s_output
 
 class Parser(argparse.ArgumentParser):
+    '''
+    argparse.ArgumentParser helper class.
 
+    - exit() is overriden to raise a SynErr ( ParserExit )
+    - _print_message prints to an outp object
+    - description formatter uses argparse.RawDescriptionHelpFormatter
+    '''
     def __init__(self, prog=None, outp=s_output.stdout, **kwargs):
-
         self.outp = outp
         self.exited = False
 

synapse/lib/layer.py

@@ -3489,7 +3489,10 @@ class Layer(s_nexus.Pusher):
 
         valt = edit[1]
         valu, stortype = valt
-        if sode.get('valu') == valt:
+
+        isarray = stortype & STOR_FLAG_ARRAY
+
+        if not isarray and sode.get('valu') == valt:
             return ()
 
         abrv = self.setPropAbrv(form, None)
@@ -3500,7 +3503,7 @@ class Layer(s_nexus.Pusher):
         sode['valu'] = valt
         self.setSodeDirty(buid, sode, form)
 
-        if stortype & STOR_FLAG_ARRAY:
+        if isarray:
 
             for indx in self.getStorIndx(stortype, valu):
                 self.layrslab.put(abrv + indx, buid, db=self.byarray)
@@ -3587,6 +3590,8 @@ class Layer(s_nexus.Pusher):
         if prop[0] == '.':  # '.' to detect universal props (as quickly as possible)
             univabrv = self.setPropAbrv(None, prop)
 
+        isarray = stortype & STOR_FLAG_ARRAY
+
         if oldv is not None:
 
             # merge intervals and min times
@@ -3599,7 +3604,7 @@ class Layer(s_nexus.Pusher):
             elif stortype == STOR_TYPE_MAXTIME:
                 valu = max(valu, oldv)
 
-            if valu == oldv and stortype == oldt:
+            if not isarray and valu == oldv and stortype == oldt:
                 return ()
 
             if oldt & STOR_FLAG_ARRAY:
@@ -3638,7 +3643,7 @@ class Layer(s_nexus.Pusher):
         sode['props'][prop] = (valu, stortype)
         self.setSodeDirty(buid, sode, form)
 
-        if stortype & STOR_FLAG_ARRAY:
+        if isarray:
 
             realtype = stortype & 0x7fff
 
@@ -3828,7 +3833,7 @@ class Layer(s_nexus.Pusher):
             kvpairs.append((tp_abrv + indx, buid))
             kvpairs.append((ftp_abrv + indx, buid))
 
-        await self.layrslab.putmulti(kvpairs, db=self.bytagprop)
+        self.layrslab._putmulti(kvpairs, db=self.bytagprop)
 
         return (
             (EDIT_TAGPROP_SET, (tag, prop, valu, oldv, stortype), ()),

synapse/lib/schemas.py

@@ -297,11 +297,18 @@ _authRulesSchema = {
         'type': 'array',
         'items': [
             {'type': 'boolean'},
-            {'type': 'array', 'items': {'type': 'string'}},
+            {
+                'type': 'array',
+                'items': {
+                    'type': 'string',
+                    'minLength': 1
+                },
+                'minItems': 1
+            },
         ],
         'minItems': 2,
         'maxItems': 2,
-    }
+    },
 }
 reqValidRules = s_config.getJsValidator(_authRulesSchema)
 

synapse/lib/snap.py

@@ -384,12 +384,22 @@ class ProtoNode:
 
         if norminfo is None:
             try:
-                valu, norminfo = prop.type.norm(valu)
+                if (isinstance(valu, dict) and isinstance(prop.type, s_types.Guid)
+                    and (form := self.ctx.snap.core.model.form(prop.type.name)) is not None):
+
+                    norms, props = await self.ctx.snap._normGuidNodeDict(form, valu)
+                    valu = await self.ctx.snap._addGuidNodeByDict(form, norms, props)
+                    norminfo = {}
+                else:
+                    valu, norminfo = prop.type.norm(valu)
+
             except s_exc.BadTypeValu as e:
-                oldm = e.get('mesg')
-                e.update({'prop': prop.name,
-                          'form': prop.form.name,
-                          'mesg': f'Bad prop value {prop.full}={valu!r} : {oldm}'})
+                if 'prop' not in e.errinfo:
+                    oldm = e.get('mesg')
+                    e.update({'prop': prop.name,
+                              'form': prop.form.name,
+                              'mesg': f'Bad prop value {prop.full}={valu!r} : {oldm}'})
+
                 if self.ctx.snap.strict:
                     raise e
                 await self.ctx.snap.warn(e)

synapse/lib/storm.py

@@ -1363,7 +1363,7 @@ class DmonManager(s_base.Base):
         '''
         Start all the dmons.
         '''
-        if self.enabled:
+        if self.enabled or self.core.safemode:
             return
         dmons = list(self.dmons.values())
         if not dmons:

synapse/lib/types.py

@@ -625,6 +625,9 @@ class Guid(Type):
         )
 
     def _normPyList(self, valu):
+        if not valu:
+            mesg = 'Guid list values cannot be empty.'
+            raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg=mesg)
         return s_common.guid(valu), {}
 
     def _normPyStr(self, valu):

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 215, 0)
+version = (2, 216, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'f4d48839f7e5f74bacdcc2d1b122d9ade6f1def7'
+commit = '2a44a91a495a7daf2e4e6e44d32c1c3030db3fd2'

synapse/lib/view.py

@@ -260,6 +260,9 @@ class View(s_nexus.Pusher):  # type: ignore
         if self.merging: # pragma: no cover
             return
 
+        if self.core.safemode:
+            return
+
         if not await self.isMergeReady():
             return
 
@@ -380,6 +383,9 @@ class View(s_nexus.Pusher):  # type: ignore
         if not await self.core.isCellActive():
             return
 
+        if self.core.safemode:
+            return
+
         self.mergetask = self.core.schedCoro(self.runViewMerge())
 
     async def finiMergeTask(self):
@@ -568,6 +574,9 @@ class View(s_nexus.Pusher):  # type: ignore
         if not await self.core.isCellActive():
             return
 
+        if self.core.safemode:
+            return
+
         self.trigtask = self.schedCoro(self._trigQueueLoop())
 
     async def finiTrigTask(self):
@@ -1577,7 +1586,7 @@ class View(s_nexus.Pusher):  # type: ignore
 
     async def runTagAdd(self, node, tag, valu):
 
-        if self.core.migration:
+        if self.core.migration or self.core.safemode:
             return
 
         # Run any trigger handlers
@@ -1585,21 +1594,21 @@ class View(s_nexus.Pusher):  # type: ignore
 
     async def runTagDel(self, node, tag, valu):
 
-        if self.core.migration:
+        if self.core.migration or self.core.safemode:
             return
 
         await self.triggers.runTagDel(node, tag)
 
     async def runNodeAdd(self, node):
 
-        if self.core.migration:
+        if self.core.migration or self.core.safemode:
             return
 
         await self.triggers.runNodeAdd(node)
 
     async def runNodeDel(self, node):
 
-        if self.core.migration:
+        if self.core.migration or self.core.safemode:
             return
 
         await self.triggers.runNodeDel(node)
@@ -1608,21 +1617,21 @@ class View(s_nexus.Pusher):  # type: ignore
         '''
         Handle when a prop set trigger event fired
         '''
-        if self.core.migration:
+        if self.core.migration or self.core.safemode:
             return
 
         await self.triggers.runPropSet(node, prop, oldv)
 
     async def runEdgeAdd(self, n1, edge, n2):
 
-        if self.core.migration:
+        if self.core.migration or self.core.safemode:
             return
 
         await self.triggers.runEdgeAdd(n1, edge, n2)
 
     async def runEdgeDel(self, n1, edge, n2):
 
-        if self.core.migration:
+        if self.core.migration or self.core.safemode:
             return
 
         await self.triggers.runEdgeDel(n1, edge, n2)

synapse/models/inet.py

@@ -1542,7 +1542,7 @@ class InetModule(s_module.CoreModule):
                     ('inet:service:app', ('guid', {}), {
                         'interfaces': ('inet:service:object',),
                         'template': {'service:base': 'application'},
-                        'doc': 'A platform specific application.'}),
+                        'doc': 'An application which is part of a service architecture.'}),
 
                     ('inet:service:instance', ('guid', {}), {
                         'doc': 'An instance of the platform such as Slack or Discord instances.'}),
@@ -3769,6 +3769,12 @@ class InetModule(s_module.CoreModule):
                         ('desc', ('str', {}), {
                             'disp': {'hint': 'text'},
                             'doc': 'A description of the platform specific application.'}),
+
+                        ('provider', ('ou:org', {}), {
+                            'doc': 'The organization which provides the application.'}),
+
+                        ('provider:name', ('ou:name', {}), {
+                            'doc': 'The name of the organization which provides the application.'}),
                     )),
 
                     ('inet:service:account', {}, (

synapse/models/risk.py

@@ -220,6 +220,8 @@ class RiskModule(s_module.CoreModule):
                     'doc': 'The threat cluster targeted the target node.'}),
                 (('risk:threat', 'uses', None), {
                     'doc': 'The threat cluster uses the target node.'}),
+                (('risk:threat', 'uses', 'inet:service:app'), {
+                    'doc': 'The threat cluster uses the online application.'}),
                 (('risk:attack', 'targets', None), {
                     'doc': 'The attack targeted the target node.'}),
                 (('risk:attack', 'uses', None), {

synapse/tests/test_cortex.py

@@ -8776,6 +8776,268 @@ class CortexBasicTest(s_t_utils.SynTest):
                 self.eq(core._test_pre_nexus_index, offs)
                 self.ge(core._test_post_nexus_index, core._test_pre_nexus_index)
 
+    async def test_cortex_safemode(self):
+        safemode = {'safemode': True}
+        nosafe = {'safemode': False}
+
+        # Cortex safemode disables the following functionality:
+        # - crons
+        # - triggers
+        # - dmons
+        # - storm package onloads
+        # - merge tasks (e.g. for quorum)
+        # - storm pools
+
+        # Make sure we're logging the message
+        with self.getStructuredAsyncLoggerStream('synapse.lib.cell', 'Booting cortex in safe-mode.') as stream:
+            async with self.getTestCore(conf=safemode) as core:
+                self.true(core.safemode)
+                self.true(await stream.wait(10))
+        msgs = stream.jsonlines()
+        self.len(1, msgs)
+        self.eq(msgs[0].get('message'), 'Booting cortex in safe-mode. Some functionality may be disabled.')
+        self.eq(msgs[0].get('level'), 'WARNING')
+
+        # Check crons, triggers, dmons in this section
+        with self.getTestDir() as dirn:
+
+            # Setup the cortex
+            async with self.getTestCore(dirn=dirn) as core:
+                await core.addCoreQueue('queue:safemode:done', {})
+                # Add a cron job and immediately disable it
+                q = '''
+                cron.add --minute +1 {
+                    $now = $lib.cast(time, now)
+                    $lib.log.warning(`SAFEMODE CRON: {$now}`)
+                    [ test:str=CRON :tick=$now ]
+                } |
+                $job = $lib.cron.list().0
+                cron.disable $job.iden
+                '''
+                await core.callStorm(q)
+                jobs = await core.listCronJobs()
+                self.len(1, jobs)
+                self.eq(jobs[0].get('enabled'), False)
+
+                # Add a regular trigger
+                q = '''
+                $lib.log.warning(`SAFEMODE TRIGGER: {$node}`)
+                $tick = :tick
+                $str = { [( test:str=TRIGGER :hehe=$tick )] }
+                $queue = $lib.queue.gen(queue:safemode)
+                $queue.put($tick)
+                '''
+                opts = {'vars': {'query': q}}
+                await core.callStorm(f'trigger.add prop:set --prop test:str:tick --query {{{q}}}')
+
+                # Add an async trigger
+                q = '''
+                $lib.log.warning(`SAFEMODE ATRIGGER: {$node}`)
+                $tick = :tick
+                $str = { [( test:str=ATRIGGER :hehe=$tick )] }
+                $queue = $lib.queue.gen(queue:safemode)
+                $queue.put($tick)
+                '''
+                opts = {'vars': {'query': q}}
+                await core.callStorm(f'trigger.add prop:set --prop test:str:tick --async --query {{{q}}}')
+
+                # Add a dmon
+                q = '''
+                $queue = $lib.queue.gen(queue:safemode)
+                $queue2 = $lib.queue.gen(queue:safemode:done)
+                while (true) {
+                    ($offs, $item) = $queue.get()
+                    $lib.log.warning(`SAFEMODE DMON: {$item}`)
+                    [ test:str=DMON :hehe=$item ]
+                    $queue2.put($item)
+                }
+                '''
+                await core.callStorm(f'$iden = $lib.dmon.add(${{{q}}}) $lib.dmon.start($iden)')
+
+                nodes = await core.nodes('test:str')
+                self.len(0, nodes)
+
+            # Run in safemode and verify cron, trigger, and dmons don't execute
+            with self.getLoggerStream('synapse.storm') as stream:
+                async with self.getTestCore(dirn=dirn, conf=safemode) as core:
+                    await core.callStorm('cron.enable $lib.cron.list().0.iden')
+                    # Increment the cron tick to get it to fire
+                    core.agenda._addTickOff(60)
+
+                    # Add a test:str:tick to get the trigger to fire
+                    await core.callStorm('[ test:str=newp :tick=1234 ]')
+
+                    # Check for test:strs
+                    nodes = await core.nodes('test:str')
+                    self.len(1, nodes)
+                    self.eq(nodes[0].repr(), 'newp')
+
+                    with self.raises(TimeoutError):
+                        await s_common.wait_for(core.coreQueueGet('queue:safemode:done', wait=True), timeout=2)
+
+                    # Add a dmon to make sure it doesn't start
+                    await core.addCoreQueue('queue:safemode:started', {})
+                    q = '''
+                    $queue = $lib.queue.gen(queue:safemode)
+                    $queue2 = $lib.queue.get(queue:safemode:started)
+                    while (true) {
+                        $queue2.put(foo)
+                        $lib.log.warning(`SAFEMODE DMON START`)
+                        ($offs, $item) = $queue.get()
+                    }
+                    '''
+                    await core.callStorm(f'$iden = $lib.dmon.add(${{{q}}}) $lib.dmon.start($iden)')
+
+                    with self.raises(TimeoutError):
+                        await s_common.wait_for(core.coreQueueGet('queue:safemode:started', wait=True), timeout=2)
+
+                stream.seek(0)
+                data = stream.read()
+                self.notin('SAFEMODE CRON', data)
+                self.notin('SAFEMODE TRIGGER', data)
+                self.notin('SAFEMODE ATRIGGER', data)
+                self.notin('SAFEMODE DMON', data)
+
+            with self.getLoggerStream('synapse.storm') as stream:
+                async with self.getTestCore(dirn=dirn) as core:
+                    core.agenda._addTickOff(60)
+
+                    item = await s_common.wait_for(core.coreQueueGet('queue:safemode:done', wait=True), timeout=5)
+                    self.len(2, item)
+
+                    nodes = await core.nodes('test:str')
+                    self.len(5, nodes)
+                    self.sorteq(
+                        ['newp', 'CRON', 'TRIGGER', 'DMON', 'ATRIGGER'],
+                        [k.repr() for k in nodes]
+                    )
+
+                stream.seek(0)
+                data = stream.read()
+                self.isin('SAFEMODE CRON', data)
+                self.isin('SAFEMODE TRIGGER', data)
+                self.isin('SAFEMODE ATRIGGER', data)
+                self.isin('SAFEMODE DMON', data)
+
+        # Check storm package onload handlers are not executed
+        with self.getTestDir() as dirn:
+            async with self.getTestCore(dirn=dirn, conf=safemode) as core:
+                pkgdef = {
+                    'name': 'foopkg',
+                    'version': (0, 0, 1),
+                    'onload': '$lib.import(foo.setup).onload()',
+                    'modules': (
+                        {
+                            'name': 'foo.setup',
+                            'storm': '''
+                                function onload() {
+                                    $lib.warn('foopkg onload')
+                                }
+                            ''',
+                        },
+                    )
+                }
+
+                waiter = core.waiter(1, 'core:pkg:onload:skipped')
+                await core.addStormPkg(pkgdef)
+                events = await waiter.wait(timeout=10)
+                self.nn(events)
+                self.len(1, events)
+                self.eq(events, (('core:pkg:onload:skipped', {'pkg': 'foopkg', 'reason': 'safemode'}),))
+
+            with self.getAsyncLoggerStream('synapse.cortex', 'foopkg onload output: foopkg onload') as stream:
+                async with self.getTestCore(dirn=dirn, conf=nosafe) as core:
+                    self.false(core.safemode)
+                    await stream.wait(timeout=10)
+
+        # Check merge tasks are not executed
+        with self.getTestDir() as dirn:
+            async with self.getTestCore(dirn=dirn, conf=safemode) as core:
+                alliden = core.auth.allrole.iden
+
+                blackout = await core.auth.addUser('blackout')
+                await blackout.allow(('view', 'read'))
+
+                await core.auth.rootuser.grant(alliden)
+
+                view = core.getView()
+                qnfo = {
+                    'count': 1,
+                    'roles': [alliden],
+                }
+                await view.setViewInfo('quorum', qnfo)
+
+                forkiden = (await view.fork()).get('iden')
+
+                opts = {'view': forkiden}
+
+                nodes = await core.nodes('[ test:str=fork ]', opts=opts)
+                self.len(1, nodes)
+
+                await core.callStorm('$lib.view.get().setMergeRequest()', opts=opts)
+                await core.callStorm('$lib.view.get().setMergeVote()', opts=opts | {'user': blackout.iden})
+
+                fork = core.getView(forkiden)
+
+                self.true(await fork.isMergeReady())
+                self.none(fork.mergetask)
+
+                self.len(0, await core.nodes('test:str'))
+
+            async with self.getTestCore(dirn=dirn, conf=nosafe) as core:
+                nodes = await core.nodes('test:str')
+                self.len(1, nodes)
+                self.eq(nodes[0].repr(), 'fork')
+
+        # Check storm pools are not enabled
+        async with self.getTestAha() as aha:
+
+            ahanet = aha.conf.req('aha:network')
+
+            async with await s_base.Base.anit() as base:
+
+                with self.getTestDir() as dirn:
+
+                    dirn00 = s_common.genpath(dirn, 'cell00')
+                    dirn01 = s_common.genpath(dirn, 'cell01')
+
+                    core00 = await base.enter_context(self.addSvcToAha(aha, '00.core', s_cortex.Cortex, dirn=dirn00,
+                                                                       conf=safemode))
+                    provinfo = {'mirror': 'core'}
+                    core01 = await base.enter_context(self.addSvcToAha(aha, '01.core', s_cortex.Cortex, dirn=dirn01,
+                                                                       provinfo=provinfo))
+
+                    msgs = await core01.stormlist('aha.pool.add pool00...')
+                    self.stormHasNoWarnErr(msgs)
+                    self.stormIsInPrint('Created AHA service pool: pool00.synapse', msgs)
+
+                    msgs = await core01.stormlist('aha.pool.svc.add pool00... 01.core...')
+                    self.stormHasNoWarnErr(msgs)
+                    self.stormIsInPrint('AHA service (01.core...) added to service pool (pool00.synapse)', msgs)
+
+                    msgs = await core00.stormlist('cortex.storm.pool.set --connection-timeout 1 --sync-timeout 1 aha://pool00...')
+                    self.stormHasNoWarnErr(msgs)
+                    self.stormIsInPrint('Storm pool configuration set.', msgs)
+
+                    # Make sure queries still work
+                    nodes = await core00.nodes('[ it:dev:str="stormpools!" ]', opts={'mirror': True})
+                    self.len(1, nodes)
+
+                    self.none(core00.stormpool)
+
+                    await core00.fini()
+
+                    # Now disable safemode and check that the stormpool is enabled
+                    core00 = await base.enter_context(self.getTestCore(dirn=dirn00, conf=nosafe))
+
+                    nodes = await core00.nodes('it:dev:str', opts={'mirror': True})
+                    self.len(1, nodes)
+
+                    self.nn(core00.stormpool)
+
+                    await core00.fini()
+                    await core01.fini()
+
     async def test_cortex_queue_mirror_authgates(self):
         async with self.getRegrCore('2.213.0-queue-authgates') as core:
             self.nn(await core.getAuthGate('queue:stillhere'))

synapse/tests/test_lib_auth.py

@@ -431,6 +431,8 @@ class AuthTest(s_test.SynTest):
                 await core.auth.allrole.setRules([(True, ('hehe', 'haha'), 'newp')])
             with self.raises(s_exc.SchemaViolation):
                 await core.auth.allrole.setRules([(True, )])
+            with self.raises(s_exc.SchemaViolation):
+                await core.auth.allrole.setRules([(True, '')])
 
     async def test_auth_archived_locked_interaction(self):