synapse 2.166.0__py311-none-any.whl → 2.168.0__py311-none-any.whl

synapse/lib/view.py

@@ -55,6 +55,7 @@ class ViewApi(s_cell.CellApi):
         layr = self.view.layers[0]
         async for item in layr.syncNodeEdits2(offs, wait=wait):
             yield item
+            await asyncio.sleep(0)
 
     @s_cell.adminapi()
     async def saveNodeEdits(self, edits, meta):
@@ -1098,7 +1099,8 @@ class View(s_nexus.Pusher):  # type: ignore
         '''
         Set a mutable view property.
         '''
-        if name not in ('name', 'desc', 'parent', 'nomerge', 'quorum'):
+        if name not in ('name', 'desc', 'parent', 'nomerge', 'protected', 'quorum'):
+            # TODO: Remove nomerge after Synapse 3.x.x
             mesg = f'{name} is not a valid view info key'
             raise s_exc.BadOptValu(mesg=mesg)
 
@@ -1346,8 +1348,8 @@ class View(s_nexus.Pusher):  # type: ignore
         if self.parent is None:
             raise s_exc.CantMergeView(mesg=f'Cannot merge view ({self.iden}) that has not been forked.')
 
-        if self.info.get('nomerge'):
-            raise s_exc.CantMergeView(mesg=f'Cannot merge view ({self.iden}) that has nomerge set.')
+        if self.info.get('protected'):
+            raise s_exc.CantMergeView(mesg=f'Cannot merge view ({self.iden}) that has protected set.')
 
         if self.parent.info.get('quorum') is not None:
             raise s_exc.CantMergeView(mesg=f'Cannot merge view({self.iden}). Parent view requires quorum voting.')

synapse/models/base.py

@@ -63,28 +63,36 @@ class BaseModule(s_module.CoreModule):
                     'doc': 'A generic rule linked to matches with -(matches)> edges.'}),
 
                 ('graph:cluster', ('guid', {}), {
+                    'deprecated': True,
                     'doc': 'A generic node, used in conjunction with Edge types, to cluster arbitrary nodes to a '
                            'single node in the model.'}),
 
                 ('graph:node', ('guid', {}), {
+                    'deprecated': True,
                     'doc': 'A generic node used to represent objects outside the model.'}),
 
                 ('graph:event', ('guid', {}), {
+                    'deprecated': True,
                     'doc': 'A generic event node to represent events outside the model.'}),
 
                 ('edge:refs', ('edge', {}), {
+                    'deprecated': True,
                     'doc': 'A digraph edge which records that N1 refers to or contains N2.'}),
 
                 ('edge:has', ('edge', {}), {
+                    'deprecated': True,
                     'doc': 'A digraph edge which records that N1 has N2.'}),
 
                 ('edge:wentto', ('timeedge', {}), {
+                    'deprecated': True,
                     'doc': 'A digraph edge which records that N1 went to N2 at a specific time.'}),
 
                 ('graph:edge', ('edge', {}), {
+                    'deprecated': True,
                     'doc': 'A generic digraph edge to show relationships outside the model.'}),
 
                 ('graph:timeedge', ('timeedge', {}), {
+                    'deprecated': True,
                     'doc': 'A generic digraph time edge to show relationships outside the model.'}),
 
                 ('meta:priority', ('int', {'enums': prioenums, 'enums:strict': False}), {

synapse/models/files.py

@@ -65,6 +65,9 @@ class FilePath(s_types.Str):
 
             path.append(part)
 
+        if len(path) == 0:
+            return '', {}
+
         fullpath = lead + '/'.join(path)
 
         base = path[-1]

synapse/models/planning.py

@@ -0,0 +1,161 @@
+import synapse.lib.module as s_module
+
+class PlanModule(s_module.CoreModule):
+
+    def getModelDefs(self):
+        return (('plan', {
+            'types': (
+                ('plan:system', ('guid', {}), {
+                    'doc': 'A planning or behavioral analysis system that defines phases and procedures.'}),
+
+                ('plan:phase', ('guid', {}), {
+                    'doc': 'A phase within a planning system which may be used to group steps within a procedure.'}),
+
+                ('plan:procedure', ('guid', {}), {
+                    'doc': 'A procedure consisting of steps.'}),
+
+                ('plan:procedure:type:taxonomy', ('taxonomy', {}), {
+                    'interfaces': ('meta:taxonomy',),
+                    'doc': 'A taxonomy of procedure types.'}),
+
+                ('plan:procedure:variable', ('guid', {}), {
+                    'doc': 'A variable used by a procedure.'}),
+
+                ('plan:procedure:step', ('guid', {}), {
+                    'doc': 'A step within a procedure.'}),
+
+                ('plan:procedure:link', ('guid', {}), {
+                    'doc': 'A link between steps in a procedure.'}),
+            ),
+
+            'forms': (
+                ('plan:system', {}, (
+
+                    ('name', ('str', {'lower': True, 'onespace': True}), {
+                        'ex': 'mitre att&ck flow',
+                        'doc': 'The name of the planning system.'}),
+
+                    ('summary', ('str', {}), {
+                        'disp': {'hint': 'text'},
+                        'doc': 'A summary of the purpose and use case for the planning system.'}),
+
+                    ('author', ('ps:contact', {}), {
+                        'doc': 'The contact of the person or organization which authored the system.'}),
+
+                    ('created', ('time', {}), {
+                        'doc': 'The time the planning system was first created.'}),
+
+                    ('updated', ('time', {}), {
+                        'doc': 'The time the planning system was last updated.'}),
+
+                    ('version', ('it:semver', {}), {
+                        'doc': 'The version of the planning system.'}),
+
+                    ('url', ('inet:url', {}), {
+                        'doc': 'The primary URL which documents the planning system.'}),
+                )),
+                ('plan:phase', {}, (
+                    ('title', ('str', {}), {
+                        'ex': 'Reconnaissance Phase',
+                        'doc': 'The title of the phase.'}),
+
+                    ('summary', ('str', {}), {
+                        'disp': {'hint': 'text'},
+                        'doc': 'A summary of the definition of the phase.'}),
+
+                    ('index', ('int', {}), {
+                        'doc': 'The index of this phase within the phases of the system.'}),
+
+                    ('url', ('inet:url', {}), {
+                        'doc': 'A URL which links to the full documentation about the phase.'}),
+
+                    ('system', ('plan:system', {}), {
+                        'doc': 'The planning system which defines this phase.'}),
+                )),
+                ('plan:procedure:type:taxonomy', {}, ()),
+                ('plan:procedure', {}, (
+
+                    ('title', ('str', {}), {
+                        'ex': 'Network Reconnaissance Procedure',
+                        'doc': 'The name of the procedure.'}),
+
+                    ('summary', ('str', {}), {
+                        'disp': {'hint': 'text'},
+                        'doc': 'A summary of the purpose and use cases for the procedure.'}),
+
+                    ('author', ('ps:contact', {}), {
+                        'doc': 'The contact of the person or organization which authored the procedure.'}),
+
+                    ('created', ('time', {}), {
+                        'doc': 'The time the procedure was created.'}),
+
+                    ('updated', ('time', {}), {
+                        'doc': 'The time the procedure was last updated.'}),
+
+                    ('version', ('it:semver', {}), {
+                        'doc': 'The version of the procedure.'}),
+
+                    ('system', ('plan:system', {}), {
+                        'doc': 'The planning system which defines this procedure.'}),
+
+                    ('type', ('plan:procedure:type:taxonomy', {}), {
+                        'doc': 'A type classification for the procedure.'}),
+
+                    ('inputs', ('array', {'type': 'plan:procedure:variable', 'uniq': True, 'sorted': True}), {
+                        'doc': 'An array of inputs required to execute the procedure.'}),
+
+                    ('firststep', ('plan:procedure:step', {}), {
+                        'doc': 'The first step in the procedure.'}),
+                )),
+                ('plan:procedure:variable', {}, (
+
+                    ('name', ('str', {}), {
+                        'doc': 'The name of the variable.'}),
+
+                    ('type', ('str', {}), {
+                        'doc': 'The type for the input. Types are specific to the planning system.'}),
+
+                    ('default', ('data', {}), {
+                        'doc': 'The optional default value if the procedure is invoked without the input.'}),
+
+                    ('procedure', ('plan:procedure', {}), {
+                        'doc': 'The procedure which defines the variable.'}),
+                )),
+                ('plan:procedure:step', {}, (
+
+                    ('phase', ('plan:phase', {}), {
+                        'doc': 'The phase that the step belongs within.'}),
+
+                    ('procedure', ('plan:procedure', {}), {
+                        'doc': 'The procedure which defines the step.'}),
+
+                    ('title', ('str', {}), {
+                        'ex': 'Scan the IPv4 address range for open ports',
+                        'doc': 'The title of the step.'}),
+
+                    ('summary', ('str', {}), {
+                        'doc': 'A summary of the tasks executed within the step.'}),
+
+                    ('outputs', ('array', {'type': 'plan:procedure:variable', 'uniq': True, 'sorted': True}), {
+                        'doc': 'An array of variables defined in this step.'}),
+
+                    ('techniques', ('array', {'type': 'ou:technique', 'uniq': True, 'sorted': True}), {
+                        'doc': 'An array of techniques used when executing this step.'}),
+
+                    ('links', ('array', {'type': 'plan:procedure:link', 'uniq': True}), {
+                        'doc': 'An array of links to subsequent steps.'}),
+
+                )),
+                ('plan:procedure:link', {}, (
+
+                    ('condition', ('bool', {}), {
+                        'doc': 'Set to true/false if this link is conditional based on a decision step.'}),
+
+                    ('next', ('plan:procedure:step', {}), {
+                        'doc': 'The next step in the plan.'}),
+
+                    ('procedure', ('plan:procedure', {}), {
+                        'doc': 'The procedure which defines the link.'}),
+                )),
+            ),
+        }),)

synapse/telepath.py

@@ -500,6 +500,7 @@ class GenrMethod(Method):
 class Pipeline(s_base.Base):
 
     async def __anit__(self, proxy, genr, name=None):
+        s_common.deprecated('Telepath.Pipeline', curv='2.167.0')
 
         await s_base.Base.__anit__(self)
 

synapse/tests/files/stormpkg/dotstorm/dotstorm.yaml

@@ -9,3 +9,6 @@ modules:
 
 commands:
     - name: dotstorm.bar
+
+onload: |
+  $lib.time.sleep(0)

synapse/tests/test_cortex.py

@@ -6337,7 +6337,7 @@ class CortexBasicTest(s_t_utils.SynTest):
                 async def action():
                     await asyncio.sleep(0.1)
                     await core.callStorm('return($lib.view.get().fork())')
-                    await core.callStorm('return($lib.cron.add(query="{graph:node=*}", hourly=30).pack())')
+                    await core.callStorm('return($lib.cron.add(query="{meta:note=*}", hourly=30).pack())')
                     tdef = {'cond': 'node:add', 'storm': '[test:str="foobar"]', 'form': 'test:int'}
                     opts = {'vars': {'tdef': tdef}}
                     trig = await core.callStorm('return($lib.trigger.add($tdef))', opts=opts)
@@ -7007,7 +7007,7 @@ class CortexBasicTest(s_t_utils.SynTest):
             # clear out the #cno.cve tags and test prune behavior.
             await core.nodes('#cno.cve [ -#cno.cve ]')
 
-            await core.nodes('[ inet:ipv4=1.2.3.4 +#cno.cve.2021.12345.foo +#cno.cve.2021.55555 ]')
+            await core.nodes('[ inet:ipv4=1.2.3.4 +#cno.cve.2021.12345.foo +#cno.cve.2021.55555.bar ]')
 
             await core.nodes('$lib.model.tags.set(cno.cve, prune, (2))')
 
@@ -7015,6 +7015,10 @@ class CortexBasicTest(s_t_utils.SynTest):
             nodes = await core.nodes('[ inet:ipv4=1.2.3.4 -#cno.cve.2021.55555 ]')
             self.sorteq(('cno', 'cno.cve', 'cno.cve.2021', 'cno.cve.2021.12345', 'cno.cve.2021.12345.foo'), [t[0] for t in nodes[0].getTags()])
 
+            # double delete shouldn't prune
+            nodes = await core.nodes('[ inet:ipv4=1.2.3.4 -#cno.cve.2021.55555 ]')
+            self.sorteq(('cno', 'cno.cve', 'cno.cve.2021', 'cno.cve.2021.12345', 'cno.cve.2021.12345.foo'), [t[0] for t in nodes[0].getTags()])
+
             # test that the pruning behavior stops at the correct level
             nodes = await core.nodes('[ inet:ipv4=1.2.3.4 -#cno.cve.2021.12345.foo ]')
             self.sorteq(('cno', 'cno.cve', 'cno.cve.2021', 'cno.cve.2021.12345'), [t[0] for t in nodes[0].getTags()])
@@ -7954,7 +7958,7 @@ class CortexBasicTest(s_t_utils.SynTest):
                     waiter = core01.stormpool.waiter(1, 'svc:del')
                     msgs = await core01.stormlist('aha.pool.svc.del pool00... 01.core...', opts={'mirror': False})
                     self.stormHasNoWarnErr(msgs)
-                    self.stormIsInPrint('AHA service (01.core...) removed from service pool (pool00.loop.vertex.link)', msgs)
+                    self.stormIsInPrint('AHA service (01.core.loop.vertex.link) removed from service pool (pool00.loop.vertex.link)', msgs)
 
                     # TODO: this wait should not return None
                     await waiter.wait(timeout=3)
@@ -8018,3 +8022,36 @@ class CortexBasicTest(s_t_utils.SynTest):
             buf = stream.read()
             self.isin('(lowuser) has a rule on the "cortex" authgate', buf)
             self.isin('(all) has a rule on the "cortex" authgate', buf)
+
+    async def test_cortex_check_nexus_init(self):
+        # This test is a simple safety net for making sure no nexus events
+        # happen before the nexus subsystem is initialized (initNexusSubsystem).
+        # It's possible for code which calls nexus APIs to run but not do
+        # anything which wouldn't be caught here. I don't think there's a good
+        # way to check for that condition though.
+
+        class Cortex(s_cortex.Cortex):
+            async def initServiceStorage(self):
+                self._test_pre_service_storage_index = await self.nexsroot.index()
+                ret = await super().initServiceStorage()
+                self._test_post_service_storage_index = await self.nexsroot.index()
+                return ret
+
+            async def initNexusSubsystem(self):
+                self._test_pre_nexus_index = await self.nexsroot.index()
+                ret = await super().initNexusSubsystem()
+                self._test_post_nexus_index = await self.nexsroot.index()
+                return ret
+
+        conf = {
+            'layer:lmdb:map_async': True,
+            'nexslog:en': True,
+            'layers:logedits': True,
+        }
+
+        with self.getTestDir() as dirn:
+            async with await Cortex.anit(dirn, conf=conf) as core:
+                offs = core._test_pre_service_storage_index
+                self.eq(core._test_post_service_storage_index, offs)
+                self.eq(core._test_pre_nexus_index, offs)
+                self.ge(core._test_post_nexus_index, core._test_pre_nexus_index)

synapse/tests/test_lib_aha.py

@@ -6,6 +6,7 @@ from unittest import mock
 import synapse.exc as s_exc
 import synapse.axon as s_axon
 import synapse.common as s_common
+import synapse.cortex as s_cortex
 import synapse.telepath as s_telepath
 
 import synapse.lib.aha as s_aha
@@ -1125,7 +1126,7 @@ class AhaTest(s_test.SynTest):
                     ready = svcinfo.get('ready')
                     online = svcinfo.get('online')
                     self.none(online)
-                    self.true(ready)  # Ready is not cleared upon restart
+                    self.false(ready)  # Ready is cleared upon restart / setting service down.
 
                     n = 3
                     if len(stack._exit_callbacks) > 0:
@@ -1146,8 +1147,6 @@ class AhaTest(s_test.SynTest):
 
         async with self.getTestAhaProv() as aha:
 
-            import synapse.cortex as s_cortex
-
             async with await s_base.Base.anit() as base:
 
                 with self.getTestDir() as dirn:
@@ -1246,7 +1245,12 @@ class AhaTest(s_test.SynTest):
 
                         msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...')
                         self.stormHasNoWarnErr(msgs)
-                        self.stormIsInPrint('AHA service (00...) removed from service pool (pool00.loop.vertex.link)', msgs)
+                        self.stormIsInPrint('AHA service (00.loop.vertex.link) removed from service pool (pool00.loop.vertex.link)',
+                                            msgs)
+
+                        msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...')
+                        self.stormHasNoWarnErr(msgs)
+                        self.stormIsInPrint('Did not remove (00...) from the service pool.', msgs)
 
                         await waiter.wait(timeout=3)
                         run00 = await (await pool.proxy(timeout=3)).getCellRunId()

synapse/tests/test_lib_cell.py

@@ -7,6 +7,7 @@ import signal
 import socket
 import asyncio
 import tarfile
+import warnings
 import collections
 import multiprocessing
 
@@ -509,8 +510,11 @@ class CellTest(s_t_utils.SynTest):
 
             s_common.yamlsave(tree, bootpath)
 
-            async with self.getTestCell(s_cell.Cell, dirn=dirn) as cell:
-                self.eq('haha', await cell.hive.get(('hehe',)))
+            with warnings.catch_warnings(record=True) as warns:
+                async with self.getTestCell(s_cell.Cell, dirn=dirn) as cell:
+                    self.eq('haha', await cell.hive.get(('hehe',)))
+
+                self.isin('Initial hive config from hiveboot.yaml', str(warns[0].message))
 
             # test that the file does not load again
             tree['kids']['redballoons'] = {'value': 99}