synapse 2.224.0__py311-none-any.whl → 2.225.0__py311-none-any.whl

synapse/axon.py

@@ -1651,8 +1651,9 @@ class Axon(s_cell.Cell):
                                    filename=field.get('filename'),
                                    content_transfer_encoding=field.get('content_transfer_encoding'))
 
-                async with sess.request(method, url, headers=headers, params=params,
-                                        data=data, ssl=ssl) as resp:
+                async with sess.request(method, url, headers=headers, params=params, data=data, ssl=ssl,
+                                        max_line_size=s_const.MAX_LINE_SIZE,
+                                        max_field_size=s_const.MAX_FIELD_SIZE) as resp:
                     info = {
                         'ok': True,
                         'url': str(resp.url),
@@ -1698,8 +1699,10 @@ class Axon(s_cell.Cell):
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
             try:
                 await self._reqHas(sha256)
-                async with sess.request(method, url, headers=headers, params=params,
-                                        data=self.get(sha256), ssl=ssl) as resp:
+                async with sess.request(method, url, headers=headers, params=params, ssl=ssl,
+                                        data=self.get(sha256),
+                                        max_line_size=s_const.MAX_LINE_SIZE,
+                                        max_field_size=s_const.MAX_FIELD_SIZE) as resp:
 
                     info = {
                         'ok': True,
@@ -1827,7 +1830,9 @@ class Axon(s_cell.Cell):
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
 
             try:
-                async with sess.request(method, url, headers=headers, params=params, json=json, data=body, ssl=ssl) as resp:
+                async with sess.request(method, url, headers=headers, params=params, json=json, data=body, ssl=ssl,
+                                        max_line_size=s_const.MAX_LINE_SIZE,
+                                        max_field_size=s_const.MAX_FIELD_SIZE) as resp:
 
                     info = self._flatten_clientresponse(resp)
 

synapse/lib/cell.py

@@ -58,7 +58,7 @@ import synapse.lib.thisplat as s_thisplat
 
 import synapse.lib.crypto.passwd as s_passwd
 
-import synapse.tools.backup as s_t_backup
+import synapse.tools.service.backup as s_t_backup
 
 logger = logging.getLogger(__name__)
 

synapse/lib/const.py

@@ -47,3 +47,7 @@ layer_pdef_csize_max = 1_000
 layer_pdef_qsize_max = 10_000
 assert layer_pdef_csize <= layer_pdef_csize_max
 assert layer_pdef_qsize <= layer_pdef_qsize_max
+
+# HTTP header constants
+MAX_LINE_SIZE = kibibyte * 64
+MAX_FIELD_SIZE = kibibyte * 64

synapse/lib/multislabseqn.py

@@ -353,7 +353,6 @@ class MultiSlabSeqn(s_base.Base):
         else:
             indx = self.indx
 
-        assert self.tailseqn
         retn = self.tailseqn.add(item, indx=indx)
 
         if advances:
@@ -363,6 +362,42 @@ class MultiSlabSeqn(s_base.Base):
 
         return retn
 
+    async def addWithPackRetn(self, item, indx=None):
+        '''
+        Add a single item to the sequence, returning the offset and packed item.
+        '''
+        advances = True
+
+        if indx is not None:
+            if indx < self.firstindx:
+                raise s_exc.BadIndxValu(mesg=f'indx lower than first index in sequence {self.firstindx}')
+
+            if indx < self._ranges[-1]:
+                ridx = self._getRangeIndx(indx)
+                assert ridx is not None
+
+                async with self._getSeqn(ridx) as seqn:
+                    retn = seqn.addWithPackRetn(item, indx=indx)
+
+                return retn
+
+            if indx >= self.indx:
+                self.indx = indx
+            else:
+                advances = False
+        else:
+            indx = self.indx
+
+        assert self.tailseqn
+        retn = self.tailseqn.addWithPackRetn(item, indx=indx)
+
+        if advances:
+            self.indx += 1
+
+            self._wake_waiters()
+
+        return retn
+
     async def last(self) -> Optional[Tuple[int, Any]]:
         ridx = self._getRangeIndx(self.indx - 1)
         if ridx is None:

synapse/lib/nexus.py

@@ -12,6 +12,9 @@ import synapse.common as s_common
 import synapse.telepath as s_telepath
 
 import synapse.lib.base as s_base
+import synapse.lib.scope as s_scope
+import synapse.lib.queue as s_queue
+import synapse.lib.msgpack as s_msgpack
 
 logger = logging.getLogger(__name__)
 
@@ -20,6 +23,9 @@ leaderversion = 'Leader is a higher version than we are.'
 # As a mirror follower, amount of time before giving up on a write request
 FOLLOWER_WRITE_WAIT_S = 30.0
 
+WINDOW_MAXSIZE = 10_000
+YIELD_PREFIX = b'\x92\xa8t2:yield\x81\xa4retn\x92\xc3\x92'
+
 class RegMethType(type):
     '''
     Metaclass that collects all methods in class with _regme prop into a class member called _regclstupls
@@ -99,6 +105,8 @@ class NexsRoot(s_base.Base):
         self._mirready = asyncio.Event()  # for testing
 
         self._mirrors: List[ChangeDist] = []
+        self._linkmirrors: List[s_queue.Window] = []
+
         self._nexskids: Dict[str, 'Pusher'] = {}
 
         # Used to match pending follower write requests with the responses arriving on the log
@@ -134,6 +142,11 @@ class NexsRoot(s_base.Base):
         self.nexslog.setIndex(maxindx)
 
         async def fini():
+            for wind in self._linkmirrors:
+                await wind.fini()
+
+            for dist in self._mirrors:
+                await dist.fini()
 
             for futu in self._futures.values():  # pragma: no cover
                 futu.cancel()
@@ -389,8 +402,16 @@ class NexsRoot(s_base.Base):
     async def _eat(self, item, indx=None):
 
         if self.donexslog:
-            saveindx = await self.nexslog.add(item, indx=indx)
-            [dist.update() for dist in tuple(self._mirrors)]
+            saveindx, packitem = await self.nexslog.addWithPackRetn(item, indx=indx)
+
+            if self._linkmirrors:
+                tupl = (saveindx, YIELD_PREFIX + s_msgpack.en(saveindx) + packitem)
+                for wind in tuple(self._linkmirrors):
+                    await wind.put(tupl)
+
+            if self._mirrors:
+                for dist in tuple(self._mirrors):
+                    dist.update()
 
         else:
             saveindx = self.nexshot.get('nexs:indx')
@@ -415,7 +436,15 @@ class NexsRoot(s_base.Base):
 
     async def iter(self, offs: int, tellready=False, wait=True) -> AsyncIterator[Any]:
         '''
-        Returns an iterator of change entries in the log
+        Returns an iterator of change entries in the log.
+
+        Notes:
+            If this method is being called in the context of a Telepath call,
+            it will directly send messages to the scope "link" object when it
+            has caught up to the realtime change window; instead of yielding
+            them as a generator. This is an optimization to avoid duplication
+            of msgpack'ing the same object over and over again as the number
+            of mirrors increases.
         '''
         if not self.donexslog:
             return
@@ -437,11 +466,41 @@ class NexsRoot(s_base.Base):
         if not wait:
             return
 
-        async with self.getChangeDist(maxoffs) as dist:
-            async for item in dist:
-                if self.isfini:
-                    raise s_exc.IsFini()
-                yield item
+        if (link := s_scope.get('link')) is None:
+            async with self.getChangeDist(maxoffs) as dist:
+                async for item in dist:
+                    yield item
+
+        else:
+            async with self.getMirrorWindow() as wind:
+
+                # Ensure we are caught up after grabbing a window
+                sync = True
+
+                async for item in self.nexslog.iter(maxoffs):
+                    maxoffs = item[0] + 1
+                    yield item
+
+                async for offs, item in wind:
+                    if sync:
+                        if offs < maxoffs:
+                            continue
+                        sync = False
+
+                    await link.send(item)
+
+    @contextlib.asynccontextmanager
+    async def getMirrorWindow(self):
+        async with await s_queue.Window.anit(maxsize=WINDOW_MAXSIZE, clearonfini=True) as wind:
+
+            async def fini():
+                self._linkmirrors.remove(wind)
+
+            wind.onfini(fini)
+
+            self._linkmirrors.append(wind)
+
+            yield wind
 
     @contextlib.asynccontextmanager
     async def getChangeDist(self, offs: int) -> AsyncIterator[ChangeDist]:

synapse/lib/queue.py

@@ -118,14 +118,17 @@ class Window(s_base.Base):
     its maxsize, it will be fini()d.  On fini(), the Window will continue to
     yield results until empty and then return.
     '''
-    async def __anit__(self, maxsize=None):
+    async def __anit__(self, maxsize=None, clearonfini=False):
         await s_base.Base.__anit__(self)
         self.maxsize = maxsize
         self.event = asyncio.Event()
 
         self.linklist = collections.deque()
+        self.clearonfini = clearonfini
 
         async def fini():
+            if self.clearonfini:
+                self.linklist.clear()
             self.event.set()
 
         self.onfini(fini)

synapse/lib/rstorm.py

@@ -25,8 +25,8 @@ import synapse.lib.stormhttp as s_stormhttp
 
 import synapse.cmds.cortex as s_cmds_cortex
 
-import synapse.tools.storm as s_storm
-import synapse.tools.genpkg as s_genpkg
+import synapse.tools.storm._cli as s_storm
+import synapse.tools.storm.pkg.gen as s_genpkg
 
 
 re_directive = regex.compile(r'^\.\.\s(shell.*|storm.*|[^:])::(?:\s(.*)$|$)')

synapse/lib/schemas.py

@@ -393,7 +393,6 @@ _changelogTypes = {'migration': 'Automatic Migrations',
                    'note': 'Notes',
                    'doc': 'Improved documentation',
                    'deprecation': 'Deprecations'}
-
 _changelogSchema = {
     'type': 'object',
     'properties': {
@@ -405,6 +404,10 @@ _changelogSchema = {
             'type': 'string',
             'minLength': 1,
         },
+        'desc:literal': {
+            'type': 'boolean',
+            'default': False,
+        },
         'prs': {
             'type': 'array',
             'items': {
@@ -948,6 +951,13 @@ _reqValidPkgdefSchema = {
                     'type': ['array', 'null'],
                     'items': {'$ref': '#/definitions/cmdinput'},
                 },
+                'cmdconf': {
+                    'type': 'object',
+                    'properties': {
+                        'svciden': {'type': 'string', 'pattern': s_config.re_iden},
+                    },
+                    'additionalProperties': True,
+                },
                 'storm': {'type': 'string'},
                 'forms': {'$ref': '#/definitions/cmdformhints'},
                 'perms': {'type': 'array',

synapse/lib/slabseqn.py

@@ -87,6 +87,34 @@ class SlabSeqn:
 
         return indx
 
+    def addWithPackRetn(self, item, indx=None):
+        '''
+        Add a single item to the sequence, returning the offset and packed item.
+        '''
+        packitem = s_msgpack.en(item)
+        if indx is not None:
+            if indx >= self.indx:
+                self.slab.put(s_common.int64en(indx), packitem, append=True, db=self.db)
+                self.indx = indx + 1
+                self.size += 1
+                self._wake_waiters()
+                return indx, packitem
+
+            oldv = self.slab.replace(s_common.int64en(indx), packitem, db=self.db)
+            if oldv is None:
+                self.size += 1
+            return indx, packitem
+
+        indx = self.indx
+        self.slab.put(s_common.int64en(indx), packitem, append=True, db=self.db)
+
+        self.indx += 1
+        self.size += 1
+
+        self._wake_waiters()
+
+        return indx, packitem
+
     def first(self):
 
         for lkey, lval in self.slab.scanByFull(db=self.db):

synapse/lib/storm.py

@@ -2545,10 +2545,14 @@ class PureCmd(Cmd):
 
         cmdopts = s_stormtypes.CmdOpts(self)
 
+        cmdconf = self.cdef.get('cmdconf', {})
+        if cmdconf:
+            cmdconf = s_msgpack.deepcopy(cmdconf, use_list=True)
+
         opts = {
             'vars': {
                 'cmdopts': cmdopts,
-                'cmdconf': self.cdef.get('cmdconf', {}),
+                'cmdconf': cmdconf,
             }
         }
 

synapse/lib/stormhttp.py

@@ -12,6 +12,7 @@ import synapse.common as s_common
 
 import synapse.lib.base as s_base
 import synapse.lib.json as s_json
+import synapse.lib.const as s_const
 import synapse.lib.msgpack as s_msgpack
 import synapse.lib.version as s_version
 import synapse.lib.stormtypes as s_stormtypes
@@ -400,7 +401,12 @@ class LibHttp(s_stormtypes.Lib):
         proxy = await s_stormtypes.toprim(proxy)
         ssl_opts = await s_stormtypes.toprim(ssl_opts)
 
-        kwargs = {'allow_redirects': allow_redirects}
+        kwargs = {
+            'max_line_size': s_const.MAX_LINE_SIZE,
+            'max_field_size': s_const.MAX_FIELD_SIZE,
+            'allow_redirects': allow_redirects,
+        }
+
         if params:
             kwargs['params'] = s_stormtypes.strifyHttpArg(params, multi=True)
 

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 224, 0)
+version = (2, 225, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = '7c6e619bf5fd9c5801891b489b11bbee7fdbebd0'
+commit = 'd2ab923e3df9fe64611ac05b6ffdd7e8c2dc1b22'

synapse/models/inet.py

@@ -3819,8 +3819,12 @@ class InetModule(s_module.CoreModule):
                     ('inet:service:account', {}, (
 
                         ('user', ('inet:user', {}), {
+                            'alts': ('users',),
                             'doc': 'The current user name of the account.'}),
 
+                        ('users', ('array', {'type': 'inet:user', 'sorted': True, 'uniq': True}), {
+                            'doc': 'An array of alternate user names for this account.'}),
+
                         ('parent', ('inet:service:account', {}), {
                             'doc': 'A parent account which owns this account.'}),
 

synapse/models/media.py

@@ -35,6 +35,10 @@ class MediaModule(s_module.CoreModule):
                     'disp': {'hint': 'text'},
                     'doc': 'A brief summary of the news item.'}),
 
+                ('body', ('str', {}), {
+                    'disp': {'hint': 'text', 'syntax': 'markdown'},
+                    'doc': 'The body of the news item.'}),
+
                 ('publisher', ('ou:org', {}), {
                     'doc': 'The organization which published the news.'}),
 

synapse/models/risk.py

@@ -420,6 +420,9 @@ class RiskModule(s_module.CoreModule):
                     ('type', ('risk:mitigation:type:taxonomy', {}), {
                         'doc': 'A taxonomy type entry for the mitigation.'}),
 
+                    ('id', ('str', {'strip': True}), {
+                        'doc': 'An identifier for the mitigation.'}),
+
                     ('desc', ('str', {}), {
                         'disp': {'hint': 'text'},
                         'doc': 'A description of the mitigation approach for the vulnerability.'}),

synapse/tests/test_cortex.py

@@ -28,8 +28,8 @@ import synapse.lib.version as s_version
 import synapse.lib.modelrev as s_modelrev
 import synapse.lib.stormsvc as s_stormsvc
 
-import synapse.tools.backup as s_tools_backup
-import synapse.tools.promote as s_tools_promote
+import synapse.tools.service.backup as s_tools_backup
+import synapse.tools.service.promote as s_tools_promote
 
 import synapse.tests.utils as s_t_utils
 from synapse.tests.utils import alist

synapse/tests/test_lib_agenda.py

@@ -9,7 +9,7 @@ import synapse.exc as s_exc
 import synapse.common as s_common
 import synapse.tests.utils as s_t_utils
 
-import synapse.tools.backup as s_tools_backup
+import synapse.tools.service.backup as s_tools_backup
 
 import synapse.lib.agenda as s_agenda
 from synapse.lib.agenda import TimeUnit as s_tu

synapse/tests/test_lib_cell.py

@@ -38,7 +38,7 @@ import synapse.lib.lmdbslab as s_lmdbslab
 import synapse.lib.crypto.passwd as s_passwd
 import synapse.lib.platforms.linux as s_linux
 
-import synapse.tools.backup as s_tools_backup
+import synapse.tools.service.backup as s_tools_backup
 
 import synapse.tests.utils as s_t_utils
 

synapse/tests/test_lib_certdir.py

@@ -9,7 +9,7 @@ import synapse.common as s_common
 import synapse.lib.certdir as s_certdir
 import synapse.lib.msgpack as s_msgpack
 import synapse.tests.utils as s_t_utils
-import synapse.tools.genpkg as s_genpkg
+import synapse.tools.storm.pkg.gen as s_genpkg
 
 import cryptography.x509 as c_x509
 import cryptography.exceptions as c_exc

synapse/tests/test_lib_httpapi.py

@@ -5,7 +5,7 @@ import aiohttp
 import aiohttp.client_exceptions as a_exc
 
 import synapse.common as s_common
-import synapse.tools.backup as s_backup
+import synapse.tools.service.backup as s_backup
 
 import synapse.exc as s_exc
 import synapse.lib.coro as s_coro

synapse/tests/test_lib_layer.py

@@ -13,7 +13,7 @@ import synapse.lib.layer as s_layer
 import synapse.lib.msgpack as s_msgpack
 import synapse.lib.spooled as s_spooled
 
-import synapse.tools.backup as s_tools_backup
+import synapse.tools.service.backup as s_tools_backup
 
 import synapse.tests.utils as s_t_utils
 

synapse/tests/test_lib_multislabseqn.py

@@ -226,6 +226,28 @@ class MultiSlabSeqn(s_t_utils.SynTest):
                 # Iterator exhausted: should have just the cache slab (10) and the tail slab (20)
                 self.len(2, msqn._openslabs)
 
+    async def test_multislabseqn_retnpack(self):
+
+        with self.getTestDir() as dirn:
+
+            async with await s_multislabseqn.MultiSlabSeqn.anit(dirn) as msqn:
+
+                self.eq((0, b'\xa4foo1'), await msqn.addWithPackRetn('foo1'))
+                self.eq((1, b'\xa4foo2'), await msqn.addWithPackRetn('foo2'))
+                self.eq((0, b'\xa4foo3'), await msqn.addWithPackRetn('foo3', indx=0))
+
+                await msqn.addWithPackRetn('foo4', indx=10)
+                await msqn.rotate()
+                await msqn.addWithPackRetn('foo5')
+                await msqn.addWithPackRetn('foo6', indx=7)
+
+                self.true(await msqn.cull(10))
+
+                with self.raises(s_exc.BadIndxValu):
+                    await msqn.addWithPackRetn('foo7', indx=7)
+
+                self.eq((12, b'\xa4foo8'), msqn.tailseqn.addWithPackRetn('foo8'))
+
     async def test_multislabseqn_cull(self):
 
         with self.getTestDir() as dirn:

synapse/tests/test_lib_nexus.py

@@ -1,4 +1,5 @@
 import asyncio
+import contextlib
 from unittest import mock
 
 import synapse.exc as s_exc
@@ -61,7 +62,7 @@ class SampleNexus2(SampleNexus, SampleMixin):
 
 class NexusTest(s_t_utils.SynTest):
 
-    async def test_nexus(self):
+    async def test_nexus_base(self):
 
         with self.getTestDir() as dirn:
             dir1 = s_common.genpath(dirn, 'nexus1')
@@ -111,6 +112,19 @@ class NexusTest(s_t_utils.SynTest):
                     stream.seek(0)
                     self.isin('while replaying log', stream.read())
 
+                    nexsindx = nexus1.nexsroot.nexslog.index()
+
+                    async def listen():
+                        async for item in nexus1.getNexusChanges(nexsindx, wait=True):
+                            break
+                        return item
+
+                    task = nexus1.schedCoro(listen())
+                    self.eq('foo', await nexus1.doathing(eventdict))
+                    offs, item = await asyncio.wait_for(task, timeout=12)
+                    self.eq(offs, nexsindx)
+                    self.eq(item[1], 'thing:doathing')
+
     async def test_nexus_modroot(self):
 
         async with self.getTestCell() as cell:
@@ -395,3 +409,30 @@ class NexusTest(s_t_utils.SynTest):
                 await core.nodes('[ it:dev:str=foo ]', opts={'view': forkiden})
 
             core.nexsroot._nexskids[layriden] = layr
+
+    async def test_nexus_iter(self):
+        async with self.getTestCell(conf={'nexslog:en': True}) as cell:
+            await cell.sync()
+
+            # Force nexus events to be added while constructing a Window
+            orig = s_nexus.NexsRoot.getMirrorWindow
+
+            @contextlib.asynccontextmanager
+            async def slowwindow(self):
+                await cell.sync()
+                async with orig(self) as wind:
+                    await cell.sync()
+                    yield wind
+
+            items = []
+            with mock.patch('synapse.lib.nexus.NexsRoot.getMirrorWindow', slowwindow):
+                async with cell.getLocalProxy() as prox:
+                    async for indx, item in prox.getNexusChanges(0):
+                        items.append(indx)
+                        if indx == 2:
+                            await cell.sync()
+                        elif indx == 3:
+                            break
+
+            # We didn't miss or duplicate items added during window construction
+            self.eq([0, 1, 2, 3], items)

synapse/tests/test_lib_slabseqn.py

@@ -15,7 +15,7 @@ class SlabSeqn(s_t_utils.SynTest):
     def chk_size(self, seqn):
         self.eq(seqn.stat()['entries'], seqn.size)
 
-    async def test_slab_seqn(self):
+    async def test_slab_seqn_base(self):
 
         with self.getTestDir() as dirn:
 
@@ -174,3 +174,32 @@ class SlabSeqn(s_t_utils.SynTest):
 
                 res = await asyncio.wait_for(genr.__anext__(), timeout=1)
                 self.eq((1, 'bar'), res)
+
+    async def test_slab_seqn_retnpack(self):
+
+        with self.getTestDir() as dirn:
+            async with await s_lmdbslab.Slab.anit(dirn, map_size=10_0000) as slab:
+
+                seqn = s_slabseqn.SlabSeqn(slab, 'seqn:test')
+
+                self.eq((0, b'\xa4foo1'), seqn.addWithPackRetn('foo1'))
+                self.eq((1, b'\xa4foo2'), seqn.addWithPackRetn('foo2'))
+                self.eq((0, b'\xa4foo3'), seqn.addWithPackRetn('foo3', indx=0))
+
+                seqn.addWithPackRetn('foo4', indx=10)
+                seqn.addWithPackRetn('foo5')
+                seqn.addWithPackRetn('foo6', indx=7)
+                seqn.addWithPackRetn('foo7', indx=7)
+
+                self.eq((12, b'\xa4foo8'), seqn.addWithPackRetn('foo8'))
+
+                valus = [valu for valu in seqn.iter(0)]
+                evals = [
+                    (0, 'foo3',),
+                    (1, 'foo2'),
+                    (7, 'foo7'),
+                    (10, 'foo4'),
+                    (11, 'foo5'),
+                    (12, 'foo8'),
+                ]
+                self.eq(valus, evals)