synapse 2.223.0__py311-none-any.whl → 2.225.0__py311-none-any.whl

synapse/axon.py

@@ -1651,8 +1651,9 @@ class Axon(s_cell.Cell):
                                    filename=field.get('filename'),
                                    content_transfer_encoding=field.get('content_transfer_encoding'))
 
-                async with sess.request(method, url, headers=headers, params=params,
-                                        data=data, ssl=ssl) as resp:
+                async with sess.request(method, url, headers=headers, params=params, data=data, ssl=ssl,
+                                        max_line_size=s_const.MAX_LINE_SIZE,
+                                        max_field_size=s_const.MAX_FIELD_SIZE) as resp:
                     info = {
                         'ok': True,
                         'url': str(resp.url),
@@ -1698,8 +1699,10 @@ class Axon(s_cell.Cell):
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
             try:
                 await self._reqHas(sha256)
-                async with sess.request(method, url, headers=headers, params=params,
-                                        data=self.get(sha256), ssl=ssl) as resp:
+                async with sess.request(method, url, headers=headers, params=params, ssl=ssl,
+                                        data=self.get(sha256),
+                                        max_line_size=s_const.MAX_LINE_SIZE,
+                                        max_field_size=s_const.MAX_FIELD_SIZE) as resp:
 
                     info = {
                         'ok': True,
@@ -1827,7 +1830,9 @@ class Axon(s_cell.Cell):
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
 
             try:
-                async with sess.request(method, url, headers=headers, params=params, json=json, data=body, ssl=ssl) as resp:
+                async with sess.request(method, url, headers=headers, params=params, json=json, data=body, ssl=ssl,
+                                        max_line_size=s_const.MAX_LINE_SIZE,
+                                        max_field_size=s_const.MAX_FIELD_SIZE) as resp:
 
                     info = self._flatten_clientresponse(resp)
 

synapse/common.py

@@ -144,7 +144,7 @@ def flatten(item):
         item: The python primitive object to normalize.
 
     Notes:
-        Only None, bool, int, bytes, strings, lists, tuples and dictionaries are acceptable input.
+        Only None, bool, int, bytes, strings, floats, lists, tuples and dictionaries are acceptable input.
         List objects will be converted to tuples.
         Dictionary objects must have keys which can be sorted.
 
@@ -155,7 +155,7 @@ def flatten(item):
     if item is None:
         return None
 
-    if isinstance(item, (str, int, bytes)):
+    if isinstance(item, (str, int, bytes, float)):
         return item
 
     if isinstance(item, (tuple, list)):

synapse/cortex.py

@@ -464,6 +464,15 @@ class CoreApi(s_cell.CellApi):
         '''
         return await self.cell.getTypeNorm(name, valu, typeopts=typeopts)
 
+    async def addType(self, typename, basetype, typeopts, typeinfo):
+        '''
+        Add an extended type to the data model.
+
+        Extended types must begin with _
+        '''
+        self.user.confirm(('model', 'type', 'add', typename))
+        return await self.cell.addType(typename, basetype, typeopts, typeinfo)
+
     async def addForm(self, formname, basetype, typeopts, typeinfo):
         '''
         Add an extended form to the data model.
@@ -535,6 +544,15 @@ class CoreApi(s_cell.CellApi):
         self.user.confirm(('model', 'tagprop', 'del'))
         return await self.cell.delTagProp(name)
 
+    async def addEdge(self, edge, edgeinfo):
+        '''
+        Add an extended edge definition to the data model.
+
+        Extended edge definitions must use a verb which begins with _
+        '''
+        self.user.confirm(('model', 'edge', 'add'))
+        return await self.cell.addEdge(edge, edgeinfo)
+
     async def addStormPkg(self, pkgdef, verify=False):
         self.user.confirm(('pkg', 'add'))
         return await self.cell.addStormPkg(pkgdef, verify=verify)
@@ -1440,6 +1458,11 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             {'perm': ('model', 'univ', 'del'), 'gate': 'cortex',
              'desc': 'Controls access to deleting extended model universal properties and values.'},
 
+            {'perm': ('model', 'edge', 'add'), 'gate': 'cortex',
+             'desc': 'Controls access to adding extended model edges.'},
+            {'perm': ('model', 'edge', 'del'), 'gate': 'cortex',
+             'desc': 'Controls access to deleting extended model edges.'},
+
             {'perm': ('node',), 'gate': 'layer',
              'desc': 'Controls all node edits in a layer.'},
             {'perm': ('node', 'add'), 'gate': 'layer',
@@ -2568,6 +2591,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         ctor.pkgname = cdef.get('pkgname')
         ctor.svciden = cdef.get('cmdconf', {}).get('svciden', '')
         ctor.forms = cdef.get('forms', {})
+        ctor.deprecated = cdef.get('deprecated', {})
 
         def getStorNode(form):
             ndef = (form.name, form.type.norm(cdef.get('name'))[0])
@@ -2596,6 +2620,18 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             if ctor.pkgname:
                 props['package'] = ctor.pkgname
 
+            if ctor.deprecated:
+                props['deprecated'] = True
+
+                if (eolvers := ctor.deprecated.get('eolvers')) is not None:
+                    props['deprecated:version'] = eolvers
+
+                if (eoldate := ctor.deprecated.get('eoldate')) is not None:
+                    props['deprecated:date'] = eoldate
+
+                if (mesg := ctor.deprecated.get('mesg')) is not None:
+                    props['deprecated:mesg'] = mesg
+
             pnorms = {}
             for prop, valu in props.items():
                 formprop = form.props.get(prop)
@@ -3662,6 +3698,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         return True
 
+    @s_cell.from_leader
     async def addUnivProp(self, name, tdef, info):
         if not isinstance(tdef, tuple):
             mesg = 'Universal property type definitions should be a tuple.'
@@ -3676,6 +3713,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             mesg = 'ext univ name must start with "_"'
             raise s_exc.BadPropDef(name=name, mesg=mesg)
 
+        self.model.getTypeClone(tdef)
+
         base = '.' + name
         if base in self.model.props:
             raise s_exc.DupPropName(mesg=f'Cannot add duplicate universal property {base}',
@@ -3697,6 +3736,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         if univ:
             await self.feedBeholder('model:univ:add', univ.pack())
 
+    @s_cell.from_leader
     async def addForm(self, formname, basetype, typeopts, typeinfo):
         if not isinstance(typeopts, dict):
             mesg = 'Form type options should be a dict.'
@@ -3718,6 +3758,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             mesg = f'Type already exists: {formname}'
             raise s_exc.DupTypeName.init(formname)
 
+        self.model.getTypeClone((basetype, typeopts))
+
         return await self._push('model:form:add', formname, basetype, typeopts, typeinfo)
 
     @s_nexus.Pusher.onPush('model:form:add')
@@ -3772,6 +3814,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         await self.fire('core:extmodel:change', form=formname, act='del', type='form')
         await self.feedBeholder('model:form:del', {'form': formname})
 
+    @s_cell.from_leader
     async def addType(self, typename, basetype, typeopts, typeinfo):
         if not isinstance(typeopts, dict):
             mesg = 'Type options should be a dict.'
@@ -3840,6 +3883,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         await self.fire('core:extmodel:change', name=typename, act='del', type='type')
         await self.feedBeholder('model:type:del', {'type': typename})
 
+    @s_cell.from_leader
     async def addFormProp(self, form, prop, tdef, info):
         if not isinstance(tdef, tuple):
             mesg = 'Form property type definitions should be a tuple.'
@@ -3858,6 +3902,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         if _form.prop(prop):
             raise s_exc.DupPropName(mesg=f'Cannot add duplicate form prop {form} {prop}',
                                      form=form, prop=prop)
+
+        self.model.getTypeClone(tdef)
+
         await self._push('model:prop:add', form, prop, tdef, info)
 
     @s_nexus.Pusher.onPush('model:prop:add')
@@ -4039,6 +4086,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         await self.fire('core:extmodel:change', name=prop, act='del', type='univ')
         await self.feedBeholder('model:univ:del', {'prop': univname})
 
+    @s_cell.from_leader
     async def addTagProp(self, name, tdef, info):
         if not isinstance(tdef, tuple):
             mesg = 'Tag property type definitions should be a tuple.'
@@ -4051,6 +4099,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         if self.exttagprops.get(name) is not None:
             raise s_exc.DupPropName(name=name)
 
+        self.model.getTypeClone(tdef)
+
         return await self._push('model:tagprop:add', name, tdef, info)
 
     @s_nexus.Pusher.onPush('model:tagprop:add')
@@ -4088,6 +4138,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         await self.fire('core:tagprop:change', name=name, act='del')
         await self.feedBeholder('model:tagprop:del', {'tagprop': name})
 
+    @s_cell.from_leader
     async def addEdge(self, edge, edgeinfo):
         if not isinstance(edgeinfo, dict):
             mesg = 'Edge info should be a dict.'
@@ -4667,6 +4718,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             s_stormlib_macro,
             s_stormlib_model,
             s_stormlib_pkg,
+            s_stormlib_task,
             s_stormlib_vault,
         ]
 
@@ -4674,9 +4726,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             for cdef in cmod.stormcmds:
                 await self._trySetStormCmd(cdef.get('name'), cdef)
 
-        for cdef in s_stormlib_task.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
     async def _initPureStormCmds(self):
         oldcmds = []
         for name, cdef in self.cmddefs.items():

synapse/datamodel.py

@@ -738,7 +738,7 @@ class Model:
 
         base = self.types.get(typedef[0])
         if base is None:
-            raise s_exc.NoSuchType(name=typedef[0])
+            raise s_exc.NoSuchType.init(typedef[0])
 
         return base.clone(typedef[1])
 

synapse/lib/cell.py

@@ -58,7 +58,7 @@ import synapse.lib.thisplat as s_thisplat
 
 import synapse.lib.crypto.passwd as s_passwd
 
-import synapse.tools.backup as s_t_backup
+import synapse.tools.service.backup as s_t_backup
 
 logger = logging.getLogger(__name__)
 

synapse/lib/const.py

@@ -47,3 +47,7 @@ layer_pdef_csize_max = 1_000
 layer_pdef_qsize_max = 10_000
 assert layer_pdef_csize <= layer_pdef_csize_max
 assert layer_pdef_qsize <= layer_pdef_qsize_max
+
+# HTTP header constants
+MAX_LINE_SIZE = kibibyte * 64
+MAX_FIELD_SIZE = kibibyte * 64

synapse/lib/layer.py

@@ -127,7 +127,7 @@ class LayerApi(s_cell.CellApi):
         Scan the full layer and yield artificial nodeedit sets.
         '''
 
-        if not self.allowed(self.liftperm):
+        if not await self.allowed(self.liftperm):
             await self._reqUserAllowed(self.readperm)
         async for item in self.layr.iterLayerNodeEdits():
             yield item
@@ -167,14 +167,14 @@ class LayerApi(s_cell.CellApi):
 
         Once caught up with storage, yield them in realtime.
         '''
-        if not self.allowed(self.liftperm):
+        if not await self.allowed(self.liftperm):
             await self._reqUserAllowed(self.readperm)
         async for item in self.layr.syncNodeEdits(offs, wait=wait, reverse=reverse):
             yield item
             await asyncio.sleep(0)
 
     async def syncNodeEdits2(self, offs, wait=True):
-        if not self.allowed(self.liftperm):
+        if not await self.allowed(self.liftperm):
             await self._reqUserAllowed(self.readperm)
         async for item in self.layr.syncNodeEdits2(offs, wait=wait):
             yield item
@@ -184,7 +184,7 @@ class LayerApi(s_cell.CellApi):
         '''
         Returns what will be the *next* nodeedit log index.
         '''
-        if not self.allowed(self.liftperm):
+        if not await self.allowed(self.liftperm):
             await self._reqUserAllowed(self.readperm)
         return await self.layr.getEditIndx()
 
@@ -192,12 +192,12 @@ class LayerApi(s_cell.CellApi):
         '''
         Return the total number of (edits, meta) pairs in the layer changelog.
         '''
-        if not self.allowed(self.liftperm):
+        if not await self.allowed(self.liftperm):
             await self._reqUserAllowed(self.readperm)
         return await self.layr.getEditSize()
 
     async def getIden(self):
-        if not self.allowed(self.liftperm):
+        if not await self.allowed(self.liftperm):
             await self._reqUserAllowed(self.readperm)
         return self.layr.iden
 

synapse/lib/multislabseqn.py

@@ -353,7 +353,6 @@ class MultiSlabSeqn(s_base.Base):
         else:
             indx = self.indx
 
-        assert self.tailseqn
         retn = self.tailseqn.add(item, indx=indx)
 
         if advances:
@@ -363,6 +362,42 @@ class MultiSlabSeqn(s_base.Base):
 
         return retn
 
+    async def addWithPackRetn(self, item, indx=None):
+        '''
+        Add a single item to the sequence, returning the offset and packed item.
+        '''
+        advances = True
+
+        if indx is not None:
+            if indx < self.firstindx:
+                raise s_exc.BadIndxValu(mesg=f'indx lower than first index in sequence {self.firstindx}')
+
+            if indx < self._ranges[-1]:
+                ridx = self._getRangeIndx(indx)
+                assert ridx is not None
+
+                async with self._getSeqn(ridx) as seqn:
+                    retn = seqn.addWithPackRetn(item, indx=indx)
+
+                return retn
+
+            if indx >= self.indx:
+                self.indx = indx
+            else:
+                advances = False
+        else:
+            indx = self.indx
+
+        assert self.tailseqn
+        retn = self.tailseqn.addWithPackRetn(item, indx=indx)
+
+        if advances:
+            self.indx += 1
+
+            self._wake_waiters()
+
+        return retn
+
     async def last(self) -> Optional[Tuple[int, Any]]:
         ridx = self._getRangeIndx(self.indx - 1)
         if ridx is None:

synapse/lib/nexus.py

@@ -12,6 +12,9 @@ import synapse.common as s_common
 import synapse.telepath as s_telepath
 
 import synapse.lib.base as s_base
+import synapse.lib.scope as s_scope
+import synapse.lib.queue as s_queue
+import synapse.lib.msgpack as s_msgpack
 
 logger = logging.getLogger(__name__)
 
@@ -20,6 +23,9 @@ leaderversion = 'Leader is a higher version than we are.'
 # As a mirror follower, amount of time before giving up on a write request
 FOLLOWER_WRITE_WAIT_S = 30.0
 
+WINDOW_MAXSIZE = 10_000
+YIELD_PREFIX = b'\x92\xa8t2:yield\x81\xa4retn\x92\xc3\x92'
+
 class RegMethType(type):
     '''
     Metaclass that collects all methods in class with _regme prop into a class member called _regclstupls
@@ -99,6 +105,8 @@ class NexsRoot(s_base.Base):
         self._mirready = asyncio.Event()  # for testing
 
         self._mirrors: List[ChangeDist] = []
+        self._linkmirrors: List[s_queue.Window] = []
+
         self._nexskids: Dict[str, 'Pusher'] = {}
 
         # Used to match pending follower write requests with the responses arriving on the log
@@ -134,6 +142,11 @@ class NexsRoot(s_base.Base):
         self.nexslog.setIndex(maxindx)
 
         async def fini():
+            for wind in self._linkmirrors:
+                await wind.fini()
+
+            for dist in self._mirrors:
+                await dist.fini()
 
             for futu in self._futures.values():  # pragma: no cover
                 futu.cancel()
@@ -389,8 +402,16 @@ class NexsRoot(s_base.Base):
     async def _eat(self, item, indx=None):
 
         if self.donexslog:
-            saveindx = await self.nexslog.add(item, indx=indx)
-            [dist.update() for dist in tuple(self._mirrors)]
+            saveindx, packitem = await self.nexslog.addWithPackRetn(item, indx=indx)
+
+            if self._linkmirrors:
+                tupl = (saveindx, YIELD_PREFIX + s_msgpack.en(saveindx) + packitem)
+                for wind in tuple(self._linkmirrors):
+                    await wind.put(tupl)
+
+            if self._mirrors:
+                for dist in tuple(self._mirrors):
+                    dist.update()
 
         else:
             saveindx = self.nexshot.get('nexs:indx')
@@ -415,7 +436,15 @@ class NexsRoot(s_base.Base):
 
     async def iter(self, offs: int, tellready=False, wait=True) -> AsyncIterator[Any]:
         '''
-        Returns an iterator of change entries in the log
+        Returns an iterator of change entries in the log.
+
+        Notes:
+            If this method is being called in the context of a Telepath call,
+            it will directly send messages to the scope "link" object when it
+            has caught up to the realtime change window; instead of yielding
+            them as a generator. This is an optimization to avoid duplication
+            of msgpack'ing the same object over and over again as the number
+            of mirrors increases.
         '''
         if not self.donexslog:
             return
@@ -437,11 +466,41 @@ class NexsRoot(s_base.Base):
         if not wait:
             return
 
-        async with self.getChangeDist(maxoffs) as dist:
-            async for item in dist:
-                if self.isfini:
-                    raise s_exc.IsFini()
-                yield item
+        if (link := s_scope.get('link')) is None:
+            async with self.getChangeDist(maxoffs) as dist:
+                async for item in dist:
+                    yield item
+
+        else:
+            async with self.getMirrorWindow() as wind:
+
+                # Ensure we are caught up after grabbing a window
+                sync = True
+
+                async for item in self.nexslog.iter(maxoffs):
+                    maxoffs = item[0] + 1
+                    yield item
+
+                async for offs, item in wind:
+                    if sync:
+                        if offs < maxoffs:
+                            continue
+                        sync = False
+
+                    await link.send(item)
+
+    @contextlib.asynccontextmanager
+    async def getMirrorWindow(self):
+        async with await s_queue.Window.anit(maxsize=WINDOW_MAXSIZE, clearonfini=True) as wind:
+
+            async def fini():
+                self._linkmirrors.remove(wind)
+
+            wind.onfini(fini)
+
+            self._linkmirrors.append(wind)
+
+            yield wind
 
     @contextlib.asynccontextmanager
     async def getChangeDist(self, offs: int) -> AsyncIterator[ChangeDist]:

synapse/lib/queue.py

@@ -118,14 +118,17 @@ class Window(s_base.Base):
     its maxsize, it will be fini()d.  On fini(), the Window will continue to
     yield results until empty and then return.
     '''
-    async def __anit__(self, maxsize=None):
+    async def __anit__(self, maxsize=None, clearonfini=False):
         await s_base.Base.__anit__(self)
         self.maxsize = maxsize
         self.event = asyncio.Event()
 
         self.linklist = collections.deque()
+        self.clearonfini = clearonfini
 
         async def fini():
+            if self.clearonfini:
+                self.linklist.clear()
             self.event.set()
 
         self.onfini(fini)

synapse/lib/rstorm.py

@@ -25,8 +25,8 @@ import synapse.lib.stormhttp as s_stormhttp
 
 import synapse.cmds.cortex as s_cmds_cortex
 
-import synapse.tools.storm as s_storm
-import synapse.tools.genpkg as s_genpkg
+import synapse.tools.storm._cli as s_storm
+import synapse.tools.storm.pkg.gen as s_genpkg
 
 
 re_directive = regex.compile(r'^\.\.\s(shell.*|storm.*|[^:])::(?:\s(.*)$|$)')

synapse/lib/schemas.py

@@ -393,7 +393,6 @@ _changelogTypes = {'migration': 'Automatic Migrations',
                    'note': 'Notes',
                    'doc': 'Improved documentation',
                    'deprecation': 'Deprecations'}
-
 _changelogSchema = {
     'type': 'object',
     'properties': {
@@ -405,6 +404,10 @@ _changelogSchema = {
             'type': 'string',
             'minLength': 1,
         },
+        'desc:literal': {
+            'type': 'boolean',
+            'default': False,
+        },
         'prs': {
             'type': 'array',
             'items': {
@@ -948,12 +951,20 @@ _reqValidPkgdefSchema = {
                     'type': ['array', 'null'],
                     'items': {'$ref': '#/definitions/cmdinput'},
                 },
+                'cmdconf': {
+                    'type': 'object',
+                    'properties': {
+                        'svciden': {'type': 'string', 'pattern': s_config.re_iden},
+                    },
+                    'additionalProperties': True,
+                },
                 'storm': {'type': 'string'},
                 'forms': {'$ref': '#/definitions/cmdformhints'},
                 'perms': {'type': 'array',
                     'items': {'type': 'array',
                         'items': {'type': 'string'}},
                 },
+                'deprecated': {'$ref': '#/definitions/deprecatedItem'},
             },
             'additionalProperties': True,
             'required': ['name', 'storm']
@@ -980,6 +991,7 @@ _reqValidPkgdefSchema = {
                             'type': 'string',
                             'enum': s_msgpack.deepcopy(datamodel_basetypes),
                         },
+                        'deprecated': {'$ref': '#/definitions/deprecatedItem'},
                     },
                 }
             ],

synapse/lib/slabseqn.py

@@ -87,6 +87,34 @@ class SlabSeqn:
 
         return indx
 
+    def addWithPackRetn(self, item, indx=None):
+        '''
+        Add a single item to the sequence, returning the offset and packed item.
+        '''
+        packitem = s_msgpack.en(item)
+        if indx is not None:
+            if indx >= self.indx:
+                self.slab.put(s_common.int64en(indx), packitem, append=True, db=self.db)
+                self.indx = indx + 1
+                self.size += 1
+                self._wake_waiters()
+                return indx, packitem
+
+            oldv = self.slab.replace(s_common.int64en(indx), packitem, db=self.db)
+            if oldv is None:
+                self.size += 1
+            return indx, packitem
+
+        indx = self.indx
+        self.slab.put(s_common.int64en(indx), packitem, append=True, db=self.db)
+
+        self.indx += 1
+        self.size += 1
+
+        self._wake_waiters()
+
+        return indx, packitem
+
     def first(self):
 
         for lkey, lval in self.slab.scanByFull(db=self.db):