synapse 2.224.0__py311-none-any.whl → 2.226.0__py311-none-any.whl

synapse/axon.py

@@ -1651,8 +1651,9 @@ class Axon(s_cell.Cell):
                                    filename=field.get('filename'),
                                    content_transfer_encoding=field.get('content_transfer_encoding'))
 
-                async with sess.request(method, url, headers=headers, params=params,
-                                        data=data, ssl=ssl) as resp:
+                async with sess.request(method, url, headers=headers, params=params, data=data, ssl=ssl,
+                                        max_line_size=s_const.MAX_LINE_SIZE,
+                                        max_field_size=s_const.MAX_FIELD_SIZE) as resp:
                     info = {
                         'ok': True,
                         'url': str(resp.url),
@@ -1698,8 +1699,10 @@ class Axon(s_cell.Cell):
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
             try:
                 await self._reqHas(sha256)
-                async with sess.request(method, url, headers=headers, params=params,
-                                        data=self.get(sha256), ssl=ssl) as resp:
+                async with sess.request(method, url, headers=headers, params=params, ssl=ssl,
+                                        data=self.get(sha256),
+                                        max_line_size=s_const.MAX_LINE_SIZE,
+                                        max_field_size=s_const.MAX_FIELD_SIZE) as resp:
 
                     info = {
                         'ok': True,
@@ -1827,7 +1830,9 @@ class Axon(s_cell.Cell):
         async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
 
             try:
-                async with sess.request(method, url, headers=headers, params=params, json=json, data=body, ssl=ssl) as resp:
+                async with sess.request(method, url, headers=headers, params=params, json=json, data=body, ssl=ssl,
+                                        max_line_size=s_const.MAX_LINE_SIZE,
+                                        max_field_size=s_const.MAX_FIELD_SIZE) as resp:
 
                     info = self._flatten_clientresponse(resp)
 

synapse/cortex.py

@@ -1538,11 +1538,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
              'desc': 'Controls access to deleting storm packages.'},
 
             {'perm': ('storm', 'asroot', 'cmd', '<cmdname>'), 'gate': 'cortex',
-            'desc': 'Controls running storm commands requiring root privileges.',
-             'ex': 'storm.asroot.cmd.movetag'},
+            'desc': 'Deprecated. Please use Storm modules to implement functionality requiring root privileges.'},
             {'perm': ('storm', 'asroot', 'mod', '<modname>'), 'gate': 'cortex',
-            'desc': 'Controls importing modules requiring root privileges.',
-             'ex': 'storm.asroot.cmd.synapse-misp.privsep'},
+            'desc': 'Deprecated. Storm modules should use the asroot:perms key to specify the permissions they require.'},
 
             {'perm': ('storm', 'graph', 'add'), 'gate': 'cortex',
              'desc': 'Controls access to add a storm graph.',
@@ -6914,7 +6912,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         Delete a cron job
 
         Args:
-            iden (bytes):  The iden of the cron job to be deleted
+            iden (str):  The iden of the cron job to be deleted
         '''
         await self._killCronTask(iden)
         try:
@@ -6931,7 +6929,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         Change an existing cron job's query
 
         Args:
-            iden (bytes):  The iden of the cron job to be changed
+            iden (str):  The iden of the cron job to be changed
         '''
         await self.agenda.mod(iden, query)
         await self.feedBeholder('cron:edit:query', {'iden': iden, 'query': query}, gates=[iden])
@@ -6942,7 +6940,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         Enable a cron job
 
         Args:
-            iden (bytes):  The iden of the cron job to be changed
+            iden (str):  The iden of the cron job to be changed
         '''
         await self.agenda.enable(iden)
         await self.feedBeholder('cron:enable', {'iden': iden}, gates=[iden])
@@ -6954,7 +6952,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         Enable a cron job
 
         Args:
-            iden (bytes):  The iden of the cron job to be changed
+            iden (str):  The iden of the cron job to be changed
         '''
         await self.agenda.disable(iden)
         await self._killCronTask(iden)

synapse/lib/agenda.py

@@ -430,6 +430,9 @@ class _Appt:
 
             if name == 'lasterrs' and not isinstance(valu, list):
                 valu = list(valu)
+            elif name == 'nexttime' and valu is None:
+                self.recs.clear()
+                self.stor._delete_appt_from_heap(self)
 
             setattr(self, name, valu)
 
@@ -469,7 +472,6 @@ class Agenda(s_base.Base):
         self.apptheap = []
         self.appts = {}
 
-        to_delete = []
         for iden, info in self.apptdefs.items():
             try:
                 appt = _Appt.unpack(self, info)
@@ -479,18 +481,30 @@ class Agenda(s_base.Base):
                 self._next_indx = max(self._next_indx, appt.indx + 1)
             except (s_exc.InconsistentStorage, s_exc.BadStorageVersion, s_exc.BadTime, TypeError, KeyError,
                     UnicodeDecodeError) as e:
-                logger.warning('Invalid appointment %r found in storage: %r.  Removing.', iden, e)
-                to_delete.append(iden)
-                continue
-
-        for iden in to_delete:
-            self.apptdefs.pop(iden)
+                logger.warning('Invalid appointment %r found in storage: %r. This appointment will be removed.', iden, e)
 
         # Make sure we don't assign the same index to 2 appointments
         if self.appts:
             maxindx = max(appt.indx for appt in self.appts.values())
             self._next_indx = maxindx + 1
 
+    async def _clear_invalid(self):
+
+        to_delete = []
+        for iden, info in self.apptdefs.items():
+            try:
+                appt = _Appt.unpack(self, info)
+                if appt.iden != iden:
+                    raise s_exc.InconsistentStorage(mesg='iden inconsistency')
+
+            except (s_exc.InconsistentStorage, s_exc.BadStorageVersion, s_exc.BadTime, TypeError, KeyError,
+                    UnicodeDecodeError) as e:
+                logger.warning('Removing invalid appointment %r.', iden)
+                to_delete.append(iden)
+
+        for iden in to_delete:
+            await self.core.delCronJob(iden)
+
     def _addappt(self, iden, appt):
         '''
         Updates the data structures to add an appointment
@@ -685,14 +699,13 @@ class Agenda(s_base.Base):
         '''
         Delete an appointment
         '''
-        appt = self.appts.get(iden)
-        if appt is None:
+        if not self.apptdefs.delete(iden):
             mesg = f'No cron job with iden: {iden}'
             raise s_exc.NoSuchIden(iden=iden, mesg=mesg)
 
-        self._delete_appt_from_heap(appt)
-        del self.appts[iden]
-        self.apptdefs.delete(iden)
+        if (appt := self.appts.get(iden)) is not None:
+            self._delete_appt_from_heap(appt)
+            del self.appts[iden]
 
     def _delete_appt_from_heap(self, appt):
         try:
@@ -729,14 +742,14 @@ class Agenda(s_base.Base):
                 await self.core.addCronEdits(appt.iden, edits)
                 await self.core.feedBeholder('cron:stop', {'iden': appt.iden})
 
-                if appt.nexttime is None:
-                    self._delete_appt_from_heap(appt)
-
     async def runloop(self):
         '''
         Task loop to issue query tasks at the right times.
         '''
         await self.clearRunningStatus()
+
+        await self._clear_invalid()
+
         while not self.isfini:
 
             timeout = None

synapse/lib/ast.py

@@ -3452,10 +3452,6 @@ class PropValue(Value):
                 raise self.kids[0].addExcInfo(exc)
 
             valu = path.node.get(name)
-            if isinstance(valu, (dict, list, tuple)):
-                # these get special cased because changing them affects the node
-                # while it's in the pipeline but the modification doesn't get stored
-                valu = s_msgpack.deepcopy(valu)
             return prop, valu
 
         # handle implicit pivot properties
@@ -3479,10 +3475,6 @@ class PropValue(Value):
                 raise self.kids[0].addExcInfo(exc)
 
             if i >= imax:
-                if isinstance(valu, (dict, list, tuple)):
-                    # these get special cased because changing them affects the node
-                    # while it's in the pipeline but the modification doesn't get stored
-                    valu = s_msgpack.deepcopy(valu)
                 return prop, valu
 
             form = runt.model.forms.get(prop.type.name)
@@ -3495,6 +3487,10 @@ class PropValue(Value):
 
     async def compute(self, runt, path):
         prop, valu = await self.getPropAndValu(runt, path)
+
+        if prop:
+            valu = await prop.type.tostorm(valu)
+
         return valu
 
 class RelPropValue(PropValue):

synapse/lib/cell.py

@@ -58,7 +58,7 @@ import synapse.lib.thisplat as s_thisplat
 
 import synapse.lib.crypto.passwd as s_passwd
 
-import synapse.tools.backup as s_t_backup
+import synapse.tools.service.backup as s_t_backup
 
 logger = logging.getLogger(__name__)
 

synapse/lib/const.py

@@ -47,3 +47,7 @@ layer_pdef_csize_max = 1_000
 layer_pdef_qsize_max = 10_000
 assert layer_pdef_csize <= layer_pdef_csize_max
 assert layer_pdef_qsize <= layer_pdef_qsize_max
+
+# HTTP header constants
+MAX_LINE_SIZE = kibibyte * 64
+MAX_FIELD_SIZE = kibibyte * 64

synapse/lib/multislabseqn.py

@@ -353,7 +353,6 @@ class MultiSlabSeqn(s_base.Base):
         else:
             indx = self.indx
 
-        assert self.tailseqn
         retn = self.tailseqn.add(item, indx=indx)
 
         if advances:
@@ -363,6 +362,42 @@ class MultiSlabSeqn(s_base.Base):
 
         return retn
 
+    async def addWithPackRetn(self, item, indx=None):
+        '''
+        Add a single item to the sequence, returning the offset and packed item.
+        '''
+        advances = True
+
+        if indx is not None:
+            if indx < self.firstindx:
+                raise s_exc.BadIndxValu(mesg=f'indx lower than first index in sequence {self.firstindx}')
+
+            if indx < self._ranges[-1]:
+                ridx = self._getRangeIndx(indx)
+                assert ridx is not None
+
+                async with self._getSeqn(ridx) as seqn:
+                    retn = seqn.addWithPackRetn(item, indx=indx)
+
+                return retn
+
+            if indx >= self.indx:
+                self.indx = indx
+            else:
+                advances = False
+        else:
+            indx = self.indx
+
+        assert self.tailseqn
+        retn = self.tailseqn.addWithPackRetn(item, indx=indx)
+
+        if advances:
+            self.indx += 1
+
+            self._wake_waiters()
+
+        return retn
+
     async def last(self) -> Optional[Tuple[int, Any]]:
         ridx = self._getRangeIndx(self.indx - 1)
         if ridx is None:

synapse/lib/nexus.py

@@ -12,6 +12,9 @@ import synapse.common as s_common
 import synapse.telepath as s_telepath
 
 import synapse.lib.base as s_base
+import synapse.lib.scope as s_scope
+import synapse.lib.queue as s_queue
+import synapse.lib.msgpack as s_msgpack
 
 logger = logging.getLogger(__name__)
 
@@ -20,6 +23,9 @@ leaderversion = 'Leader is a higher version than we are.'
 # As a mirror follower, amount of time before giving up on a write request
 FOLLOWER_WRITE_WAIT_S = 30.0
 
+WINDOW_MAXSIZE = 10_000
+YIELD_PREFIX = b'\x92\xa8t2:yield\x81\xa4retn\x92\xc3\x92'
+
 class RegMethType(type):
     '''
     Metaclass that collects all methods in class with _regme prop into a class member called _regclstupls
@@ -99,6 +105,8 @@ class NexsRoot(s_base.Base):
         self._mirready = asyncio.Event()  # for testing
 
         self._mirrors: List[ChangeDist] = []
+        self._linkmirrors: List[s_queue.Window] = []
+
         self._nexskids: Dict[str, 'Pusher'] = {}
 
         # Used to match pending follower write requests with the responses arriving on the log
@@ -134,6 +142,11 @@ class NexsRoot(s_base.Base):
         self.nexslog.setIndex(maxindx)
 
         async def fini():
+            for wind in self._linkmirrors:
+                await wind.fini()
+
+            for dist in self._mirrors:
+                await dist.fini()
 
             for futu in self._futures.values():  # pragma: no cover
                 futu.cancel()
@@ -389,8 +402,16 @@ class NexsRoot(s_base.Base):
     async def _eat(self, item, indx=None):
 
         if self.donexslog:
-            saveindx = await self.nexslog.add(item, indx=indx)
-            [dist.update() for dist in tuple(self._mirrors)]
+            saveindx, packitem = await self.nexslog.addWithPackRetn(item, indx=indx)
+
+            if self._linkmirrors:
+                tupl = (saveindx, YIELD_PREFIX + s_msgpack.en(saveindx) + packitem)
+                for wind in tuple(self._linkmirrors):
+                    await wind.put(tupl)
+
+            if self._mirrors:
+                for dist in tuple(self._mirrors):
+                    dist.update()
 
         else:
             saveindx = self.nexshot.get('nexs:indx')
@@ -415,7 +436,15 @@ class NexsRoot(s_base.Base):
 
     async def iter(self, offs: int, tellready=False, wait=True) -> AsyncIterator[Any]:
         '''
-        Returns an iterator of change entries in the log
+        Returns an iterator of change entries in the log.
+
+        Notes:
+            If this method is being called in the context of a Telepath call,
+            it will directly send messages to the scope "link" object when it
+            has caught up to the realtime change window; instead of yielding
+            them as a generator. This is an optimization to avoid duplication
+            of msgpack'ing the same object over and over again as the number
+            of mirrors increases.
         '''
         if not self.donexslog:
             return
@@ -437,11 +466,41 @@ class NexsRoot(s_base.Base):
         if not wait:
             return
 
-        async with self.getChangeDist(maxoffs) as dist:
-            async for item in dist:
-                if self.isfini:
-                    raise s_exc.IsFini()
-                yield item
+        if (link := s_scope.get('link')) is None:
+            async with self.getChangeDist(maxoffs) as dist:
+                async for item in dist:
+                    yield item
+
+        else:
+            async with self.getMirrorWindow() as wind:
+
+                # Ensure we are caught up after grabbing a window
+                sync = True
+
+                async for item in self.nexslog.iter(maxoffs):
+                    maxoffs = item[0] + 1
+                    yield item
+
+                async for offs, item in wind:
+                    if sync:
+                        if offs < maxoffs:
+                            continue
+                        sync = False
+
+                    await link.send(item)
+
+    @contextlib.asynccontextmanager
+    async def getMirrorWindow(self):
+        async with await s_queue.Window.anit(maxsize=WINDOW_MAXSIZE, clearonfini=True) as wind:
+
+            async def fini():
+                self._linkmirrors.remove(wind)
+
+            wind.onfini(fini)
+
+            self._linkmirrors.append(wind)
+
+            yield wind
 
     @contextlib.asynccontextmanager
     async def getChangeDist(self, offs: int) -> AsyncIterator[ChangeDist]:

synapse/lib/platforms/linux.py

@@ -112,7 +112,9 @@ def getSysctls():
         ('vm.dirty_expire_centisecs', '/proc/sys/vm/dirty_expire_centisecs', int),
         ('vm.dirty_writeback_centisecs', '/proc/sys/vm/dirty_writeback_centisecs', int),
         ('vm.dirty_background_ratio', '/proc/sys/vm/dirty_background_ratio', int),
+        ('vm.dirty_background_bytes', '/proc/sys/vm/dirty_background_bytes', int),
         ('vm.dirty_ratio', '/proc/sys/vm/dirty_ratio', int),
+        ('vm.dirty_bytes', '/proc/sys/vm/dirty_bytes', int),
     )
     ret = {}
     for key, fp, func in _sysctls:

synapse/lib/queue.py

@@ -118,14 +118,17 @@ class Window(s_base.Base):
     its maxsize, it will be fini()d.  On fini(), the Window will continue to
     yield results until empty and then return.
     '''
-    async def __anit__(self, maxsize=None):
+    async def __anit__(self, maxsize=None, clearonfini=False):
         await s_base.Base.__anit__(self)
         self.maxsize = maxsize
         self.event = asyncio.Event()
 
         self.linklist = collections.deque()
+        self.clearonfini = clearonfini
 
         async def fini():
+            if self.clearonfini:
+                self.linklist.clear()
             self.event.set()
 
         self.onfini(fini)

synapse/lib/rstorm.py

@@ -25,8 +25,8 @@ import synapse.lib.stormhttp as s_stormhttp
 
 import synapse.cmds.cortex as s_cmds_cortex
 
-import synapse.tools.storm as s_storm
-import synapse.tools.genpkg as s_genpkg
+import synapse.tools.storm._cli as s_storm
+import synapse.tools.storm.pkg.gen as s_genpkg
 
 
 re_directive = regex.compile(r'^\.\.\s(shell.*|storm.*|[^:])::(?:\s(.*)$|$)')

synapse/lib/schemas.py

@@ -393,7 +393,6 @@ _changelogTypes = {'migration': 'Automatic Migrations',
                    'note': 'Notes',
                    'doc': 'Improved documentation',
                    'deprecation': 'Deprecations'}
-
 _changelogSchema = {
     'type': 'object',
     'properties': {
@@ -405,6 +404,10 @@ _changelogSchema = {
             'type': 'string',
             'minLength': 1,
         },
+        'desc:literal': {
+            'type': 'boolean',
+            'default': False,
+        },
         'prs': {
             'type': 'array',
             'items': {
@@ -948,6 +951,13 @@ _reqValidPkgdefSchema = {
                     'type': ['array', 'null'],
                     'items': {'$ref': '#/definitions/cmdinput'},
                 },
+                'cmdconf': {
+                    'type': 'object',
+                    'properties': {
+                        'svciden': {'type': 'string', 'pattern': s_config.re_iden},
+                    },
+                    'additionalProperties': True,
+                },
                 'storm': {'type': 'string'},
                 'forms': {'$ref': '#/definitions/cmdformhints'},
                 'perms': {'type': 'array',

synapse/lib/slabseqn.py

@@ -87,6 +87,34 @@ class SlabSeqn:
 
         return indx
 
+    def addWithPackRetn(self, item, indx=None):
+        '''
+        Add a single item to the sequence, returning the offset and packed item.
+        '''
+        packitem = s_msgpack.en(item)
+        if indx is not None:
+            if indx >= self.indx:
+                self.slab.put(s_common.int64en(indx), packitem, append=True, db=self.db)
+                self.indx = indx + 1
+                self.size += 1
+                self._wake_waiters()
+                return indx, packitem
+
+            oldv = self.slab.replace(s_common.int64en(indx), packitem, db=self.db)
+            if oldv is None:
+                self.size += 1
+            return indx, packitem
+
+        indx = self.indx
+        self.slab.put(s_common.int64en(indx), packitem, append=True, db=self.db)
+
+        self.indx += 1
+        self.size += 1
+
+        self._wake_waiters()
+
+        return indx, packitem
+
     def first(self):
 
         for lkey, lval in self.slab.scanByFull(db=self.db):

synapse/lib/storm.py

@@ -2521,9 +2521,17 @@ class PureCmd(Cmd):
         perm = ('storm', 'asroot', 'cmd') + tuple(name.split('.'))
 
         asroot = runt.allowed(perm)
-        if self.asroot and not asroot:
-            mesg = f'Command ({name}) elevates privileges.  You need perm: storm.asroot.cmd.{name}'
-            raise s_exc.AuthDeny(mesg=mesg, user=runt.user.iden, username=runt.user.name)
+        if self.asroot:
+            mesg = f'Command ({name}) requires asroot permission which is deprecated and will be removed in v3.0.0. ' \
+                    'Functionality which requires elevated permissions should be implemented in Storm modules and use ' \
+                    'asroot:perms to specify the required permissions.'
+
+            s_common.deprecated('Storm command asroot key', curv='2.226.0', eolv='3.0.0')
+            await runt.warnonce(mesg, log=False)
+
+            if not asroot:
+                mesg = f'Command ({name}) elevates privileges.  You need perm: storm.asroot.cmd.{name}'
+                raise s_exc.AuthDeny(mesg=mesg, user=runt.user.iden, username=runt.user.name)
 
         # if a command requires perms, check em!
         # ( used to create more intuitive perm boundaries )
@@ -2545,10 +2553,14 @@ class PureCmd(Cmd):
 
         cmdopts = s_stormtypes.CmdOpts(self)
 
+        cmdconf = self.cdef.get('cmdconf', {})
+        if cmdconf:
+            cmdconf = s_msgpack.deepcopy(cmdconf, use_list=True)
+
         opts = {
             'vars': {
                 'cmdopts': cmdopts,
-                'cmdconf': self.cdef.get('cmdconf', {}),
+                'cmdconf': cmdconf,
             }
         }
 

synapse/lib/stormhttp.py

@@ -12,6 +12,7 @@ import synapse.common as s_common
 
 import synapse.lib.base as s_base
 import synapse.lib.json as s_json
+import synapse.lib.const as s_const
 import synapse.lib.msgpack as s_msgpack
 import synapse.lib.version as s_version
 import synapse.lib.stormtypes as s_stormtypes
@@ -400,7 +401,12 @@ class LibHttp(s_stormtypes.Lib):
         proxy = await s_stormtypes.toprim(proxy)
         ssl_opts = await s_stormtypes.toprim(ssl_opts)
 
-        kwargs = {'allow_redirects': allow_redirects}
+        kwargs = {
+            'max_line_size': s_const.MAX_LINE_SIZE,
+            'max_field_size': s_const.MAX_FIELD_SIZE,
+            'allow_redirects': allow_redirects,
+        }
+
         if params:
             kwargs['params'] = s_stormtypes.strifyHttpArg(params, multi=True)
 

synapse/lib/stormlib/aha.py

@@ -740,10 +740,10 @@ The ready column indicates that a service has entered into the realtime change w
                     $info = $cell_infos.$svcname
                     $cell_info = $info.cell
                     $status.nexs_indx = $cell_info.nexsindx
-                    if ($cell_info.uplink) {
-                        $status.role = 'follower'
-                    } else {
+                    if ($cell_info.active) {
                         $status.role = 'leader'
+                    } else {
+                        $status.role = 'follower'
                     }
                     $status.version = $info.cell.verstring
                     $status.synapse_version = $info.synapse.verstring

synapse/lib/stormtypes.py

@@ -1499,9 +1499,16 @@ class LibBase(Lib):
             perm = ('storm', 'asroot', 'mod') + tuple(name.split('.'))
             asroot = self.runt.allowed(perm)
 
-            if mdef.get('asroot', False) and not asroot:
-                mesg = f'Module ({name}) elevates privileges.  You need perm: storm.asroot.mod.{name}'
-                raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
+            if mdef.get('asroot', False):
+                mesg = f'Module ({name}) requires asroot permission but does not specify any asroot:perms. ' \
+                        'storm.asroot.mod.<modname> style permissons are deprecated and will be removed in v3.0.0.'
+
+                s_common.deprecated('Storm module asroot key', curv='2.226.0', eolv='3.0.0')
+                await self.runt.warnonce(mesg, log=False)
+
+                if not asroot:
+                    mesg = f'Module ({name}) elevates privileges.  You need perm: storm.asroot.mod.{name}'
+                    raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
 
         modr = await self.runt.getModRuntime(query, opts={'vars': {'modconf': modconf}})
         modr.asroot = asroot

synapse/lib/types.py

@@ -32,6 +32,8 @@ class Type:
     # ( due to hot-loop needs in the storm runtime )
     isarray = False
 
+    ismutable = False
+
     def __init__(self, modl, name, info, opts):
         '''
         Construct a new Type object.
@@ -369,6 +371,17 @@ class Type:
         topt.update(opts)
         return self.__class__(self.modl, self.name, self.info, topt)
 
+    async def tostorm(self, valu):
+        '''
+        Allows type-specific modifications to values to make them safe for use in the runtime.
+
+        Args:
+            valu (any): The valu to update.
+        '''
+        if self.ismutable:
+            return s_msgpack.deepcopy(valu, use_list=True)
+        return valu
+
     def __eq__(self, othr):
         if self.name != othr.name:
             return False
@@ -415,6 +428,7 @@ class Bool(Type):
 class Array(Type):
 
     isarray = True
+    ismutable = True
 
     def postTypeInit(self):
 
@@ -534,6 +548,9 @@ class Comp(Type):
 
             _type = self.tcache[name]
 
+            if _type.ismutable:
+                self.ismutable = True
+
             norm, info = _type.norm(valu[i])
 
             subs[name] = norm
@@ -1623,6 +1640,8 @@ class TimeEdge(Edge):
 
 class Data(Type):
 
+    ismutable = True
+
     stortype = s_layer.STOR_TYPE_MSGP
 
     def postTypeInit(self):
@@ -1643,6 +1662,7 @@ class Data(Type):
 
 class NodeProp(Type):
 
+    ismutable = True
     stortype = s_layer.STOR_TYPE_MSGP
 
     def postTypeInit(self):