synapse 2.178.0__py311-none-any.whl → 2.180.0__py311-none-any.whl

synapse/cortex.py

@@ -1354,20 +1354,20 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
              'desc': 'Controls access to adding specific extended model properties.',
              'ex': 'model.prop.add._foo:bar'},
             {'perm': ('model', 'prop', 'del'), 'gate': 'cortex',
-             'desc': 'Controls access to deleting extended model properties.'},
+             'desc': 'Controls access to deleting extended model properties and values.'},
             {'perm': ('model', 'prop', 'del', '<form>'), 'gate': 'cortex',
-             'desc': 'Controls access to deleting specific extended model properties.',
+             'desc': 'Controls access to deleting specific extended model properties and values.',
              'ex': 'model.prop.del._foo:bar'},
 
             {'perm': ('model', 'tagprop', 'add'), 'gate': 'cortex',
-             'desc': 'Controls access to adding extended model tag properties.'},
+             'desc': 'Controls access to adding extended model tag properties and values.'},
             {'perm': ('model', 'tagprop', 'del'), 'gate': 'cortex',
-             'desc': 'Controls access to deleting extended model tag properties.'},
+             'desc': 'Controls access to deleting extended model tag properties and values.'},
 
             {'perm': ('model', 'univ', 'add'), 'gate': 'cortex',
              'desc': 'Controls access to adding extended model universal properties.'},
             {'perm': ('model', 'univ', 'del'), 'gate': 'cortex',
-             'desc': 'Controls access to deleting extended model universal properties.'},
+             'desc': 'Controls access to deleting extended model universal properties and values.'},
 
             {'perm': ('node',), 'gate': 'layer',
              'desc': 'Controls all node edits in a layer.'},
@@ -1626,6 +1626,25 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         if self.isactive:
             await self.finiStormPool()
 
+    @s_nexus.Pusher.onPushAuto('model:lock:prop')
+    async def setPropLocked(self, name, locked):
+        prop = self.model.reqProp(name)
+        self.modellocks.set(f'prop/{name}', locked)
+        prop.locked = locked
+
+    @s_nexus.Pusher.onPushAuto('model:lock:univ')
+    async def setUnivLocked(self, name, locked):
+        prop = self.model.reqUniv(name)
+        self.modellocks.set(f'univ/{name}', locked)
+        for prop in self.model.getAllUnivs(name):
+            prop.locked = locked
+
+    @s_nexus.Pusher.onPushAuto('model:lock:tagprop')
+    async def setTagPropLocked(self, name, locked):
+        prop = self.model.reqTagProp(name)
+        self.modellocks.set(f'tagprop/{name}', locked)
+        prop.locked = locked
+
     @s_nexus.Pusher.onPushAuto('model:depr:lock')
     async def setDeprLock(self, name, locked):
 
@@ -2093,17 +2112,15 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         async def onSetTrigDoc(node, prop, valu):
             valu = str(valu)
             iden = node.ndef[1]
-            trig = node.snap.view.triggers.get(iden)
             node.snap.user.confirm(('trigger', 'set', 'doc'), gateiden=iden)
-            await trig.set('doc', valu)
+            await node.snap.view.setTriggerInfo(iden, 'doc', valu)
             node.props[prop.name] = valu
 
         async def onSetTrigName(node, prop, valu):
             valu = str(valu)
             iden = node.ndef[1]
-            trig = node.snap.view.triggers.get(iden)
             node.snap.user.confirm(('trigger', 'set', 'name'), gateiden=iden)
-            await trig.set('name', valu)
+            await node.snap.view.setTriggerInfo(iden, 'name', valu)
             node.props[prop.name] = valu
 
         async def onSetCronDoc(node, prop, valu):
@@ -3437,7 +3454,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         for layr in self.layers.values():
             async for item in layr.iterFormRows(formname):
-                mesg = f'Nodes still exist with form: {formname}'
+                mesg = f'Nodes still exist with form: {formname} in layer {layr.iden}'
                 raise s_exc.CantDelForm(mesg=mesg)
 
         self.model.delForm(formname)
@@ -3486,14 +3503,116 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             await self.feedBeholder('model:prop:add', {'form': form, 'prop': prop.pack()})
 
     async def delFormProp(self, form, prop):
+        self.reqExtProp(form, prop)
+        return await self._push('model:prop:del', form, prop)
+
+    async def _delAllFormProp(self, formname, propname, meta):
+        '''
+        Delete all instances of a property from all layers.
+
+        NOTE: This does not fire triggers.
+        '''
+        self.reqExtProp(formname, propname)
+
+        fullname = f'{formname}:{propname}'
+        prop = self.model.prop(fullname)
+
+        await self.setPropLocked(fullname, True)
+
+        for layr in list(self.layers.values()):
+
+            genr = layr.iterPropRows(formname, propname)
+
+            async for rows in s_coro.chunks(genr):
+                nodeedits = []
+                for buid, valu in rows:
+                    nodeedits.append((buid, prop.form.name, (
+                        (s_layer.EDIT_PROP_DEL, (prop.name, None, prop.type.stortype), ()),
+                    )))
+
+                await layr.saveNodeEdits(nodeedits, meta)
+                await asyncio.sleep(0)
+
+    async def _delAllUnivProp(self, propname, meta):
+        '''
+        Delete all instances of a universal property from all layers.
+
+        NOTE: This does not fire triggers.
+        '''
+        self.reqExtUniv(propname)
+
+        full = f'.{propname}'
+        prop = self.model.univ(full)
+
+        await self.setUnivLocked(full, True)
+
+        for layr in list(self.layers.values()):
+
+            genr = layr.iterUnivRows(full)
+
+            async for rows in s_coro.chunks(genr):
+                nodeedits = []
+                for buid, valu in rows:
+                    sode = await layr.getStorNode(buid)
+                    nodeedits.append((buid, sode.get('form'), (
+                        (s_layer.EDIT_PROP_DEL, (prop.name, None, prop.type.stortype), ()),
+                    )))
+
+                await layr.saveNodeEdits(nodeedits, meta)
+                await asyncio.sleep(0)
+
+    async def _delAllTagProp(self, propname, meta):
+        '''
+        Delete all instances of a tag property from all layers.
+
+        NOTE: This does not fire triggers.
+        '''
+        self.reqExtTagProp(propname)
+        prop = self.model.getTagProp(propname)
+
+        await self.setTagPropLocked(propname, True)
+
+        for layr in list(self.layers.values()):
+
+            for form, tag, tagprop in layr.getTagProps():
+
+                if tagprop != propname: # pragma: no cover
+                    await asyncio.sleep(0)
+                    continue
+
+                genr = layr.iterTagPropRows(tag, tagprop, form)
+
+                async for rows in s_coro.chunks(genr):
+                    nodeedits = []
+                    for buid, valu in rows:
+                        nodeedits.append((buid, form, (
+                            (s_layer.EDIT_TAGPROP_DEL, (tag, prop.name, None, prop.type.stortype), ()),
+                        )))
+
+                    await layr.saveNodeEdits(nodeedits, meta)
+                    await asyncio.sleep(0)
+
+    def reqExtProp(self, form, prop):
         full = f'{form}:{prop}'
         pdef = self.extprops.get(full)
-
         if pdef is None:
             mesg = f'No ext prop named {full}'
             raise s_exc.NoSuchProp(form=form, prop=prop, mesg=mesg)
+        return pdef
 
-        return await self._push('model:prop:del', form, prop)
+    def reqExtUniv(self, prop):
+        udef = self.extunivs.get(prop)
+        if udef is None:
+            mesg = f'No ext univ named {prop}'
+            raise s_exc.NoSuchUniv(name=prop, mesg=mesg)
+        return udef
+
+    def reqExtTagProp(self, name):
+        pdef = self.exttagprops.get(name)
+        if pdef is None:
+            mesg = f'No tag prop named {name}'
+            raise s_exc.NoSuchTagProp(mesg=mesg, name=name)
+        return pdef
 
     @s_nexus.Pusher.onPush('model:prop:del')
     async def _delFormProp(self, form, prop):
@@ -3508,22 +3627,19 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         for layr in self.layers.values():
             async for item in layr.iterPropRows(form, prop):
-                mesg = f'Nodes still exist with prop: {form}:{prop}'
+                mesg = f'Nodes still exist with prop: {form}:{prop} in layer {layr.iden}'
                 raise s_exc.CantDelProp(mesg=mesg)
 
         self.model.delFormProp(form, prop)
         self.extprops.pop(full, None)
+        self.modellocks.pop(f'prop/{full}', None)
         await self.fire('core:extmodel:change',
                         form=form, prop=prop, act='del', type='formprop')
 
         await self.feedBeholder('model:prop:del', {'form': form, 'prop': prop})
 
     async def delUnivProp(self, prop):
-        udef = self.extunivs.get(prop)
-        if udef is None:
-            mesg = f'No ext univ named {prop}'
-            raise s_exc.NoSuchUniv(name=prop, mesg=mesg)
-
+        self.reqExtUniv(prop)
         return await self._push('model:univ:del', prop)
 
     @s_nexus.Pusher.onPush('model:univ:del')
@@ -3538,11 +3654,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         univname = '.' + prop
         for layr in self.layers.values():
             async for item in layr.iterUnivRows(univname):
-                mesg = f'Nodes still exist with universal prop: {prop}'
+                mesg = f'Nodes still exist with universal prop: {prop} in layer {layr.iden}'
                 raise s_exc.CantDelUniv(mesg=mesg)
 
         self.model.delUnivProp(prop)
         self.extunivs.pop(prop, None)
+        self.modellocks.pop(f'univ/{prop}', None)
         await self.fire('core:extmodel:change', name=prop, act='del', type='univ')
         await self.feedBeholder('model:univ:del', {'prop': univname})
 
@@ -3574,11 +3691,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             await self.feedBeholder('model:tagprop:add', tagp.pack())
 
     async def delTagProp(self, name):
-        pdef = self.exttagprops.get(name)
-        if pdef is None:
-            mesg = f'No tag prop named {name}'
-            raise s_exc.NoSuchProp(mesg=mesg, name=name)
-
+        self.reqExtTagProp(name)
         return await self._push('model:tagprop:del', name)
 
     @s_nexus.Pusher.onPush('model:tagprop:del')
@@ -3589,12 +3702,13 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         for layr in self.layers.values():
             if await layr.hasTagProp(name):
-                mesg = f'Nodes still exist with tagprop: {name}'
+                mesg = f'Nodes still exist with tagprop: {name} in layer {layr.iden}'
                 raise s_exc.CantDelProp(mesg=mesg)
 
         self.model.delTagProp(name)
 
         self.exttagprops.pop(name, None)
+        self.modellocks.pop(f'tagprop/{name}', None)
         await self.fire('core:tagprop:change', name=name, act='del')
         await self.feedBeholder('model:tagprop:del', {'tagprop': name})
 
@@ -3941,7 +4055,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             self.stormvars.set(s_stormlib_cell.runtime_fixes_key, s_stormlib_cell.getMaxHotFixes())
 
     async def _initDeprLocks(self):
+
         self.deprlocks = self.cortexdata.getSubKeyVal('model:deprlocks:')
+        self.modellocks = self.cortexdata.getSubKeyVal('model:locks:')
+
         # TODO: 3.0.0 conversion will truncate this hive key
 
         if self.inaugural:
@@ -3967,6 +4084,24 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             if _type is not None:
                 _type.locked = locked
 
+        for name, locked in self.modellocks.items():
+
+            prop = None
+            elemtype, elemname = name.split('/', 1)
+
+            if elemtype == 'prop':
+                prop = self.model.prop(elemname)
+            elif elemtype == 'univ':
+                prop = self.model.univ(elemname)
+                if prop is not None:
+                    for univ in self.model.getAllUnivs(elemname):
+                        univ.locked = locked
+            elif elemtype == 'tagprop':
+                prop = self.model.getTagProp(elemname)
+
+            if prop is not None:
+                prop.locked = locked
+
     async def _initJsonStor(self):
 
         self.jsonurl = self.conf.get('jsonstor')
@@ -5595,7 +5730,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         if (nexsoffs := opts.get('nexsoffs')) is not None:
             if not await self.waitNexsOffs(nexsoffs, timeout=opts.get('nexstimeout')):
-                raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs}.')
+                raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs} in count()')
 
         view = self._viewFromOpts(opts)
 
@@ -5659,7 +5794,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         if (nexsoffs := opts.get('nexsoffs')) is not None:
             if not await self.waitNexsOffs(nexsoffs, timeout=opts.get('nexstimeout')):
-                raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs}.')
+                raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs} in storm().')
 
         view = self._viewFromOpts(opts)
         async for mesg in view.storm(text, opts=opts):
@@ -5686,7 +5821,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         if (nexsoffs := opts.get('nexsoffs')) is not None:
             if not await self.waitNexsOffs(nexsoffs, timeout=opts.get('nexstimeout')):
-                raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs}.')
+                raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs} in callStorm().')
 
         view = self._viewFromOpts(opts)
         return await view.callStorm(text, opts=opts)
@@ -5714,7 +5849,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         if (nexsoffs := opts.get('nexsoffs')) is not None:
             if not await self.waitNexsOffs(nexsoffs, timeout=opts.get('nexstimeout')):
-                raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs}.')
+                raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs} in exportStorm().')
 
         user = self._userFromOpts(opts)
         view = self._viewFromOpts(opts)
@@ -7072,6 +7207,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             raise s_exc.BadArg('Only vault names and permissions can be changed.')
 
         vault = self.reqVault(iden)
+        oldv = vault.get(key)
         vault[key] = valu
 
         s_schemas.reqValidVault(vault)
@@ -7079,8 +7215,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         bidn = s_common.uhex(iden)
 
         if key == 'name':
-            name = vault.get('name')
-            self.slab.delete(name.encode(), db=self.vaultsbynamedb)
+            self.slab.delete(oldv.encode(), db=self.vaultsbynamedb)
             self.slab.put(valu.encode(), bidn, db=self.vaultsbynamedb)
 
         self.slab.put(bidn, s_msgpack.en(vault), db=self.vaultsdb)

synapse/datamodel.py

@@ -30,6 +30,7 @@ class TagProp:
         self.info = info
         self.tdef = tdef
         self.model = model
+        self.locked = False
 
         self.utf8 = name.encode()
         self.nenc = name.encode() + b'\x00'
@@ -127,6 +128,10 @@ class Prop:
         self.deprecated = self.info.get('deprecated', False)
 
         self.type = self.modl.getTypeClone(typedef)
+        self.typehash = self.type.typehash
+
+        if self.type.isarray:
+            self.arraytypehash = self.type.arraytype.typehash
 
         if form is not None:
             form.setProp(name, self)
@@ -269,6 +274,11 @@ class Form:
         if self.type is None:
             raise s_exc.NoSuchType(name=name)
 
+        self.typehash = self.type.typehash
+
+        if self.type.isarray:
+            self.arraytypehash = self.type.arraytype.typehash
+
         self.form = self
 
         self.props = {}     # name: Prop()
@@ -462,6 +472,7 @@ class Model:
         self.modeldefs = []
 
         self.univs = {}
+        self.allunivs = collections.defaultdict(list)
 
         self.propsbytype = collections.defaultdict(dict)  # name: Prop()
         self.arraysbytype = collections.defaultdict(dict)
@@ -620,6 +631,30 @@ class Model:
             self.formprefixcache[prefix] = forms
         return forms
 
+    def reqProp(self, name):
+        prop = self.prop(name)
+        if prop is not None:
+            return prop
+
+        mesg = f'No property named {name}.'
+        raise s_exc.NoSuchProp(mesg=mesg, name=name)
+
+    def reqUniv(self, name):
+        prop = self.univ(name)
+        if prop is not None:
+            return prop
+
+        mesg = f'No universal property named {name}.'
+        raise s_exc.NoSuchUniv(mesg=mesg, name=name)
+
+    def reqTagProp(self, name):
+        prop = self.getTagProp(name)
+        if prop is not None:
+            return prop
+
+        mesg = f'No tag property named {name}.'
+        raise s_exc.NoSuchTagProp(mesg=mesg, name=name)
+
     def reqFormsByPrefix(self, prefix, extra=None):
         forms = self.getFormsByPrefix(prefix)
         if not forms:
@@ -965,13 +1000,18 @@ class Model:
 
     def _addFormUniv(self, form, name, tdef, info):
 
+        univ = self.reqUniv(name)
+
         prop = Prop(self, form, name, tdef, info)
+        prop.locked = univ.locked
 
         full = f'{form.name}{name}'
 
         self.props[full] = prop
         self.props[(form.name, name)] = prop
 
+        self.allunivs[name].append(prop)
+
     def addUnivProp(self, name, tdef, info):
 
         base = '.' + name
@@ -985,8 +1025,13 @@ class Model:
         self.props[base] = univ
         self.univs[base] = univ
 
+        self.allunivs[base].append(univ)
+
         for form in self.forms.values():
-            self._addFormUniv(form, base, tdef, info)
+            prop = self._addFormUniv(form, base, tdef, info)
+
+    def getAllUnivs(self, name):
+        return list(self.allunivs.get(name, ()))
 
     def addFormProp(self, formname, propname, tdef, info):
         form = self.forms.get(formname)
@@ -1090,6 +1135,7 @@ class Model:
             raise s_exc.NoSuchUniv(name=propname)
 
         self.univs.pop(univname, None)
+        self.allunivs.pop(univname, None)
 
         for form in self.forms.values():
             self.delFormProp(form.name, univname)

synapse/exc.py

@@ -77,6 +77,7 @@ class BackupAlreadyRunning(SynErr):
 class StormPkgRequires(SynErr): pass
 class StormPkgConflicts(SynErr): pass
 
+class BadName(SynErr): pass
 class BadPkgDef(SynErr): pass
 class BadCmdName(SynErr): pass
 class BadCmprValu(SynErr): pass

synapse/lib/aha.py

@@ -733,7 +733,7 @@ class AhaCell(s_cell.Cell):
         # that do not intend to listen for provisioning.
         hostname = self.conf.get('dns:name')
         if hostname is not None:
-            return f'ssl://{hostname}:27272'
+            return f'ssl://0.0.0.0:27272?hostname={hostname}'
 
     def _getDmonListen(self):
 
@@ -776,6 +776,7 @@ class AhaCell(s_cell.Cell):
         if provurl is not None:
             self.provdmon = await ProvDmon.anit(self)
             self.onfini(self.provdmon)
+            logger.info(f'provision listening: {provurl}')
             self.provaddr = await self.provdmon.listen(provurl)
 
     async def _clearInactiveSessions(self):