synapse 2.168.0__py311-none-any.whl → 2.170.0__py311-none-any.whl

synapse/lib/layer.py

@@ -107,6 +107,7 @@ reqValidLdef = s_config.getJsValidator({
 })
 
 WINDOW_MAXSIZE = 10_000
+MIGR_COMMIT_SIZE = 1_000
 
 class LayerApi(s_cell.CellApi):
 
@@ -2612,7 +2613,7 @@ class Layer(s_nexus.Pusher):
             venc = lkey[32:]
 
             putkeys.append((n1buid + n2buid, venc))
-            if len(putkeys) > 1000:
+            if len(putkeys) > MIGR_COMMIT_SIZE:
                 commit()
 
         if len(putkeys):
@@ -2623,6 +2624,33 @@ class Layer(s_nexus.Pusher):
 
         logger.warning(f'...complete!')
 
+    async def _layrV10toV11(self):
+
+        logger.warning(f'Adding byform index to layer {self.iden}')
+
+        def commit():
+            self.layrslab.putmulti(putkeys, db=self.byform)
+            putkeys.clear()
+
+        putkeys = []
+        async for buid, sode in self.getStorNodes():
+            if not (form := sode.get('form')):
+                continue
+
+            abrv = self.setPropAbrv(form, None)
+            putkeys.append((abrv, buid))
+
+            if len(putkeys) > MIGR_COMMIT_SIZE:
+                commit()
+
+        if putkeys:
+            commit()
+
+        self.meta.set('version', 11)
+        self.layrvers = 11
+
+        logger.warning('...complete!')
+
     async def _initSlabs(self, slabopts):
 
         otherslabopts = {
@@ -2659,6 +2687,7 @@ class Layer(s_nexus.Pusher):
         self.edgesn1n2 = self.layrslab.initdb('edgesn1n2', dupsort=True)
 
         self.bytag = self.layrslab.initdb('bytag', dupsort=True)
+        self.byform = self.layrslab.initdb('byform', dupsort=True)
         self.byprop = self.layrslab.initdb('byprop', dupsort=True)
         self.byarray = self.layrslab.initdb('byarray', dupsort=True)
         self.bytagprop = self.layrslab.initdb('bytagprop', dupsort=True)
@@ -2684,7 +2713,7 @@ class Layer(s_nexus.Pusher):
         await self._initSlabs(slabopts)
 
         if self.fresh:
-            self.meta.set('version', 10)
+            self.meta.set('version', 11)
 
         self.layrslab.addResizeCallback(self.core.checkFreeSpace)
         self.dataslab.addResizeCallback(self.core.checkFreeSpace)
@@ -2719,8 +2748,11 @@ class Layer(s_nexus.Pusher):
         if self.layrvers < 10:
             await self._layrV9toV10()
 
-        if self.layrvers != 10:
-            mesg = f'Got layer version {self.layrvers}.  Expected 10.  Accidental downgrade?'
+        if self.layrvers < 11:
+            await self._layrV10toV11()
+
+        if self.layrvers != 11:
+            mesg = f'Got layer version {self.layrvers}.  Expected 11.  Accidental downgrade?'
             raise s_exc.BadStorageVersion(mesg=mesg)
 
     async def getLayerSize(self):
@@ -3323,6 +3355,11 @@ class Layer(s_nexus.Pusher):
             return
 
         # no more refs in this layer.  time to pop it...
+        try:
+            abrv = self.getPropAbrv(sode.get('form'), None)
+            self.layrslab.delete(abrv, val=buid, db=self.byform)
+        except s_exc.NoSuchAbrv:
+            pass
         self.dirty.pop(buid, None)
         self.buidcache.pop(buid, None)
         self.layrslab.delete(buid, db=self.bybuidv3)
@@ -3343,11 +3380,14 @@ class Layer(s_nexus.Pusher):
         if sode.get('valu') == valt:
             return ()
 
+        abrv = self.setPropAbrv(form, None)
+
+        if sode.get('form') is None:
+            self.layrslab.put(abrv, buid, db=self.byform)
+
         sode['valu'] = valt
         self.setSodeDirty(buid, sode, form)
 
-        abrv = self.setPropAbrv(form, None)
-
         if stortype & STOR_FLAG_ARRAY:
 
             for indx in self.getStorIndx(stortype, valu):
@@ -3467,6 +3507,10 @@ class Layer(s_nexus.Pusher):
                     if univabrv is not None:
                         self.layrslab.delete(univabrv + oldi, buid, db=self.byprop)
 
+        if sode.get('form') is None:
+            formabrv = self.setPropAbrv(form, None)
+            self.layrslab.put(formabrv, buid, db=self.byform)
+
         sode['props'][prop] = (valu, stortype)
         self.setSodeDirty(buid, sode, form)
 
@@ -3561,6 +3605,9 @@ class Layer(s_nexus.Pusher):
             if oldv == valu:
                 return ()
 
+        if sode.get('form') is None:
+            self.layrslab.put(formabrv, buid, db=self.byform)
+
         sode['tags'][tag] = valu
         self.setSodeDirty(buid, sode, form)
 
@@ -3624,6 +3671,10 @@ class Layer(s_nexus.Pusher):
                     self.layrslab.delete(tp_abrv + oldi, buid, db=self.bytagprop)
                     self.layrslab.delete(ftp_abrv + oldi, buid, db=self.bytagprop)
 
+        if sode.get('form') is None:
+            formabrv = self.setPropAbrv(form, None)
+            self.layrslab.put(formabrv, buid, db=self.byform)
+
         if tag not in sode['tagprops']:
             sode['tagprops'][tag] = {}
         sode['tagprops'][tag][prop] = (valu, stortype)
@@ -3682,6 +3733,8 @@ class Layer(s_nexus.Pusher):
         # a bit of special case...
         if sode.get('form') is None:
             self.setSodeDirty(buid, sode, form)
+            formabrv = self.setPropAbrv(form, None)
+            self.layrslab.put(formabrv, buid, db=self.byform)
 
         if oldb is not None:
             oldv = s_msgpack.un(oldb)
@@ -3729,6 +3782,8 @@ class Layer(s_nexus.Pusher):
         # a bit of special case...
         if sode.get('form') is None:
             self.setSodeDirty(buid, sode, form)
+            formabrv = self.setPropAbrv(form, None)
+            self.layrslab.put(formabrv, buid, db=self.byform)
 
         self.layrslab.put(venc, buid + n2buid, db=self.byverb)
         self.layrslab.put(n1key, n2buid, db=self.edgesn1)
@@ -4208,6 +4263,20 @@ class Layer(s_nexus.Pusher):
             yield buid, s_msgpack.un(byts)
             await asyncio.sleep(0)
 
+    async def getStorNodesByForm(self, form):
+        '''
+        Yield (buid, sode) tuples for nodes of a given form with props/tags/tagprops/edges/nodedata in this layer.
+        '''
+        try:
+            abrv = self.getPropAbrv(form, None)
+        except s_exc.NoSuchAbrv:
+            return
+
+        for _, buid in self.layrslab.scanByDups(abrv, db=self.byform):
+            sode = await self.getStorNode(buid)
+            yield buid, sode
+            await asyncio.sleep(0)
+
     async def iterNodeEditLog(self, offs=0):
         '''
         Iterate the node edit log and yield (offs, edits, meta) tuples.

synapse/lib/node.py

@@ -237,6 +237,13 @@ class Node:
             for item in valu:
                 retn.append((name, (dest, item)))
 
+        for name in refs.get('ndefarray', ()):
+            if (valu := self.props.get(name)) is None:
+                continue
+
+            for item in valu:
+                retn.append((name, item))
+
         return retn
 
     async def set(self, name, valu, init=False):

synapse/lib/snap.py

@@ -183,6 +183,15 @@ class ProtoNode:
 
         return s_common.novalu
 
+    async def hasData(self, name):
+        if name in self.nodedata:
+            return True
+
+        if self.node is not None:
+            return await self.node.hasData(name)
+
+        return False
+
     async def setData(self, name, valu):
 
         if await self.getData(name) == valu:
@@ -287,6 +296,15 @@ class ProtoNode:
         if self.node is not None:
             return self.node.getTagProp(tag, name)
 
+    def hasTagProp(self, tag, name):
+        if (tag, name) in self.tagprops:
+            return True
+
+        if self.node is not None:
+            return self.node.hasTagProp(tag, name)
+
+        return False
+
     async def setTagProp(self, tag, name, valu):
 
         tagnode = await self.addTag(tag)
@@ -322,7 +340,7 @@ class ProtoNode:
         if self.node is not None:
             return self.node.get(name)
 
-    async def _set(self, prop, valu, norminfo=None):
+    async def _set(self, prop, valu, norminfo=None, ignore_ro=False):
 
         if prop.locked:
             mesg = f'Prop {prop.full} is locked due to deprecation.'
@@ -360,7 +378,7 @@ class ProtoNode:
         if curv == valu:
             return False
 
-        if prop.info.get('ro') and curv is not None:
+        if not ignore_ro and prop.info.get('ro') and curv is not None:
             mesg = f'Property is read only: {prop.full}.'
             await self.ctx.snap._raiseOnStrict(s_exc.ReadOnlyProp, mesg)
             return False
@@ -372,12 +390,12 @@ class ProtoNode:
 
         return valu, norminfo
 
-    async def set(self, name, valu, norminfo=None):
+    async def set(self, name, valu, norminfo=None, ignore_ro=False):
         prop = self.form.props.get(name)
         if prop is None:
             return False
 
-        retn = await self._set(prop, valu, norminfo=norminfo)
+        retn = await self._set(prop, valu, norminfo=norminfo, ignore_ro=ignore_ro)
         if retn is False:
             return False
 
@@ -717,7 +735,9 @@ class Snap(s_base.Base):
 
         show_storage = False
 
-        self.core._logStormQuery(text, user, info={'mode': opts.get('mode', 'storm'), 'view': self.view.iden})
+        info = opts.get('_loginfo', {})
+        info.update({'mode': opts.get('mode', 'storm'), 'view': self.view.iden})
+        self.core._logStormQuery(text, user, info=info)
 
         # { form: ( embedprop, ... ) }
         embeds = opts.get('embeds')

synapse/lib/storm.py

@@ -2351,7 +2351,7 @@ class Runtime(s_base.Base):
                 if view is None:
                     raise s_exc.NoSuchView(mesg=f'No such view iden={viewiden}', iden=viewiden)
 
-                self.user.confirm(('view', 'read'), gateiden=viewiden)
+                self.confirm(('view', 'read'), gateiden=viewiden)
                 snap = await view.snap(self.user)
 
         return snap

synapse/lib/stormlib/cortex.py

@@ -980,7 +980,7 @@ class HttpReq(s_stormtypes.StormType):
         try:
             return json.loads(self.rnfo.get('body'))
         except (UnicodeDecodeError, json.JSONDecodeError) as e:
-            raise s_exc.StormRuntimeError(mesg='Failed to decode request body as JSON: {e}') from None
+            raise s_exc.StormRuntimeError(mesg=f'Failed to decode request body as JSON: {e}') from None
 
     @s_stormtypes.stormfunc(readonly=True)
     async def _methSendCode(self, code):