synapse 2.155.0__py311-none-any.whl → 2.156.0__py311-none-any.whl

synapse/lib/stormtypes.py

@@ -6317,6 +6317,29 @@ class Layer(Prim):
                       {'name': 'propname', 'type': 'str', 'desc': 'The property or form name to look up.', },
                       {'name': 'maxsize', 'type': 'int', 'desc': 'The maximum number of rows to look up.',
                        'default': None, },
+                      {'name': 'valu', 'type': 'any', 'default': '$lib.undef',
+                       'desc': 'A specific value of the property to look up.', },
+                  ),
+                  'returns': {'type': 'int', 'desc': 'The count of rows.', }}},
+        {'name': 'getPropArrayCount',
+         'desc': 'Get the number of individual value rows in the layer for the given array property name.',
+         'type': {'type': 'function', '_funcname': '_methGetPropArrayCount',
+                  'args': (
+                      {'name': 'propname', 'type': 'str', 'desc': 'The property name to look up.', },
+                      {'name': 'valu', 'type': 'any', 'default': '$lib.undef',
+                       'desc': 'A specific value in the array property to look up.', },
+                  ),
+                  'returns': {'type': 'int', 'desc': 'The count of rows.', }}},
+        {'name': 'getTagPropCount',
+         'desc': 'Get the number of rows in the layer for the given tag property.',
+         'type': {'type': 'function', '_funcname': '_methGetTagPropCount',
+                  'args': (
+                      {'name': 'tag', 'type': 'str', 'desc': 'The tag to look up.', },
+                      {'name': 'propname', 'type': 'str', 'desc': 'The property name to look up.', },
+                      {'name': 'form', 'type': 'str', 'default': None,
+                       'desc': 'The optional form to look up.', },
+                      {'name': 'valu', 'type': 'any', 'default': '$lib.undef',
+                       'desc': 'A specific value of the property to look up.', },
                   ),
                   'returns': {'type': 'int', 'desc': 'The count of rows.', }}},
         {'name': 'getFormCounts', 'desc': '''
@@ -6514,6 +6537,8 @@ class Layer(Prim):
             'liftByProp': self.liftByProp,
             'getTagCount': self._methGetTagCount,
             'getPropCount': self._methGetPropCount,
+            'getTagPropCount': self._methGetTagPropCount,
+            'getPropArrayCount': self._methGetPropArrayCount,
             'getFormCounts': self._methGetFormcount,
             'getStorNode': self.getStorNode,
             'getStorNodes': self.getStorNodes,
@@ -6678,7 +6703,7 @@ class Layer(Prim):
         return await layr.getTagCount(tagname, formname=formname)
 
     @stormfunc(readonly=True)
-    async def _methGetPropCount(self, propname, maxsize=None):
+    async def _methGetPropCount(self, propname, maxsize=None, valu=undef):
         propname = await tostr(propname)
         maxsize = await toint(maxsize, noneok=True)
 
@@ -6691,13 +6716,86 @@ class Layer(Prim):
         await self.runt.reqUserCanReadLayer(layriden)
         layr = self.runt.snap.core.getLayer(layriden)
 
+        if valu is undef:
+            if prop.isform:
+                return await layr.getPropCount(prop.name, None, maxsize=maxsize)
+
+            if prop.isuniv:
+                return await layr.getPropCount(None, prop.name, maxsize=maxsize)
+
+            return await layr.getPropCount(prop.form.name, prop.name, maxsize=maxsize)
+
+        valu = await toprim(valu)
+        norm, info = prop.type.norm(valu)
+
+        if prop.isform:
+            return layr.getPropValuCount(prop.name, None, prop.type.stortype, norm)
+
+        if prop.isuniv:
+            return layr.getPropValuCount(None, prop.name, prop.type.stortype, norm)
+
+        return layr.getPropValuCount(prop.form.name, prop.name, prop.type.stortype, norm)
+
+    @stormfunc(readonly=True)
+    async def _methGetPropArrayCount(self, propname, valu=undef):
+        propname = await tostr(propname)
+
+        prop = self.runt.snap.core.model.prop(propname)
+        if prop is None:
+            mesg = f'No property named {propname}'
+            raise s_exc.NoSuchProp(mesg=mesg)
+
+        if not prop.type.isarray:
+            mesg = f'Property is not an array type: {prop.type.name}.'
+            raise s_exc.BadTypeValu(mesg=mesg)
+
+        layriden = self.valu.get('iden')
+        await self.runt.reqUserCanReadLayer(layriden)
+        layr = self.runt.snap.core.getLayer(layriden)
+
+        if valu is undef:
+            if prop.isform:
+                return await layr.getPropArrayCount(prop.name, None)
+
+            if prop.isuniv:
+                return await layr.getPropArrayCount(None, prop.name)
+
+            return await layr.getPropArrayCount(prop.form.name, prop.name)
+
+        valu = await toprim(valu)
+        atyp = prop.type.arraytype
+        norm, info = atyp.norm(valu)
+
         if prop.isform:
-            return await layr.getPropCount(prop.name, None, maxsize=maxsize)
+            return layr.getPropArrayValuCount(prop.name, None, atyp.stortype, norm)
 
         if prop.isuniv:
-            return await layr.getUnivPropCount(prop.name, maxsize=maxsize)
+            return layr.getPropArrayValuCount(None, prop.name, atyp.stortype, norm)
+
+        return layr.getPropArrayValuCount(prop.form.name, prop.name, atyp.stortype, norm)
+
+    @stormfunc(readonly=True)
+    async def _methGetTagPropCount(self, tag, propname, form=None, valu=undef):
+        tag = await tostr(tag)
+        propname = await tostr(propname)
+        form = await tostr(form, noneok=True)
+
+        prop = self.runt.snap.core.model.getTagProp(propname)
+        if prop is None:
+            mesg = f'No tag property named {propname}'
+            raise s_exc.NoSuchTagProp(name=propname, mesg=mesg)
 
-        return await layr.getPropCount(prop.form.name, prop.name, maxsize=maxsize)
+        layriden = self.valu.get('iden')
+        await self.runt.reqUserCanReadLayer(layriden)
+        layr = self.runt.snap.core.getLayer(layriden)
+
+        if valu is undef:
+            return await layr.getTagPropCount(form, tag, prop.name)
+
+        valu = await toprim(valu)
+        norm, info = prop.type.norm(valu)
+
+        return layr.getTagPropValuCount(form, tag, prop.name, prop.type.stortype, norm)
 
     @stormfunc(readonly=True)
     async def _methLayerEdits(self, offs=0, wait=True, size=None):
@@ -7022,6 +7120,63 @@ class View(Prim):
                       {'type': 'dict',
                        'desc': "Dictionary containing form names and the count of the nodes in the View's Layers.", }}},
 
+        {'name': 'getPropCount',
+         'desc': '''
+            Get the number of nodes in the View with a specific property and optional value.
+
+            Notes:
+               This is a fast approximate count calculated by summing the number of
+               nodes with the property value in each layer of the view. Property values
+               which are overwritten by different values in higher layers will still
+               be included in the count.
+            ''',
+         'type': {'type': 'function', '_funcname': '_methGetPropCount',
+                  'args': (
+                      {'name': 'propname', 'type': 'str', 'desc': 'The property name to look up.', },
+                      {'name': 'valu', 'type': 'any', 'default': '$lib.undef',
+                       'desc': 'The value of the property to look up.', },
+                  ),
+                  'returns': {'type': 'int', 'desc': 'The count of nodes.', }}},
+
+        {'name': 'getPropArrayCount',
+         'desc': '''
+            Get the number of invidivual array property values in the View for the given array property name.
+
+            Notes:
+               This is a fast approximate count calculated by summing the number of
+               array property values in each layer of the view. Property values
+               which are overwritten by different values in higher layers will
+               still be included in the count.
+            ''',
+         'type': {'type': 'function', '_funcname': '_methGetPropArrayCount',
+                  'args': (
+                      {'name': 'propname', 'type': 'str', 'desc': 'The property name to look up.', },
+                      {'name': 'valu', 'type': 'any', 'default': '$lib.undef',
+                       'desc': 'The value in the array property to look up.', },
+                  ),
+                  'returns': {'type': 'int', 'desc': 'The count of nodes.', }}},
+
+        {'name': 'getTagPropCount',
+         'desc': '''
+            Get the number of nodes in the View with the given tag property and optional value.
+
+            Notes:
+               This is a fast approximate count calculated by summing the number of
+               nodes with the tag property value in each layer of the view.
+               Values which are overwritten by different values in higher layers
+               will still be included in the count.
+            ''',
+         'type': {'type': 'function', '_funcname': '_methGetTagPropCount',
+                  'args': (
+                      {'name': 'tag', 'type': 'str', 'desc': 'The tag to look up.', },
+                      {'name': 'propname', 'type': 'str', 'desc': 'The property name to look up.', },
+                      {'name': 'form', 'type': 'str', 'default': None,
+                       'desc': 'The optional form to look up.', },
+                      {'name': 'valu', 'type': 'any', 'default': '$lib.undef',
+                       'desc': 'The value of the property to look up.', },
+                  ),
+                  'returns': {'type': 'int', 'desc': 'The count of nodes.', }}},
+
         {'name': 'detach', 'desc': 'Detach the view from its parent. WARNING: This cannot be reversed.',
          'type': {'type': 'function', '_funcname': 'detach',
                   'args': (),
@@ -7059,6 +7214,9 @@ class View(Prim):
             'addNodeEdits': self._methAddNodeEdits,
             'getEdgeVerbs': self._methGetEdgeVerbs,
             'getFormCounts': self._methGetFormcount,
+            'getPropCount': self._methGetPropCount,
+            'getTagPropCount': self._methGetTagPropCount,
+            'getPropArrayCount': self._methGetPropArrayCount,
         }
 
     async def addNode(self, form, valu, props=None):
@@ -7112,6 +7270,53 @@ class View(Prim):
         todo = s_common.todo('getFormCounts')
         return await self.viewDynCall(todo, ('view', 'read'))
 
+    @stormfunc(readonly=True)
+    async def _methGetPropCount(self, propname, valu=undef):
+        propname = await tostr(propname)
+
+        if valu is undef:
+            valu = s_common.novalu
+        else:
+            valu = await toprim(valu)
+
+        viewiden = self.valu.get('iden')
+        self.runt.confirm(('view', 'read'), gateiden=viewiden)
+        view = self.runt.snap.core.getView(viewiden)
+
+        return await view.getPropCount(propname, valu=valu)
+
+    @stormfunc(readonly=True)
+    async def _methGetTagPropCount(self, tag, propname, form=None, valu=undef):
+        tag = await tostr(tag)
+        propname = await tostr(propname)
+        form = await tostr(form, noneok=True)
+
+        if valu is undef:
+            valu = s_common.novalu
+        else:
+            valu = await toprim(valu)
+
+        viewiden = self.valu.get('iden')
+        self.runt.confirm(('view', 'read'), gateiden=viewiden)
+        view = self.runt.snap.core.getView(viewiden)
+
+        return await view.getTagPropCount(form, tag, propname, valu=valu)
+
+    @stormfunc(readonly=True)
+    async def _methGetPropArrayCount(self, propname, valu=undef):
+        propname = await tostr(propname)
+
+        if valu is undef:
+            valu = s_common.novalu
+        else:
+            valu = await toprim(valu)
+
+        viewiden = self.valu.get('iden')
+        self.runt.confirm(('view', 'read'), gateiden=viewiden)
+        view = self.runt.snap.core.getView(viewiden)
+
+        return await view.getPropArrayCount(propname, valu=valu)
+
     @stormfunc(readonly=True)
     async def _methGetEdges(self, verb=None):
         verb = await toprim(verb)
@@ -7155,8 +7360,14 @@ class View(Prim):
             else:
                 valu = await tostr(await toprim(valu), noneok=True)
 
+            if name == 'parent' and valu is not None:
+                self.runt.snap.core.reqView(valu, mesg='The parent view must already exist.')
+                self.runt.confirm(('view', 'read'), gateiden=valu)
+                self.runt.confirm(('view', 'fork'), gateiden=valu)
+
         elif name == 'nomerge':
             valu = await tobool(valu)
+
         elif name == 'layers':
 
             view = self._reqView()
@@ -7185,8 +7396,11 @@ class View(Prim):
             mesg = f'View does not support setting: {name}'
             raise s_exc.BadOptValu(mesg=mesg)
 
-        todo = s_common.todo('setViewInfo', name, valu)
-        valu = await self.viewDynCall(todo, ('view', 'set', name))
+        view = self.runt.snap.core.reqView(self.valu.get('iden'))
+
+        self.runt.confirm(('view', 'set', name), gateiden=view.iden)
+        await view.setViewInfo(name, valu)
+
         self.valu[name] = valu
 
     @stormfunc(readonly=True)
@@ -7217,10 +7431,9 @@ class View(Prim):
         useriden = self.runt.user.iden
         viewiden = self.valu.get('iden')
 
-        gatekeys = (
-            (useriden, ('view', 'add'), None),
-            (useriden, ('view', 'read'), viewiden),
-        )
+        self.runt.confirm(('view', 'add'))
+        self.runt.confirm(('view', 'read'), gateiden=viewiden)
+        self.runt.confirm(('view', 'fork'), gateiden=viewiden)
 
         ldef = {'creator': self.runt.user.iden}
         vdef = {'creator': self.runt.user.iden}
@@ -7228,9 +7441,9 @@ class View(Prim):
         if name is not None:
             vdef['name'] = name
 
-        todo = s_common.todo('fork', ldef=ldef, vdef=vdef)
+        view = self.runt.snap.core.reqView(viewiden)
 
-        newv = await self.runt.dyncall(viewiden, todo, gatekeys=gatekeys)
+        newv = await view.fork(ldef=ldef, vdef=vdef)
 
         return View(self.runt, newv, path=self.path)
 

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 155, 0)
+version = (2, 156, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'e20efe6ec519c7379f76589d3cdf782549337879'
+commit = '918a46d769574c99d1510654c1c2aaa13c3fda2c'

synapse/lib/view.py

@@ -398,6 +398,95 @@ class View(s_nexus.Pusher):  # type: ignore
                 counts[name] += valu
         return counts
 
+    async def getPropCount(self, propname, valu=s_common.novalu):
+        prop = self.core.model.prop(propname)
+        if prop is None:
+            mesg = f'No property named {propname}'
+            raise s_exc.NoSuchProp(mesg=mesg)
+
+        count = 0
+        formname = None
+        propname = None
+
+        if prop.isform:
+            formname = prop.name
+        else:
+            propname = prop.name
+            if not prop.isuniv:
+                formname = prop.form.name
+
+        if valu is s_common.novalu:
+            for layr in self.layers:
+                await asyncio.sleep(0)
+                count += await layr.getPropCount(formname, propname)
+            return count
+
+        norm, info = prop.type.norm(valu)
+
+        for layr in self.layers:
+            await asyncio.sleep(0)
+            count += layr.getPropValuCount(formname, propname, prop.type.stortype, norm)
+
+        return count
+
+    async def getTagPropCount(self, form, tag, propname, valu=s_common.novalu):
+        prop = self.core.model.getTagProp(propname)
+        if prop is None:
+            mesg = f'No tag property named {propname}'
+            raise s_exc.NoSuchTagProp(name=propname, mesg=mesg)
+
+        count = 0
+
+        if valu is s_common.novalu:
+            for layr in self.layers:
+                await asyncio.sleep(0)
+                count += await layr.getTagPropCount(form, tag, prop.name)
+            return count
+
+        norm, info = prop.type.norm(valu)
+
+        for layr in self.layers:
+            await asyncio.sleep(0)
+            count += layr.getTagPropValuCount(form, tag, prop.name, prop.type.stortype, norm)
+
+        return count
+
+    async def getPropArrayCount(self, propname, valu=s_common.novalu):
+        prop = self.core.model.prop(propname)
+        if prop is None:
+            mesg = f'No property named {propname}'
+            raise s_exc.NoSuchProp(mesg=mesg)
+
+        if not prop.type.isarray:
+            mesg = f'Property is not an array type: {prop.type.name}.'
+            raise s_exc.BadTypeValu(mesg=mesg)
+
+        count = 0
+        formname = None
+        propname = None
+
+        if prop.isform:
+            formname = prop.name
+        else:
+            propname = prop.name
+            if not prop.isuniv:
+                formname = prop.form.name
+
+        if valu is s_common.novalu:
+            for layr in self.layers:
+                await asyncio.sleep(0)
+                count += await layr.getPropArrayCount(formname, propname)
+            return count
+
+        atyp = prop.type.arraytype
+        norm, info = atyp.norm(valu)
+
+        for layr in self.layers:
+            await asyncio.sleep(0)
+            count += layr.getPropArrayValuCount(formname, propname, atyp.stortype, norm)
+
+        return count
+
     async def getEdgeVerbs(self):
 
         async with await s_spooled.Set.anit(dirn=self.core.dirn, cell=self.core) as vset:
@@ -531,11 +620,8 @@ class View(s_nexus.Pusher):  # type: ignore
 
         mode = opts.get('mode', 'storm')
         editformat = opts.get('editformat', 'nodeedits')
-        if editformat not in ('nodeedits', 'splices', 'count', 'none'):
-            raise s_exc.BadConfValu(mesg='editformat')
-
-        if editformat == 'splices':
-            s_common.deprdate('storm option editformat=splices', s_common._splicedepr)
+        if editformat not in ('nodeedits', 'count', 'none'):
+            raise s_exc.BadConfValu(mesg=f'invalid edit format, got {editformat}', name='editformat', valu=editformat)
 
         texthash = hashlib.md5(text.encode(errors='surrogatepass'), usedforsecurity=False).hexdigest()
 
@@ -629,18 +715,11 @@ class View(s_nexus.Pusher):  # type: ignore
                 if editformat == 'none':
                     continue
 
-                if editformat == 'count':
-                    count = sum(len(edit[2]) for edit in mesg[1].get('edits', ()))
-                    mesg = ('node:edits:count', {'count': count})
-                    yield mesg
-                    continue
-
-                assert editformat == 'splices'
+                assert editformat == 'count'
 
-                nodeedits = mesg[1].get('edits', [()])
-                async for _, splice in self.layers[0].makeSplices(0, nodeedits, None):
-                    if not show or splice[0] in show:
-                        yield splice
+                count = sum(len(edit[2]) for edit in mesg[1].get('edits', ()))
+                mesg = ('node:edits:count', {'count': count})
+                yield mesg
                 continue
 
             if kind == 'fini':
@@ -690,11 +769,7 @@ class View(s_nexus.Pusher):  # type: ignore
 
         if name == 'parent':
 
-            parent = self.core.getView(valu)
-            if parent is None:
-                mesg = 'The parent view must already exist.'
-                raise s_exc.NoSuchView(mesg=mesg)
-
+            parent = self.core.reqView(valu, mesg='The parent view must already exist.')
             if parent.iden == self.iden:
                 mesg = 'A view may not have parent set to itself.'
                 raise s_exc.BadArg(mesg=mesg)

synapse/models/inet.py

@@ -1,4 +1,5 @@
 import socket
+import asyncio
 import hashlib
 import logging
 import ipaddress
@@ -7,6 +8,7 @@ import urllib.parse
 
 import idna
 import regex
+import collections
 import unicodedata
 
 import synapse.exc as s_exc
@@ -479,10 +481,20 @@ class IPv4(s_types.Type):
         return minv, maxv
 
     def getCidrRange(self, text):
-        addr, mask = text.split('/', 1)
+        addr, mask_str = text.split('/', 1)
         norm, info = self.norm(addr)
 
-        mask = cidrmasks[int(mask)]
+        try:
+            mask_int = int(mask_str)
+        except ValueError:
+            raise s_exc.BadTypeValu(valu=text, name=self.name,
+                                    mesg=f'Invalid CIDR Mask "{text}"')
+
+        if mask_int > 32 or mask_int < 0:
+            raise s_exc.BadTypeValu(valu=text, name=self.name,
+                                    mesg=f'Invalid CIDR Mask "{text}"')
+
+        mask = cidrmasks[mask_int]
 
         minv = norm & mask[0]
         return minv, minv + mask[1]
@@ -1008,39 +1020,50 @@ class InetModule(s_module.CoreModule):
         fqdn = node.ndef[1]
         domain = node.get('domain')
 
-        if domain is None:
-            await node.set('iszone', False)
-            await node.set('issuffix', True)
-            return
+        async with node.snap.getEditor() as editor:
+            protonode = editor.loadNode(node)
+            if domain is None:
+                await protonode.set('iszone', False)
+                await protonode.set('issuffix', True)
+                return
 
-        if node.get('issuffix') is None:
-            await node.set('issuffix', False)
+            if protonode.get('issuffix') is None:
+                await protonode.set('issuffix', False)
 
-        # almost certainly in the cache anyway....
-        parent = await node.snap.addNode('inet:fqdn', domain)
+            parent = await node.snap.getNodeByNdef(('inet:fqdn', domain))
+            if parent is None:
+                parent = await editor.addNode('inet:fqdn', domain)
 
-        if parent.get('issuffix'):
-            await node.set('iszone', True)
-            await node.set('zone', fqdn)
-            return
+            if parent.get('issuffix'):
+                await protonode.set('iszone', True)
+                await protonode.set('zone', fqdn)
+                return
 
-        await node.set('iszone', False)
+            await protonode.set('iszone', False)
 
-        if parent.get('iszone'):
-            await node.set('zone', domain)
-            return
+            if parent.get('iszone'):
+                await protonode.set('zone', domain)
+                return
 
-        zone = parent.get('zone')
-        if zone is not None:
-            await node.set('zone', zone)
+            zone = parent.get('zone')
+            if zone is not None:
+                await protonode.set('zone', zone)
 
     async def _onSetFqdnIsSuffix(self, node, oldv):
 
         fqdn = node.ndef[1]
 
         issuffix = node.get('issuffix')
-        async for child in node.snap.nodesByPropValu('inet:fqdn:domain', '=', fqdn):
-            await child.set('iszone', issuffix)
+
+        async with node.snap.getEditor() as editor:
+            async for child in node.snap.nodesByPropValu('inet:fqdn:domain', '=', fqdn):
+                await asyncio.sleep(0)
+
+                if child.get('iszone') == issuffix:
+                    continue
+
+                protonode = editor.loadNode(child)
+                await protonode.set('iszone', issuffix)
 
     async def _onSetFqdnIsZone(self, node, oldv):
 
@@ -1069,17 +1092,24 @@ class InetModule(s_module.CoreModule):
 
     async def _onSetFqdnZone(self, node, oldv):
 
-        fqdn = node.ndef[1]
+        todo = collections.deque([node.ndef[1]])
         zone = node.get('zone')
 
-        async for child in node.snap.nodesByPropValu('inet:fqdn:domain', '=', fqdn):
+        async with node.snap.getEditor() as editor:
+            while todo:
+                fqdn = todo.pop()
+                async for child in node.snap.nodesByPropValu('inet:fqdn:domain', '=', fqdn):
+                    await asyncio.sleep(0)
+
+                    # if they are their own zone level, skip
+                    if child.get('iszone') or child.get('zone') == zone:
+                        continue
 
-            # if they are their own zone level, skip
-            if child.get('iszone'):
-                continue
+                    # the have the same zone we do
+                    protonode = editor.loadNode(child)
+                    await protonode.set('zone', zone)
 
-            # the have the same zone we do
-            await child.set('zone', zone)
+                    todo.append(child.ndef[1])
 
     def getModelDefs(self):
         return (