synapse 2.218.1__py311-none-any.whl → 2.219.0__py311-none-any.whl

synapse/lib/stormtypes.py

@@ -2,7 +2,6 @@ import bz2
 import copy
 import gzip
 import time
-
 import regex
 import types
 import base64
@@ -10,6 +9,7 @@ import pprint
 import struct
 import asyncio
 import decimal
+import hashlib
 import inspect
 import logging
 import binascii
@@ -730,6 +730,18 @@ class LibPkg(Lib):
                       {'name': 'pkgdef', 'type': 'dict', 'desc': 'A Storm Package definition.', },
                   ),
                   'returns': {'type': 'dict', 'desc': 'A dictionary listing dependencies and if they are met.', }}},
+        {'name': 'vars',
+         'desc': "Get a dictionary representing the package's persistent variables.",
+         'type': {'type': 'function', '_funcname': '_libPkgVars',
+                  'args': (
+                      {'name': 'name', 'type': 'str',
+                       'desc': 'A Storm Package name to get vars for.', },
+                  ),
+                  'returns': {'type': 'pkg:vars', 'desc': 'A dictionary representing the package variables.', }}},
+    )
+    _storm_lib_perms = (
+        {'perm': ('power-ups', '<name>', 'admin'), 'gate': 'cortex',
+         'desc': 'Controls the ability to interact with the vars for a Storm Package by name.'},
     )
     _storm_lib_path = ('pkg',)
 
@@ -741,6 +753,7 @@ class LibPkg(Lib):
             'del': self._libPkgDel,
             'list': self._libPkgList,
             'deps': self._libPkgDeps,
+            'vars': self._libPkgVars,
         }
 
     async def _libPkgAdd(self, pkgdef, verify=False):
@@ -780,6 +793,11 @@ class LibPkg(Lib):
         pkgdef = await toprim(pkgdef)
         return await self.runt.snap.core.verifyStormPkgDeps(pkgdef)
 
+    async def _libPkgVars(self, name):
+        name = await tostr(name)
+        confirm(('power-ups', name, 'admin'))
+        return PkgVars(self.runt, name)
+
 @registry.registerLib
 class LibDmon(Lib):
     '''
@@ -2745,9 +2763,13 @@ class LibAxon(Lib):
 
         self.runt.confirm(('axon', 'upload'))
 
+        sha256 = hashlib.sha256(byts).digest()
+
         await self.runt.snap.core.getAxon()
-        size, sha256 = await self.runt.snap.core.axon.put(byts)
+        if await self.runt.snap.core.axon.has(sha256):
+            return (len(byts), s_common.ehex(sha256))
 
+        size, sha256 = await self.runt.snap.core.axon.put(byts)
         return (size, s_common.ehex(sha256))
 
     @stormfunc(readonly=True)
@@ -6015,6 +6037,45 @@ class LibVars(Lib):
     async def _libVarsType(self, valu):
         return await totype(valu)
 
+@registry.registerType
+class PkgVars(Prim):
+    '''
+    The Storm deref/setitem/iter convention on top of pkg vars information.
+    '''
+    _storm_typename = 'pkg:vars'
+    _ismutable = True
+
+    def __init__(self, runt, valu, path=None):
+        Prim.__init__(self, valu, path=path)
+        self.runt = runt
+
+    def _reqPkgAdmin(self):
+        confirm(('power-ups', self.valu, 'admin'))
+
+    @stormfunc(readonly=True)
+    async def deref(self, name):
+        self._reqPkgAdmin()
+        name = await tostr(name)
+        return await self.runt.snap.core.getStormPkgVar(self.valu, name)
+
+    async def setitem(self, name, valu):
+        self._reqPkgAdmin()
+        name = await tostr(name)
+
+        if valu is undef:
+            await self.runt.snap.core.popStormPkgVar(self.valu, name)
+            return
+
+        valu = await toprim(valu)
+        await self.runt.snap.core.setStormPkgVar(self.valu, name, valu)
+
+    @stormfunc(readonly=True)
+    async def iter(self):
+        self._reqPkgAdmin()
+        async for name, valu in self.runt.snap.core.iterStormPkgVars(self.valu):
+            yield name, valu
+            await asyncio.sleep(0)
+
 @registry.registerType
 class Query(Prim):
     '''
@@ -7069,6 +7130,37 @@ class Layer(Prim):
                        'desc': 'The name of the form to get storage nodes for.'},
                    ),
                   'returns': {'name': 'Yields', 'type': 'list', 'desc': 'Tuple of buid, sode values.', }}},
+        {'name': 'getStorNodesByProp', 'desc': '''
+            Get buid, sode tuples representing the data stored in the layer for a given property.
+            Notes:
+                The storage nodes represent **only** the data stored in the layer
+                and may not represent whole nodes.
+            ''',
+         'type': {'type': 'function', '_funcname': 'getStorNodesByProp',
+                  'args': (
+                      {'name': 'propname', 'type': 'str', 'desc': 'The full property name to lift by.'},
+                      {'name': 'propvalu', 'type': 'obj', 'desc': 'The value for the property.', 'default': None},
+                      {'name': 'propcmpr', 'type': 'str', 'desc': 'The comparison operation to use on the value.',
+                       'default': '='},
+                  ),
+                  'returns': {'name': 'Yields', 'type': 'list', 'desc': 'Tuple of buid, sode values.', }}},
+        {'name': 'setStorNodeProp',
+         'desc': 'Set a property on a node in this layer.',
+         'type': {'type': 'function', '_funcname': 'setStorNodeProp',
+                  'args': (
+                      {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node iden.'},
+                      {'name': 'prop', 'type': 'str', 'desc': 'The property name to set.'},
+                      {'name': 'valu', 'type': 'any', 'desc': 'The value to set.'},
+                  ),
+                  'returns': {'type': 'boolean', 'desc': 'Returns true if edits were made.'}}},
+        {'name': 'delStorNodeProp',
+         'desc': 'Delete a property from a node in this layer.',
+         'type': {'type': 'function', '_funcname': 'delStorNodeProp',
+                  'args': (
+                      {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node iden.'},
+                      {'name': 'prop', 'type': 'str', 'desc': 'The property name to delete.'},
+                  ),
+                  'returns': {'type': 'boolean', 'desc': 'Returns true if edits were made.'}}},
         {'name': 'getMirrorStatus', 'desc': '''
             Return a dictionary of the mirror synchronization status for the layer.
             ''',
@@ -7287,10 +7379,13 @@ class Layer(Prim):
             'getStorNode': self.getStorNode,
             'getStorNodes': self.getStorNodes,
             'getStorNodesByForm': self.getStorNodesByForm,
+            'getStorNodesByProp': self.getStorNodesByProp,
             'getEdgesByN1': self.getEdgesByN1,
             'getEdgesByN2': self.getEdgesByN2,
             'getNodeData': self.getNodeData,
             'getMirrorStatus': self.getMirrorStatus,
+            'setStorNodeProp': self.setStorNodeProp,
+            'delStorNodeProp': self.delStorNodeProp,
         }
 
     @stormfunc(readonly=True)
@@ -7364,6 +7459,25 @@ class Layer(Prim):
         layr = self.runt.snap.core.getLayer(iden)
         return await layr.getMirrorStatus()
 
+    async def setStorNodeProp(self, nodeid, prop, valu):
+        iden = self.valu.get('iden')
+        layr = self.runt.snap.core.getLayer(iden)
+        buid = s_common.uhex(await tostr(nodeid))
+        prop = await tostr(prop)
+        valu = await tostor(valu)
+        self.runt.reqAdmin(mesg='setStorNodeProp() requires admin privileges.')
+        meta = {'time': s_common.now(), 'user': self.runt.user.iden}
+        return await layr.setStorNodeProp(buid, prop, valu, meta=meta)
+
+    async def delStorNodeProp(self, nodeid, prop):
+        iden = self.valu.get('iden')
+        layr = self.runt.snap.core.getLayer(iden)
+        buid = s_common.uhex(await tostr(nodeid))
+        prop = await tostr(prop)
+        self.runt.reqAdmin(mesg='delStorNodeProp() requires admin privileges.')
+        meta = {'time': s_common.now(), 'user': self.runt.user.iden}
+        return await layr.delStorNodeProp(buid, prop, meta=meta)
+
     async def _addPull(self, url, offs=0, queue_size=s_const.layer_pdef_qsize, chunk_size=s_const.layer_pdef_csize):
         url = await tostr(url)
         offs = await toint(offs)
@@ -7646,6 +7760,28 @@ class Layer(Prim):
         async for item in layr.getStorNodesByForm(form):
             yield item
 
+    @stormfunc(readonly=True)
+    async def getStorNodesByProp(self, propname, propvalu=None, propcmpr='='):
+        propname = await tostr(propname)
+        propvalu = await tostor(propvalu)
+        propcmpr = await tostr(propcmpr)
+
+        layriden = self.valu.get('iden')
+        await self.runt.reqUserCanReadLayer(layriden)
+        layr = self.runt.snap.core.getLayer(layriden)
+
+        prop = self.runt.snap.core.model.reqProp(propname)
+
+        if propvalu is not None:
+            norm, info = prop.type.norm(propvalu)
+            cmprvals = prop.type.getStorCmprs(propcmpr, norm)
+            async for _, buid, sode in layr.liftByPropValu(prop.form.name, prop.name, cmprvals):
+                yield (s_common.ehex(buid), sode)
+            return
+
+        async for _, buid, sode in layr.liftByProp(prop.form.name, prop.name):
+            yield (s_common.ehex(buid), sode)
+
     @stormfunc(readonly=True)
     async def getEdges(self):
         layriden = self.valu.get('iden')

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 218, 1)
+version = (2, 219, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'f91b8902226a2b8d984804f164c4c65e8b7e5ce3'
+commit = '9e442b257a7ddfa8d476eafdcad21c3a0440cc83'

synapse/models/inet.py

@@ -3796,6 +3796,9 @@ class InetModule(s_module.CoreModule):
                         ('user', ('inet:user', {}), {
                             'doc': 'The current user name of the account.'}),
 
+                        ('parent', ('inet:service:account', {}), {
+                            'doc': 'A parent account which owns this account.'}),
+
                         ('email', ('inet:email', {}), {
                             'doc': 'The current email address associated with the account.'}),
 

synapse/tests/files/stormpkg/badinits.yaml

@@ -0,0 +1,12 @@
+name: badinits
+version: 0.0.1
+
+inits:
+  name: badinits:version
+  versions:
+    - name: first
+      version: 7
+      query: ""
+    - name: second
+      version: 2
+      query: ""

synapse/tests/files/stormpkg/testpkg.yaml

@@ -45,6 +45,18 @@ onload: |
   $lib.time.sleep($lib.globals.get(onload_sleep, 0))
   $lib.globals.set(testpkg, testpkg-done)
 
+inits:
+  key: testpkg:version
+  versions:
+    - name: first
+      version: 0
+      inaugural: true
+      query: "$lib.globals.set(testpkg-first, (true))"
+    - name: second
+      desc: Another init
+      version: 2
+      query: "$lib.globals.set(testpkg-second, (true))"
+
 external_modules:
     - name: testext
       package: synapse.tests.files

synapse/tests/test_lib_layer.py

@@ -2283,3 +2283,122 @@ class LayerTest(s_t_utils.SynTest):
 
             sodes = await s_t_utils.alist(layr00.getStorNodesByForm('inet:ipv4'))
             self.len(0, sodes)
+
+    async def test_layer_migrate_props_fork(self):
+
+        async with self.getTestCore() as core:
+
+            iden = (await core.addUser('lowuser')).get('iden')
+            lowuser = {'user': iden}
+
+            fork00 = await core.view.fork()
+            layr00 = core.getLayer(fork00['layers'][0]['iden'])
+
+            await core.nodes('''
+                for $prop in (_custom:risk:level, _custom:risk:severity) {
+                    $lib.model.ext.addFormProp(
+                        test:guid,
+                        $prop,
+                        (["int", {"enums": [[10, "low"], [20, "medium"], [30, "high"]]}]),
+                        ({"doc": "hey now"}),
+                    )
+                }
+
+            ''')
+            self.len(1, await core.nodes('syn:prop=test:guid:_custom:risk:level'))
+            self.len(1, await core.nodes('syn:prop=test:guid:_custom:risk:severity'))
+
+            await core.nodes('[ test:guid=* :name=test1 :_custom:risk:level=low ]', opts={'view': fork00['iden']})
+
+            await core.getView(fork00['iden']).delete()
+
+            with self.raises(s_exc.CantDelProp) as cm:
+                await core.callStorm('''
+                    $fullprop = "test:guid:_custom:risk:level"
+                    for $view in $lib.view.list(deporder=$lib.true) {
+                        view.exec $view.iden {
+                            yield $lib.layer.get().liftByProp($fullprop)
+                            $repr = $node.repr("_custom:risk:level")
+                            [ :severity=$repr -:_custom:risk:level ]
+                        }
+                    }
+                    $lib.model.ext.delFormProp("test:guid", "_custom:risk:level")
+                ''')
+            self.isin('Nodes still exist with prop: test:guid:_custom:risk:level', str(cm.exception))
+            self.len(1, await core.nodes('syn:prop=test:guid:_custom:risk:level'))
+
+            with self.raises(s_exc.NoSuchProp) as cm:
+                await core.callStorm('''
+                    $layer = $lib.layer.get()
+                    $layer.getStorNodesByProp("foo:bar:_custom:risk:level")
+                ''')
+            self.isin('No property named', str(cm.exception))
+
+            with self.raises(s_exc.NoSuchProp):
+                await core.callStorm('''
+                    $fullprop = "test:guid:_custom:risk:level"
+                    for $layer in $lib.layer.list() {
+                        for ($buid, $sode) in $layer.getStorNodesByProp($fullprop) {
+                            $oldv = $sode.props."_custom:risk:level"
+                            $layer.setStorNodeProp($buid, "foo:bar:severity", $oldv.0)
+                        }
+                    }
+                ''')
+
+            with self.raises(s_exc.BadTypeValu):
+                await core.callStorm('''
+                    $fullprop = "test:guid:_custom:risk:level"
+                    for $layer in $lib.layer.list() {
+                        for ($buid, $sode) in $layer.getStorNodesByProp($fullprop) {
+                            $layer.setStorNodeProp($buid, $fullprop, "newp")
+                        }
+                    }
+                ''')
+
+            with self.raises(s_exc.NoSuchProp):
+                await core.callStorm('''
+                    $fullprop = "test:guid:_custom:risk:level"
+                    for $layer in $lib.layer.list() {
+                        for ($buid, $sode) in $layer.getStorNodesByProp($fullprop) {
+                            $layer.delStorNodeProp($buid, "foo:bar:severity")
+                        }
+                    }
+                ''')
+
+            with self.raises(s_exc.AuthDeny) as cm:
+                await core.callStorm('''
+                    $buid = "8c454b27df9c0ba109c123265b50869759bccac5bbec83b41992b4e91207f4a4"
+                    $layer = $lib.layer.get()
+                    $layer.setStorNodeProp($buid, "foo:bar:severity", "newp")
+                ''', opts=lowuser)
+            self.isin('requires admin privileges', str(cm.exception))
+
+            with self.raises(s_exc.AuthDeny) as cm:
+                await core.callStorm('''
+                    $buid = "8c454b27df9c0ba109c123265b50869759bccac5bbec83b41992b4e91207f4a4"
+                    $layer = $lib.layer.get()
+                    $layer.delStorNodeProp($buid, "foo:bar:severity")
+                ''', opts=lowuser)
+            self.isin('requires admin privileges', str(cm.exception))
+
+            await core.callStorm('''
+                $fullprop = "test:guid:_custom:risk:level"
+                for $layer in $lib.layer.list() {
+                    if $layer.getPropCount($fullprop, maxsize=1) {
+                        for ($buid, $sode) in $layer.getStorNodesByProp($fullprop, (10), "=") {
+                            $oldv = $sode.props."_custom:risk:level"
+                            $layer.setStorNodeProp($buid, "test:guid:_custom:risk:severity", $oldv.0)
+                            $layer.delStorNodeProp($buid, $fullprop)
+                        }
+                    }
+                }
+                $lib.model.ext.delFormProp("test:guid", "_custom:risk:level")
+            ''')
+            self.len(0, await core.nodes('syn:prop=test:guid:_custom:risk:level'))
+            self.len(0, await core.nodes('test:guid:_custom:risk:severity'))
+
+            view00 = (await core.addView(vdef={'layers': [layr00.iden]}))['iden']
+            nodes = await core.nodes('test:guid', opts={'view': view00})
+            self.len(1, nodes)
+            self.none(nodes[0].props.get('_custom:risk:level'))
+            self.eq(nodes[0].props.get('_custom:risk:severity'), 10)

synapse/tests/test_lib_storm.py

@@ -3105,6 +3105,185 @@ class StormTest(s_t_utils.SynTest):
 
             await core00.waitfini()
 
+    async def test_storm_pkg_inits(self):
+
+        async def loadPkg(core, pkg):
+            waiter = core.waiter(2, 'core:pkg:onload:start', 'core:pkg:onload:complete')
+
+            await core.addStormPkg(pkg)
+
+            events = await waiter.wait(timeout=10)
+            self.eq(events, [
+                ('core:pkg:onload:start', {'pkg': 'testload'}),
+                ('core:pkg:onload:complete', {'pkg': 'testload'}),
+            ])
+
+        with self.getTestDir() as dirn:
+
+            async with self.getTestCore(dirn=dirn) as core:
+
+                pkg = {
+                    'name': 'testload',
+                    'version': '0.1.0',
+                    'inits': {
+                        'key': 'testload:version',
+                        'versions': [
+                            {
+                                'version': 0,
+                                'name': 'init00',
+                                'query': '$lib.globals.set(init00, $lib.time.now())',
+                            },
+                            {
+                                'version': 1,
+                                'name': 'init01',
+                                'inaugural': True,
+                                'query': '$lib.globals.set(init01, $lib.time.now())',
+                            },
+                        ]
+                    },
+                }
+
+                # bad init queries fail on load
+
+                pkg['inits']['versions'].append({
+                    'version': 2,
+                    'name': 'bad',
+                    'query': '...',
+                })
+
+                await self.asyncraises(s_exc.BadSyntax, core.addStormPkg(pkg))
+
+                pkg['inits']['versions'].pop(2)
+
+                # non-increasing init versions fail on load
+
+                pkg['inits']['versions'].append({
+                    'version': 0,
+                    'name': 'bad',
+                    'query': '$lib.print("")',
+                })
+
+                await self.asyncraises(s_exc.BadPkgDef, core.addStormPkg(pkg))
+
+                pkg['inits']['versions'].pop(2)
+
+                # only inaugural inits run on first load
+
+                await loadPkg(core, pkg)
+
+                self.eq(1, await core.getStormPkgVar('testload', 'testload:version'))
+                self.none(await core.getStormVar('init00'))
+                self.nn(init01 := await core.getStormVar('init01'))
+
+                # non-inaugural inits run on reload
+                # inits always run before onload
+
+                pkg['version'] = '0.2.0'
+                pkg['onload'] = '$lib.time.sleep((0.1)) $lib.globals.set(onload, $lib.time.now())'
+                pkg['inits']['versions'].append({
+                    'version': 2,
+                    'name': 'init02',
+                    'query': '$lib.globals.set(init02, $lib.time.now())',
+                })
+
+                await loadPkg(core, pkg)
+
+                self.eq(2, await core.getStormPkgVar('testload', 'testload:version'))
+                self.none(await core.getStormVar('init00'))
+                self.eq(init01, await core.getStormVar('init01'))
+                self.nn(init02 := await core.getStormVar('init02'))
+                self.nn(onload := await core.getStormVar('onload'))
+                self.gt(onload, init02)
+
+                # inits run even if onload fails
+                # prior inits do not re-run
+
+                pkg['version'] = '0.3.0'
+                pkg['onload'] = '$lib.raise(SynErr, whoopsie)'
+                pkg['inits']['versions'].append({
+                    'version': 3,
+                    'name': 'init03',
+                    'inaugural': True,
+                    'query': '$lib.globals.set(init03, $lib.time.now())',
+                })
+
+                await loadPkg(core, pkg)
+
+                self.eq(3, await core.getStormPkgVar('testload', 'testload:version'))
+                self.eq(init02, await core.getStormVar('init02'))
+                self.nn(await core.getStormVar('init03'))
+
+                # init failure stops progression
+
+                await core.setStormVar('dofail', True)
+
+                pkg['version'] = '0.4.0'
+                pkg['onload'] = '$lib.globals.set(onload, $lib.time.now()) $lib.time.sleep((0.1))'
+                pkg['inits']['versions'].extend([
+                    {
+                        'version': 4,
+                        'name': 'init04',
+                        'query': '''
+                            if $lib.globals.get(dofail) { $lib.raise(SynErr, newp) }
+                            $lib.globals.set(init04, $lib.time.now())
+                        ''',
+                    },
+                    {
+                        'version': 6,
+                        'name': 'init06',
+                        'query': '$lib.globals.set(init06, $lib.time.now())',
+                    },
+                ])
+
+                mesg = 'testload init vers=4 output: (\'SynErr\''
+                with self.getAsyncLoggerStream('synapse.cortex', mesg) as stream:
+                    await loadPkg(core, pkg)
+                    self.eq(3, await core.getStormPkgVar('testload', 'testload:version'))
+                    await stream.wait(timeout=10)
+
+                self.none(await core.getStormVar('init04'))
+                self.none(await core.getStormVar('init06'))
+
+                await core.setStormVar('dofail', False)
+
+            with self.getAsyncLoggerStream('synapse.cortex', 'testload finished onload') as stream:
+                async with self.getTestCore(dirn=dirn) as core:
+                    await stream.wait(timeout=10)
+
+                    # prior versions dont re-run, but a failed one does
+
+                    self.eq(6, await core.getStormPkgVar('testload', 'testload:version'))
+                    self.gt(await core.getStormVar('onload'), onload)
+                    self.eq(init02, await core.getStormVar('init02'))
+                    self.nn(await core.getStormVar('init04'))
+                    self.nn(await core.getStormVar('init06'))
+
+                    # coverage for prints and warns
+
+                    pkg['version'] = '0.5.0'
+                    pkg['inits']['versions'].append({
+                        'version': 7,
+                        'name': 'init07',
+                        'query': '$lib.print("doing a print")',
+                    })
+
+                    with self.getAsyncLoggerStream('synapse.cortex', 'doing a print') as stream:
+                        await loadPkg(core, pkg)
+                        await stream.wait(timeout=10)
+                        self.eq(7, await core.getStormPkgVar('testload', 'testload:version'))
+
+                    pkg['version'] = '0.6.0'
+                    pkg['inits']['versions'].append({
+                        'version': 8,
+                        'name': 'init08',
+                        'query': '$lib.warn("doing a warn")',
+                    })
+
+                    with self.getAsyncLoggerStream('synapse.cortex', 'doing a warn') as stream:
+                        await loadPkg(core, pkg)
+                        await stream.wait(timeout=10)
+                        self.eq(8, await core.getStormPkgVar('testload', 'testload:version'))
+
     async def test_storm_tree(self):
 
         async with self.getTestCore() as core: