synapse 2.218.1__py311-none-any.whl → 2.220.0__py311-none-any.whl

synapse/lib/stormtypes.py

@@ -2,7 +2,6 @@ import bz2
 import copy
 import gzip
 import time
-
 import regex
 import types
 import base64
@@ -10,6 +9,7 @@ import pprint
 import struct
 import asyncio
 import decimal
+import hashlib
 import inspect
 import logging
 import binascii
@@ -730,6 +730,18 @@ class LibPkg(Lib):
                       {'name': 'pkgdef', 'type': 'dict', 'desc': 'A Storm Package definition.', },
                   ),
                   'returns': {'type': 'dict', 'desc': 'A dictionary listing dependencies and if they are met.', }}},
+        {'name': 'vars',
+         'desc': "Get a dictionary representing the package's persistent variables.",
+         'type': {'type': 'function', '_funcname': '_libPkgVars',
+                  'args': (
+                      {'name': 'name', 'type': 'str',
+                       'desc': 'A Storm Package name to get vars for.', },
+                  ),
+                  'returns': {'type': 'pkg:vars', 'desc': 'A dictionary representing the package variables.', }}},
+    )
+    _storm_lib_perms = (
+        {'perm': ('power-ups', '<name>', 'admin'), 'gate': 'cortex',
+         'desc': 'Controls the ability to interact with the vars for a Storm Package by name.'},
     )
     _storm_lib_path = ('pkg',)
 
@@ -741,6 +753,7 @@ class LibPkg(Lib):
             'del': self._libPkgDel,
             'list': self._libPkgList,
             'deps': self._libPkgDeps,
+            'vars': self._libPkgVars,
         }
 
     async def _libPkgAdd(self, pkgdef, verify=False):
@@ -780,6 +793,11 @@ class LibPkg(Lib):
         pkgdef = await toprim(pkgdef)
         return await self.runt.snap.core.verifyStormPkgDeps(pkgdef)
 
+    async def _libPkgVars(self, name):
+        name = await tostr(name)
+        confirm(('power-ups', name, 'admin'))
+        return PkgVars(self.runt, name)
+
 @registry.registerLib
 class LibDmon(Lib):
     '''
@@ -2745,9 +2763,13 @@ class LibAxon(Lib):
 
         self.runt.confirm(('axon', 'upload'))
 
+        sha256 = hashlib.sha256(byts).digest()
+
         await self.runt.snap.core.getAxon()
-        size, sha256 = await self.runt.snap.core.axon.put(byts)
+        if await self.runt.snap.core.axon.has(sha256):
+            return (len(byts), s_common.ehex(sha256))
 
+        size, sha256 = await self.runt.snap.core.axon.put(byts)
         return (size, s_common.ehex(sha256))
 
     @stormfunc(readonly=True)
@@ -4945,6 +4967,24 @@ class Bytes(Prim):
                       {'name': 'offset', 'type': 'int', 'desc': 'An offset to begin unpacking from.', 'default': 0},
                   ),
                   'returns': {'type': 'list', 'desc': 'The unpacked primitive values.', }}},
+        {'name': 'xor', 'desc': '''
+            Perform an "exclusive or" bitwise operation on the bytes and another set of bytes.
+
+            Notes:
+                The key bytes provided as an argument will be repeated as needed until all bytes have been
+                xor'd.
+
+                If a string is provided as the key argument, it will be utf8 encoded before being xor'd.
+
+            Examples:
+                Perform an xor operation on the bytes in $encoded using the bytes in $key::
+
+                    $decoded = $encoded.xor($key)''',
+         'type': {'type': 'function', '_funcname': '_methXor',
+                  'args': (
+                      {'name': 'key', 'type': ['str', 'bytes'], 'desc': 'The key bytes to perform the xor operation with.'},
+                  ),
+                  'returns': {'type': 'bytes', 'desc': "The xor'd bytes."}}},
     )
     _storm_typename = 'bytes'
     _ismutable = False
@@ -4955,6 +4995,7 @@ class Bytes(Prim):
 
     def getObjLocals(self):
         return {
+            'xor': self._methXor,
             'decode': self._methDecode,
             'bunzip': self._methBunzip,
             'gunzip': self._methGunzip,
@@ -5043,6 +5084,26 @@ class Bytes(Prim):
         except UnicodeDecodeError as e:
             raise s_exc.StormRuntimeError(mesg=f'{e}: {s_common.trimText(repr(valu))}') from None
 
+    @stormfunc(readonly=True)
+    async def _methXor(self, key):
+        key = await toprim(key)
+        if isinstance(key, str):
+            key = key.encode()
+
+        if not isinstance(key, bytes):
+            raise s_exc.BadArg(mesg='$bytes.xor() key argument must be bytes or a str.')
+
+        if len(key) == 0:
+            raise s_exc.BadArg(mesg='$bytes.xor() key length must be greater than 0.')
+
+        arry = bytearray(self.valu)
+        keylen = len(key)
+
+        for i in range(len(arry)):
+            arry[i] ^= key[i % keylen]
+
+        return bytes(arry)
+
 @registry.registerType
 class Dict(Prim):
     '''
@@ -6015,6 +6076,45 @@ class LibVars(Lib):
     async def _libVarsType(self, valu):
         return await totype(valu)
 
+@registry.registerType
+class PkgVars(Prim):
+    '''
+    The Storm deref/setitem/iter convention on top of pkg vars information.
+    '''
+    _storm_typename = 'pkg:vars'
+    _ismutable = True
+
+    def __init__(self, runt, valu, path=None):
+        Prim.__init__(self, valu, path=path)
+        self.runt = runt
+
+    def _reqPkgAdmin(self):
+        confirm(('power-ups', self.valu, 'admin'))
+
+    @stormfunc(readonly=True)
+    async def deref(self, name):
+        self._reqPkgAdmin()
+        name = await tostr(name)
+        return await self.runt.snap.core.getStormPkgVar(self.valu, name)
+
+    async def setitem(self, name, valu):
+        self._reqPkgAdmin()
+        name = await tostr(name)
+
+        if valu is undef:
+            await self.runt.snap.core.popStormPkgVar(self.valu, name)
+            return
+
+        valu = await toprim(valu)
+        await self.runt.snap.core.setStormPkgVar(self.valu, name, valu)
+
+    @stormfunc(readonly=True)
+    async def iter(self):
+        self._reqPkgAdmin()
+        async for name, valu in self.runt.snap.core.iterStormPkgVars(self.valu):
+            yield name, valu
+            await asyncio.sleep(0)
+
 @registry.registerType
 class Query(Prim):
     '''
@@ -7069,6 +7169,37 @@ class Layer(Prim):
                        'desc': 'The name of the form to get storage nodes for.'},
                    ),
                   'returns': {'name': 'Yields', 'type': 'list', 'desc': 'Tuple of buid, sode values.', }}},
+        {'name': 'getStorNodesByProp', 'desc': '''
+            Get buid, sode tuples representing the data stored in the layer for a given property.
+            Notes:
+                The storage nodes represent **only** the data stored in the layer
+                and may not represent whole nodes.
+            ''',
+         'type': {'type': 'function', '_funcname': 'getStorNodesByProp',
+                  'args': (
+                      {'name': 'propname', 'type': 'str', 'desc': 'The full property name to lift by.'},
+                      {'name': 'propvalu', 'type': 'obj', 'desc': 'The value for the property.', 'default': None},
+                      {'name': 'propcmpr', 'type': 'str', 'desc': 'The comparison operation to use on the value.',
+                       'default': '='},
+                  ),
+                  'returns': {'name': 'Yields', 'type': 'list', 'desc': 'Tuple of buid, sode values.', }}},
+        {'name': 'setStorNodeProp',
+         'desc': 'Set a property on a node in this layer.',
+         'type': {'type': 'function', '_funcname': 'setStorNodeProp',
+                  'args': (
+                      {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node iden.'},
+                      {'name': 'prop', 'type': 'str', 'desc': 'The property name to set.'},
+                      {'name': 'valu', 'type': 'any', 'desc': 'The value to set.'},
+                  ),
+                  'returns': {'type': 'boolean', 'desc': 'Returns true if edits were made.'}}},
+        {'name': 'delStorNodeProp',
+         'desc': 'Delete a property from a node in this layer.',
+         'type': {'type': 'function', '_funcname': 'delStorNodeProp',
+                  'args': (
+                      {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node iden.'},
+                      {'name': 'prop', 'type': 'str', 'desc': 'The property name to delete.'},
+                  ),
+                  'returns': {'type': 'boolean', 'desc': 'Returns true if edits were made.'}}},
         {'name': 'getMirrorStatus', 'desc': '''
             Return a dictionary of the mirror synchronization status for the layer.
             ''',
@@ -7287,10 +7418,13 @@ class Layer(Prim):
             'getStorNode': self.getStorNode,
             'getStorNodes': self.getStorNodes,
             'getStorNodesByForm': self.getStorNodesByForm,
+            'getStorNodesByProp': self.getStorNodesByProp,
             'getEdgesByN1': self.getEdgesByN1,
             'getEdgesByN2': self.getEdgesByN2,
             'getNodeData': self.getNodeData,
             'getMirrorStatus': self.getMirrorStatus,
+            'setStorNodeProp': self.setStorNodeProp,
+            'delStorNodeProp': self.delStorNodeProp,
         }
 
     @stormfunc(readonly=True)
@@ -7364,6 +7498,25 @@ class Layer(Prim):
         layr = self.runt.snap.core.getLayer(iden)
         return await layr.getMirrorStatus()
 
+    async def setStorNodeProp(self, nodeid, prop, valu):
+        iden = self.valu.get('iden')
+        layr = self.runt.snap.core.getLayer(iden)
+        buid = s_common.uhex(await tostr(nodeid))
+        prop = await tostr(prop)
+        valu = await tostor(valu)
+        self.runt.reqAdmin(mesg='setStorNodeProp() requires admin privileges.')
+        meta = {'time': s_common.now(), 'user': self.runt.user.iden}
+        return await layr.setStorNodeProp(buid, prop, valu, meta=meta)
+
+    async def delStorNodeProp(self, nodeid, prop):
+        iden = self.valu.get('iden')
+        layr = self.runt.snap.core.getLayer(iden)
+        buid = s_common.uhex(await tostr(nodeid))
+        prop = await tostr(prop)
+        self.runt.reqAdmin(mesg='delStorNodeProp() requires admin privileges.')
+        meta = {'time': s_common.now(), 'user': self.runt.user.iden}
+        return await layr.delStorNodeProp(buid, prop, meta=meta)
+
     async def _addPull(self, url, offs=0, queue_size=s_const.layer_pdef_qsize, chunk_size=s_const.layer_pdef_csize):
         url = await tostr(url)
         offs = await toint(offs)
@@ -7646,6 +7799,28 @@ class Layer(Prim):
         async for item in layr.getStorNodesByForm(form):
             yield item
 
+    @stormfunc(readonly=True)
+    async def getStorNodesByProp(self, propname, propvalu=None, propcmpr='='):
+        propname = await tostr(propname)
+        propvalu = await tostor(propvalu)
+        propcmpr = await tostr(propcmpr)
+
+        layriden = self.valu.get('iden')
+        await self.runt.reqUserCanReadLayer(layriden)
+        layr = self.runt.snap.core.getLayer(layriden)
+
+        prop = self.runt.snap.core.model.reqProp(propname)
+
+        if propvalu is not None:
+            norm, info = prop.type.norm(propvalu)
+            cmprvals = prop.type.getStorCmprs(propcmpr, norm)
+            async for _, buid, sode in layr.liftByPropValu(prop.form.name, prop.name, cmprvals):
+                yield (s_common.ehex(buid), sode)
+            return
+
+        async for _, buid, sode in layr.liftByProp(prop.form.name, prop.name):
+            yield (s_common.ehex(buid), sode)
+
     @stormfunc(readonly=True)
     async def getEdges(self):
         layriden = self.valu.get('iden')

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 218, 1)
+version = (2, 220, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'f91b8902226a2b8d984804f164c4c65e8b7e5ce3'
+commit = '913156d6c195af916a32860341f285954cb4e882'

synapse/models/inet.py

@@ -3796,6 +3796,9 @@ class InetModule(s_module.CoreModule):
                         ('user', ('inet:user', {}), {
                             'doc': 'The current user name of the account.'}),
 
+                        ('parent', ('inet:service:account', {}), {
+                            'doc': 'A parent account which owns this account.'}),
+
                         ('email', ('inet:email', {}), {
                             'doc': 'The current email address associated with the account.'}),
 

synapse/tests/files/stormpkg/badinits.yaml

@@ -0,0 +1,12 @@
+name: badinits
+version: 0.0.1
+
+inits:
+  name: badinits:version
+  versions:
+    - name: first
+      version: 7
+      query: ""
+    - name: second
+      version: 2
+      query: ""

synapse/tests/files/stormpkg/testpkg.yaml

@@ -45,6 +45,18 @@ onload: |
   $lib.time.sleep($lib.globals.get(onload_sleep, 0))
   $lib.globals.set(testpkg, testpkg-done)
 
+inits:
+  key: testpkg:version
+  versions:
+    - name: first
+      version: 0
+      inaugural: true
+      query: "$lib.globals.set(testpkg-first, (true))"
+    - name: second
+      desc: Another init
+      version: 2
+      query: "$lib.globals.set(testpkg-second, (true))"
+
 external_modules:
     - name: testext
       package: synapse.tests.files

synapse/tests/test_lib_grammar.py

@@ -1,9 +1,9 @@
 import lark  # type: ignore
 
 import synapse.exc as s_exc
+import synapse.data as s_data
 
 import synapse.lib.parser as s_parser
-import synapse.lib.datfile as s_datfile
 import synapse.lib.grammar as s_grammar
 
 import synapse.tests.utils as s_t_utils
@@ -1414,9 +1414,7 @@ class GrammarTest(s_t_utils.SynTest):
         '''
         Validates that we have no grammar ambiguities
         '''
-        with s_datfile.openDatFile('synapse.lib/storm.lark') as larkf:
-            grammar = larkf.read().decode()
-
+        grammar = s_data.getLark('storm')
         parser = lark.Lark(grammar, start='query', debug=True, regex=True, parser='lalr',
                            keep_all_tokens=True, maybe_placeholders=False,
                            propagate_positions=True)

synapse/tests/test_lib_json.py

@@ -125,6 +125,35 @@ class JsonTest(s_test.SynTest):
 
         self.eq(valu, s_json.loads(s_json.dumps(valu)))
 
+    async def test_lib_json_control_strings(self):
+        valus = [
+            'line1"line2',
+            'line1/line2',
+            'line1\\line2',
+            'line1\bline2',
+            'line1\fline2',
+            'line1\nline2',
+            'line1\rline2',
+            'line1\tline2',
+            'line1\u0009line2',
+            'line1\u1000line2',
+            'line1\u2000line2',
+            'line1\u3000line2',
+        ]
+
+        with self.getLoggerStream('synapse.lib.json') as stream:
+            async with self.getTestCore() as core:
+                for valu in valus:
+                    q = '$lib.print($valu) $lib.print($lib.json.save($valu))'
+                    msgs = await core.stormlist(q, opts={'vars': {'valu': valu}})
+                    self.stormHasNoWarnErr(msgs)
+
+                    self.eq(s_json.loads(s_json.dumps(valu)), valu)
+
+        stream.seek(0)
+        data = stream.read()
+        self.notin('fallback JSON', data)
+
     async def test_jsload(self):
         with self.getTestDir() as dirn:
             with s_common.genfile(dirn, 'jsload.json') as fp:

synapse/tests/test_lib_layer.py

@@ -2283,3 +2283,122 @@ class LayerTest(s_t_utils.SynTest):
 
             sodes = await s_t_utils.alist(layr00.getStorNodesByForm('inet:ipv4'))
             self.len(0, sodes)
+
+    async def test_layer_migrate_props_fork(self):
+
+        async with self.getTestCore() as core:
+
+            iden = (await core.addUser('lowuser')).get('iden')
+            lowuser = {'user': iden}
+
+            fork00 = await core.view.fork()
+            layr00 = core.getLayer(fork00['layers'][0]['iden'])
+
+            await core.nodes('''
+                for $prop in (_custom:risk:level, _custom:risk:severity) {
+                    $lib.model.ext.addFormProp(
+                        test:guid,
+                        $prop,
+                        (["int", {"enums": [[10, "low"], [20, "medium"], [30, "high"]]}]),
+                        ({"doc": "hey now"}),
+                    )
+                }
+
+            ''')
+            self.len(1, await core.nodes('syn:prop=test:guid:_custom:risk:level'))
+            self.len(1, await core.nodes('syn:prop=test:guid:_custom:risk:severity'))
+
+            await core.nodes('[ test:guid=* :name=test1 :_custom:risk:level=low ]', opts={'view': fork00['iden']})
+
+            await core.getView(fork00['iden']).delete()
+
+            with self.raises(s_exc.CantDelProp) as cm:
+                await core.callStorm('''
+                    $fullprop = "test:guid:_custom:risk:level"
+                    for $view in $lib.view.list(deporder=$lib.true) {
+                        view.exec $view.iden {
+                            yield $lib.layer.get().liftByProp($fullprop)
+                            $repr = $node.repr("_custom:risk:level")
+                            [ :severity=$repr -:_custom:risk:level ]
+                        }
+                    }
+                    $lib.model.ext.delFormProp("test:guid", "_custom:risk:level")
+                ''')
+            self.isin('Nodes still exist with prop: test:guid:_custom:risk:level', str(cm.exception))
+            self.len(1, await core.nodes('syn:prop=test:guid:_custom:risk:level'))
+
+            with self.raises(s_exc.NoSuchProp) as cm:
+                await core.callStorm('''
+                    $layer = $lib.layer.get()
+                    $layer.getStorNodesByProp("foo:bar:_custom:risk:level")
+                ''')
+            self.isin('No property named', str(cm.exception))
+
+            with self.raises(s_exc.NoSuchProp):
+                await core.callStorm('''
+                    $fullprop = "test:guid:_custom:risk:level"
+                    for $layer in $lib.layer.list() {
+                        for ($buid, $sode) in $layer.getStorNodesByProp($fullprop) {
+                            $oldv = $sode.props."_custom:risk:level"
+                            $layer.setStorNodeProp($buid, "foo:bar:severity", $oldv.0)
+                        }
+                    }
+                ''')
+
+            with self.raises(s_exc.BadTypeValu):
+                await core.callStorm('''
+                    $fullprop = "test:guid:_custom:risk:level"
+                    for $layer in $lib.layer.list() {
+                        for ($buid, $sode) in $layer.getStorNodesByProp($fullprop) {
+                            $layer.setStorNodeProp($buid, $fullprop, "newp")
+                        }
+                    }
+                ''')
+
+            with self.raises(s_exc.NoSuchProp):
+                await core.callStorm('''
+                    $fullprop = "test:guid:_custom:risk:level"
+                    for $layer in $lib.layer.list() {
+                        for ($buid, $sode) in $layer.getStorNodesByProp($fullprop) {
+                            $layer.delStorNodeProp($buid, "foo:bar:severity")
+                        }
+                    }
+                ''')
+
+            with self.raises(s_exc.AuthDeny) as cm:
+                await core.callStorm('''
+                    $buid = "8c454b27df9c0ba109c123265b50869759bccac5bbec83b41992b4e91207f4a4"
+                    $layer = $lib.layer.get()
+                    $layer.setStorNodeProp($buid, "foo:bar:severity", "newp")
+                ''', opts=lowuser)
+            self.isin('requires admin privileges', str(cm.exception))
+
+            with self.raises(s_exc.AuthDeny) as cm:
+                await core.callStorm('''
+                    $buid = "8c454b27df9c0ba109c123265b50869759bccac5bbec83b41992b4e91207f4a4"
+                    $layer = $lib.layer.get()
+                    $layer.delStorNodeProp($buid, "foo:bar:severity")
+                ''', opts=lowuser)
+            self.isin('requires admin privileges', str(cm.exception))
+
+            await core.callStorm('''
+                $fullprop = "test:guid:_custom:risk:level"
+                for $layer in $lib.layer.list() {
+                    if $layer.getPropCount($fullprop, maxsize=1) {
+                        for ($buid, $sode) in $layer.getStorNodesByProp($fullprop, (10), "=") {
+                            $oldv = $sode.props."_custom:risk:level"
+                            $layer.setStorNodeProp($buid, "test:guid:_custom:risk:severity", $oldv.0)
+                            $layer.delStorNodeProp($buid, $fullprop)
+                        }
+                    }
+                }
+                $lib.model.ext.delFormProp("test:guid", "_custom:risk:level")
+            ''')
+            self.len(0, await core.nodes('syn:prop=test:guid:_custom:risk:level'))
+            self.len(0, await core.nodes('test:guid:_custom:risk:severity'))
+
+            view00 = (await core.addView(vdef={'layers': [layr00.iden]}))['iden']
+            nodes = await core.nodes('test:guid', opts={'view': view00})
+            self.len(1, nodes)
+            self.none(nodes[0].props.get('_custom:risk:level'))
+            self.eq(nodes[0].props.get('_custom:risk:severity'), 10)