synapse 2.222.0__py311-none-any.whl → 2.223.0__py311-none-any.whl

synapse/cortex.py

@@ -61,6 +61,7 @@ import synapse.lib.stormlib.gen as s_stormlib_gen  # NOQA
 import synapse.lib.stormlib.gis as s_stormlib_gis  # NOQA
 import synapse.lib.stormlib.hex as s_stormlib_hex  # NOQA
 import synapse.lib.stormlib.log as s_stormlib_log  # NOQA
+import synapse.lib.stormlib.pkg as s_stormlib_pkg  # NOQA
 import synapse.lib.stormlib.xml as s_stormlib_xml  # NOQA
 import synapse.lib.stormlib.auth as s_stormlib_auth  # NOQA
 import synapse.lib.stormlib.cell as s_stormlib_cell  # NOQA
@@ -72,6 +73,7 @@ import synapse.lib.stormlib.mime as s_stormlib_mime  # NOQA
 import synapse.lib.stormlib.pack as s_stormlib_pack  # NOQA
 import synapse.lib.stormlib.smtp as s_stormlib_smtp  # NOQA
 import synapse.lib.stormlib.stix as s_stormlib_stix  # NOQA
+import synapse.lib.stormlib.task as s_stormlib_task  # NOQA
 import synapse.lib.stormlib.yaml as s_stormlib_yaml  # NOQA
 import synapse.lib.stormlib.basex as s_stormlib_basex  # NOQA
 import synapse.lib.stormlib.cache as s_stormlib_cache  # NOQA
@@ -2262,6 +2264,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         self.onfini(slab.fini)
 
         self.multiqueue = await slab.getMultiQueue('cortex:queue', nexsroot=self.nexsroot)
+        self.stormpkgqueue = await slab.getMultiQueue('storm:pkg:queue', nexsroot=self.nexsroot)
 
     async def _initStormGraphs(self):
         path = os.path.join(self.dirn, 'slabs', 'graphs.lmdb')
@@ -3062,8 +3065,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                         if not ok:
                             break
 
-                        curvers = vers
-                        await self.setStormPkgVar(name, varname, vers)
+                        curvers = max(vers, await self.getStormPkgVar(name, varname, default=-1))
+                        await self.setStormPkgVar(name, varname, curvers)
                         logger.info(f'{name} finished init vers={vers}: {vname}', extra=logextra)
 
                 if onload is not None:
@@ -3348,6 +3351,91 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         for item in pkgvars.items():
             yield item
 
+    async def addStormPkgQueue(self, pkgname, name):
+        guid = s_common.guid((pkgname, name))
+        if self.stormpkgqueue.exists(guid):
+            mesg = f'Queue named {name} already exists for package {pkgname}!'
+            raise s_exc.DupName(mesg=mesg)
+
+        info = {
+            'iden': guid,
+            'name': name,
+            'pkgname': pkgname,
+            'created': s_common.now()
+        }
+
+        await self._push('storm:pkg:queue:add', pkgname, name, info)
+
+    @s_nexus.Pusher.onPush('storm:pkg:queue:add')
+    async def _addStormPkgQueue(self, pkgname, name, info):
+        guid = s_common.guid((pkgname, name))
+        if self.stormpkgqueue.exists(guid):
+            return
+        await self.stormpkgqueue.add(guid, info)
+
+    async def listStormPkgQueues(self, pkgname=None):
+        for pkginfo in self.stormpkgqueue.list():
+            if pkgname is None or pkginfo['meta'].get('pkgname') == pkgname:
+                yield pkginfo
+
+    async def getStormPkgQueue(self, pkgname, name):
+        guid = s_common.guid((pkgname, name))
+        return self.stormpkgqueue.status(guid)
+
+    async def delStormPkgQueue(self, pkgname, name):
+        guid = s_common.guid((pkgname, name))
+        if not self.stormpkgqueue.exists(guid):
+            mesg = f'No queue named {name} exists for package {pkgname}!'
+            raise s_exc.NoSuchName(mesg=mesg)
+
+        await self._push('storm:pkg:queue:del', pkgname, name)
+
+    @s_nexus.Pusher.onPush('storm:pkg:queue:del')
+    async def _delStormPkgQueue(self, pkgname, name):
+        guid = s_common.guid((pkgname, name))
+        if not self.stormpkgqueue.exists(guid):
+            return
+        await self.stormpkgqueue.rem(guid)
+
+    async def stormPkgQueueGet(self, pkgname, name, offs=0, wait=False):
+        guid = s_common.guid((pkgname, name))
+        async for item in self.stormpkgqueue.gets(guid, offs, cull=False, wait=wait):
+            return item
+
+    async def stormPkgQueueGets(self, pkgname, name, offs=0, wait=False, size=None):
+        count = 0
+        guid = s_common.guid((pkgname, name))
+        async for item in self.stormpkgqueue.gets(guid, offs, cull=False, wait=wait):
+
+            yield item
+
+            count += 1
+            if size is not None and count >= size:
+                return
+
+    async def stormPkgQueuePuts(self, pkgname, name, items):
+        return await self._push('storm:pkg:queue:puts', pkgname, name, items)
+
+    @s_nexus.Pusher.onPush('storm:pkg:queue:puts', passitem=True)
+    async def _stormPkgQueuePuts(self, pkgname, name, items, nexsitem):
+        nexsoff, nexsmesg = nexsitem
+        guid = s_common.guid((pkgname, name))
+        return await self.stormpkgqueue.puts(guid, items, reqid=nexsoff)
+
+    @s_nexus.Pusher.onPushAuto('storm:pkg:queue:cull')
+    async def stormPkgQueueCull(self, pkgname, name, offs):
+        guid = s_common.guid((pkgname, name))
+        await self.stormpkgqueue.cull(guid, offs)
+
+    @s_nexus.Pusher.onPushAuto('storm:pkg:queue:pop')
+    async def stormPkgQueuePop(self, pkgname, name, offs):
+        guid = s_common.guid((pkgname, name))
+        return await self.stormpkgqueue.pop(guid, offs)
+
+    async def stormPkgQueueSize(self, pkgname, name):
+        guid = s_common.guid((pkgname, name))
+        return self.stormpkgqueue.size(guid)
+
     async def _cortexHealth(self, health):
         health.update('cortex', 'nominal')
 
@@ -4568,34 +4656,25 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         self.addStormCmd(s_stormlib_cortex.StormPoolGetCmd)
         self.addStormCmd(s_stormlib_cortex.StormPoolSetCmd)
 
-        for cdef in s_stormsvc.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_storm.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_aha.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_gen.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_auth.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_macro.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_model.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_cortex.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_vault.stormcmds:
-            await self._trySetStormCmd(cdef.get('name'), cdef)
-
-        for cdef in s_stormlib_index.stormcmds:
+        cmdmods = [
+            s_storm,
+            s_stormsvc,
+            s_stormlib_aha,
+            s_stormlib_auth,
+            s_stormlib_cortex,
+            s_stormlib_gen,
+            s_stormlib_index,
+            s_stormlib_macro,
+            s_stormlib_model,
+            s_stormlib_pkg,
+            s_stormlib_vault,
+        ]
+
+        for cmod in cmdmods:
+            for cdef in cmod.stormcmds:
+                await self._trySetStormCmd(cdef.get('name'), cdef)
+
+        for cdef in s_stormlib_task.stormcmds:
             await self._trySetStormCmd(cdef.get('name'), cdef)
 
     async def _initPureStormCmds(self):

synapse/cryotank.py

@@ -39,7 +39,7 @@ class CryoTank(s_base.Base):
     A CryoTank implements a stream of structured data.
     '''
     async def __anit__(self, dirn, iden, conf=None):
-
+        s_common.deprecated('synapse.cryotank.CryoTank', curv='2.223.0')
         await s_base.Base.__anit__(self)
 
         if conf is None:

synapse/lib/ast.py

@@ -4527,9 +4527,6 @@ class N1Walk(Oper):
 
     def buildfilter(self, runt, destforms, cmpr):
 
-        if not isinstance(destforms, (tuple, list)):
-            destforms = (destforms,)
-
         if '*' in destforms:
             if cmpr is not None:
                 mesg = 'Wild card walk operations do not support comparison.'
@@ -4614,6 +4611,11 @@ class N1Walk(Oper):
                 dest = await self.kids[1].compute(runt, path)
                 dest = await s_stormtypes.toprim(dest)
 
+                if isinstance(dest, (tuple, list)):
+                    dest = [await s_stormtypes.tostr(form) for form in dest]
+                else:
+                    dest = (await s_stormtypes.tostr(dest),)
+
                 destfilt = self.buildfilter(runt, dest, cmpr)
 
             for verb in verbs:

synapse/lib/nexus.py

@@ -572,7 +572,7 @@ class NexsRoot(s_base.Base):
 
                     offs, args = item
                     if offs != self.nexslog.index():
-                        logger.error('Local Nexus offset is out of sync from remote cell! Aborting mirror sync')
+                        logger.error(f'Local Nexus offset is out of sync from remote cell! Aborting mirror sync. Local offs={self.nexslog.index()}, Remote {offs=}')
                         await self.fini()
                         return
 

synapse/lib/snap.py

@@ -1305,6 +1305,11 @@ class Snap(s_base.Base):
             mesg = 'The snapshot is in read-only mode.'
             raise s_exc.IsReadOnly(mesg=mesg)
 
+        useriden = meta.get('user')
+        if useriden is None:
+            mesg = 'meta is missing user key. Cannot process edits.'
+            raise s_exc.BadArg(mesg=mesg, name='user')
+
         wlyr = self.wlyr
         nodes = []
         callbacks = []
@@ -1344,12 +1349,12 @@ class Snap(s_base.Base):
                 if etyp == s_layer.EDIT_NODE_ADD:
                     node.bylayer['ndef'] = wlyr.iden
                     callbacks.append((node.form.wasAdded, (node,)))
-                    callbacks.append((self.view.runNodeAdd, (node,)))
+                    callbacks.append((self.view.runNodeAdd, (node, useriden)))
                     continue
 
                 if etyp == s_layer.EDIT_NODE_DEL:
                     callbacks.append((node.form.wasDeleted, (node,)))
-                    callbacks.append((self.view.runNodeDel, (node,)))
+                    callbacks.append((self.view.runNodeDel, (node, useriden)))
                     continue
 
                 if etyp == s_layer.EDIT_PROP_SET:
@@ -1365,7 +1370,7 @@ class Snap(s_base.Base):
                     node.bylayer['props'][name] = wlyr.iden
 
                     callbacks.append((prop.wasSet, (node, oldv)))
-                    callbacks.append((self.view.runPropSet, (node, prop, oldv)))
+                    callbacks.append((self.view.runPropSet, (node, prop, oldv, useriden)))
                     continue
 
                 if etyp == s_layer.EDIT_PROP_DEL:
@@ -1381,7 +1386,7 @@ class Snap(s_base.Base):
                     node.bylayer['props'].pop(name, None)
 
                     callbacks.append((prop.wasDel, (node, oldv)))
-                    callbacks.append((self.view.runPropSet, (node, prop, oldv)))
+                    callbacks.append((self.view.runPropSet, (node, prop, oldv, useriden)))
                     continue
 
                 if etyp == s_layer.EDIT_TAG_SET:
@@ -1391,7 +1396,7 @@ class Snap(s_base.Base):
                     node.tags[tag] = valu
                     node.bylayer['tags'][tag] = wlyr.iden
 
-                    callbacks.append((self.view.runTagAdd, (node, tag, valu)))
+                    callbacks.append((self.view.runTagAdd, (node, tag, valu, useriden,)))
                     continue
 
                 if etyp == s_layer.EDIT_TAG_DEL:
@@ -1401,7 +1406,7 @@ class Snap(s_base.Base):
                     node.tags.pop(tag, None)
                     node.bylayer['tags'].pop(tag, None)
 
-                    callbacks.append((self.view.runTagDel, (node, tag, oldv)))
+                    callbacks.append((self.view.runTagDel, (node, tag, oldv, useriden)))
                     continue
 
                 if etyp == s_layer.EDIT_TAGPROP_SET:
@@ -1436,14 +1441,15 @@ class Snap(s_base.Base):
                 if etyp == s_layer.EDIT_EDGE_ADD:
                     verb, n2iden = parms
                     n2 = await self.getNodeByBuid(s_common.uhex(n2iden))
-                    callbacks.append((self.view.runEdgeAdd, (node, verb, n2)))
+                    callbacks.append((self.view.runEdgeAdd, (node, verb, n2, useriden)))
 
                 if etyp == s_layer.EDIT_EDGE_DEL:
                     verb, n2iden = parms
                     n2 = await self.getNodeByBuid(s_common.uhex(n2iden))
-                    callbacks.append((self.view.runEdgeDel, (node, verb, n2)))
+                    callbacks.append((self.view.runEdgeDel, (node, verb, n2, useriden)))
 
-        [await func(*args) for (func, args) in callbacks]
+        for func, args in callbacks:
+            await func(*args)
 
         if actualedits:
             await self.fire('node:edits', edits=actualedits)

synapse/lib/storm.py

@@ -439,196 +439,6 @@ stormcmds = (
             }
         ''',
     },
-    {
-        'name': 'pkg.list',
-        'descr': 'List the storm packages loaded in the cortex.',
-        'cmdargs': (
-            ('--verbose', {'default': False, 'action': 'store_true',
-                'help': 'Display build time for each package.'}),
-        ),
-        'storm': '''
-            init {
-                $conf = ({
-                    "columns": [
-                        {"name": "name", "width": 40},
-                        {"name": "vers", "width": 10},
-                    ],
-                    "separators": {
-                        "row:outline": false,
-                        "column:outline": false,
-                        "header:row": "#",
-                        "data:row": "",
-                        "column": "",
-                    },
-                })
-                if $cmdopts.verbose {
-                    $conf.columns.append(({"name": "time", "width": 20}))
-                }
-                $printer = $lib.tabular.printer($conf)
-            }
-
-            $pkgs = $lib.pkg.list()
-
-            if $($pkgs.size() > 0) {
-                $lib.print('Loaded storm packages:')
-                $lib.print($printer.header())
-                for $pkg in $pkgs {
-                    $row = (
-                        $pkg.name, $pkg.version,
-                    )
-                    if $cmdopts.verbose {
-                        try {
-                            $row.append($lib.time.format($pkg.build.time, '%Y-%m-%d %H:%M:%S'))
-                        } catch StormRuntimeError as _ {
-                            $row.append('not available')
-                        }
-                    }
-                    $lib.print($printer.row($row))
-                }
-            } else {
-                $lib.print('No storm packages installed.')
-            }
-        '''
-    },
-    {
-        'name': 'pkg.perms.list',
-        'descr': 'List any permissions declared by the package.',
-        'cmdargs': (
-            ('name', {'help': 'The name (or name prefix) of the package.', 'type': 'str'}),
-        ),
-        'storm': '''
-            $pdef = $lib.null
-            for $pkg in $lib.pkg.list() {
-                if $pkg.name.startswith($cmdopts.name) {
-                    $pdef = $pkg
-                    break
-                }
-            }
-
-            if (not $pdef) {
-                $lib.warn(`Package ({$cmdopts.name}) not found!`)
-            } else {
-                if $pdef.perms {
-                    $lib.print(`Package ({$cmdopts.name}) defines the following permissions:`)
-                    for $permdef in $pdef.perms {
-                        $defv = $permdef.default
-                        if ( $defv = $lib.null ) {
-                            $defv = $lib.false
-                        }
-                        $text = `{('.').join($permdef.perm).ljust(32)} : {$permdef.desc} ( default: {$defv} )`
-                        $lib.print($text)
-                    }
-                } else {
-                    $lib.print(`Package ({$cmdopts.name}) contains no permissions definitions.`)
-                }
-            }
-        '''
-    },
-    {
-        'name': 'pkg.del',
-        'descr': 'Remove a storm package from the cortex.',
-        'cmdargs': (
-            ('name', {'help': 'The name (or name prefix) of the package to remove.'}),
-        ),
-        'storm': '''
-
-            $pkgs = $lib.set()
-
-            for $pkg in $lib.pkg.list() {
-                if $pkg.name.startswith($cmdopts.name) {
-                    $pkgs.add($pkg.name)
-                }
-            }
-
-            if $($pkgs.size() = 0) {
-
-                $lib.print('No package names match "{name}". Aborting.', name=$cmdopts.name)
-
-            } elif $($pkgs.size() = 1) {
-
-                $name = $pkgs.list().index(0)
-                $lib.print('Removing package: {name}', name=$name)
-                $lib.pkg.del($name)
-
-            } else {
-
-                $lib.print('Multiple package names match "{name}". Aborting.', name=$cmdopts.name)
-
-            }
-        '''
-    },
-    {
-        'name': 'pkg.docs',
-        'descr': 'Display documentation included in a storm package.',
-        'cmdargs': (
-            ('name', {'help': 'The name (or name prefix) of the package.'}),
-        ),
-        'storm': '''
-            $pdef = $lib.null
-            for $pkg in $lib.pkg.list() {
-                if $pkg.name.startswith($cmdopts.name) {
-                    $pdef = $pkg
-                    break
-                }
-            }
-
-            if (not $pdef) {
-                $lib.warn("Package ({name}) not found!", name=$cmdopts.name)
-            } else {
-                if $pdef.docs {
-                    for $doc in $pdef.docs {
-                        $lib.print($doc.content)
-                    }
-                } else {
-                    $lib.print("Package ({name}) contains no documentation.", name=$cmdopts.name)
-                }
-            }
-        '''
-    },
-    {
-        'name': 'pkg.load',
-        'descr': 'Load a storm package from an HTTP URL.',
-        'cmdargs': (
-            ('url', {'help': 'The HTTP URL to load the package from.'}),
-            ('--raw', {'default': False, 'action': 'store_true',
-                'help': 'Response JSON is a raw package definition without an envelope.'}),
-            ('--verify', {'default': False, 'action': 'store_true',
-                'help': 'Enforce code signature verification on the storm package.'}),
-            ('--ssl-noverify', {'default': False, 'action': 'store_true',
-                'help': 'Specify to disable SSL verification of the server.'}),
-        ),
-        'storm': '''
-            init {
-                $ssl = $lib.true
-                if $cmdopts.ssl_noverify { $ssl = $lib.false }
-
-                $headers = ({'X-Synapse-Version': ('.').join($lib.version.synapse())})
-
-                $resp = $lib.inet.http.get($cmdopts.url, ssl_verify=$ssl, headers=$headers)
-
-                if ($resp.code != 200) {
-                    $lib.warn("pkg.load got HTTP code: {code} for URL: {url}", code=$resp.code, url=$cmdopts.url)
-                    $lib.exit()
-                }
-
-                $reply = $resp.json()
-                if $cmdopts.raw {
-                    $pkg = $reply
-                } else {
-                    if ($reply.status != "ok") {
-                        $lib.warn("pkg.load got JSON error: {code} for URL: {url}", code=$reply.code, url=$cmdopts.url)
-                        $lib.exit()
-                    }
-
-                    $pkg = $reply.result
-                }
-
-                $pkd = $lib.pkg.add($pkg, verify=$cmdopts.verify)
-
-                $lib.print("Loaded Package: {name} @{version}", name=$pkg.name, version=$pkg.version)
-            }
-        ''',
-    },
     {
         'name': 'version',
         'descr': 'Show version metadata relating to Synapse.',

synapse/lib/stormlib/auth.py

@@ -1810,7 +1810,7 @@ class LibUser(s_stormtypes.Lib):
                   'returns': {'type': 'boolean',
                               'desc': 'True if the user has the requested permission, false otherwise.', }}},
         {'name': 'vars', 'desc': "Get a dictionary representing the current user's persistent variables.",
-         'type': 'auth:user:vars', },
+         'type': 'user:vars:dict', },
         {'name': 'profile', 'desc': "Get a dictionary representing the current user's profile information.",
          'type': 'auth:user:profile', },
         {'name': 'iden', 'desc': 'The user GUID for the current storm user.', 'type': 'str'},

synapse/lib/stormlib/mime.py

@@ -3,9 +3,9 @@ import bs4
 import synapse.lib.coro as s_coro
 import synapse.lib.stormtypes as s_stormtypes
 
-def htmlToText(html):
+def htmlToText(html, separator='\n', strip=True):
     soup = bs4.BeautifulSoup(html, 'html5lib')
-    return soup.get_text(separator='\n', strip=True)
+    return soup.get_text(separator=separator, strip=strip)
 
 @s_stormtypes.registry.registerLib
 class LibMimeHtml(s_stormtypes.Lib):
@@ -17,8 +17,12 @@ class LibMimeHtml(s_stormtypes.Lib):
          'type': {'type': 'function', '_funcname': 'totext',
                   'args': (
                       {'name': 'html', 'type': 'str', 'desc': 'The HTML text to be parsed.'},
+                      {'name': 'separator', 'type': 'str', 'default': '\n',
+                       'desc': 'The string used to join text.'},
+                      {'name': 'strip', 'type': 'boolean', 'default': True,
+                       'desc': 'Strip whitespace from the beginning and end of tag text.'},
                   ),
-                  'returns': {'type': 'str', 'desc': 'The newline-joined inner HTML text.', }
+                  'returns': {'type': 'str', 'desc': 'The separator-joined inner HTML text.', }
         }},
     )
 
@@ -30,6 +34,12 @@ class LibMimeHtml(s_stormtypes.Lib):
         }
 
     @s_stormtypes.stormfunc(readonly=True)
-    async def totext(self, html):
+    async def totext(self, html, separator='\n', strip=True):
         html = await s_stormtypes.tostr(html)
-        return await s_coro.semafork(htmlToText, html)
+        separator = await s_stormtypes.tostr(separator, noneok=True)
+        strip = await s_stormtypes.tobool(strip)
+
+        if separator is None:
+            separator = ''
+
+        return await s_coro.semafork(htmlToText, html, separator, strip)