synapse 2.212.0__py311-none-any.whl → 2.214.0__py311-none-any.whl

synapse/lib/certdir.py

@@ -1,7 +1,6 @@
 import io
 import os
 import ssl
-import time
 import shutil
 import socket
 import logging
@@ -397,6 +396,28 @@ class CertDir:
         '''
         return self._genPkeyCsr(name, 'hosts', outp=outp)
 
+    def delHostCsr(self, name: str, outp: OutPutOrNone = None) -> bool:
+        '''
+        Delete an existing host CSR.
+
+        Args:
+            name: The name of the host CSR.
+            outp: The output buffer.
+
+        Returns:
+            bool: True if the CSR is deleted, False if it did not exist.
+        '''
+        path = self.getHostCsrPath(name)
+        if path is None:
+            return False
+        try:
+            os.unlink(path)
+        except Exception as e:  # pragma: no cover
+            raise s_exc.SynErr(mesg=f'Failed to delete CSR {path} - {e}') from e
+        if outp:
+            outp.printf(f'Deleted CSR at {path}')
+        return True
+
     def genUserCert(self,
                     name: str,
                     signas: StrOrNone = None,
@@ -697,6 +718,28 @@ class CertDir:
         '''
         return self._genPkeyCsr(name, 'users', outp=outp)
 
+    def delUserCsr(self, name: str, outp: OutPutOrNone = None) -> bool:
+        '''
+        Delete an existing user CSR.
+
+        Args:
+            name: The name of the user CSR.
+            outp: The output buffer.
+
+        Returns:
+            bool: True if the CSR is deleted, False if it did not exist.
+        '''
+        path = self.getUserCsrPath(name)
+        if path is None:
+            return False
+        try:
+            os.unlink(path)
+        except Exception as e:  # pragma: no cover
+            raise s_exc.SynErr(mesg=f'Failed to delete CSR {path} - {e}') from e
+        if outp:
+            outp.printf(f'Deleted CSR at {path}')
+        return True
+
     def getCaCert(self, name: str) -> CertOrNone:
         '''
         Loads the X509 object for a given CA.

synapse/lib/cmd.py

@@ -1,6 +1,11 @@
+import sys
+import asyncio
 import argparse
 
 import synapse.exc as s_exc
+import synapse.common as s_common
+
+import synapse.lib.coro as s_coro
 import synapse.lib.output as s_output
 
 class Parser(argparse.ArgumentParser):
@@ -25,6 +30,7 @@ class Parser(argparse.ArgumentParser):
 
         if message is not None:
             self.outp.printf(message)
+
         raise s_exc.ParserExit(mesg=message, status=status)
 
     def _print_message(self, text, fd=None):
@@ -32,3 +38,21 @@ class Parser(argparse.ArgumentParser):
         Note:  this overrides an existing method in ArgumentParser
         '''
         self.outp.printf(text)
+
+async def wrapmain(func): # pragma: no cover
+
+    try:
+        return await func(sys.argv[1:])
+
+    except s_exc.ParserExit:
+        return 1
+
+    except Exception as e:
+        print(f'ERROR: {s_exc.reprexc(e)}')
+        return 1
+
+    finally:
+        await s_coro.await_bg_tasks(timeout=10)
+
+def exitmain(func): # pragma: no cover
+    sys.exit(asyncio.run(wrapmain(func)))

synapse/lib/coro.py

@@ -175,10 +175,16 @@ def create_task(coro):
 
     return task
 
-async def await_bg_tasks():
+async def await_bg_tasks(timeout=None):
+
     if not bgtasks:
         return []
-    return await asyncio.gather(*tuple(bgtasks), return_exceptions=True)
+
+    coro = asyncio.gather(*tuple(bgtasks), return_exceptions=True)
+    try:
+        return await s_common.wait_for(coro, timeout)
+    except (asyncio.CancelledError, asyncio.TimeoutError):
+        return []
 
 class GenrHelp:
 

synapse/lib/drive.py

@@ -69,6 +69,9 @@ class Drive(s_base.Base):
 
     def getItemInfo(self, iden, typename=None):
         info = self._getItemInfo(s_common.uhex(iden))
+        if not info:
+            return
+
         if typename is not None:
             self._reqInfoType(info, typename)
         return info
@@ -210,7 +213,7 @@ class Drive(s_base.Base):
 
     def _setItemPerm(self, bidn, perm):
         info = self._reqItemInfo(bidn)
-        info['perm'] = perm
+        info['permissions'] = perm
         s_schemas.reqValidDriveInfo(info)
         self.slab.put(LKEY_INFO + bidn, s_msgpack.en(info), db=self.dbname)
         return info
@@ -286,7 +289,7 @@ class Drive(s_base.Base):
         info['kids'] = 0
         info['parent'] = pariden
 
-        info.setdefault('perm', {'users': {}, 'roles': {}})
+        info.setdefault('permissions', {'users': {}, 'roles': {}})
         info.setdefault('version', (0, 0, 0))
 
         s_schemas.reqValidDriveInfo(info)
@@ -447,6 +450,8 @@ class Drive(s_base.Base):
 
         if vers is None:
             info = self._getItemInfo(bidn)
+            if info is None:
+                return None
             vers = info.get('version')
 
         versindx = getVersIndx(vers)

synapse/lib/link.py

@@ -61,7 +61,15 @@ async def unixconnect(path):
     '''
     Connect to a PF_UNIX server listening on the given path.
     '''
-    reader, writer = await asyncio.open_unix_connection(path=path)
+    try:
+        reader, writer = await asyncio.open_unix_connection(path=path)
+    except ConnectionRefusedError as e:
+        mesg = f'Cell path is not listening: {path}'
+        raise s_exc.LinkErr(mesg=mesg) from e
+    except FileNotFoundError as e:
+        mesg = f'Cell path does not exist: {path}'
+        raise s_exc.NoSuchPath(mesg=mesg) from e
+
     info = {'path': path, 'unix': True}
     return await Link.anit(reader, writer, info=info)
 
@@ -321,9 +329,9 @@ class Link(s_base.Base):
                 raise
 
             except Exception as e:
-                mesg = f'rx error {e} link={self.getAddrInfo()}'
+                mesg = f'rx closed unexpectedly {e} link={self.getAddrInfo()}'
                 if isinstance(e, (BrokenPipeError, ConnectionResetError)):
-                    logger.warning(mesg)
+                    logger.debug(mesg)
                 else:
                     logger.exception(mesg)
                 await self.fini()

synapse/lib/schemas.py

@@ -469,7 +469,7 @@ driveInfoSchema = {
         'parent': {'type': 'string', 'pattern': s_config.re_iden},
         'type': {'type': 'string', 'pattern': re_drivename},
         'name': {'type': 'string', 'pattern': re_drivename},
-        'perm': s_msgpack.deepcopy(easyPermSchema),
+        'permissions': s_msgpack.deepcopy(easyPermSchema),
         'kids': {'type': 'number', 'minimum': 0},
         'created': {'type': 'number'},
         'creator': {'type': 'string', 'pattern': s_config.re_iden},

synapse/lib/scrape.py

@@ -65,6 +65,8 @@ inverse_prefixs = {
     '<': '>',
     '{': '}',
     '(': ')',
+    '\u2018': '\u2019',  # Unicode Left Single Quotation Mark and Right Single Quotation Mark
+    '\u201c': '\u201d',  # Unicode Left Double Quotation Mark and Right Double Quotation Mark
 }
 
 cve_dashes = ''.join(('-',) + s_chop.unicode_dashes)
@@ -287,7 +289,7 @@ def url_scheme_check(match: regex.Match):
 scrape_types = [  # type: ignore
     ('file:path', linux_path_regex, {'callback': linux_path_check, 'flags': regex.VERBOSE}),
     ('file:path', windows_path_regex, {'callback': windows_path_check, 'flags': regex.VERBOSE}),
-    ('inet:url', r'(?P<prefix>[\\{<\(\[]?)(?P<valu>[a-zA-Z][a-zA-Z0-9]*://(?(?=[,.]+[ \'\"\t\n\r\f\v])|[^ \'\"\t\n\r\f\v])+)',
+    ('inet:url', r'(?P<prefix>[‘“\\{<\(\[]?)(?P<valu>[a-zA-Z][a-zA-Z0-9]*://(?(?=[,.]+[ \'\"\t\n\r\f\v])|[^ \'\"\t\n\r\f\v])+)',
      {'callback': url_scheme_check}),
     ('inet:url', r'(["\'])?(?P<valu>\\[^\n]+?)(?(1)\1|\s)', {'callback': unc_path_check}),
     ('inet:email', r'(?=(?:[^a-z0-9_.+-]|^)(?P<valu>[a-z0-9_\.\-+]{1,256}@(?:[a-z0-9_-]{1,63}\.){1,10}(?:%s))(?:[^a-z0-9_.-]|[.\s]|$))' % tldcat, {}),

synapse/lib/snap.py

@@ -1325,13 +1325,13 @@ class Snap(s_base.Base):
 
                 if etyp == s_layer.EDIT_NODE_ADD:
                     node.bylayer['ndef'] = wlyr.iden
-                    callbacks.append((node.form.wasAdded, (node,), {}))
-                    callbacks.append((self.view.runNodeAdd, (node,), {}))
+                    callbacks.append((node.form.wasAdded, (node,)))
+                    callbacks.append((self.view.runNodeAdd, (node,)))
                     continue
 
                 if etyp == s_layer.EDIT_NODE_DEL:
-                    callbacks.append((node.form.wasDeleted, (node,), {}))
-                    callbacks.append((self.view.runNodeDel, (node,), {}))
+                    callbacks.append((node.form.wasDeleted, (node,)))
+                    callbacks.append((self.view.runNodeDel, (node,)))
                     continue
 
                 if etyp == s_layer.EDIT_PROP_SET:
@@ -1346,8 +1346,8 @@ class Snap(s_base.Base):
                     node.props[name] = valu
                     node.bylayer['props'][name] = wlyr.iden
 
-                    callbacks.append((prop.wasSet, (node, oldv), {}))
-                    callbacks.append((self.view.runPropSet, (node, prop, oldv), {}))
+                    callbacks.append((prop.wasSet, (node, oldv)))
+                    callbacks.append((self.view.runPropSet, (node, prop, oldv)))
                     continue
 
                 if etyp == s_layer.EDIT_PROP_DEL:
@@ -1362,8 +1362,8 @@ class Snap(s_base.Base):
                     node.props.pop(name, None)
                     node.bylayer['props'].pop(name, None)
 
-                    callbacks.append((prop.wasDel, (node, oldv), {}))
-                    callbacks.append((self.view.runPropSet, (node, prop, oldv), {}))
+                    callbacks.append((prop.wasDel, (node, oldv)))
+                    callbacks.append((self.view.runPropSet, (node, prop, oldv)))
                     continue
 
                 if etyp == s_layer.EDIT_TAG_SET:
@@ -1373,8 +1373,7 @@ class Snap(s_base.Base):
                     node.tags[tag] = valu
                     node.bylayer['tags'][tag] = wlyr.iden
 
-                    callbacks.append((self.view.runTagAdd, (node, tag, valu), {}))
-                    callbacks.append((self.wlyr.fire, ('tag:add', ), {'tag': tag, 'node': node.iden()}))
+                    callbacks.append((self.view.runTagAdd, (node, tag, valu)))
                     continue
 
                 if etyp == s_layer.EDIT_TAG_DEL:
@@ -1384,8 +1383,7 @@ class Snap(s_base.Base):
                     node.tags.pop(tag, None)
                     node.bylayer['tags'].pop(tag, None)
 
-                    callbacks.append((self.view.runTagDel, (node, tag, oldv), {}))
-                    callbacks.append((self.wlyr.fire, ('tag:del', ), {'tag': tag, 'node': node.iden()}))
+                    callbacks.append((self.view.runTagDel, (node, tag, oldv)))
                     continue
 
                 if etyp == s_layer.EDIT_TAGPROP_SET:
@@ -1420,14 +1418,14 @@ class Snap(s_base.Base):
                 if etyp == s_layer.EDIT_EDGE_ADD:
                     verb, n2iden = parms
                     n2 = await self.getNodeByBuid(s_common.uhex(n2iden))
-                    callbacks.append((self.view.runEdgeAdd, (node, verb, n2), {}))
+                    callbacks.append((self.view.runEdgeAdd, (node, verb, n2)))
 
                 if etyp == s_layer.EDIT_EDGE_DEL:
                     verb, n2iden = parms
                     n2 = await self.getNodeByBuid(s_common.uhex(n2iden))
-                    callbacks.append((self.view.runEdgeDel, (node, verb, n2), {}))
+                    callbacks.append((self.view.runEdgeDel, (node, verb, n2)))
 
-        [await func(*args, **kwargs) for (func, args, kwargs) in callbacks]
+        [await func(*args) for (func, args) in callbacks]
 
         if actualedits:
             await self.fire('node:edits', edits=actualedits)
@@ -1456,7 +1454,9 @@ class Snap(s_base.Base):
         if isinstance(valu, dict):
             form = self.core.model.reqForm(name)
             if isinstance(form.type, s_types.Guid):
-                return await self._addGuidNodeByDict(form, valu, props=props)
+                norms, props = await self._normGuidNodeDict(form, valu, props=props)
+                valu = await self._addGuidNodeByDict(form, norms, props)
+                return await self.getNodeByNdef((name, valu))
 
         async with self.getEditor() as editor:
             protonode = await editor.addNode(name, valu, props=props, norminfo=norminfo)
@@ -1466,100 +1466,109 @@ class Snap(s_base.Base):
         # the newly constructed node is cached
         return await self.getNodeByBuid(protonode.buid)
 
-    async def _addGuidNodeByDict(self, form, vals, props=None):
+    async def _addGuidNodeByDict(self, form, norms, props):
 
-        if props is None:
-            props = {}
-
-        trycast = vals.pop('$try', False)
-        addprops = vals.pop('$props', None)
-
-        if not vals:
-            mesg = f'No values provided for form {form.full}'
-            raise s_exc.BadTypeValu(mesg=mesg)
-
-        for name, valu in list(props.items()):
-            try:
-                props[name] = form.reqProp(name).type.norm(valu)
-            except s_exc.BadTypeValu as e:
-                mesg = e.get('mesg')
-                e.update({
-                    'prop': name,
-                    'form': form.name,
-                    'mesg': f'Bad value for prop {form.name}:{name}: {mesg}',
-                })
-                raise e
-
-        if addprops is not None:
-            for name, valu in addprops.items():
-                try:
-                    props[name] = form.reqProp(name).type.norm(valu)
-                except s_exc.BadTypeValu as e:
-                    mesg = e.get("mesg")
-                    if not trycast:
-                        e.update({
-                            'prop': name,
-                            'form': form.name,
-                            'mesg': f'Bad value for prop {form.name}:{name}: {mesg}'
-                        })
-                        raise e
-                    await self.warn(f'Skipping bad value for prop {form.name}:{name}: {mesg}')
+        for name, info in norms.items():
+            if info[0].isform:
+                valu = await self._addGuidNodeByDict(*info)
+                norms[name] = (form.prop(name), valu, {})
 
-        norms, proplist = self._normGuidNodeDict(form, vals)
+        for name, info in props.items():
+            if info[0].isform:
+                valu = await self._addGuidNodeByDict(*info)
+                props[name] = (form.prop(name), valu, {})
 
-        iden = s_common.guid(proplist)
-        node = await self._getGuidNodeByNorms(form, iden, norms)
+        node = await self._getGuidNodeByNorms(form, norms)
 
         async with self.getEditor() as editor:
 
             if node is not None:
                 proto = editor.loadNode(node)
             else:
-                proto = await editor.addNode(form.name, iden)
+                proplist = [(name, info[1]) for name, info in norms.items()]
+                proplist.sort()
+
+                proto = await editor.addNode(form.name, proplist)
                 for name, (prop, valu, info) in norms.items():
                     await proto.set(name, valu, norminfo=info)
 
             # ensure the non-deconf props are set
-            for name, (valu, info) in props.items():
+            for name, (prop, valu, info) in props.items():
                 await proto.set(name, valu, norminfo=info)
 
-        return await self.getNodeByBuid(proto.buid)
+        return proto.valu
+
+    async def _normGuidNodeDict(self, form, vals, props=None):
+
+        if props is None:
+            props = {}
+
+        trycast = vals.pop('$try', False)
+        addprops = vals.pop('$props', None)
+
+        if not vals:
+            mesg = f'No values provided for form {form.full}'
+            raise s_exc.BadTypeValu(mesg=mesg)
+
+        props |= await self._normGuidNodeProps(form, props)
 
-    def _normGuidNodeDict(self, form, props):
+        if addprops:
+            props |= await self._normGuidNodeProps(form, addprops, trycast=trycast)
+
+        norms = await self._normGuidNodeProps(form, vals)
+
+        return norms, props
+
+    async def _normGuidNodeProps(self, form, props, trycast=False):
 
         norms = {}
-        proplist = []
 
-        for name, valu in props.items():
+        for name, valu in list(props.items()):
+            prop = form.reqProp(name)
+
+            if isinstance(valu, dict) and isinstance(prop.type, s_types.Guid):
+                pform = self.core.model.reqForm(prop.type.name)
+                gnorm, gprop = await self._normGuidNodeDict(pform, valu)
+                norms[name] = (pform, gnorm, gprop)
+                continue
 
             try:
-                prop = form.reqProp(name)
-                norm, norminfo = prop.type.norm(valu)
+                norms[name] = (prop, *prop.type.norm(valu))
 
-                norms[name] = (prop, norm, norminfo)
-                proplist.append((name, norm))
             except s_exc.BadTypeValu as e:
-                mesg = e.get('mesg')
-                e.update({
-                    'prop': name,
-                    'form': form.name,
-                    'mesg': f'Bad value for prop {form.name}:{name}: {mesg}',
-                })
-                raise e
-
-        proplist.sort()
+                if not trycast:
+                    if 'prop' not in e.errinfo:
+                        mesg = e.get('mesg')
+                        e.update({
+                            'prop': name,
+                            'form': form.name,
+                            'mesg': f'Bad value for prop {form.name}:{name}: {mesg}',
+                        })
+                    raise e
 
-        return norms, proplist
+        return norms
 
     async def _getGuidNodeByDict(self, form, props):
-        norms, proplist = self._normGuidNodeDict(form, props)
-        return await self._getGuidNodeByNorms(form, s_common.guid(proplist), norms)
+        norms, _ = await self._normGuidNodeDict(form, props)
+        return await self._getGuidNodeByNorms(form, norms)
+
+    async def _getGuidNodeByNorms(self, form, norms):
 
-    async def _getGuidNodeByNorms(self, form, iden, norms):
+        proplist = []
+        for name, info in norms.items():
+            if info[0].isform:
+                if (node := await self._getGuidNodeByNorms(*info[:2])) is None:
+                    return
+                valu = node.ndef[1]
+                norms[name] = (form.prop(name), valu, {})
+                proplist.append((name, valu))
+            else:
+                proplist.append((name, info[1]))
 
         # check first for an exact match via our same deconf strategy
+        proplist.sort()
 
-        node = await self.getNodeByNdef((form.full, iden))
+        node = await self.getNodeByNdef((form.full, s_common.guid(proplist)))
         if node is not None:
 
             # ensure we still match the property deconf criteria

synapse/lib/storm.py

@@ -1454,7 +1454,8 @@ class StormDmon(s_base.Base):
         viewiden = opts.get('view')
 
         info = {'iden': self.iden, 'name': self.ddef.get('name', 'storm dmon'), 'view': viewiden}
-        await self.core.boss.promote('storm:dmon', user=self.user, info=info)
+
+        await self.core.boss.promote('storm:dmon', user=self.user, info=info, background=True)
 
         def dmonPrint(evnt):
             self._runLogAdd(evnt)

synapse/lib/stormlib/imap.py

@@ -4,6 +4,7 @@ import aioimaplib
 
 import synapse.exc as s_exc
 import synapse.common as s_common
+import synapse.lib.coro as s_coro
 import synapse.lib.stormtypes as s_stormtypes
 
 async def run_imap_coro(coro):
@@ -88,8 +89,8 @@ class ImapLib(s_stormtypes.Lib):
             imap_cli = aioimaplib.IMAP4(host=host, port=port, timeout=timeout)
 
         async def fini():
-            # call protocol.logout() so fini() doesn't hang
-            await s_common.wait_for(imap_cli.protocol.logout(), 5)
+            # call protocol.logout() via a background task
+            s_coro.create_task(s_common.wait_for(imap_cli.protocol.logout(), 5))
 
         self.runt.snap.onfini(fini)
 

synapse/lib/stormlib/spooled.py

@@ -37,6 +37,7 @@ class LibSpooled(s_stormtypes.Lib):
     async def _methSet(self, *vals):
         core = self.runt.snap.core
         spool = await s_spooled.Set.anit(dirn=core.dirn, cell=core, size=1000)
+        self.runt.snap.onfini(spool)
 
         valu = list(vals)
         for item in valu:
@@ -107,3 +108,6 @@ class SpooledSet(s_stormtypes.Set):
 
     async def value(self):
         return set([x async for x in self.valu])
+
+    async def bool(self):
+        return bool(len(self.valu))

synapse/lib/stormtypes.py

@@ -4731,6 +4731,9 @@ class Str(Prim):
             return str(self) == str(othr)
         return False
 
+    async def bool(self):
+        return bool(self.valu)
+
     @stormfunc(readonly=True)
     async def _methStrFind(self, valu):
         text = await tostr(valu)
@@ -4970,6 +4973,9 @@ class Bytes(Prim):
             return self.valu == othr.valu
         return False
 
+    async def bool(self):
+        return bool(self.valu)
+
     async def _storm_copy(self):
         item = await s_coro.ornot(self.value)
         return s_msgpack.deepcopy(item, use_list=True)
@@ -5042,6 +5048,9 @@ class Dict(Prim):
     def __len__(self):
         return len(self.valu)
 
+    async def bool(self):
+        return bool(self.valu)
+
     async def _storm_copy(self):
         item = await s_coro.ornot(self.value)
         return s_msgpack.deepcopy(item, use_list=True)
@@ -5194,6 +5203,9 @@ class Set(Prim):
     def __len__(self):
         return len(self.valu)
 
+    async def bool(self):
+        return bool(self.valu)
+
     async def _methSetSize(self):
         return len(self)
 
@@ -5384,6 +5396,9 @@ class List(Prim):
     def __len__(self):
         return len(self.valu)
 
+    async def bool(self):
+        return bool(self.valu)
+
     @stormfunc(readonly=True)
     async def _methListHas(self, valu):
         if valu in self.valu:
@@ -5519,6 +5534,9 @@ class Bool(Prim):
     def __hash__(self):
         return hash((self._storm_typename, self.value()))
 
+    async def bool(self):
+        return bool(self.valu)
+
 @registry.registerType
 class Number(Prim):
     '''

synapse/lib/task.py

@@ -22,6 +22,7 @@ class Task(s_base.Base):
             info = {}
 
         self.boss = boss
+        self.background = False
 
         task._syn_task = self