synapse 2.213.0__py311-none-any.whl → 2.215.0__py311-none-any.whl

synapse/lib/cmd.py

@@ -1,6 +1,11 @@
+import sys
+import asyncio
 import argparse
 
 import synapse.exc as s_exc
+import synapse.common as s_common
+
+import synapse.lib.coro as s_coro
 import synapse.lib.output as s_output
 
 class Parser(argparse.ArgumentParser):
@@ -25,6 +30,7 @@ class Parser(argparse.ArgumentParser):
 
         if message is not None:
             self.outp.printf(message)
+
         raise s_exc.ParserExit(mesg=message, status=status)
 
     def _print_message(self, text, fd=None):
@@ -32,3 +38,21 @@ class Parser(argparse.ArgumentParser):
         Note:  this overrides an existing method in ArgumentParser
         '''
         self.outp.printf(text)
+
+async def wrapmain(func): # pragma: no cover
+
+    try:
+        return await func(sys.argv[1:])
+
+    except s_exc.ParserExit:
+        return 1
+
+    except Exception as e:
+        print(f'ERROR: {s_exc.reprexc(e)}')
+        return 1
+
+    finally:
+        await s_coro.await_bg_tasks(timeout=10)
+
+def exitmain(func): # pragma: no cover
+    sys.exit(asyncio.run(wrapmain(func)))

synapse/lib/coro.py

@@ -175,10 +175,16 @@ def create_task(coro):
 
     return task
 
-async def await_bg_tasks():
+async def await_bg_tasks(timeout=None):
+
     if not bgtasks:
         return []
-    return await asyncio.gather(*tuple(bgtasks), return_exceptions=True)
+
+    coro = asyncio.gather(*tuple(bgtasks), return_exceptions=True)
+    try:
+        return await s_common.wait_for(coro, timeout)
+    except (asyncio.CancelledError, asyncio.TimeoutError):
+        return []
 
 class GenrHelp:
 

synapse/lib/drive.py

@@ -69,6 +69,9 @@ class Drive(s_base.Base):
 
     def getItemInfo(self, iden, typename=None):
         info = self._getItemInfo(s_common.uhex(iden))
+        if not info:
+            return
+
         if typename is not None:
             self._reqInfoType(info, typename)
         return info
@@ -210,7 +213,7 @@ class Drive(s_base.Base):
 
     def _setItemPerm(self, bidn, perm):
         info = self._reqItemInfo(bidn)
-        info['perm'] = perm
+        info['permissions'] = perm
         s_schemas.reqValidDriveInfo(info)
         self.slab.put(LKEY_INFO + bidn, s_msgpack.en(info), db=self.dbname)
         return info
@@ -286,7 +289,7 @@ class Drive(s_base.Base):
         info['kids'] = 0
         info['parent'] = pariden
 
-        info.setdefault('perm', {'users': {}, 'roles': {}})
+        info.setdefault('permissions', {'users': {}, 'roles': {}})
         info.setdefault('version', (0, 0, 0))
 
         s_schemas.reqValidDriveInfo(info)
@@ -447,6 +450,8 @@ class Drive(s_base.Base):
 
         if vers is None:
             info = self._getItemInfo(bidn)
+            if info is None:
+                return None
             vers = info.get('version')
 
         versindx = getVersIndx(vers)

synapse/lib/jsonstor.py

@@ -220,6 +220,7 @@ class JsonStor(s_base.Base):
 
         step[name] = valu
         self.dirty[buid] = item
+        self.slab.dirty = True
         return True
 
     async def delPathObjProp(self, path, prop):
@@ -241,6 +242,7 @@ class JsonStor(s_base.Base):
         step.pop(names[-1], None)
 
         self.dirty[buid] = item
+        self.slab.dirty = True
         return True
 
     async def cmpDelPathObjProp(self, path, prop, valu):
@@ -264,6 +266,7 @@ class JsonStor(s_base.Base):
 
         step.pop(name, None)
         self.dirty[buid] = item
+        self.slab.dirty = True
         return True
 
     async def popPathObjProp(self, path, prop, defv=None):
@@ -285,7 +288,7 @@ class JsonStor(s_base.Base):
 
         retn = step.pop(names[-1], defv)
         self.dirty[buid] = item
-
+        self.slab.dirty = True
         return retn
 
 class JsonStorApi(s_cell.CellApi):

synapse/lib/layer.py

@@ -3569,6 +3569,7 @@ class Layer(s_nexus.Pusher):
 
         if not self.mayDelBuid(buid, sode):
             self.setSodeDirty(buid, sode, form)
+            self.layrslab.dirty = True
 
         return (
             (EDIT_NODE_DEL, (valu, stortype), ()),
@@ -3842,8 +3843,9 @@ class Layer(s_nexus.Pusher):
             return ()
 
         oldv, oldt = tp_dict.pop(prop, (None, None))
-        if not tp_dict.get(tag):
+        if not tp_dict:
             sode['tagprops'].pop(tag, None)
+
         if oldv is None:
             self.mayDelBuid(buid, sode)
             return ()

synapse/lib/link.py

@@ -61,7 +61,15 @@ async def unixconnect(path):
     '''
     Connect to a PF_UNIX server listening on the given path.
     '''
-    reader, writer = await asyncio.open_unix_connection(path=path)
+    try:
+        reader, writer = await asyncio.open_unix_connection(path=path)
+    except ConnectionRefusedError as e:
+        mesg = f'Cell path is not listening: {path}'
+        raise s_exc.LinkErr(mesg=mesg) from e
+    except FileNotFoundError as e:
+        mesg = f'Cell path does not exist: {path}'
+        raise s_exc.NoSuchPath(mesg=mesg) from e
+
     info = {'path': path, 'unix': True}
     return await Link.anit(reader, writer, info=info)
 
@@ -321,9 +329,9 @@ class Link(s_base.Base):
                 raise
 
             except Exception as e:
-                mesg = f'rx error {e} link={self.getAddrInfo()}'
+                mesg = f'rx closed unexpectedly {e} link={self.getAddrInfo()}'
                 if isinstance(e, (BrokenPipeError, ConnectionResetError)):
-                    logger.warning(mesg)
+                    logger.debug(mesg)
                 else:
                     logger.exception(mesg)
                 await self.fini()

synapse/lib/schemas.py

@@ -469,7 +469,7 @@ driveInfoSchema = {
         'parent': {'type': 'string', 'pattern': s_config.re_iden},
         'type': {'type': 'string', 'pattern': re_drivename},
         'name': {'type': 'string', 'pattern': re_drivename},
-        'perm': s_msgpack.deepcopy(easyPermSchema),
+        'permissions': s_msgpack.deepcopy(easyPermSchema),
         'kids': {'type': 'number', 'minimum': 0},
         'created': {'type': 'number'},
         'creator': {'type': 'string', 'pattern': s_config.re_iden},

synapse/lib/snap.py

@@ -1454,7 +1454,9 @@ class Snap(s_base.Base):
         if isinstance(valu, dict):
             form = self.core.model.reqForm(name)
             if isinstance(form.type, s_types.Guid):
-                return await self._addGuidNodeByDict(form, valu, props=props)
+                norms, props = await self._normGuidNodeDict(form, valu, props=props)
+                valu = await self._addGuidNodeByDict(form, norms, props)
+                return await self.getNodeByNdef((name, valu))
 
         async with self.getEditor() as editor:
             protonode = await editor.addNode(name, valu, props=props, norminfo=norminfo)
@@ -1464,100 +1466,109 @@ class Snap(s_base.Base):
         # the newly constructed node is cached
         return await self.getNodeByBuid(protonode.buid)
 
-    async def _addGuidNodeByDict(self, form, vals, props=None):
+    async def _addGuidNodeByDict(self, form, norms, props):
 
-        if props is None:
-            props = {}
-
-        trycast = vals.pop('$try', False)
-        addprops = vals.pop('$props', None)
-
-        if not vals:
-            mesg = f'No values provided for form {form.full}'
-            raise s_exc.BadTypeValu(mesg=mesg)
-
-        for name, valu in list(props.items()):
-            try:
-                props[name] = form.reqProp(name).type.norm(valu)
-            except s_exc.BadTypeValu as e:
-                mesg = e.get('mesg')
-                e.update({
-                    'prop': name,
-                    'form': form.name,
-                    'mesg': f'Bad value for prop {form.name}:{name}: {mesg}',
-                })
-                raise e
-
-        if addprops is not None:
-            for name, valu in addprops.items():
-                try:
-                    props[name] = form.reqProp(name).type.norm(valu)
-                except s_exc.BadTypeValu as e:
-                    mesg = e.get("mesg")
-                    if not trycast:
-                        e.update({
-                            'prop': name,
-                            'form': form.name,
-                            'mesg': f'Bad value for prop {form.name}:{name}: {mesg}'
-                        })
-                        raise e
-                    await self.warn(f'Skipping bad value for prop {form.name}:{name}: {mesg}')
+        for name, info in norms.items():
+            if info[0].isform:
+                valu = await self._addGuidNodeByDict(*info)
+                norms[name] = (form.prop(name), valu, {})
 
-        norms, proplist = self._normGuidNodeDict(form, vals)
+        for name, info in props.items():
+            if info[0].isform:
+                valu = await self._addGuidNodeByDict(*info)
+                props[name] = (form.prop(name), valu, {})
 
-        iden = s_common.guid(proplist)
-        node = await self._getGuidNodeByNorms(form, iden, norms)
+        node = await self._getGuidNodeByNorms(form, norms)
 
         async with self.getEditor() as editor:
 
             if node is not None:
                 proto = editor.loadNode(node)
             else:
-                proto = await editor.addNode(form.name, iden)
+                proplist = [(name, info[1]) for name, info in norms.items()]
+                proplist.sort()
+
+                proto = await editor.addNode(form.name, proplist)
                 for name, (prop, valu, info) in norms.items():
                     await proto.set(name, valu, norminfo=info)
 
             # ensure the non-deconf props are set
-            for name, (valu, info) in props.items():
+            for name, (prop, valu, info) in props.items():
                 await proto.set(name, valu, norminfo=info)
 
-        return await self.getNodeByBuid(proto.buid)
+        return proto.valu
+
+    async def _normGuidNodeDict(self, form, vals, props=None):
+
+        if props is None:
+            props = {}
+
+        trycast = vals.pop('$try', False)
+        addprops = vals.pop('$props', None)
+
+        if not vals:
+            mesg = f'No values provided for form {form.full}'
+            raise s_exc.BadTypeValu(mesg=mesg)
+
+        props |= await self._normGuidNodeProps(form, props)
 
-    def _normGuidNodeDict(self, form, props):
+        if addprops:
+            props |= await self._normGuidNodeProps(form, addprops, trycast=trycast)
+
+        norms = await self._normGuidNodeProps(form, vals)
+
+        return norms, props
+
+    async def _normGuidNodeProps(self, form, props, trycast=False):
 
         norms = {}
-        proplist = []
 
-        for name, valu in props.items():
+        for name, valu in list(props.items()):
+            prop = form.reqProp(name)
+
+            if isinstance(valu, dict) and isinstance(prop.type, s_types.Guid):
+                pform = self.core.model.reqForm(prop.type.name)
+                gnorm, gprop = await self._normGuidNodeDict(pform, valu)
+                norms[name] = (pform, gnorm, gprop)
+                continue
 
             try:
-                prop = form.reqProp(name)
-                norm, norminfo = prop.type.norm(valu)
+                norms[name] = (prop, *prop.type.norm(valu))
 
-                norms[name] = (prop, norm, norminfo)
-                proplist.append((name, norm))
             except s_exc.BadTypeValu as e:
-                mesg = e.get('mesg')
-                e.update({
-                    'prop': name,
-                    'form': form.name,
-                    'mesg': f'Bad value for prop {form.name}:{name}: {mesg}',
-                })
-                raise e
-
-        proplist.sort()
+                if not trycast:
+                    if 'prop' not in e.errinfo:
+                        mesg = e.get('mesg')
+                        e.update({
+                            'prop': name,
+                            'form': form.name,
+                            'mesg': f'Bad value for prop {form.name}:{name}: {mesg}',
+                        })
+                    raise e
 
-        return norms, proplist
+        return norms
 
     async def _getGuidNodeByDict(self, form, props):
-        norms, proplist = self._normGuidNodeDict(form, props)
-        return await self._getGuidNodeByNorms(form, s_common.guid(proplist), norms)
+        norms, _ = await self._normGuidNodeDict(form, props)
+        return await self._getGuidNodeByNorms(form, norms)
+
+    async def _getGuidNodeByNorms(self, form, norms):
 
-    async def _getGuidNodeByNorms(self, form, iden, norms):
+        proplist = []
+        for name, info in norms.items():
+            if info[0].isform:
+                if (node := await self._getGuidNodeByNorms(*info[:2])) is None:
+                    return
+                valu = node.ndef[1]
+                norms[name] = (form.prop(name), valu, {})
+                proplist.append((name, valu))
+            else:
+                proplist.append((name, info[1]))
 
         # check first for an exact match via our same deconf strategy
+        proplist.sort()
 
-        node = await self.getNodeByNdef((form.full, iden))
+        node = await self.getNodeByNdef((form.full, s_common.guid(proplist)))
         if node is not None:
 
             # ensure we still match the property deconf criteria

synapse/lib/storm.py

@@ -1454,7 +1454,8 @@ class StormDmon(s_base.Base):
         viewiden = opts.get('view')
 
         info = {'iden': self.iden, 'name': self.ddef.get('name', 'storm dmon'), 'view': viewiden}
-        await self.core.boss.promote('storm:dmon', user=self.user, info=info)
+
+        await self.core.boss.promote('storm:dmon', user=self.user, info=info, background=True)
 
         def dmonPrint(evnt):
             self._runLogAdd(evnt)

synapse/lib/stormlib/imap.py

@@ -4,6 +4,7 @@ import aioimaplib
 
 import synapse.exc as s_exc
 import synapse.common as s_common
+import synapse.lib.coro as s_coro
 import synapse.lib.stormtypes as s_stormtypes
 
 async def run_imap_coro(coro):
@@ -88,8 +89,8 @@ class ImapLib(s_stormtypes.Lib):
             imap_cli = aioimaplib.IMAP4(host=host, port=port, timeout=timeout)
 
         async def fini():
-            # call protocol.logout() so fini() doesn't hang
-            await s_common.wait_for(imap_cli.protocol.logout(), 5)
+            # call protocol.logout() via a background task
+            s_coro.create_task(s_common.wait_for(imap_cli.protocol.logout(), 5))
 
         self.runt.snap.onfini(fini)
 

synapse/lib/stormlib/spooled.py

@@ -37,6 +37,7 @@ class LibSpooled(s_stormtypes.Lib):
     async def _methSet(self, *vals):
         core = self.runt.snap.core
         spool = await s_spooled.Set.anit(dirn=core.dirn, cell=core, size=1000)
+        self.runt.snap.onfini(spool)
 
         valu = list(vals)
         for item in valu:

synapse/lib/task.py

@@ -22,6 +22,7 @@ class Task(s_base.Base):
             info = {}
 
         self.boss = boss
+        self.background = False
 
         task._syn_task = self
 

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 213, 0)
+version = (2, 215, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'e3d843a470b137e9d2094b69a7878101afb4f8b1'
+commit = 'f4d48839f7e5f74bacdcc2d1b122d9ade6f1def7'

synapse/models/inet.py

@@ -1780,6 +1780,8 @@ class InetModule(s_module.CoreModule):
                             ('remover', ('inet:service:account', {}), {
                                 'doc': 'The service account which removed or decommissioned the {service:base}.'}),
 
+                            ('app', ('inet:service:app', {}), {
+                                'doc': 'The app which contains the {service:base}.'}),
                         ),
                     }),
 
@@ -3748,6 +3750,9 @@ class InetModule(s_module.CoreModule):
 
                         ('tenant', ('inet:service:tenant', {}), {
                             'doc': 'The tenant which contains the instance.'}),
+
+                        ('app', ('inet:service:app', {}), {
+                            'doc': 'The app which contains the instance.'}),
                     )),
 
                     ('inet:service:app', {}, (

synapse/models/infotech.py

@@ -849,6 +849,14 @@ class ItModule(s_module.CoreModule):
                 ('it:adid', ('str', {'lower': True, 'strip': True}), {
                     'doc': 'An advertising identification string.'}),
 
+                # https://learn.microsoft.com/en-us/windows-hardware/drivers/install/hklm-system-currentcontrolset-services-registry-tree
+                ('it:os:windows:service', ('guid', {}), {
+                    'doc': 'A Microsoft Windows service configuration on a host.'}),
+
+                # TODO
+                # ('it:os:windows:task', ('guid', {}), {
+                #     'doc': 'A Microsoft Windows scheduled task configuration.'}),
+
                 # https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-dtyp/c92a27b1-c772-4fa7-a432-15df5f1b66a1
                 ('it:os:windows:sid', ('str', {'regex': r'^S-1-(?:\d{1,10}|0x[0-9a-fA-F]{12})(?:-(?:\d+|0x[0-9a-fA-F]{2,}))*$'}), {
                     'doc': 'A Microsoft Windows Security Identifier.',
@@ -2582,7 +2590,44 @@ class ItModule(s_module.CoreModule):
                     ('sandbox:file', ('file:bytes', {}), {
                         'doc': 'The initial sample given to a sandbox environment to analyze.'
                     }),
+
+                    # TODO
+                    # ('windows:task', ('it:os:windows:task', {}), {
+                    #     'doc': 'The Microsoft Windows scheduled task responsible for starting the process.'}),
+
+                    ('windows:service', ('it:os:windows:service', {}), {
+                        'doc': 'The Microsoft Windows service responsible for starting the process.'}),
+                )),
+
+                ('it:os:windows:service', {}, (
+
+                    ('host', ('it:host', {}), {
+                        'doc': 'The host that the service was configured on.'}),
+
+                    ('name', ('str', {'lower': True, 'onespace': True}), {
+                        'doc': 'The name of the service from the registry key within Services.'}),
+
+                    # TODO flags...
+                    ('type', ('int', {'min': 0}), {
+                        'doc': 'The type of service from the Type registry key.'}),
+
+                    ('start', ('int', {'min': 0}), {
+                        'doc': 'The start configuration of the service from the Start registry key.'}),
+
+                    ('errorcontrol', ('int', {'min': 0}), {
+                        'doc': 'The service error handling behavior from the ErrorControl registry key.'}),
+
+                    ('displayname', ('str', {'lower': True, 'onespace': True}), {
+                        'doc': 'The friendly name of the service from the DisplayName registry key.'}),
+
+                    # TODO 3.0 text
+                    ('description', ('str', {}), {
+                        'doc': 'The description of the service from the Description registry key.'}),
+
+                    ('imagepath', ('file:path', {}), {
+                        'doc': 'The path to the service binary from the ImagePath registry key.'}),
                 )),
+
                 ('it:query', {}, ()),
                 ('it:exec:query', {}, (
 

synapse/tests/files/testpkg_build_docs/docs/bar.rst

@@ -0,0 +1,15 @@
+:tocdepth: 2
+
+:orphan:
+
+.. highlight:: none
+.. storm-cortex:: default
+
+Bar
+---
+
+A wild node appears!
+
+.. storm-cli:: [inet:asn=1]
+
+Words!

synapse/tests/files/testpkg_build_docs/docs/foo.rst

@@ -0,0 +1,4 @@
+Hello world
+-----------
+
+Words

synapse/tests/files/testpkg_build_docs/storm/testcmd.storm

@@ -0,0 +1,5 @@
+$lib.debug = $cmdopts.debug
+
+$lib.print('Testcmd!')
+
+if $lib.debug { $lib.print('Debug enabled!') }

synapse/tests/files/testpkg_build_docs/testpkg.yaml

@@ -0,0 +1,69 @@
+name: testpkg
+version: 1.2.3
+synapse_version: ">=2.64.0,<3.0.0"
+
+genopts:
+  dotstorm: true
+
+author:
+  url: https://vertex.link
+  name: The Vertex Project, LLC.
+
+desc: The test package does things
+
+docs:
+    - title: Foo
+      path: docs/_build/foo.md
+    - title: Bar
+      path: docs/_build/bar.md
+    - title: Package Documentation
+      path: docs/_build/stormpackage.md
+
+modules:
+  - name: testmod
+  - name: apimod
+    apidefs:
+      - name: search
+        desc: |
+          Execute a search
+          
+          This API will foo the bar.
+          
+          Examples:
+              Foo the ``bar``::
+          
+                yield $lib.import(apimod).search(bar)
+          
+              Baz the bam::
+          
+                yield $lib.import(apimod).search(bam)
+
+        type:
+          type: function
+          args:
+            - { name: text, type: str, desc: "The text." }
+            - { name: mintime, type: [str, int], desc: "The mintime.", default: "-30days" }
+            - { name: foo, type: str, desc: "The foo." }
+            - { name: bar, type: str, desc: "The bar." }
+            - { name: baz, type: str, desc: "The baz." }
+          returns:
+            name: yields
+            type: node
+            desc: Yields it:dev:str nodes.
+      - name: status
+        desc: Get the status of the foo.
+        type:
+          type: function
+          returns:
+            type: dict
+            desc: A status dictionary.
+
+commands:
+  - name: testcmd
+    descr: |
+      A testcmd!
+    cmdargs:
+      - - --debug
+        - default: false
+          action: store_true
+          help: Show verbose debug output.

synapse/tests/test_cortex.py

@@ -5785,6 +5785,12 @@ class CortexBasicTest(s_t_utils.SynTest):
                     ddef = await core01.callStorm('return($lib.dmon.get($iden))', opts=opts)
                     self.none(ddef)
 
+                    await core00.callStorm('queue.del hehe')
+                    await core01.sync()
+
+                    self.none(await core00.getAuthGate('queue:hehe'))
+                    self.none(await core01.getAuthGate('queue:hehe'))
+
             # now lets start up in the opposite order...
             async with self.getTestCore(dirn=path01, conf=core01conf) as core01:
 
@@ -5889,7 +5895,7 @@ class CortexBasicTest(s_t_utils.SynTest):
 
                     # consumer offline
                     await asyncio.sleep(0)
-                    await self.asyncraises(ConnectionRefusedError, core00.callStorm(strim, opts=opts))
+                    await self.asyncraises(s_exc.LinkErr, core00.callStorm(strim, opts=opts))
 
                     # admin can still cull and break the mirror
                     await core00.nodes('[ inet:ipv4=127.0.0.1/28 ]')
@@ -8478,6 +8484,14 @@ class CortexBasicTest(s_t_utils.SynTest):
                     self.eq(msgs[1].get('hash'), qhash)
                     self.eq(msgs[1].get('pool:from'), f'00.core.{ahanet}')
 
+                    with self.getLoggerStream('synapse') as stream:
+                        core01.boss.is_shutdown = True
+                        self.stormHasNoWarnErr(await core00.stormlist('inet:asn=0'))
+                        core01.boss.is_shutdown = False
+
+                    stream.seek(0)
+                    self.isin('Proxy for pool mirror [01.core.synapse] is shutting down. Skipping.', stream.read())
+
                     with patch('synapse.cortex.CoreApi.getNexsIndx', _hang):
 
                         with self.getLoggerStream('synapse') as stream:
@@ -8761,3 +8775,8 @@ class CortexBasicTest(s_t_utils.SynTest):
                 self.eq(core._test_post_service_storage_index, offs)
                 self.eq(core._test_pre_nexus_index, offs)
                 self.ge(core._test_post_nexus_index, core._test_pre_nexus_index)
+
+    async def test_cortex_queue_mirror_authgates(self):
+        async with self.getRegrCore('2.213.0-queue-authgates') as core:
+            self.nn(await core.getAuthGate('queue:stillhere'))
+            self.none(await core.getAuthGate('queue:authtest'))