synapse 2.177.0__py311-none-any.whl → 2.179.0__py311-none-any.whl

synapse/lib/nexus.py

@@ -91,7 +91,6 @@ class NexsRoot(s_base.Base):
         self.writeholds = set()
 
         self.applytask = None
-        self.applylock = asyncio.Lock()
 
         self.ready = asyncio.Event()
         self.donexslog = self.cell.conf.get('nexslog:en')
@@ -110,9 +109,7 @@ class NexsRoot(s_base.Base):
 
         logpath = s_common.genpath(self.dirn, 'slabs', 'nexuslog')
 
-        self.map_async = self.cell.conf.get('nexslog:async')
-        self.nexsslab = await s_lmdbslab.Slab.anit(path, map_async=self.map_async)
-        self.nexsslab.addResizeCallback(cell.checkFreeSpace)
+        self.nexsslab = await cell._initSlabFile(path)
 
         self.nexshot = await self.nexsslab.getHotCount('nexs:indx')
 
@@ -126,8 +123,7 @@ class NexsRoot(s_base.Base):
         elif vers != 2:
             raise s_exc.BadStorageVersion(mesg=f'Got nexus log version {vers}.  Expected 2.  Accidental downgrade?')
 
-        slabopts = {'map_async': self.map_async}
-        self.nexslog = await s_multislabseqn.MultiSlabSeqn.anit(logpath, slabopts=slabopts, cell=cell)
+        self.nexslog = await s_multislabseqn.MultiSlabSeqn.anit(logpath, cell=cell)
 
         # just in case were previously configured differently
         logindx = self.nexslog.index()
@@ -147,6 +143,9 @@ class NexsRoot(s_base.Base):
 
         self.onfini(fini)
 
+    def getNexsKids(self):
+        return list(self._nexskids.values())
+
     async def _migrateV1toV2(self, nexspath, logpath):
         '''
         Close the slab, move it to the new multislab location, then copy out the nexshot
@@ -195,8 +194,7 @@ class NexsRoot(s_base.Base):
 
         # Open a fresh slab where the old one used to be
         logger.warning(f'Re-opening fresh nexslog slab at {nexspath} for nexshot')
-        self.nexsslab = await s_lmdbslab.Slab.anit(nexspath, map_async=self.map_async)
-        self.nexsslab.addResizeCallback(self.cell.checkFreeSpace)
+        self.nexsslab = await self.cell._initSlabFile(nexspath)
 
         self.nexshot = await self.nexsslab.getHotCount('nexs:indx')
 
@@ -230,7 +228,7 @@ class NexsRoot(s_base.Base):
 
     async def enNexsLog(self):
 
-        async with self.applylock:
+        async with self.cell.nexslock:
 
             if self.donexslog:
                 return
@@ -309,7 +307,6 @@ class NexsRoot(s_base.Base):
         If I'm not a follower, mutate, otherwise, ask the leader to make the change and wait for the follower loop
         to hand me the result through a future.
         '''
-
         # pick up a reference to avoid race when we eventually can promote
         client = self.client
 
@@ -344,7 +341,7 @@ class NexsRoot(s_base.Base):
         if meta is None:
             meta = {}
 
-        async with self.applylock:
+        async with self.cell.nexslock:
             self.reqNotReadOnly()
             # Keep a reference to the shielded task to ensure it isn't GC'd
             self.applytask = asyncio.create_task(self._eat((nexsiden, event, args, kwargs, meta)))
@@ -577,6 +574,9 @@ class NexsRoot(s_base.Base):
                         if respfutu is not None:
                             respfutu.set_result(retn)
 
+            except s_exc.LinkShutDown:
+                logger.warning(f'mirror loop: leader closed the connection.')
+
             except Exception as exc:  # pragma: no cover
                 logger.exception(f'error in mirror loop: {exc}')
 
@@ -634,9 +634,21 @@ class Pusher(s_base.Base, metaclass=RegMethType):
             assert prev is not None, f'Failed removing {self.nexsiden}'
 
         self.onfini(onfini)
-
         self.nexsroot = nexsroot
 
+    async def modNexsRoot(self, ctor):
+
+        kids = [self]
+        if self.nexsroot is not None:
+            kids = self.nexsroot.getNexsKids()
+            await self.nexsroot.fini()
+
+        nexsroot = await ctor()
+
+        [kid.setNexsRoot(nexsroot) for kid in kids]
+
+        await nexsroot.startup()
+
     @classmethod
     def onPush(cls, event: str, passitem=False) -> Callable:
         '''

synapse/lib/schemas.py

@@ -274,6 +274,7 @@ reqValidSslCtxOpts = s_config.getJsValidator({
         'verify': {'type': 'boolean', 'default': True},
         'client_cert': {'type': ['string', 'null'], 'default': None},
         'client_key': {'type': ['string', 'null'], 'default': None},
+        'ca_cert': {'type': ['string', 'null'], 'default': None},
     },
     'additionalProperties': False,
 })
@@ -454,3 +455,41 @@ tabularConfSchema = {
 }
 
 reqValidTabularConf = s_config.getJsValidator(tabularConfSchema)
+
+emptySchema = {'object': {}, 'additionalProperties': False}
+re_drivename = r'^[\w_.-]{1,128}$'
+
+driveInfoSchema = {
+    'type': 'object',
+    'properties': {
+        'iden': {'type': 'string', 'pattern': s_config.re_iden},
+        'parent': {'type': 'string', 'pattern': s_config.re_iden},
+        'type': {'type': 'string', 'pattern': re_drivename},
+        'name': {'type': 'string', 'pattern': re_drivename},
+        'perm': s_msgpack.deepcopy(easyPermSchema),
+        'kids': {'type': 'number', 'minimum': 0},
+        'created': {'type': 'number'},
+        'creator': {'type': 'string', 'pattern': s_config.re_iden},
+        # these are also data version info...
+        'size': {'type': 'number', 'minimum': 0},
+        'updated': {'type': 'number'},
+        'updater': {'type': 'string', 'pattern': s_config.re_iden},
+        'version': {'type': 'array', 'items': {'type': 'number', 'minItems': 3, 'maxItems': 3}},
+    },
+    'required': ('iden', 'parent', 'name', 'created', 'creator', 'kids'),
+    'additionalProperties': False,
+}
+reqValidDriveInfo = s_config.getJsValidator(driveInfoSchema)
+
+driveDataVersSchema = {
+    'type': 'object',
+    'properties': {
+        'size': {'type': 'number', 'minimum': 0},
+        'updated': {'type': 'number'},
+        'updater': {'type': 'string', 'pattern': s_config.re_iden},
+        'version': {'type': 'array', 'items': {'type': 'number', 'minItems': 3, 'maxItems': 3}},
+    },
+    'required': ('size', 'version', 'updated', 'updater'),
+    'additionalProperties': False,
+}
+reqValidDriveDataVers = s_config.getJsValidator(driveDataVersSchema)

synapse/lib/snap.py

@@ -316,6 +316,10 @@ class ProtoNode:
             mesg = f'Tagprop {name} does not exist in this Cortex.'
             return await self.ctx.snap._raiseOnStrict(s_exc.NoSuchTagProp, mesg)
 
+        if prop.locked:
+            mesg = f'Tagprop {name} is locked.'
+            return await self.ctx.snap._raiseOnStrict(s_exc.IsDeprLocked, mesg)
+
         try:
             norm, info = prop.type.norm(valu)
         except s_exc.BadTypeValu as e:
@@ -1029,7 +1033,7 @@ class Snap(s_base.Base):
             if node is not None:
                 yield node
 
-    async def nodesByPropValu(self, full, cmpr, valu, reverse=False):
+    async def nodesByPropValu(self, full, cmpr, valu, reverse=False, norm=True):
         if cmpr == 'type=':
             if reverse:
                 async for node in self.nodesByPropTypeValu(full, valu, reverse=reverse):
@@ -1050,10 +1054,13 @@ class Snap(s_base.Base):
             mesg = f'No property named "{full}".'
             raise s_exc.NoSuchProp(mesg=mesg)
 
-        cmprvals = prop.type.getStorCmprs(cmpr, valu)
-        # an empty return probably means ?= with invalid value
-        if not cmprvals:
-            return
+        if norm:
+            cmprvals = prop.type.getStorCmprs(cmpr, valu)
+            # an empty return probably means ?= with invalid value
+            if not cmprvals:
+                return
+        else:
+            cmprvals = ((cmpr, valu, prop.type.stortype),)
 
         if prop.isrunt:
             for storcmpr, storvalu, _ in cmprvals:
@@ -1108,7 +1115,7 @@ class Snap(s_base.Base):
             async for node in self.nodesByPropArray(prop.full, '=', valu, reverse=reverse):
                 yield node
 
-    async def nodesByPropArray(self, full, cmpr, valu, reverse=False):
+    async def nodesByPropArray(self, full, cmpr, valu, reverse=False, norm=True):
 
         prop = self.core.model.prop(full)
         if prop is None:
@@ -1119,7 +1126,10 @@ class Snap(s_base.Base):
             mesg = f'Array syntax is invalid on non array type: {prop.type.name}.'
             raise s_exc.BadTypeValu(mesg=mesg)
 
-        cmprvals = prop.type.arraytype.getStorCmprs(cmpr, valu)
+        if norm:
+            cmprvals = prop.type.arraytype.getStorCmprs(cmpr, valu)
+        else:
+            cmprvals = ((cmpr, valu, prop.type.arraytype.stortype),)
 
         if prop.isform:
             async for (buid, sodes) in self.core._liftByPropArray(prop.name, None, cmprvals, self.layers, reverse=reverse):

synapse/lib/storm.py

@@ -53,7 +53,9 @@ When condition is tag:add or tag:del, you may optionally provide a form name
 to restrict the trigger to fire only on tags added or deleted from nodes of
 those forms.
 
-The added tag is provided to the query as an embedded variable '$tag'.
+The added tag is provided to the query in the ``$auto`` dictionary variable under
+``$auto.opts.tag``. Usage of the ``$tag`` variable is deprecated and it will no longer
+be populated in Synapse v3.0.0.
 
 Simple one level tag globbing is supported, only at the end after a period,
 that is aka.* matches aka.foo and aka.bar but not aka.foo.bar. aka* is not

synapse/lib/stormhttp.py

@@ -95,6 +95,7 @@ class LibHttp(s_stormtypes.Lib):
             'verify': <bool> - Perform SSL/TLS verification. Is overridden by the ssl_verify argument.
             'client_cert': <str> - PEM encoded full chain certificate for use in mTLS.
             'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
+            'ca_cert': <str> - A PEM encoded full chain CA certificate for use when verifying the request.
         }
     '''
     _storm_locals = (

synapse/lib/stormlib/imap.py

@@ -50,6 +50,8 @@ class ImapLib(s_stormtypes.Lib):
                      'desc': 'The time to wait for all commands on the server to execute.'},
                     {'type': 'bool', 'name': 'ssl', 'default': True,
                      'desc': 'Use SSL to connect to the IMAP server.'},
+                    {'type': 'bool', 'name': 'ssl_verify', 'default': True,
+                     'desc': 'Perform SSL/TLS verification.'},
                 ),
                 'returns': {
                     'type': 'inet:imap:server',
@@ -69,17 +71,19 @@ class ImapLib(s_stormtypes.Lib):
             'connect': self.connect,
         }
 
-    async def connect(self, host, port=993, timeout=30, ssl=True):
+    async def connect(self, host, port=993, timeout=30, ssl=True, ssl_verify=True):
 
         self.runt.confirm(('storm', 'inet', 'imap', 'connect'))
 
         ssl = await s_stormtypes.tobool(ssl)
         host = await s_stormtypes.tostr(host)
         port = await s_stormtypes.toint(port)
+        ssl_verify = await s_stormtypes.tobool(ssl_verify)
         timeout = await s_stormtypes.toint(timeout, noneok=True)
 
         if ssl:
-            imap_cli = aioimaplib.IMAP4_SSL(host=host, port=port, timeout=timeout)
+            ctx = self.runt.snap.core.getCachedSslCtx(opts=None, verify=ssl_verify)
+            imap_cli = aioimaplib.IMAP4_SSL(host=host, port=port, timeout=timeout, ssl_context=ctx)
         else:
             imap_cli = aioimaplib.IMAP4(host=host, port=port, timeout=timeout)
 

synapse/lib/stormlib/modelext.py

@@ -1,4 +1,5 @@
 import synapse.exc as s_exc
+import synapse.common as s_common
 import synapse.lib.grammar as s_grammar
 
 import synapse.lib.stormtypes as s_stormtypes
@@ -54,6 +55,8 @@ class LibModelExt(s_stormtypes.Lib):
                   'args': (
                       {'name': 'formname', 'type': 'str', 'desc': 'The form with the extended property.', },
                       {'name': 'propname', 'type': 'str', 'desc': 'The extended property to remove.', },
+                      {'name': 'force', 'type': 'boolean', 'default': False,
+                       'desc': 'Delete the property from all nodes before removing the definition.', },
                   ),
                   'returns': {'type': 'null', }}},
         {'name': 'delUnivProp',
@@ -61,12 +64,16 @@ class LibModelExt(s_stormtypes.Lib):
          'type': {'type': 'function', '_funcname': 'delUnivProp',
                   'args': (
                       {'name': 'propname', 'type': 'str', 'desc': 'Name of the universal property to remove.', },
+                      {'name': 'force', 'type': 'boolean', 'default': False,
+                       'desc': 'Delete the property from all nodes before removing the definition.', },
                   ),
                   'returns': {'type': 'null', }}},
         {'name': 'delTagProp', 'desc': 'Remove an extended tag property definition from the model.',
          'type': {'type': 'function', '_funcname': 'delTagProp',
                   'args': (
                       {'name': 'propname', 'type': 'str', 'desc': 'Name of the tag property to remove.', },
+                      {'name': 'force', 'type': 'boolean', 'default': False,
+                       'desc': 'Delete the tag property from all nodes before removing the definition.', },
                   ),
                   'returns': {'type': 'null', }}},
         {'name': 'getExtModel', 'desc': 'Get all extended model elements.',
@@ -163,20 +170,39 @@ class LibModelExt(s_stormtypes.Lib):
         s_stormtypes.confirm(('model', 'form', 'del', formname))
         await self.runt.snap.core.delForm(formname)
 
-    async def delFormProp(self, formname, propname):
+    async def delFormProp(self, formname, propname, force=False):
         formname = await s_stormtypes.tostr(formname)
         propname = await s_stormtypes.tostr(propname)
+        force = await s_stormtypes.tobool(force)
         s_stormtypes.confirm(('model', 'prop', 'del', formname))
+
+        if force is True:
+            meta = {'user': self.runt.snap.user.iden, 'time': s_common.now()}
+            await self.runt.snap.core._delAllFormProp(formname, propname, meta)
+
         await self.runt.snap.core.delFormProp(formname, propname)
 
-    async def delUnivProp(self, propname):
+    async def delUnivProp(self, propname, force=False):
         propname = await s_stormtypes.tostr(propname)
+        force = await s_stormtypes.tobool(force)
         s_stormtypes.confirm(('model', 'univ', 'del'))
+
+        if force:
+            meta = {'user': self.runt.snap.user.iden, 'time': s_common.now()}
+            await self.runt.snap.core._delAllUnivProp(propname, meta)
+
         await self.runt.snap.core.delUnivProp(propname)
 
-    async def delTagProp(self, propname):
+    async def delTagProp(self, propname, force=False):
         propname = await s_stormtypes.tostr(propname)
+        force = await s_stormtypes.tobool(force)
+
         s_stormtypes.confirm(('model', 'tagprop', 'del'))
+
+        if force:
+            meta = {'user': self.runt.snap.user.iden, 'time': s_common.now()}
+            await self.runt.snap.core._delAllTagProp(propname, meta)
+
         await self.runt.snap.core.delTagProp(propname)
 
     @s_stormtypes.stormfunc(readonly=True)

synapse/lib/stormlib/smtp.py

@@ -96,6 +96,8 @@ class SmtpMessage(s_stormtypes.StormType):
                         'desc': 'Use the STARTTLS directive with the SMTP server.'},
                     {'name': 'timeout', 'type': 'int', 'default': 60,
                         'desc': 'The timeout (in seconds) to wait for message delivery.'},
+                    {'type': 'bool', 'name': 'ssl_verify', 'default': True,
+                     'desc': 'Perform SSL/TLS verification.'},
                   ),
                   'returns': {'type': 'list', 'desc': 'An ($ok, $valu) tuple.'}}},
 
@@ -148,7 +150,8 @@ class SmtpMessage(s_stormtypes.StormType):
     async def _getEmailHtml(self):
         return self.bodyhtml
 
-    async def send(self, host, port=25, user=None, passwd=None, usetls=False, starttls=False, timeout=60):
+    async def send(self, host, port=25, user=None, passwd=None, usetls=False, starttls=False, timeout=60,
+                   ssl_verify=True):
 
         self.runt.confirm(('storm', 'inet', 'smtp', 'send'))
 
@@ -161,6 +164,7 @@ class SmtpMessage(s_stormtypes.StormType):
             port = await s_stormtypes.toint(port)
             usetls = await s_stormtypes.tobool(usetls)
             starttls = await s_stormtypes.tobool(starttls)
+            ssl_verify = await s_stormtypes.tobool(ssl_verify)
 
             if usetls and starttls:
                 raise s_exc.BadArg(mesg='usetls and starttls are mutually exclusive arguments.')
@@ -183,6 +187,10 @@ class SmtpMessage(s_stormtypes.StormType):
 
             recipients = [await s_stormtypes.tostr(e) for e in self.recipients]
 
+            ctx = None
+            if usetls or starttls:
+                ctx = self.runt.snap.core.getCachedSslCtx(opts=None, verify=ssl_verify)
+
             futu = aiosmtplib.send(message,
                                    port=port,
                                    hostname=host,
@@ -191,7 +199,9 @@ class SmtpMessage(s_stormtypes.StormType):
                                    use_tls=usetls,
                                    start_tls=starttls,
                                    username=user,
-                                   password=passwd)
+                                   password=passwd,
+                                   tls_context=ctx,
+                                   )
 
             await s_common.wait_for(futu, timeout=timeout)
 

synapse/lib/stormlib/stix.py

@@ -875,14 +875,19 @@ class LibStixImport(s_stormtypes.Lib):
         bundle = await s_stormtypes.toprim(bundle)
         config = await s_stormtypes.toprim(config)
 
+        if not isinstance(config, dict):
+            mesg = 'STIX ingest config must be a dictionary.'
+            raise s_exc.BadArg(mesg=mesg)
+
         config.setdefault('bundle', {})
         config.setdefault('objects', {})
         config.setdefault('relationships', [])
 
-        rellook = {r['type']: r for r in config.get('relationships', ())}
-
-        nodesbyid = {}
-        relationships = []
+        try:
+            rellook = {r['type']: r for r in config.get('relationships', ())}
+        except Exception as e:
+            mesg = f'Error processing relationships in STIX bundle ingest config: {e}.'
+            raise s_exc.BadArg(mesg=mesg)
 
         bundlenode = None
 
@@ -890,13 +895,41 @@ class LibStixImport(s_stormtypes.Lib):
         if bundleconf is None:
             bundleconf = {}
 
+        if not isinstance(bundleconf, dict):
+            mesg = 'STIX ingest config bundle value must be a dictionary.'
+            raise s_exc.BadArg(mesg=mesg)
+
         bundlestorm = bundleconf.get('storm')
         if bundlestorm:
+            if not isinstance(bundlestorm, str):
+                mesg = 'STIX ingest config storm query must be a string.'
+                raise s_exc.BadArg(mesg=mesg)
+
             bundlenode = await self._callStorm(bundlestorm, {'bundle': bundle})
 
         self.runt.layerConfirm(('node', 'edge', 'add', 'refs'))
 
-        for obj in bundle.get('objects'):
+        try:
+            nodesbyid = await self._ingestObjects(bundle, config, rellook)
+        except Exception as e:
+            mesg = f'Error processing objects in STIX bundle ingest: {e}.'
+            raise s_exc.BadArg(mesg=mesg)
+
+        if bundlenode is not None:
+            for node in nodesbyid.values():
+                await bundlenode.addEdge('refs', node.iden())
+                await asyncio.sleep(0)
+            yield bundlenode
+
+        for node in nodesbyid.values():
+            yield node
+
+    async def _ingestObjects(self, bundle, config, rellook):
+
+        nodesbyid = {}
+        relationships = []
+
+        for obj in bundle.get('objects', ()):
 
             objtype = obj.get('type')
 
@@ -971,7 +1004,7 @@ class LibStixImport(s_stormtypes.Lib):
                 await self.runt.snap.warnonce(f'STIX bundle ingest has no relationship definition for: {reltype}.')
 
         # attempt to resolve object_refs
-        for obj in bundle.get('objects'):
+        for obj in bundle.get('objects', ()):
 
             node = nodesbyid.get(obj.get('id'))
             if node is None:
@@ -984,17 +1017,7 @@ class LibStixImport(s_stormtypes.Lib):
 
                 await node.addEdge('refs', refsnode.iden())
 
-        if bundlenode is not None:
-            for node in nodesbyid.values():
-                await bundlenode.addEdge('refs', node.iden())
-                await asyncio.sleep(0)
-            yield bundlenode
-
-        for node in nodesbyid.values():
-            yield node
-
-        if bundlenode:
-            yield bundlenode
+        return nodesbyid
 
     async def _callStorm(self, text, varz):
 

synapse/lib/stormlib/vault.py

@@ -649,7 +649,7 @@ class Vault(s_stormtypes.Prim):
          'type': {'type': 'function', '_funcname': '_methSetPerm',
                   'args': (
                       {'name': 'iden', 'type': 'str', 'desc': 'The user or role to modify.'},
-                      {'name': 'level', 'type': 'str', 'desc': 'The easyperm level for the iden. $lib.undef to remove an existing permission.'},
+                      {'name': 'level', 'type': 'str', 'desc': 'The easyperm level for the iden. $lib.null to remove an existing permission.'},
                   ),
                   'returns': {'type': 'boolean', 'desc': '$lib.true if the permission was set, $lib.false otherwise.', }}},
 
@@ -744,7 +744,7 @@ class Vault(s_stormtypes.Prim):
         s_stormtypes.confirmEasyPerm(vault, s_cell.PERM_ADMIN, mesg=mesg)
 
         iden = await s_stormtypes.tostr(iden)
-        level = await s_stormtypes.toint(level)
+        level = await s_stormtypes.toint(level, noneok=True)
 
         return await self.runt.snap.core.setVaultPerm(self.valu, iden, level)
 

synapse/lib/stormtypes.py

@@ -5277,7 +5277,7 @@ class Number(Prim):
     def __init__(self, valu, path=None):
         try:
             valu = s_common.hugenum(valu)
-        except decimal.DecimalException as e:
+        except (TypeError, decimal.DecimalException) as e:
             mesg = f'Failed to make number from {valu!r}'
             raise s_exc.BadCast(mesg=mesg) from e
 

synapse/lib/types.py

@@ -1,3 +1,4 @@
+import sys
 import asyncio
 import decimal
 import logging
@@ -78,6 +79,14 @@ class Type:
 
         self.postTypeInit()
 
+        normopts = dict(self.opts)
+        for optn, valu in normopts.items():
+            if isinstance(valu, float):
+                normopts[optn] = str(valu)
+
+        ctor = '.'.join([self.__class__.__module__, self.__class__.__qualname__])
+        self.typehash = sys.intern(s_common.guid((ctor, s_common.flatten(normopts))))
+
     def _storLiftSafe(self, cmpr, valu):
         try:
             return self.storlifts['=']('=', valu)

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 177, 0)
+version = (2, 179, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'b7da795aeb690020e38e99dc5aa1f505cdc2d659'
+commit = '1c51c58523e9012655d8d18bfa1ad149e5536c25'