synapse 2.178.0__py311-none-any.whl → 2.180.0__py311-none-any.whl

synapse/lib/schemas.py

@@ -274,6 +274,7 @@ reqValidSslCtxOpts = s_config.getJsValidator({
         'verify': {'type': 'boolean', 'default': True},
         'client_cert': {'type': ['string', 'null'], 'default': None},
         'client_key': {'type': ['string', 'null'], 'default': None},
+        'ca_cert': {'type': ['string', 'null'], 'default': None},
     },
     'additionalProperties': False,
 })
@@ -454,3 +455,99 @@ tabularConfSchema = {
 }
 
 reqValidTabularConf = s_config.getJsValidator(tabularConfSchema)
+
+emptySchema = {'object': {}, 'additionalProperties': False}
+re_drivename = r'^[\w_.-]{1,128}$'
+
+driveInfoSchema = {
+    'type': 'object',
+    'properties': {
+        'iden': {'type': 'string', 'pattern': s_config.re_iden},
+        'parent': {'type': 'string', 'pattern': s_config.re_iden},
+        'type': {'type': 'string', 'pattern': re_drivename},
+        'name': {'type': 'string', 'pattern': re_drivename},
+        'perm': s_msgpack.deepcopy(easyPermSchema),
+        'kids': {'type': 'number', 'minimum': 0},
+        'created': {'type': 'number'},
+        'creator': {'type': 'string', 'pattern': s_config.re_iden},
+        # these are also data version info...
+        'size': {'type': 'number', 'minimum': 0},
+        'updated': {'type': 'number'},
+        'updater': {'type': 'string', 'pattern': s_config.re_iden},
+        'version': {'type': 'array', 'items': {'type': 'number', 'minItems': 3, 'maxItems': 3}},
+    },
+    'required': ('iden', 'parent', 'name', 'created', 'creator', 'kids'),
+    'additionalProperties': False,
+}
+reqValidDriveInfo = s_config.getJsValidator(driveInfoSchema)
+
+driveDataVersSchema = {
+    'type': 'object',
+    'properties': {
+        'size': {'type': 'number', 'minimum': 0},
+        'updated': {'type': 'number'},
+        'updater': {'type': 'string', 'pattern': s_config.re_iden},
+        'version': {'type': 'array', 'items': {'type': 'number', 'minItems': 3, 'maxItems': 3}},
+    },
+    'required': ('size', 'version', 'updated', 'updater'),
+    'additionalProperties': False,
+}
+reqValidDriveDataVers = s_config.getJsValidator(driveDataVersSchema)
+
+stixIngestConfigSchema = {
+    'type': 'object',
+    'properties': {
+        'bundle': {
+            'type': ['object', 'null'],
+            'properties': {'storm': {'type': 'string'}},
+        },
+        'objects': {
+            'type': 'object',
+            'properties': {'storm': {'type': 'string'}},
+        },
+        'relationships': {
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                    'type': {
+                        'type': 'array',
+                        'items': {
+                            'type': ['string', 'null'],
+                            'minItems': 3,
+                            'maxItems': 3,
+                        },
+                    },
+                    'storm': {'type': 'string'},
+                },
+                'required': ['type'],
+            },
+        },
+    },
+    'required': ['bundle', 'objects'],
+}
+reqValidStixIngestConfig = s_config.getJsValidator(stixIngestConfigSchema)
+
+stixIngestBundleSchema = {
+    'type': 'object',
+    'properties': {
+        'objects': {
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                    'id': {'type': 'string'},
+                    'type': {'type': 'string'},
+                    'object_refs': {'type': 'array', 'items': {'type': 'string'}},
+                    'relationship_type': {'type': 'string'},
+                    'source_ref': {'type': 'string'},
+                    'target_ref': {'type': 'string'},
+                },
+                'required': ['id', 'type'],
+                'if': {'properties': {'type': {'const': 'relationship'}}},
+                'then': {'required': ['source_ref', 'target_ref']},
+            }
+        },
+    },
+}
+reqValidStixIngestBundle = s_config.getJsValidator(stixIngestBundleSchema)

synapse/lib/snap.py

@@ -316,6 +316,10 @@ class ProtoNode:
             mesg = f'Tagprop {name} does not exist in this Cortex.'
             return await self.ctx.snap._raiseOnStrict(s_exc.NoSuchTagProp, mesg)
 
+        if prop.locked:
+            mesg = f'Tagprop {name} is locked.'
+            return await self.ctx.snap._raiseOnStrict(s_exc.IsDeprLocked, mesg)
+
         try:
             norm, info = prop.type.norm(valu)
         except s_exc.BadTypeValu as e:
@@ -732,6 +736,7 @@ class Snap(s_base.Base):
 
         dorepr = False
         dopath = False
+        dolink = False
 
         show_storage = False
 
@@ -750,6 +755,7 @@ class Snap(s_base.Base):
         if opts is not None:
             dorepr = opts.get('repr', False)
             dopath = opts.get('path', False)
+            dolink = opts.get('links', False)
             show_storage = opts.get('show:storage', False)
 
         async for node, path in self.storm(text, opts=opts, user=user):
@@ -757,6 +763,9 @@ class Snap(s_base.Base):
             pode = node.pack(dorepr=dorepr)
             pode[1]['path'] = await path.pack(path=dopath)
 
+            if dolink:
+                pode[1]['links'] = path.links
+
             if show_storage:
                 pode[1]['storage'] = await node.getStorNodes()
 
@@ -1029,7 +1038,7 @@ class Snap(s_base.Base):
             if node is not None:
                 yield node
 
-    async def nodesByPropValu(self, full, cmpr, valu, reverse=False):
+    async def nodesByPropValu(self, full, cmpr, valu, reverse=False, norm=True):
         if cmpr == 'type=':
             if reverse:
                 async for node in self.nodesByPropTypeValu(full, valu, reverse=reverse):
@@ -1050,10 +1059,13 @@ class Snap(s_base.Base):
             mesg = f'No property named "{full}".'
             raise s_exc.NoSuchProp(mesg=mesg)
 
-        cmprvals = prop.type.getStorCmprs(cmpr, valu)
-        # an empty return probably means ?= with invalid value
-        if not cmprvals:
-            return
+        if norm:
+            cmprvals = prop.type.getStorCmprs(cmpr, valu)
+            # an empty return probably means ?= with invalid value
+            if not cmprvals:
+                return
+        else:
+            cmprvals = ((cmpr, valu, prop.type.stortype),)
 
         if prop.isrunt:
             for storcmpr, storvalu, _ in cmprvals:
@@ -1108,7 +1120,7 @@ class Snap(s_base.Base):
             async for node in self.nodesByPropArray(prop.full, '=', valu, reverse=reverse):
                 yield node
 
-    async def nodesByPropArray(self, full, cmpr, valu, reverse=False):
+    async def nodesByPropArray(self, full, cmpr, valu, reverse=False, norm=True):
 
         prop = self.core.model.prop(full)
         if prop is None:
@@ -1119,7 +1131,10 @@ class Snap(s_base.Base):
             mesg = f'Array syntax is invalid on non array type: {prop.type.name}.'
             raise s_exc.BadTypeValu(mesg=mesg)
 
-        cmprvals = prop.type.arraytype.getStorCmprs(cmpr, valu)
+        if norm:
+            cmprvals = prop.type.arraytype.getStorCmprs(cmpr, valu)
+        else:
+            cmprvals = ((cmpr, valu, prop.type.arraytype.stortype),)
 
         if prop.isform:
             async for (buid, sodes) in self.core._liftByPropArray(prop.name, None, cmprvals, self.layers, reverse=reverse):
@@ -1602,18 +1617,28 @@ class Snap(s_base.Base):
             last = verb
             yield verb
 
-    async def getNdefRefs(self, buid):
+    async def _getLayrNdefProp(self, layr, buid):
+        async for refsbuid, refsabrv in layr.getNdefRefs(buid):
+            yield refsbuid, layr.getAbrvProp(refsabrv)
+
+    async def getNdefRefs(self, buid, props=False):
         last = None
-        gens = [layr.getNdefRefs(buid) for layr in self.layers]
+        if props:
+            gens = [self._getLayrNdefProp(layr, buid) for layr in self.layers]
+        else:
+            gens = [layr.getNdefRefs(buid) for layr in self.layers]
 
-        async for refsbuid, _ in s_common.merggenr2(gens):
+        async for refsbuid, xtra in s_common.merggenr2(gens):
             if refsbuid == last:
                 continue
 
             await asyncio.sleep(0)
             last = refsbuid
 
-            yield refsbuid
+            if props:
+                yield refsbuid, xtra[1]
+            else:
+                yield refsbuid
 
     async def hasNodeData(self, buid, name):
         '''

synapse/lib/storm.py

@@ -53,7 +53,9 @@ When condition is tag:add or tag:del, you may optionally provide a form name
 to restrict the trigger to fire only on tags added or deleted from nodes of
 those forms.
 
-The added tag is provided to the query as an embedded variable '$tag'.
+The added tag is provided to the query in the ``$auto`` dictionary variable under
+``$auto.opts.tag``. Usage of the ``$tag`` variable is deprecated and it will no longer
+be populated in Synapse v3.0.0.
 
 Simple one level tag globbing is supported, only at the end after a period,
 that is aka.* matches aka.foo and aka.bar but not aka.foo.bar. aka* is not
@@ -5375,7 +5377,7 @@ class TeeCmd(Cmd):
 
                     outq = asyncio.Queue(maxsize=outq_size)
                     for subr in runts:
-                        subg = s_common.agen((node, path.fork(node)))
+                        subg = s_common.agen((node, path.fork(node, None)))
                         self.runt.snap.schedCoro(self.pipeline(subr, outq, genr=subg))
 
                     exited = 0
@@ -5397,7 +5399,7 @@ class TeeCmd(Cmd):
                 else:
 
                     for subr in runts:
-                        subg = s_common.agen((node, path.fork(node)))
+                        subg = s_common.agen((node, path.fork(node, None)))
                         async for subitem in subr.execute(genr=subg):
                             yield subitem
 
@@ -5551,6 +5553,7 @@ class ScrapeCmd(Cmd):
             if not todo:
                 todo = list(node.props.values())
 
+            link = {'type': 'scrape'}
             for text in todo:
 
                 text = str(text)
@@ -5560,7 +5563,7 @@ class ScrapeCmd(Cmd):
                         continue
 
                     nnode = await node.snap.addNode(form, valu)
-                    npath = path.fork(nnode)
+                    npath = path.fork(nnode, link)
 
                     if refs:
                         if node.form.isrunt:
@@ -5664,8 +5667,9 @@ class LiftByVerb(Cmd):
 
                     yield _node, _path
 
+                    link = {'type': 'runtime'}
                     async for node in self.iterEdgeNodes(verb, idenset, n2):
-                        yield node, _path.fork(node)
+                        yield node, _path.fork(node, link)
 
 class EdgesDelCmd(Cmd):
     '''

synapse/lib/stormhttp.py

@@ -95,6 +95,7 @@ class LibHttp(s_stormtypes.Lib):
             'verify': <bool> - Perform SSL/TLS verification. Is overridden by the ssl_verify argument.
             'client_cert': <str> - PEM encoded full chain certificate for use in mTLS.
             'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
+            'ca_cert': <str> - A PEM encoded full chain CA certificate for use when verifying the request.
         }
     '''
     _storm_locals = (

synapse/lib/stormlib/modelext.py

@@ -1,4 +1,5 @@
 import synapse.exc as s_exc
+import synapse.common as s_common
 import synapse.lib.grammar as s_grammar
 
 import synapse.lib.stormtypes as s_stormtypes
@@ -54,6 +55,8 @@ class LibModelExt(s_stormtypes.Lib):
                   'args': (
                       {'name': 'formname', 'type': 'str', 'desc': 'The form with the extended property.', },
                       {'name': 'propname', 'type': 'str', 'desc': 'The extended property to remove.', },
+                      {'name': 'force', 'type': 'boolean', 'default': False,
+                       'desc': 'Delete the property from all nodes before removing the definition.', },
                   ),
                   'returns': {'type': 'null', }}},
         {'name': 'delUnivProp',
@@ -61,12 +64,16 @@ class LibModelExt(s_stormtypes.Lib):
          'type': {'type': 'function', '_funcname': 'delUnivProp',
                   'args': (
                       {'name': 'propname', 'type': 'str', 'desc': 'Name of the universal property to remove.', },
+                      {'name': 'force', 'type': 'boolean', 'default': False,
+                       'desc': 'Delete the property from all nodes before removing the definition.', },
                   ),
                   'returns': {'type': 'null', }}},
         {'name': 'delTagProp', 'desc': 'Remove an extended tag property definition from the model.',
          'type': {'type': 'function', '_funcname': 'delTagProp',
                   'args': (
                       {'name': 'propname', 'type': 'str', 'desc': 'Name of the tag property to remove.', },
+                      {'name': 'force', 'type': 'boolean', 'default': False,
+                       'desc': 'Delete the tag property from all nodes before removing the definition.', },
                   ),
                   'returns': {'type': 'null', }}},
         {'name': 'getExtModel', 'desc': 'Get all extended model elements.',
@@ -163,20 +170,39 @@ class LibModelExt(s_stormtypes.Lib):
         s_stormtypes.confirm(('model', 'form', 'del', formname))
         await self.runt.snap.core.delForm(formname)
 
-    async def delFormProp(self, formname, propname):
+    async def delFormProp(self, formname, propname, force=False):
         formname = await s_stormtypes.tostr(formname)
         propname = await s_stormtypes.tostr(propname)
+        force = await s_stormtypes.tobool(force)
         s_stormtypes.confirm(('model', 'prop', 'del', formname))
+
+        if force is True:
+            meta = {'user': self.runt.snap.user.iden, 'time': s_common.now()}
+            await self.runt.snap.core._delAllFormProp(formname, propname, meta)
+
         await self.runt.snap.core.delFormProp(formname, propname)
 
-    async def delUnivProp(self, propname):
+    async def delUnivProp(self, propname, force=False):
         propname = await s_stormtypes.tostr(propname)
+        force = await s_stormtypes.tobool(force)
         s_stormtypes.confirm(('model', 'univ', 'del'))
+
+        if force:
+            meta = {'user': self.runt.snap.user.iden, 'time': s_common.now()}
+            await self.runt.snap.core._delAllUnivProp(propname, meta)
+
         await self.runt.snap.core.delUnivProp(propname)
 
-    async def delTagProp(self, propname):
+    async def delTagProp(self, propname, force=False):
         propname = await s_stormtypes.tostr(propname)
+        force = await s_stormtypes.tobool(force)
+
         s_stormtypes.confirm(('model', 'tagprop', 'del'))
+
+        if force:
+            meta = {'user': self.runt.snap.user.iden, 'time': s_common.now()}
+            await self.runt.snap.core._delAllTagProp(propname, meta)
+
         await self.runt.snap.core.delTagProp(propname)
 
     @s_stormtypes.stormfunc(readonly=True)

synapse/lib/stormlib/stix.py

@@ -11,6 +11,7 @@ import synapse.common as s_common
 import synapse.lib.coro as s_coro
 import synapse.lib.node as s_node
 import synapse.lib.msgpack as s_msgpack
+import synapse.lib.schemas as s_schemas
 import synapse.lib.stormctrl as s_stormctrl
 import synapse.lib.stormtypes as s_stormtypes
 
@@ -875,14 +876,19 @@ class LibStixImport(s_stormtypes.Lib):
         bundle = await s_stormtypes.toprim(bundle)
         config = await s_stormtypes.toprim(config)
 
+        if not isinstance(config, dict):
+            mesg = 'STIX ingest config must be a dictionary.'
+            raise s_exc.BadArg(mesg=mesg)
+
         config.setdefault('bundle', {})
         config.setdefault('objects', {})
         config.setdefault('relationships', [])
 
-        rellook = {r['type']: r for r in config.get('relationships', ())}
-
-        nodesbyid = {}
-        relationships = []
+        try:
+            rellook = {r['type']: r for r in config.get('relationships', ())}
+        except Exception as e:
+            mesg = f'Error processing relationships in STIX bundle ingest config: {e}.'
+            raise s_exc.BadArg(mesg=mesg)
 
         bundlenode = None
 
@@ -890,13 +896,44 @@ class LibStixImport(s_stormtypes.Lib):
         if bundleconf is None:
             bundleconf = {}
 
+        if not isinstance(bundleconf, dict):
+            mesg = 'STIX ingest config bundle value must be a dictionary.'
+            raise s_exc.BadArg(mesg=mesg)
+
         bundlestorm = bundleconf.get('storm')
         if bundlestorm:
+            if not isinstance(bundlestorm, str):
+                mesg = 'STIX ingest config storm query must be a string.'
+                raise s_exc.BadArg(mesg=mesg)
+
             bundlenode = await self._callStorm(bundlestorm, {'bundle': bundle})
 
         self.runt.layerConfirm(('node', 'edge', 'add', 'refs'))
 
-        for obj in bundle.get('objects'):
+        config = s_schemas.reqValidStixIngestConfig(config)
+        bundle = s_schemas.reqValidStixIngestBundle(bundle)
+
+        try:
+            nodesbyid = await self._ingestObjects(bundle, config, rellook)
+        except Exception as e:
+            mesg = f'Error processing objects in STIX bundle ingest: {e}.'
+            raise s_exc.BadArg(mesg=mesg)
+
+        if bundlenode is not None:
+            for node in nodesbyid.values():
+                await bundlenode.addEdge('refs', node.iden())
+                await asyncio.sleep(0)
+            yield bundlenode
+
+        for node in nodesbyid.values():
+            yield node
+
+    async def _ingestObjects(self, bundle, config, rellook):
+
+        nodesbyid = {}
+        relationships = []
+
+        for obj in bundle.get('objects', ()):
 
             objtype = obj.get('type')
 
@@ -971,7 +1008,7 @@ class LibStixImport(s_stormtypes.Lib):
                 await self.runt.snap.warnonce(f'STIX bundle ingest has no relationship definition for: {reltype}.')
 
         # attempt to resolve object_refs
-        for obj in bundle.get('objects'):
+        for obj in bundle.get('objects', ()):
 
             node = nodesbyid.get(obj.get('id'))
             if node is None:
@@ -984,17 +1021,7 @@ class LibStixImport(s_stormtypes.Lib):
 
                 await node.addEdge('refs', refsnode.iden())
 
-        if bundlenode is not None:
-            for node in nodesbyid.values():
-                await bundlenode.addEdge('refs', node.iden())
-                await asyncio.sleep(0)
-            yield bundlenode
-
-        for node in nodesbyid.values():
-            yield node
-
-        if bundlenode:
-            yield bundlenode
+        return nodesbyid
 
     async def _callStorm(self, text, varz):
 

synapse/lib/stormlib/vault.py

@@ -649,7 +649,7 @@ class Vault(s_stormtypes.Prim):
          'type': {'type': 'function', '_funcname': '_methSetPerm',
                   'args': (
                       {'name': 'iden', 'type': 'str', 'desc': 'The user or role to modify.'},
-                      {'name': 'level', 'type': 'str', 'desc': 'The easyperm level for the iden. $lib.undef to remove an existing permission.'},
+                      {'name': 'level', 'type': 'str', 'desc': 'The easyperm level for the iden. $lib.null to remove an existing permission.'},
                   ),
                   'returns': {'type': 'boolean', 'desc': '$lib.true if the permission was set, $lib.false otherwise.', }}},
 
@@ -744,7 +744,7 @@ class Vault(s_stormtypes.Prim):
         s_stormtypes.confirmEasyPerm(vault, s_cell.PERM_ADMIN, mesg=mesg)
 
         iden = await s_stormtypes.tostr(iden)
-        level = await s_stormtypes.toint(level)
+        level = await s_stormtypes.toint(level, noneok=True)
 
         return await self.runt.snap.core.setVaultPerm(self.valu, iden, level)
 

synapse/lib/stormtypes.py

@@ -5277,7 +5277,7 @@ class Number(Prim):
     def __init__(self, valu, path=None):
         try:
             valu = s_common.hugenum(valu)
-        except decimal.DecimalException as e:
+        except (TypeError, decimal.DecimalException) as e:
             mesg = f'Failed to make number from {valu!r}'
             raise s_exc.BadCast(mesg=mesg) from e
 

synapse/lib/types.py

@@ -1,3 +1,4 @@
+import sys
 import asyncio
 import decimal
 import logging
@@ -78,6 +79,14 @@ class Type:
 
         self.postTypeInit()
 
+        normopts = dict(self.opts)
+        for optn, valu in normopts.items():
+            if isinstance(valu, float):
+                normopts[optn] = str(valu)
+
+        ctor = '.'.join([self.__class__.__module__, self.__class__.__qualname__])
+        self.typehash = sys.intern(s_common.guid((ctor, s_common.flatten(normopts))))
+
     def _storLiftSafe(self, cmpr, valu):
         try:
             return self.storlifts['=']('=', valu)

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 178, 0)
+version = (2, 180, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = '56bad29e241893b702ea437d500a0229e8e1956b'
+commit = 'aecbb8c0839f3ad763076dd7acae968fae6ba8b7'