synapse 2.197.0__py311-none-any.whl → 2.199.0__py311-none-any.whl

synapse/lib/snap.py

@@ -138,10 +138,30 @@ class ProtoNode:
 
         if not await self.ctx.snap.hasNodeEdge(self.buid, verb, s_common.uhex(n2iden)):
             self.edges.add(tupl)
+            if len(self.edges) >= 1000:
+                await self.flushEdits()
             return True
 
         return False
 
+    async def flushEdits(self):
+        if (nodeedit := self.getNodeEdit()) is not None:
+            nodecache = {self.buid: self.node}
+            nodes = await self.ctx.snap.applyNodeEdits((nodeedit,), nodecache=nodecache, meta=self.ctx.meta)
+
+            if self.node is None:
+                if nodes and nodes[0].buid == self.buid:
+                    self.node = nodes[0]
+                else:  # pragma: no cover
+                    self.node = await self.ctx.snap.getNodeByBuid(self.buid)
+
+            self.tags.clear()
+            self.props.clear()
+            self.tagprops.clear()
+            self.edges.clear()
+            self.edgedels.clear()
+            self.nodedata.clear()
+
     async def delEdge(self, verb, n2iden):
 
         if not isinstance(verb, str):
@@ -169,6 +189,8 @@ class ProtoNode:
 
         if await self.ctx.snap.layers[-1].hasNodeEdge(self.buid, verb, s_common.uhex(n2iden)):
             self.edgedels.add(tupl)
+            if len(self.edgedels) >= 1000:
+                await self.flushEdits()
             return True
 
         return False
@@ -417,6 +439,9 @@ class ProtoNode:
                 full = f'{prop.name}:{subname}'
                 subprop = self.form.props.get(full)
                 if subprop is not None and not subprop.locked:
+                    if subprop.deprecated:
+                        self.ctx.snap._skipPropDeprWarn(subprop.full)
+
                     await self.set(full, subvalu)
 
         propadds = norminfo.get('adds')
@@ -426,11 +451,14 @@ class ProtoNode:
 
         return True
 
-    async def getSetOps(self, name, valu, norminfo=None):
+    async def getSetSubOps(self, name, valu, norminfo=None):
         prop = self.form.props.get(name)
-        if prop is None:
+        if prop is None or prop.locked:
             return ()
 
+        if prop.deprecated:
+            self.ctx.snap._skipPropDeprWarn(prop.full)
+
         retn = await self._set(prop, valu, norminfo=norminfo)
         if retn is False:
             return ()
@@ -447,9 +475,7 @@ class ProtoNode:
         if propsubs is not None:
             for subname, subvalu in propsubs.items():
                 full = f'{prop.name}:{subname}'
-                subprop = self.form.props.get(full)
-                if subprop is not None and not subprop.locked:
-                    ops.append(self.getSetOps(full, subvalu))
+                ops.append(self.getSetSubOps(full, subvalu))
 
         propadds = norminfo.get('adds')
         if propadds is not None:
@@ -462,7 +488,8 @@ class SnapEditor:
     '''
     A SnapEditor allows tracking node edits with subs/deps as a transaction.
     '''
-    def __init__(self, snap):
+    def __init__(self, snap, meta=None):
+        self.meta = meta
         self.snap = snap
         self.protonodes = {}
         self.maxnodes = snap.core.maxnodes
@@ -480,6 +507,19 @@ class SnapEditor:
                 nodeedits.append(nodeedit)
         return nodeedits
 
+    async def flushEdits(self):
+        nodecache = {}
+        nodeedits = []
+        for protonode in self.protonodes.values():
+            if (nodeedit := protonode.getNodeEdit()) is not None:
+                nodeedits.append(nodeedit)
+                nodecache[protonode.buid] = protonode.node
+
+        if nodeedits:
+            await self.snap.applyNodeEdits(nodeedits, nodecache=nodecache, meta=self.meta)
+
+        self.protonodes.clear()
+
     async def _addNode(self, form, valu, props=None, norminfo=None):
 
         self.snap.core._checkMaxNodes()
@@ -556,7 +596,7 @@ class SnapEditor:
         subs = norminfo.get('subs')
         if subs is not None:
             for prop, valu in subs.items():
-                ops.append(protonode.getSetOps(prop, valu))
+                ops.append(protonode.getSetSubOps(prop, valu))
 
         adds = norminfo.get('adds')
         if adds is not None:
@@ -592,7 +632,7 @@ class SnapEditor:
         subs = norminfo.get('subs')
         if subs is not None:
             for prop, valu in subs.items():
-                ops.append(protonode.getSetOps(prop, valu))
+                ops.append(protonode.getSetSubOps(prop, valu))
 
             while ops:
                 oset = ops.popleft()
@@ -849,6 +889,10 @@ class Snap(s_base.Base):
         self._warnonce_keys.add(mesg)
         await self.warn(mesg, log, **info)
 
+    def _skipPropDeprWarn(self, name):
+        mesg = f'The property {name} is deprecated or using a deprecated type and will be removed in 3.0.0'
+        self._warnonce_keys.add(mesg)
+
     async def getNodeByBuid(self, buid):
         '''
         Retrieve a node tuple by binary id.
@@ -1062,6 +1106,23 @@ class Snap(s_base.Base):
             mesg = f'No property named "{full}".'
             raise s_exc.NoSuchProp(mesg=mesg)
 
+        if isinstance(valu, dict) and isinstance(prop.type, s_types.Guid) and cmpr == '=':
+            if prop.isform:
+                if (node := await self._getGuidNodeByDict(prop, valu)) is not None:
+                    yield node
+                return
+
+            fname = prop.type.name
+            if (form := prop.modl.form(fname)) is None:
+                mesg = f'The property "{full}" type "{fname}" is not a form and cannot be lifted using a dictionary.'
+                raise s_exc.BadTypeValu(mesg=mesg)
+
+            if (node := await self._getGuidNodeByDict(form, valu)) is None:
+                return
+
+            norm = False
+            valu = node.ndef[1]
+
         if norm:
             cmprvals = prop.type.getStorCmprs(cmpr, valu)
             # an empty return probably means ?= with invalid value
@@ -1189,11 +1250,13 @@ class Snap(s_base.Base):
         if nodes:
             return nodes[0]
 
-    async def applyNodeEdits(self, edits, nodecache=None):
+    async def applyNodeEdits(self, edits, nodecache=None, meta=None):
         '''
         Sends edits to the write layer and evaluates the consequences (triggers, node object updates)
         '''
-        meta = await self.getSnapMeta()
+        if meta is None:
+            meta = await self.getSnapMeta()
+
         saveoff, changes, nodes = await self._applyNodeEdits(edits, meta, nodecache=nodecache)
         return nodes
 
@@ -1398,10 +1461,6 @@ class Snap(s_base.Base):
 
     async def _addGuidNodeByDict(self, form, vals, props=None):
 
-        norms = {}
-        counts = []
-        proplist = []
-
         if props is None:
             props = {}
 
@@ -1439,7 +1498,32 @@ class Snap(s_base.Base):
                         raise e
                     await self.warn(f'Skipping bad value for prop {form.name}:{name}: {mesg}')
 
-        for name, valu in vals.items():
+        norms, proplist = self._normGuidNodeDict(form, vals)
+
+        iden = s_common.guid(proplist)
+        node = await self._getGuidNodeByNorms(form, iden, norms)
+
+        async with self.getEditor() as editor:
+
+            if node is not None:
+                proto = editor.loadNode(node)
+            else:
+                proto = await editor.addNode(form.name, iden)
+                for name, (prop, valu, info) in norms.items():
+                    await proto.set(name, valu, norminfo=info)
+
+            # ensure the non-deconf props are set
+            for name, (valu, info) in props.items():
+                await proto.set(name, valu, norminfo=info)
+
+        return await self.getNodeByBuid(proto.buid)
+
+    def _normGuidNodeDict(self, form, props):
+
+        norms = {}
+        proplist = []
+
+        for name, valu in props.items():
 
             try:
                 prop = form.reqProp(name)
@@ -1458,22 +1542,24 @@ class Snap(s_base.Base):
 
         proplist.sort()
 
+        return norms, proplist
+
+    async def _getGuidNodeByDict(self, form, props):
+        norms, proplist = self._normGuidNodeDict(form, props)
+        return await self._getGuidNodeByNorms(form, s_common.guid(proplist), norms)
+
+    async def _getGuidNodeByNorms(self, form, iden, norms):
+
         # check first for an exact match via our same deconf strategy
-        iden = s_common.guid(proplist)
 
         node = await self.getNodeByNdef((form.full, iden))
         if node is not None:
 
             # ensure we still match the property deconf criteria
-            for name, (prop, norm, info) in norms.items():
+            for (prop, norm, info) in norms.values():
                 if not self._filtByPropAlts(node, prop, norm):
                     break
             else:
-                # ensure the non-deconf props are set
-                async with self.getEditor() as editor:
-                    proto = editor.loadNode(node)
-                    for name, (valu, info) in props.items():
-                        await proto.set(name, valu, norminfo=info)
                 return node
 
         # TODO there is an opportunity here to populate
@@ -1482,8 +1568,10 @@ class Snap(s_base.Base):
         # if we lookup a node and it no longer passes the
         # filter...
 
+        counts = []
+
         # no exact match. lets do some counting.
-        for name, (prop, norm, info) in norms.items():
+        for (prop, norm, info) in norms.values():
             count = await self._getPropAltCount(prop, norm)
             counts.append((count, prop, norm))
 
@@ -1499,21 +1587,9 @@ class Snap(s_base.Base):
                 if not self._filtByPropAlts(node, prop, norm):
                     break
             else:
-                # ensure the non-deconf props are set
-                async with self.getEditor() as editor:
-                    proto = editor.loadNode(node)
-                    for name, (valu, info) in props.items():
-                        await proto.set(name, valu, norminfo=info)
                 return node
 
-        async with self.getEditor() as editor:
-            proto = await editor.addNode(form.name, iden)
-            for name, (prop, valu, info) in norms.items():
-                await proto.set(name, valu, norminfo=info)
-            for name, (valu, info) in props.items():
-                await proto.set(name, valu, norminfo=info)
-
-        return await self.getNodeByBuid(proto.buid)
+        return None
 
     async def _getPropAltCount(self, prop, valu):
         count = 0

synapse/lib/storm.lark

@@ -40,7 +40,7 @@ _editblock: "[" _editoper* "]"
 // A single edit operation
 _editoper: editnodeadd
             | editpropset | editunivset | edittagpropset | edittagadd | editcondpropset
-            | editpropdel | editunivdel | edittagpropdel | edittagdel
+            | editpropsetmulti | editpropdel | editunivdel | edittagpropdel | edittagdel
             | editparens | edgeaddn1 | edgedeln1 | edgeaddn2 | edgedeln2
 
 // Parenthesis in an edit block don't have incoming nodes
@@ -48,18 +48,20 @@ editparens: "(" editnodeadd _editoper* ")"
 edittagadd: "+" [SETTAGOPER] tagname [(EQSPACE | EQNOSPACE) _valu]
 editunivdel: EXPRMINUS univprop
 edittagdel: EXPRMINUS tagname
-editpropset: relprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu
+editpropset: relprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYMODSET) _valu
 editcondpropset: relprop condsetoper _valu
+editpropsetmulti: relprop (MODSETMULTI | TRYMODSETMULTI) _valu
 editpropdel: EXPRMINUS relprop
-editunivset: univprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu
-editnodeadd: formname (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu
-edittagpropset: "+" tagprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu
+editunivset: univprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYMODSET) _valu
+editnodeadd: formname (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYMODSET) _valu
+edittagpropset: "+" tagprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYMODSET) _valu
 edittagpropdel: EXPRMINUS tagprop
 
 EQSPACE: /((?<=\s)=|=(?=\s))/
 MODSET.4: "+=" | "-="
-TRYSETPLUS.1: "?+="
-TRYSETMINUS.1: "?-="
+TRYMODSET.1: "?+=" | "?-="
+MODSETMULTI.4: "++=" | "--="
+TRYMODSETMULTI.1: "?++=" | "?--="
 TRYSET.1: "?="
 SETTAGOPER: "?"
 
@@ -226,10 +228,10 @@ _EDGEN2JOININIT: "<+("
 // comparisons to an expression like '<(2)'
 _EDGEN2INIT: /\<\((?=(?>(?<EDGEN2INITRECUR>\(((?>[^()"'`]+|(?&EDGEN2INITRECUR)|`(?:[^`\\]|\\.)*`|"(?:[^"\\]|\\.)*"|'''.*?'''|'[^']*'(?!'))*)\))|'''.*?'''|`(?:[^`\\]|\\.)*`|"(?:[^"\\]|\\.)*"|'[^']*'(?!')|[^)])*\)[\-\+])/
 
-edgeaddn1: _EDGEADDN1INIT _valu _EDGEN1FINI baresubquery
-edgedeln1: _EDGEN1INIT _valu _EDGEN1FINI baresubquery
-edgeaddn2: _EDGEN2INIT _valu _EDGEADDN2FINI baresubquery
-edgedeln2: _EDGEN2INIT _valu _EDGEN2FINI baresubquery
+edgeaddn1: _EDGEADDN1INIT _valu _EDGEN1FINI (baresubquery | _varvalu)
+edgedeln1: _EDGEN1INIT _valu _EDGEN1FINI (baresubquery | _varvalu)
+edgeaddn2: _EDGEN2INIT _valu _EDGEADDN2FINI (baresubquery | _varvalu)
+edgedeln2: _EDGEN2INIT _valu _EDGEN2FINI (baresubquery | _varvalu)
 
 _REVERSE: /reverse(?=[\s\(])/
 liftreverse: _REVERSE "(" (liftformtag | liftpropby | liftprop | liftbyarray | lifttagtag | liftbytag | liftbytagprop | liftbyformtagprop) ")"

synapse/lib/storm.py

@@ -1299,7 +1299,7 @@ stormcmds = (
 
 @s_cache.memoize(size=1024)
 def queryhash(text):
-    return hashlib.md5(text.encode(errors='surrogatepass'), usedforsecurity=False).hexdigest()
+    return s_common.queryhash(text)
 
 class DmonManager(s_base.Base):
     '''
@@ -3653,23 +3653,26 @@ class MergeCmd(Cmd):
 
             genr = diffgenr()
 
-        async with await runt.snap.view.parent.snap(user=runt.user.iden) as snap:
+        async with await runt.snap.view.parent.snap(user=runt.user) as snap:
             snap.strict = False
 
             snap.on('warn', runt.snap.dist)
 
+            meta = {'user': runt.user.iden}
+
+            if doapply:
+                editor = s_snap.SnapEditor(snap, meta=meta)
+
             async for node, path in genr:
 
                 # the timestamp for the adds/subs of each node merge will match
                 nodeiden = node.iden()
-                meta = {'user': runt.user.iden, 'time': s_common.now()}
+
+                meta['time'] = s_common.now()
 
                 sodes = await node.getStorNodes()
                 sode = sodes[0]
 
-                if doapply:
-                    editor = s_snap.SnapEditor(snap)
-
                 subs = []
 
                 # check all node perms first
@@ -3818,13 +3821,11 @@ class MergeCmd(Cmd):
                     subs.append((s_layer.EDIT_NODE_DEL, valu, ()))
 
                 if doapply:
-                    addedits = editor.getNodeEdits()
-                    if addedits:
-                        await runt.snap.view.parent.storNodeEdits(addedits, meta=meta)
+                    await editor.flushEdits()
 
                     if subs:
                         subedits = [(node.buid, node.form.name, subs)]
-                        await runt.snap.view.storNodeEdits(subedits, meta=meta)
+                        await runt.snap.applyNodeEdits(subedits, nodecache={node.buid: node}, meta=meta)
 
                 runt.snap.clearCachedNode(node.buid)
                 yield await runt.snap.getNodeByBuid(node.buid), path

synapse/lib/storm_format.py

@@ -55,6 +55,7 @@ TerminalPygMap = {
     'LSQB': p_t.Punctuation,
     'MCASEBARE': p_t.Literal.String,
     'MODSET': p_t.Operator,
+    'MODSETMULTI': p_t.Operator,
     'NONQUOTEWORD': p_t.Literal,
     'NOT': p_t.Keyword,
     'NULL': p_t.Keyword,
@@ -74,8 +75,8 @@ TerminalPygMap = {
     'TAGSEGNOVAR': p_t.Name,
     'TRIPLEQUOTEDSTRING': p_t.Literal.String,
     'TRYSET': p_t.Operator,
-    'TRYSETMINUS': p_t.Operator,
-    'TRYSETPLUS': p_t.Operator,
+    'TRYMODSET': p_t.Operator,
+    'TRYMODSETMULTI': p_t.Operator,
     'UNIVNAME': p_t.Name,
     'UNSET': p_t.Operator,
     'EXPRUNIVNAME': p_t.Name,

synapse/lib/stormtypes.py

@@ -3490,7 +3490,11 @@ class LibRegx(Lib):
         lkey = (pattern, flags)
         regx = self.compiled.get(lkey)
         if regx is None:
-            regx = self.compiled[lkey] = regex.compile(pattern, flags=flags)
+            try:
+                regx = self.compiled[lkey] = regex.compile(pattern, flags=flags)
+            except (regex.error, ValueError) as e:
+                mesg = f'Error compiling regex pattern: {e}: pattern="{s_common.trimText(pattern)}"'
+                raise s_exc.BadArg(mesg=mesg) from None
         return regx
 
     @stormfunc(readonly=True)
@@ -3500,7 +3504,12 @@ class LibRegx(Lib):
         pattern = await tostr(pattern)
         replace = await tostr(replace)
         regx = await self._getRegx(pattern, flags)
-        return regx.sub(replace, text)
+
+        try:
+            return regx.sub(replace, text)
+        except (regex.error, IndexError) as e:
+            mesg = f'$lib.regex.replace() error: {e}'
+            raise s_exc.BadArg(mesg=mesg) from None
 
     @stormfunc(readonly=True)
     async def matches(self, pattern, text, flags=0):
@@ -9868,7 +9877,7 @@ async def tostor(valu, isndef=False):
         retn = []
         for v in valu:
             try:
-                retn.append(await tostor(v))
+                retn.append(await tostor(v, isndef=isndef))
             except s_exc.NoSuchType:
                 pass
         return tuple(retn)
@@ -9877,7 +9886,7 @@ async def tostor(valu, isndef=False):
         retn = {}
         for k, v in valu.items():
             try:
-                retn[k] = await tostor(v)
+                retn[k] = await tostor(v, isndef=isndef)
             except s_exc.NoSuchType:
                 pass
         return retn

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 197, 0)
+version = (2, 199, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'c8c758131cae47ef21e23ac8242012459ebfce9d'
+commit = 'c256bd46a9df8621fd05fd6a0f36ca476ad29942'

synapse/lib/view.py

@@ -1474,7 +1474,8 @@ class View(s_nexus.Pusher):  # type: ignore
 
             meta = await snap.getSnapMeta()
             async for nodeedits in fromlayr.iterLayerNodeEdits():
-                await self.parent.storNodeEdits([nodeedits], meta)
+                meta['time'] = s_common.now()
+                await snap.saveNodeEdits([nodeedits], meta)
 
     async def swapLayer(self):
         oldlayr = self.layers[0]

synapse/models/infotech.py

@@ -1252,6 +1252,15 @@ class ItModule(s_module.CoreModule):
                     ('product', ('it:prod:softver', {}), {
                         'doc': 'The software which produced the log entry.'}),
 
+                    ('service:platform', ('inet:service:platform', {}), {
+                        'doc': 'The service platform which generated the log event.'}),
+
+                    ('service:instance', ('inet:service:instance', {}), {
+                        'doc': 'The service instance which generated the log event.'}),
+
+                    ('service:account', ('inet:service:account', {}), {
+                        'doc': 'The service account which generated the log event.'}),
+
                 )),
                 ('it:domain', {}, (
                     ('name', ('str', {'lower': True, 'onespace': True}), {
@@ -2587,6 +2596,15 @@ class ItModule(s_module.CoreModule):
 
                     ('synuser', ('syn:user', {}), {
                         'doc': 'The synapse user who executed the query.'}),
+
+                    ('service:platform', ('inet:service:platform', {}), {
+                        'doc': 'The service platform which was queried.'}),
+
+                    ('service:instance', ('inet:service:instance', {}), {
+                        'doc': 'The service instance which was queried.'}),
+
+                    ('service:account', ('inet:service:account', {}), {
+                        'doc': 'The service account which ran the query.'}),
                 )),
                 ('it:exec:thread', {}, (
                     ('proc', ('it:exec:proc', {}), {

synapse/models/risk.py

@@ -810,6 +810,15 @@ class RiskModule(s_module.CoreModule):
 
                     ('host', ('it:host', {}), {
                         'doc': 'The host which generated the alert.'}),
+
+                    ('service:platform', ('inet:service:platform', {}), {
+                        'doc': 'The service platform which generated the alert.'}),
+
+                    ('service:instance', ('inet:service:instance', {}), {
+                        'doc': 'The service instance which generated the alert.'}),
+
+                    ('service:account', ('inet:service:account', {}), {
+                        'doc': 'The service account which generated the alert.'}),
                 )),
                 ('risk:compromisetype', {}, ()),
                 ('risk:compromise', {}, (

synapse/models/syn.py

@@ -23,7 +23,15 @@ class SynUser(s_types.Guid):
             if user is not None:
                 return user.iden, {}
 
-        return s_types.Guid._normPyStr(self, text)
+        if text == '*':
+            mesg = f'{self.name} values must be a valid username or a guid.'
+            raise s_exc.BadTypeValu(mesg=mesg, name=self.name, valu=text)
+
+        try:
+            return s_types.Guid._normPyStr(self, text)
+        except s_exc.BadTypeValu:
+            mesg = f'No user named {text} and value is not a guid.'
+            raise s_exc.BadTypeValu(mesg=mesg, name=self.name, valu=text) from None
 
     def repr(self, iden):
 
@@ -51,7 +59,15 @@ class SynRole(s_types.Guid):
             if role is not None:
                 return role.iden, {}
 
-        return s_types.Guid._normPyStr(self, text)
+        if text == '*':
+            mesg = f'{self.name} values must be a valid rolename or a guid.'
+            raise s_exc.BadTypeValu(mesg=mesg, name=self.name, valu=text)
+
+        try:
+            return s_types.Guid._normPyStr(self, text)
+        except s_exc.BadTypeValu:
+            mesg = f'No role named {text} and value is not a guid.'
+            raise s_exc.BadTypeValu(mesg=mesg, name=self.name, valu=text) from None
 
     def repr(self, iden):
 

synapse/tests/files/stormpkg/badendpoints.yaml

@@ -0,0 +1,7 @@
+name: test
+version: 0.0.1
+commands:
+  - name: 'foo.bar'
+    storm: (null)
+    endpoints:
+      - host: vertex.link

synapse/tests/files/stormpkg/testpkg.yaml

@@ -113,6 +113,14 @@ commands:
           - help: Help on foo opt
         - - --bar
           - help: Help on bar opt
+      storm: |
+        test:str
+      endpoints:
+        - path: /v1/test/one
+        - path: /v1/test/two
+          host: vertex.link
+        - path: /v1/test/three
+          desc: endpoint three
 
     - name: testpkg.baz
       descr: |