synapse 2.196.0__py311-none-any.whl → 2.198.0__py311-none-any.whl

synapse/lib/schemas.py

@@ -1,7 +1,8 @@
 import synapse.lib.const as s_const
 import synapse.lib.config as s_config
+import synapse.lib.grammar as s_grammar
 import synapse.lib.msgpack as s_msgpack
-
+import synapse.lib.version as s_version
 
 easyPermSchema = {
     'type': 'object',
@@ -551,3 +552,492 @@ stixIngestBundleSchema = {
     },
 }
 reqValidStixIngestBundle = s_config.getJsValidator(stixIngestBundleSchema)
+
+_reqValidGdefSchema = {
+    'type': 'object',
+    'properties': {
+        'iden': {'type': 'string', 'pattern': s_config.re_iden},
+        'name': {'type': 'string', 'minLength': 1},
+        'desc': {'type': 'string', 'default': ''},
+        'scope': {'type': 'string', 'enum': ['user', 'power-up']},
+        'creator': {'type': 'string', 'pattern': s_config.re_iden},
+        'power-up': {'type': 'string', 'minLength': 1},
+        'maxsize': {'type': 'number', 'minimum': 0},
+        'existing': {'type': 'array', 'items': {'type': 'string'}},
+        'created': {'type': 'number'},
+        'updated': {'type': 'number'},
+        'refs': {'type': 'boolean', 'default': False},
+        'edges': {'type': 'boolean', 'default': True},
+        'edgelimit': {'type': 'number', 'default': 3000},
+        'degrees': {'type': ['integer', 'null'], 'minimum': 0},
+        'filterinput': {'type': 'boolean', 'default': True},
+        'yieldfiltered': {'type': 'boolean', 'default': False},
+        'filters': {
+            'type': ['array', 'null'],
+            'items': {'type': 'string'}
+        },
+        'pivots': {
+            'type': ['array', 'null'],
+            'items': {'type': 'string'}
+        },
+        'forms': {
+            'type': 'object',
+            'patternProperties': {
+                '^.*$': {
+                    'type': 'object',
+                    'properties': {
+                        'filters': {
+                            'type': ['array', 'null'],
+                            'items': {'type': 'string'}
+                        },
+                        'pivots': {
+                            'type': ['array', 'null'],
+                            'items': {'type': 'string'}
+                        }
+                    },
+                    'additionalProperties': False,
+                }
+            }
+        },
+        'permissions': s_msgpack.deepcopy(easyPermSchema)
+    },
+    'additionalProperties': False,
+    'required': ['iden', 'name', 'scope'],
+    'allOf': [
+        {
+            'if': {'properties': {'scope': {'const': 'power-up'}}},
+            'then': {'required': ['power-up']},
+            'else': {'required': ['creator']},
+        }
+    ]
+}
+reqValidGdef = s_config.getJsValidator(_reqValidGdefSchema)
+
+_reqValidPermDefSchema = {
+    'type': 'object',
+    'properties': {
+        'perm': {'type': 'array', 'items': {'type': 'string'}},
+        'desc': {'type': 'string'},
+        'gate': {'type': 'string'},
+        'ex': {'type': 'string'},  # Example string
+        'workflowconfig': {'type': 'boolean'},
+        'default': {'type': 'boolean', 'default': False},
+    },
+    'required': ['perm', 'desc', 'gate'],
+}
+
+reqValidPermDef = s_config.getJsValidator(_reqValidPermDefSchema)
+
+# N.B. This is kept in sync with s_datamodel.Datamodel().types
+# with the DatamodelTest.test_datamodel_schema_basetypes test.
+datamodel_basetypes = [
+    'int',
+    'float',
+    'range',
+    'str',
+    'hex',
+    'bool',
+    'time',
+    'duration',
+    'ival',
+    'guid',
+    'syn:tag:part',
+    'syn:tag',
+    'comp',
+    'loc',
+    'ndef',
+    'array',
+    'edge',
+    'timeedge',
+    'data',
+    'nodeprop',
+    'hugenum',
+    'taxon',
+    'taxonomy',
+    'velocity',
+]
+
+_reqValidPkgdefSchema = {
+    'type': 'object',
+    'properties': {
+        'name': {'type': 'string'},
+        'version': {
+            'type': 'string',
+            'pattern': s_version.semverstr,
+        },
+        'build': {
+            'type' 'object'
+            'properties': {
+                'time': {'type': 'number'},
+            },
+            'required': ['time'],
+        },
+        'codesign': {
+            'type': 'object',
+            'properties': {
+                'sign': {'type': 'string'},
+                'cert': {'type': 'string'},
+            },
+            'required': ['cert', 'sign'],
+        },
+        # TODO: Remove me after Synapse 3.0.0.
+        'synapse_minversion': {
+            'type': ['array', 'null'],
+            'items': {'type': 'number'}
+        },
+        'synapse_version': {
+            'type': 'string',
+        },
+        'modules': {
+            'type': ['array', 'null'],
+            'items': {'$ref': '#/definitions/module'}
+        },
+        'docs': {
+            'type': ['array', 'null'],
+            'items': {'$ref': '#/definitions/doc'},
+        },
+        'logo': {
+            'type': 'object',
+            'properties': {
+                'mime': {'type': 'string'},
+                'file': {'type': 'string'},
+            },
+            'additionalProperties': True,
+            'required': ['mime', 'file'],
+        },
+        'commands': {
+            'type': ['array', 'null'],
+            'items': {'$ref': '#/definitions/command'},
+        },
+        'graphs': {
+            'type': ['array', 'null'],
+            'items': s_msgpack.deepcopy(_reqValidGdefSchema, use_list=True),
+        },
+        'desc': {'type': 'string'},
+        'svciden': {'type': ['string', 'null'], 'pattern': s_config.re_iden},
+        'onload': {'type': 'string'},
+        'author': {
+            'type': 'object',
+            'properties': {
+                'url': {'type': 'string'},
+                'name': {'type': 'string'},
+            },
+            'required': ['name', 'url'],
+        },
+        'depends': {
+            'properties': {
+                'requires': {'type': 'array', 'items': {'$ref': '#/definitions/require'}},
+                'conflicts': {'type': 'array', 'items': {'$ref': '#/definitions/conflict'}},
+            },
+            'additionalProperties': True,
+        },
+        'perms': {
+            'type': 'array',
+            'items': s_msgpack.deepcopy(_reqValidPermDefSchema),
+        },
+        'configvars': {
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                    'name': {'type': 'string'},
+                    'varname': {'type': 'string'},
+                    'desc': {'type': 'string'},
+                    'default': {},
+                    'workflowconfig': {'type': 'boolean'},
+                    'type': {'$ref': '#/definitions/configvartype'},
+                    'scopes': {
+                        'type': 'array',
+                        'items': {
+                            'type': 'string',
+                            'enum': ['global', 'self']
+                        },
+                    },
+                },
+                'required': ['name', 'varname', 'desc', 'type', 'scopes'],
+            },
+        },
+    },
+    'additionalProperties': True,
+    'required': ['name', 'version'],
+    'definitions': {
+        'doc': {
+            'type': 'object',
+            'properties': {
+                'title': {'type': 'string'},
+                'content': {'type': 'string'},
+            },
+            'additionalProperties': True,
+            'required': ['title', 'content'],
+        },
+        'module': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'storm': {'type': 'string'},
+                'modconf': {'type': 'object'},
+                'apidefs': {
+                    'type': ['array', 'null'],
+                    'items': {'$ref': '#/definitions/apidef'},
+                },
+                'asroot': {'type': 'boolean'},
+                'asroot:perms': {'type': 'array',
+                    'items': {'type': 'array',
+                        'items': {'type': 'string'}},
+                },
+            },
+            'additionalProperties': True,
+            'required': ['name', 'storm']
+        },
+        'apidef': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'desc': {'type': 'string'},
+                'deprecated': {'$ref': '#/definitions/deprecatedItem'},
+                'type': {
+                    'type': 'object',
+                    'properties': {
+                        'type': {
+                            'type': 'string',
+                            'enum': ['function']
+                        },
+                        'args': {
+                            'type': 'array',
+                            'items': {'$ref': '#/definitions/apiarg'},
+                        },
+                        'returns': {
+                            'type': 'object',
+                            'properties': {
+                                'name': {
+                                    'type': 'string',
+                                    'enum': ['yields'],
+                                },
+                                'desc': {'type': 'string'},
+                                'type': {
+                                    'oneOf': [
+                                        {'$ref': '#/definitions/apitype'},
+                                        {'type': 'array', 'items': {'$ref': '#/definitions/apitype'}},
+                                    ],
+                                },
+                            },
+                            'additionalProperties': False,
+                            'required': ['type', 'desc']
+                        },
+                    },
+                    'additionalProperties': False,
+                    'required': ['type', 'returns'],
+                },
+            },
+            'additionalProperties': False,
+            'required': ['name', 'desc', 'type']
+        },
+        'apiarg': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'desc': {'type': 'string'},
+                'type': {
+                    'oneOf': [
+                        {'$ref': '#/definitions/apitype'},
+                        {'type': 'array', 'items': {'$ref': '#/definitions/apitype'}},
+                    ],
+                },
+                'default': {'type': ['boolean', 'integer', 'string', 'null']},
+            },
+            'additionalProperties': False,
+            'required': ['name', 'desc', 'type']
+        },
+        'deprecatedItem': {
+            'type': 'object',
+            'properties': {
+                'eolvers': {'type': 'string', 'minLength': 1,
+                            'description': "The version which will not longer support the item."},
+                'eoldate': {'type': 'string', 'minLength': 1,
+                            'description': 'Optional string indicating Synapse releases after this date may no longer support the item.'},
+                'mesg': {'type': ['string', 'null'], 'default': None,
+                         'description': 'Optional message to include in the warning text.'}
+            },
+            'oneOf': [
+                {
+                    'required': ['eolvers'],
+                    'not': {'required': ['eoldate']}
+                },
+                {
+                    'required': ['eoldate'],
+                    'not': {'required': ['eolvers']}
+                }
+            ],
+            'additionalProperties': False,
+        },
+        'apitype': {
+            'type': 'string',
+        },
+        'command': {
+            'type': 'object',
+            'properties': {
+                'name': {
+                    'type': 'string',
+                    'pattern': s_grammar.re_scmd
+                },
+                'endpoints': {
+                    'type': 'array',
+                    'items': {
+                        'type': 'object',
+                        'properties': {
+                            'path': {'type': 'string'},
+                            'host': {'type': 'string'},
+                            'desc': {'type': 'string'},
+                        },
+                        'required': ['path'],
+                        'additionalProperties': False
+                    }
+                },
+                'cmdargs': {
+                    'type': ['array', 'null'],
+                    'items': {'$ref': '#/definitions/cmdarg'},
+                },
+                'cmdinputs': {
+                    'type': ['array', 'null'],
+                    'items': {'$ref': '#/definitions/cmdinput'},
+                },
+                'storm': {'type': 'string'},
+                'forms': {'$ref': '#/definitions/cmdformhints'},
+                'perms': {'type': 'array',
+                    'items': {'type': 'array',
+                        'items': {'type': 'string'}},
+                },
+            },
+            'additionalProperties': True,
+            'required': ['name', 'storm']
+        },
+        'cmdarg': {
+            'type': 'array',
+            'items': [
+                {'type': 'string'},
+                {
+                    'type': 'object',
+                    'properties': {
+                        'help': {'type': 'string'},
+                        'default': {},
+                        'dest': {'type': 'string'},
+                        'required': {'type': 'boolean'},
+                        'action': {'type': 'string'},
+                        'nargs': {'type': ['string', 'integer']},
+                        'choices': {
+                            'type': 'array',
+                            'uniqueItems': True,
+                            'minItems': 1,
+                        },
+                        'type': {
+                            'type': 'string',
+                            'enum': s_msgpack.deepcopy(datamodel_basetypes),
+                        },
+                    },
+                }
+            ],
+            'additionalItems': False,
+        },
+        'cmdinput': {
+            'type': 'object',
+            'properties': {
+                'form': {'type': 'string'},
+                'help': {'type': 'string'},
+            },
+            'additionalProperties': True,
+            'required': ['form'],
+        },
+        'configvartype': {
+            'anyOf': [
+                {'type': 'array', 'items': {'$ref': '#/definitions/configvartype'}},
+                {'type': 'string'},
+            ]
+        },
+        # deprecated
+        'cmdformhints': {
+            'type': 'object',
+            'properties': {
+                'input': {
+                    'type': 'array',
+                    'uniqueItems': True,
+                    'items': {
+                        'type': 'string',
+                    }
+                },
+                'output': {
+                    'type': 'array',
+                    'uniqueItems': True,
+                    'items': {
+                        'type': 'string',
+                    }
+                },
+                'nodedata': {
+                    'type': 'array',
+                    'uniqueItems': True,
+                    'items': {
+                        'type': 'array',
+                        'items': [
+                            {'type': 'string'},
+                            {'type': 'string'},
+                        ],
+                        'additionalItems': False,
+                    },
+                },
+            }
+        },
+        'require': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'version': {'type': 'string'},
+                'desc': {'type': 'string'},
+                'optional': {'type': 'boolean'},
+            },
+            'additionalItems': True,
+            'required': ('name', 'version'),
+        },
+        'conflict': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'version': {'type': 'string'},
+                'desc': {'type': 'string'},
+            },
+            'additionalItems': True,
+            'required': ('name',),
+        },
+    }
+}
+reqValidPkgdef = s_config.getJsValidator(_reqValidPkgdefSchema)
+
+_reqValidDdefSchema = {
+    'type': 'object',
+    'properties': {
+        'name': {'type': 'string'},
+        'storm': {'type': 'string'},
+        'view': {'type': 'string', 'pattern': s_config.re_iden},
+        'user': {'type': 'string', 'pattern': s_config.re_iden},
+        'iden': {'type': 'string', 'pattern': s_config.re_iden},
+        'enabled': {'type': 'boolean', 'default': True},
+        'stormopts': {
+            'oneOf': [
+                {'type': 'null'},
+                {'$ref': '#/definitions/stormopts'}
+            ]
+        }
+    },
+    'additionalProperties': True,
+    'required': ['iden', 'user', 'storm'],
+    'definitions': {
+        'stormopts': {
+            'type': 'object',
+            'properties': {
+                'repr': {'type': 'boolean'},
+                'path': {'type': 'string'},
+                'show': {'type': 'array', 'items': {'type': 'string'}}
+            },
+            'additionalProperties': True,
+        },
+    }
+}
+reqValidDdef = s_config.getJsValidator(_reqValidDdefSchema)

synapse/lib/snap.py

@@ -138,6 +138,12 @@ class ProtoNode:
 
         if not await self.ctx.snap.hasNodeEdge(self.buid, verb, s_common.uhex(n2iden)):
             self.edges.add(tupl)
+
+            if len(self.edges) >= 1000:
+                nodeedits = self.ctx.getNodeEdits()
+                await self.ctx.snap.applyNodeEdits(nodeedits)
+                self.edges.clear()
+
             return True
 
         return False
@@ -169,6 +175,12 @@ class ProtoNode:
 
         if await self.ctx.snap.layers[-1].hasNodeEdge(self.buid, verb, s_common.uhex(n2iden)):
             self.edgedels.add(tupl)
+
+            if len(self.edgedels) >= 1000:
+                nodeedits = self.ctx.getNodeEdits()
+                await self.ctx.snap.applyNodeEdits(nodeedits)
+                self.edgedels.clear()
+
             return True
 
         return False
@@ -1062,6 +1074,23 @@ class Snap(s_base.Base):
             mesg = f'No property named "{full}".'
             raise s_exc.NoSuchProp(mesg=mesg)
 
+        if isinstance(valu, dict) and isinstance(prop.type, s_types.Guid) and cmpr == '=':
+            if prop.isform:
+                if (node := await self._getGuidNodeByDict(prop, valu)) is not None:
+                    yield node
+                return
+
+            fname = prop.type.name
+            if (form := prop.modl.form(fname)) is None:
+                mesg = f'The property "{full}" type "{fname}" is not a form and cannot be lifted using a dictionary.'
+                raise s_exc.BadTypeValu(mesg=mesg)
+
+            if (node := await self._getGuidNodeByDict(form, valu)) is None:
+                return
+
+            norm = False
+            valu = node.ndef[1]
+
         if norm:
             cmprvals = prop.type.getStorCmprs(cmpr, valu)
             # an empty return probably means ?= with invalid value
@@ -1398,10 +1427,6 @@ class Snap(s_base.Base):
 
     async def _addGuidNodeByDict(self, form, vals, props=None):
 
-        norms = {}
-        counts = []
-        proplist = []
-
         if props is None:
             props = {}
 
@@ -1439,7 +1464,32 @@ class Snap(s_base.Base):
                         raise e
                     await self.warn(f'Skipping bad value for prop {form.name}:{name}: {mesg}')
 
-        for name, valu in vals.items():
+        norms, proplist = self._normGuidNodeDict(form, vals)
+
+        iden = s_common.guid(proplist)
+        node = await self._getGuidNodeByNorms(form, iden, norms)
+
+        async with self.getEditor() as editor:
+
+            if node is not None:
+                proto = editor.loadNode(node)
+            else:
+                proto = await editor.addNode(form.name, iden)
+                for name, (prop, valu, info) in norms.items():
+                    await proto.set(name, valu, norminfo=info)
+
+            # ensure the non-deconf props are set
+            for name, (valu, info) in props.items():
+                await proto.set(name, valu, norminfo=info)
+
+        return await self.getNodeByBuid(proto.buid)
+
+    def _normGuidNodeDict(self, form, props):
+
+        norms = {}
+        proplist = []
+
+        for name, valu in props.items():
 
             try:
                 prop = form.reqProp(name)
@@ -1458,22 +1508,24 @@ class Snap(s_base.Base):
 
         proplist.sort()
 
+        return norms, proplist
+
+    async def _getGuidNodeByDict(self, form, props):
+        norms, proplist = self._normGuidNodeDict(form, props)
+        return await self._getGuidNodeByNorms(form, s_common.guid(proplist), norms)
+
+    async def _getGuidNodeByNorms(self, form, iden, norms):
+
         # check first for an exact match via our same deconf strategy
-        iden = s_common.guid(proplist)
 
         node = await self.getNodeByNdef((form.full, iden))
         if node is not None:
 
             # ensure we still match the property deconf criteria
-            for name, (prop, norm, info) in norms.items():
+            for (prop, norm, info) in norms.values():
                 if not self._filtByPropAlts(node, prop, norm):
                     break
             else:
-                # ensure the non-deconf props are set
-                async with self.getEditor() as editor:
-                    proto = editor.loadNode(node)
-                    for name, (valu, info) in props.items():
-                        await proto.set(name, valu, norminfo=info)
                 return node
 
         # TODO there is an opportunity here to populate
@@ -1482,8 +1534,10 @@ class Snap(s_base.Base):
         # if we lookup a node and it no longer passes the
         # filter...
 
+        counts = []
+
         # no exact match. lets do some counting.
-        for name, (prop, norm, info) in norms.items():
+        for (prop, norm, info) in norms.values():
             count = await self._getPropAltCount(prop, norm)
             counts.append((count, prop, norm))
 
@@ -1499,21 +1553,9 @@ class Snap(s_base.Base):
                 if not self._filtByPropAlts(node, prop, norm):
                     break
             else:
-                # ensure the non-deconf props are set
-                async with self.getEditor() as editor:
-                    proto = editor.loadNode(node)
-                    for name, (valu, info) in props.items():
-                        await proto.set(name, valu, norminfo=info)
                 return node
 
-        async with self.getEditor() as editor:
-            proto = await editor.addNode(form.name, iden)
-            for name, (prop, valu, info) in norms.items():
-                await proto.set(name, valu, norminfo=info)
-            for name, (valu, info) in props.items():
-                await proto.set(name, valu, norminfo=info)
-
-        return await self.getNodeByBuid(proto.buid)
+        return None
 
     async def _getPropAltCount(self, prop, valu):
         count = 0