synapse 2.196.0__py311-none-any.whl → 2.198.0__py311-none-any.whl

synapse/lib/storm.lark

@@ -40,7 +40,7 @@ _editblock: "[" _editoper* "]"
 // A single edit operation
 _editoper: editnodeadd
             | editpropset | editunivset | edittagpropset | edittagadd | editcondpropset
-            | editpropdel | editunivdel | edittagpropdel | edittagdel
+            | editpropsetmulti | editpropdel | editunivdel | edittagpropdel | edittagdel
             | editparens | edgeaddn1 | edgedeln1 | edgeaddn2 | edgedeln2
 
 // Parenthesis in an edit block don't have incoming nodes
@@ -48,18 +48,20 @@ editparens: "(" editnodeadd _editoper* ")"
 edittagadd: "+" [SETTAGOPER] tagname [(EQSPACE | EQNOSPACE) _valu]
 editunivdel: EXPRMINUS univprop
 edittagdel: EXPRMINUS tagname
-editpropset: relprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu
+editpropset: relprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYMODSET) _valu
 editcondpropset: relprop condsetoper _valu
+editpropsetmulti: relprop (MODSETMULTI | TRYMODSETMULTI) _valu
 editpropdel: EXPRMINUS relprop
-editunivset: univprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu
-editnodeadd: formname (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu
-edittagpropset: "+" tagprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu
+editunivset: univprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYMODSET) _valu
+editnodeadd: formname (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYMODSET) _valu
+edittagpropset: "+" tagprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYMODSET) _valu
 edittagpropdel: EXPRMINUS tagprop
 
 EQSPACE: /((?<=\s)=|=(?=\s))/
 MODSET.4: "+=" | "-="
-TRYSETPLUS.1: "?+="
-TRYSETMINUS.1: "?-="
+TRYMODSET.1: "?+=" | "?-="
+MODSETMULTI.4: "++=" | "--="
+TRYMODSETMULTI.1: "?++=" | "?--="
 TRYSET.1: "?="
 SETTAGOPER: "?"
 
@@ -226,10 +228,10 @@ _EDGEN2JOININIT: "<+("
 // comparisons to an expression like '<(2)'
 _EDGEN2INIT: /\<\((?=(?>(?<EDGEN2INITRECUR>\(((?>[^()"'`]+|(?&EDGEN2INITRECUR)|`(?:[^`\\]|\\.)*`|"(?:[^"\\]|\\.)*"|'''.*?'''|'[^']*'(?!'))*)\))|'''.*?'''|`(?:[^`\\]|\\.)*`|"(?:[^"\\]|\\.)*"|'[^']*'(?!')|[^)])*\)[\-\+])/
 
-edgeaddn1: _EDGEADDN1INIT _valu _EDGEN1FINI baresubquery
-edgedeln1: _EDGEN1INIT _valu _EDGEN1FINI baresubquery
-edgeaddn2: _EDGEN2INIT _valu _EDGEADDN2FINI baresubquery
-edgedeln2: _EDGEN2INIT _valu _EDGEN2FINI baresubquery
+edgeaddn1: _EDGEADDN1INIT _valu _EDGEN1FINI (baresubquery | _varvalu)
+edgedeln1: _EDGEN1INIT _valu _EDGEN1FINI (baresubquery | _varvalu)
+edgeaddn2: _EDGEN2INIT _valu _EDGEADDN2FINI (baresubquery | _varvalu)
+edgedeln2: _EDGEN2INIT _valu _EDGEN2FINI (baresubquery | _varvalu)
 
 _REVERSE: /reverse(?=[\s\(])/
 liftreverse: _REVERSE "(" (liftformtag | liftpropby | liftprop | liftbyarray | lifttagtag | liftbytag | liftbytagprop | liftbyformtagprop) ")"

synapse/lib/storm.py

@@ -26,14 +26,13 @@ import synapse.lib.config as s_config
 import synapse.lib.autodoc as s_autodoc
 import synapse.lib.grammar as s_grammar
 import synapse.lib.msgpack as s_msgpack
+import synapse.lib.schemas as s_schemas
 import synapse.lib.spooled as s_spooled
 import synapse.lib.version as s_version
 import synapse.lib.hashitem as s_hashitem
 import synapse.lib.stormctrl as s_stormctrl
 import synapse.lib.stormtypes as s_stormtypes
 
-import synapse.lib.stormlib.graph as s_stormlib_graph
-
 logger = logging.getLogger(__name__)
 
 addtriggerdescr = '''
@@ -190,391 +189,6 @@ Examples:
     wget https://vertex.link https://vtx.lk
 '''
 
-permdef_schema = {
-    'type': 'object',
-    'properties': {
-        'perm': {'type': 'array', 'items': {'type': 'string'}},
-        'desc': {'type': 'string'},
-        'gate': {'type': 'string'},
-        'ex': {'type': 'string'},  # Example string
-        'workflowconfig': {'type': 'boolean'},
-        'default': {'type': 'boolean', 'default': False},
-    },
-    'required': ['perm', 'desc', 'gate'],
-}
-
-reqValidPermDef = s_config.getJsValidator(permdef_schema)
-
-reqValidPkgdef = s_config.getJsValidator({
-    'type': 'object',
-    'properties': {
-        'name': {'type': 'string'},
-        'version': {
-            'type': 'string',
-            'pattern': s_version.semverstr,
-        },
-        'build': {
-            'type' 'object'
-            'properties': {
-                'time': {'type': 'number'},
-            },
-            'required': ['time'],
-        },
-        'codesign': {
-            'type': 'object',
-            'properties': {
-                'sign': {'type': 'string'},
-                'cert': {'type': 'string'},
-            },
-            'required': ['cert', 'sign'],
-        },
-        # TODO: Remove me after Synapse 3.0.0.
-        'synapse_minversion': {
-            'type': ['array', 'null'],
-            'items': {'type': 'number'}
-        },
-        'synapse_version': {
-            'type': 'string',
-        },
-        'modules': {
-            'type': ['array', 'null'],
-            'items': {'$ref': '#/definitions/module'}
-        },
-        'docs': {
-            'type': ['array', 'null'],
-            'items': {'$ref': '#/definitions/doc'},
-        },
-        'logo': {
-            'type': 'object',
-            'properties': {
-                'mime': {'type': 'string'},
-                'file': {'type': 'string'},
-            },
-            'additionalProperties': True,
-            'required': ['mime', 'file'],
-        },
-        'commands': {
-            'type': ['array', 'null'],
-            'items': {'$ref': '#/definitions/command'},
-        },
-        'graphs': {
-            'type': ['array', 'null'],
-            'items': s_stormlib_graph.gdefSchema,
-        },
-        'desc': {'type': 'string'},
-        'svciden': {'type': ['string', 'null'], 'pattern': s_config.re_iden},
-        'onload': {'type': 'string'},
-        'author': {
-            'type': 'object',
-            'properties': {
-                'url': {'type': 'string'},
-                'name': {'type': 'string'},
-            },
-            'required': ['name', 'url'],
-        },
-        'depends': {
-            'properties': {
-                'requires': {'type': 'array', 'items': {'$ref': '#/definitions/require'}},
-                'conflicts': {'type': 'array', 'items': {'$ref': '#/definitions/conflict'}},
-            },
-            'additionalProperties': True,
-        },
-        'perms': {
-            'type': 'array',
-            'items': permdef_schema,
-        },
-        'configvars': {
-            'type': 'array',
-            'items': {
-                'type': 'object',
-                'properties': {
-                    'name': {'type': 'string'},
-                    'varname': {'type': 'string'},
-                    'desc': {'type': 'string'},
-                    'default': {},
-                    'workflowconfig': {'type': 'boolean'},
-                    'type': {'$ref': '#/definitions/configvartype'},
-                    'scopes': {
-                        'type': 'array',
-                        'items': {
-                            'type': 'string',
-                            'enum': ['global', 'self']
-                        },
-                    },
-                },
-                'required': ['name', 'varname', 'desc', 'type', 'scopes'],
-            },
-        },
-    },
-    'additionalProperties': True,
-    'required': ['name', 'version'],
-    'definitions': {
-        'doc': {
-            'type': 'object',
-            'properties': {
-                'title': {'type': 'string'},
-                'content': {'type': 'string'},
-            },
-            'additionalProperties': True,
-            'required': ['title', 'content'],
-        },
-        'module': {
-            'type': 'object',
-            'properties': {
-                'name': {'type': 'string'},
-                'storm': {'type': 'string'},
-                'modconf': {'type': 'object'},
-                'apidefs': {
-                    'type': ['array', 'null'],
-                    'items': {'$ref': '#/definitions/apidef'},
-                },
-                'asroot': {'type': 'boolean'},
-                'asroot:perms': {'type': 'array',
-                    'items': {'type': 'array',
-                        'items': {'type': 'string'}},
-                },
-            },
-            'additionalProperties': True,
-            'required': ['name', 'storm']
-        },
-        'apidef': {
-            'type': 'object',
-            'properties': {
-                'name': {'type': 'string'},
-                'desc': {'type': 'string'},
-                'deprecated': {'$ref': '#/definitions/deprecatedItem'},
-                'type': {
-                    'type': 'object',
-                    'properties': {
-                        'type': {
-                            'type': 'string',
-                            'enum': ['function']
-                        },
-                        'args': {
-                            'type': 'array',
-                            'items': {'$ref': '#/definitions/apiarg'},
-                        },
-                        'returns': {
-                            'type': 'object',
-                            'properties': {
-                                'name': {
-                                    'type': 'string',
-                                    'enum': ['yields'],
-                                },
-                                'desc': {'type': 'string'},
-                                'type': {
-                                    'oneOf': [
-                                        {'$ref': '#/definitions/apitype'},
-                                        {'type': 'array', 'items': {'$ref': '#/definitions/apitype'}},
-                                    ],
-                                },
-                            },
-                            'additionalProperties': False,
-                            'required': ['type', 'desc']
-                        },
-                    },
-                    'additionalProperties': False,
-                    'required': ['type', 'returns'],
-                },
-            },
-            'additionalProperties': False,
-            'required': ['name', 'desc', 'type']
-        },
-        'apiarg': {
-            'type': 'object',
-            'properties': {
-                'name': {'type': 'string'},
-                'desc': {'type': 'string'},
-                'type': {
-                    'oneOf': [
-                        {'$ref': '#/definitions/apitype'},
-                        {'type': 'array', 'items': {'$ref': '#/definitions/apitype'}},
-                    ],
-                },
-                'default': {'type': ['boolean', 'integer', 'string', 'null']},
-            },
-            'additionalProperties': False,
-            'required': ['name', 'desc', 'type']
-        },
-        'deprecatedItem': {
-            'type': 'object',
-            'properties': {
-                'eolvers': {'type': 'string', 'minLength': 1,
-                            'description': "The version which will not longer support the item."},
-                'eoldate': {'type': 'string', 'minLength': 1,
-                            'description': 'Optional string indicating Synapse releases after this date may no longer support the item.'},
-                'mesg': {'type': ['string', 'null'], 'default': None,
-                         'description': 'Optional message to include in the warning text.'}
-            },
-            'oneOf': [
-                {
-                    'required': ['eolvers'],
-                    'not': {'required': ['eoldate']}
-                },
-                {
-                    'required': ['eoldate'],
-                    'not': {'required': ['eolvers']}
-                }
-            ],
-            'additionalProperties': False,
-        },
-        'apitype': {
-            'type': 'string',
-        },
-        'command': {
-            'type': 'object',
-            'properties': {
-                'name': {
-                    'type': 'string',
-                    'pattern': s_grammar.re_scmd
-                },
-                'cmdargs': {
-                    'type': ['array', 'null'],
-                    'items': {'$ref': '#/definitions/cmdarg'},
-                },
-                'cmdinputs': {
-                    'type': ['array', 'null'],
-                    'items': {'$ref': '#/definitions/cmdinput'},
-                },
-                'storm': {'type': 'string'},
-                'forms': {'$ref': '#/definitions/cmdformhints'},
-                'perms': {'type': 'array',
-                    'items': {'type': 'array',
-                        'items': {'type': 'string'}},
-                },
-            },
-            'additionalProperties': True,
-            'required': ['name', 'storm']
-        },
-        'cmdarg': {
-            'type': 'array',
-            'items': [
-                {'type': 'string'},
-                {
-                    'type': 'object',
-                    'properties': {
-                        'help': {'type': 'string'},
-                        'default': {},
-                        'dest': {'type': 'string'},
-                        'required': {'type': 'boolean'},
-                        'action': {'type': 'string'},
-                        'nargs': {'type': ['string', 'integer']},
-                        'choices': {
-                            'type': 'array',
-                            'uniqueItems': True,
-                            'minItems': 1,
-                        },
-                        'type': {
-                            'type': 'string',
-                            'enum': list(s_datamodel.Model().types)
-                        },
-                    },
-                }
-            ],
-            'additionalItems': False,
-        },
-        'cmdinput': {
-            'type': 'object',
-            'properties': {
-                'form': {'type': 'string'},
-                'help': {'type': 'string'},
-            },
-            'additionalProperties': True,
-            'required': ['form'],
-        },
-        'configvartype': {
-            'anyOf': [
-                {'type': 'array', 'items': {'$ref': '#/definitions/configvartype'}},
-                {'type': 'string'},
-            ]
-        },
-        # deprecated
-        'cmdformhints': {
-            'type': 'object',
-            'properties': {
-                'input': {
-                    'type': 'array',
-                    'uniqueItems': True,
-                    'items': {
-                        'type': 'string',
-                    }
-                },
-                'output': {
-                    'type': 'array',
-                    'uniqueItems': True,
-                    'items': {
-                        'type': 'string',
-                    }
-                },
-                'nodedata': {
-                    'type': 'array',
-                    'uniqueItems': True,
-                    'items': {
-                        'type': 'array',
-                        'items': [
-                            {'type': 'string'},
-                            {'type': 'string'},
-                        ],
-                        'additionalItems': False,
-                    },
-                },
-            }
-        },
-        'require': {
-            'type': 'object',
-            'properties': {
-                'name': {'type': 'string'},
-                'version': {'type': 'string'},
-                'desc': {'type': 'string'},
-                'optional': {'type': 'boolean'},
-            },
-            'additionalItems': True,
-            'required': ('name', 'version'),
-        },
-        'conflict': {
-            'type': 'object',
-            'properties': {
-                'name': {'type': 'string'},
-                'version': {'type': 'string'},
-                'desc': {'type': 'string'},
-            },
-            'additionalItems': True,
-            'required': ('name',),
-        },
-    }
-})
-
-reqValidDdef = s_config.getJsValidator({
-    'type': 'object',
-    'properties': {
-        'name': {'type': 'string'},
-        'storm': {'type': 'string'},
-        'view': {'type': 'string', 'pattern': s_config.re_iden},
-        'user': {'type': 'string', 'pattern': s_config.re_iden},
-        'iden': {'type': 'string', 'pattern': s_config.re_iden},
-        'enabled': {'type': 'boolean', 'default': True},
-        'stormopts': {
-            'oneOf': [
-                {'type': 'null'},
-                {'$ref': '#/definitions/stormopts'}
-            ]
-        }
-    },
-    'additionalProperties': True,
-    'required': ['iden', 'user', 'storm'],
-    'definitions': {
-        'stormopts': {
-            'type': 'object',
-            'properties': {
-                'repr': {'type': 'boolean'},
-                'path': {'type': 'string'},
-                'show': {'type': 'array', 'items': {'type': 'string'}}
-            },
-            'additionalProperties': True,
-        },
-    }
-})
-
 stormcmds = (
     {
         'name': 'queue.add',
@@ -1685,7 +1299,7 @@ stormcmds = (
 
 @s_cache.memoize(size=1024)
 def queryhash(text):
-    return hashlib.md5(text.encode(errors='surrogatepass'), usedforsecurity=False).hexdigest()
+    return s_common.queryhash(text)
 
 class DmonManager(s_base.Base):
     '''
@@ -2463,11 +2077,15 @@ class Runtime(s_base.Base):
 
 class Parser:
 
-    def __init__(self, prog=None, descr=None, root=None):
+    def __init__(self, prog=None, descr=None, root=None, model=None):
 
         if root is None:
             root = self
 
+        if model is None:
+            model = s_datamodel.Model()
+        self.model = model
+
         self.prog = prog
         self.descr = descr
 
@@ -2495,7 +2113,7 @@ class Parser:
         assert len(names)
 
         argtype = opts.get('type')
-        if argtype is not None and argtype not in s_datamodel.Model().types:
+        if argtype is not None and argtype not in s_schemas.datamodel_basetypes:
             mesg = f'Argument type "{argtype}" is not a valid model type name'
             raise s_exc.BadArg(mesg=mesg, argtype=str(argtype))
 
@@ -2643,7 +2261,7 @@ class Parser:
             valu = todo.popleft()
             if argtype is not None:
                 try:
-                    valu = s_datamodel.Model().type(argtype).norm(valu)[0]
+                    valu = self.model.type(argtype).norm(valu)[0]
                 except Exception:
                     mesg = f'Invalid value for type ({argtype}): {valu}'
                     return self.help(mesg=mesg)
@@ -2666,7 +2284,7 @@ class Parser:
                 valu = todo.popleft()
                 if argtype is not None:
                     try:
-                        valu = s_datamodel.Model().type(argtype).norm(valu)[0]
+                        valu = self.model.type(argtype).norm(valu)[0]
                     except Exception:
                         mesg = f'Invalid value for type ({argtype}): {valu}'
                         return self.help(mesg=mesg)
@@ -2689,7 +2307,7 @@ class Parser:
 
                 if argtype is not None:
                     try:
-                        valu = s_datamodel.Model().type(argtype).norm(valu)[0]
+                        valu = self.model.type(argtype).norm(valu)[0]
                     except Exception:
                         mesg = f'Invalid value for type ({argtype}): {valu}'
                         return self.help(mesg=mesg)
@@ -2718,7 +2336,7 @@ class Parser:
             valu = todo.popleft()
             if argtype is not None:
                 try:
-                    valu = s_datamodel.Model().type(argtype).norm(valu)[0]
+                    valu = self.model.type(argtype).norm(valu)[0]
                 except Exception:
                     mesg = f'Invalid value for type ({argtype}): {valu}'
                     return self.help(mesg=mesg)
@@ -2945,7 +2563,7 @@ class Cmd:
         return self.__class__.__doc__
 
     def getArgParser(self):
-        return Parser(prog=self.getName(), descr=self.getDescr())
+        return Parser(prog=self.getName(), descr=self.getDescr(), model=self.runt.model)
 
     async def setArgv(self, argv):
 

synapse/lib/storm_format.py

@@ -55,6 +55,7 @@ TerminalPygMap = {
     'LSQB': p_t.Punctuation,
     'MCASEBARE': p_t.Literal.String,
     'MODSET': p_t.Operator,
+    'MODSETMULTI': p_t.Operator,
     'NONQUOTEWORD': p_t.Literal,
     'NOT': p_t.Keyword,
     'NULL': p_t.Keyword,
@@ -74,8 +75,8 @@ TerminalPygMap = {
     'TAGSEGNOVAR': p_t.Name,
     'TRIPLEQUOTEDSTRING': p_t.Literal.String,
     'TRYSET': p_t.Operator,
-    'TRYSETMINUS': p_t.Operator,
-    'TRYSETPLUS': p_t.Operator,
+    'TRYMODSET': p_t.Operator,
+    'TRYMODSETMULTI': p_t.Operator,
     'UNIVNAME': p_t.Name,
     'UNSET': p_t.Operator,
     'EXPRUNIVNAME': p_t.Name,

synapse/lib/stormlib/graph.py

@@ -5,67 +5,6 @@ import synapse.lib.msgpack as s_msgpack
 import synapse.lib.schemas as s_schemas
 import synapse.lib.stormtypes as s_stormtypes
 
-gdefSchema = {
-    'type': 'object',
-    'properties': {
-        'iden': {'type': 'string', 'pattern': s_config.re_iden},
-        'name': {'type': 'string', 'minLength': 1},
-        'desc': {'type': 'string', 'default': ''},
-        'scope': {'type': 'string', 'enum': ['user', 'power-up']},
-        'creator': {'type': 'string', 'pattern': s_config.re_iden},
-        'power-up': {'type': 'string', 'minLength': 1},
-        'maxsize': {'type': 'number', 'minimum': 0},
-        'existing': {'type': 'array', 'items': {'type': 'string'}},
-        'created': {'type': 'number'},
-        'updated': {'type': 'number'},
-        'refs': {'type': 'boolean', 'default': False},
-        'edges': {'type': 'boolean', 'default': True},
-        'edgelimit': {'type': 'number', 'default': 3000},
-        'degrees': {'type': ['integer', 'null'], 'minimum': 0},
-        'filterinput': {'type': 'boolean', 'default': True},
-        'yieldfiltered': {'type': 'boolean', 'default': False},
-        'filters': {
-            'type': ['array', 'null'],
-            'items': {'type': 'string'}
-        },
-        'pivots': {
-            'type': ['array', 'null'],
-            'items': {'type': 'string'}
-        },
-        'forms': {
-            'type': 'object',
-            'patternProperties': {
-                '^.*$': {
-                    'type': 'object',
-                    'properties': {
-                        'filters': {
-                            'type': ['array', 'null'],
-                            'items': {'type': 'string'}
-                        },
-                        'pivots': {
-                            'type': ['array', 'null'],
-                            'items': {'type': 'string'}
-                        }
-                    },
-                    'additionalProperties': False,
-                }
-            }
-        },
-        'permissions': s_msgpack.deepcopy(s_schemas.easyPermSchema)
-    },
-    'additionalProperties': False,
-    'required': ['iden', 'name', 'scope'],
-    'allOf': [
-        {
-            'if': {'properties': {'scope': {'const': 'power-up'}}},
-            'then': {'required': ['power-up']},
-            'else': {'required': ['creator']},
-        }
-    ]
-}
-
-reqValidGdef = s_config.getJsValidator(gdefSchema)
-
 USER_EDITABLE = {
     'desc',
     'name',

synapse/lib/stormlib/index.py

@@ -0,0 +1,52 @@
+import synapse.lib.stormtypes as s_stormtypes
+
+stormcmds = (
+    {
+        'name': 'index.count.prop',
+        'descr': '''
+            Display the number of properties or property values in the view.
+
+            Examples:
+
+                // Display the number of file:path:ext properties in the view.
+                index.count.prop file:path:ext
+
+                // Display the number of file:path:ext properties with the value
+                // "exe" in the view.
+                index.count.prop file:path:ext --value exe
+
+        ''',
+        'cmdargs': (
+            ('prop', {'help': 'The name of the form or property to count.'}),
+            ('--value', {'help': 'The specific value to count instances of.',
+                         'default': s_stormtypes.undef}),
+        ),
+        'storm': '''
+
+        $conf = (
+            {"columns": [
+                {"name": "Count", "width": 12},
+                {"name": "Layer Iden", "width": 32},
+                {"name": "Layer Name"},
+            ],
+            "separators": {
+                'data:row': ''
+            },
+        })
+        $printer = $lib.tabular.printer($conf)
+
+        $lib.print($printer.header())
+
+        $total = (0)
+        for $layer in $lib.view.get().layers {
+
+            $count = $layer.getPropCount($cmdopts.prop, valu=$cmdopts.value)
+
+            $total = ($total + $count)
+
+            $lib.print($printer.row(($count, $layer.iden, $layer.name)))
+        }
+        $lib.print(`Total: {$total}`)
+        ''',
+    },
+)