synapse 2.196.0__py311-none-any.whl → 2.197.0__py311-none-any.whl

synapse/lib/storm.py

@@ -26,14 +26,13 @@ import synapse.lib.config as s_config
 import synapse.lib.autodoc as s_autodoc
 import synapse.lib.grammar as s_grammar
 import synapse.lib.msgpack as s_msgpack
+import synapse.lib.schemas as s_schemas
 import synapse.lib.spooled as s_spooled
 import synapse.lib.version as s_version
 import synapse.lib.hashitem as s_hashitem
 import synapse.lib.stormctrl as s_stormctrl
 import synapse.lib.stormtypes as s_stormtypes
 
-import synapse.lib.stormlib.graph as s_stormlib_graph
-
 logger = logging.getLogger(__name__)
 
 addtriggerdescr = '''
@@ -190,391 +189,6 @@ Examples:
     wget https://vertex.link https://vtx.lk
 '''
 
-permdef_schema = {
-    'type': 'object',
-    'properties': {
-        'perm': {'type': 'array', 'items': {'type': 'string'}},
-        'desc': {'type': 'string'},
-        'gate': {'type': 'string'},
-        'ex': {'type': 'string'},  # Example string
-        'workflowconfig': {'type': 'boolean'},
-        'default': {'type': 'boolean', 'default': False},
-    },
-    'required': ['perm', 'desc', 'gate'],
-}
-
-reqValidPermDef = s_config.getJsValidator(permdef_schema)
-
-reqValidPkgdef = s_config.getJsValidator({
-    'type': 'object',
-    'properties': {
-        'name': {'type': 'string'},
-        'version': {
-            'type': 'string',
-            'pattern': s_version.semverstr,
-        },
-        'build': {
-            'type' 'object'
-            'properties': {
-                'time': {'type': 'number'},
-            },
-            'required': ['time'],
-        },
-        'codesign': {
-            'type': 'object',
-            'properties': {
-                'sign': {'type': 'string'},
-                'cert': {'type': 'string'},
-            },
-            'required': ['cert', 'sign'],
-        },
-        # TODO: Remove me after Synapse 3.0.0.
-        'synapse_minversion': {
-            'type': ['array', 'null'],
-            'items': {'type': 'number'}
-        },
-        'synapse_version': {
-            'type': 'string',
-        },
-        'modules': {
-            'type': ['array', 'null'],
-            'items': {'$ref': '#/definitions/module'}
-        },
-        'docs': {
-            'type': ['array', 'null'],
-            'items': {'$ref': '#/definitions/doc'},
-        },
-        'logo': {
-            'type': 'object',
-            'properties': {
-                'mime': {'type': 'string'},
-                'file': {'type': 'string'},
-            },
-            'additionalProperties': True,
-            'required': ['mime', 'file'],
-        },
-        'commands': {
-            'type': ['array', 'null'],
-            'items': {'$ref': '#/definitions/command'},
-        },
-        'graphs': {
-            'type': ['array', 'null'],
-            'items': s_stormlib_graph.gdefSchema,
-        },
-        'desc': {'type': 'string'},
-        'svciden': {'type': ['string', 'null'], 'pattern': s_config.re_iden},
-        'onload': {'type': 'string'},
-        'author': {
-            'type': 'object',
-            'properties': {
-                'url': {'type': 'string'},
-                'name': {'type': 'string'},
-            },
-            'required': ['name', 'url'],
-        },
-        'depends': {
-            'properties': {
-                'requires': {'type': 'array', 'items': {'$ref': '#/definitions/require'}},
-                'conflicts': {'type': 'array', 'items': {'$ref': '#/definitions/conflict'}},
-            },
-            'additionalProperties': True,
-        },
-        'perms': {
-            'type': 'array',
-            'items': permdef_schema,
-        },
-        'configvars': {
-            'type': 'array',
-            'items': {
-                'type': 'object',
-                'properties': {
-                    'name': {'type': 'string'},
-                    'varname': {'type': 'string'},
-                    'desc': {'type': 'string'},
-                    'default': {},
-                    'workflowconfig': {'type': 'boolean'},
-                    'type': {'$ref': '#/definitions/configvartype'},
-                    'scopes': {
-                        'type': 'array',
-                        'items': {
-                            'type': 'string',
-                            'enum': ['global', 'self']
-                        },
-                    },
-                },
-                'required': ['name', 'varname', 'desc', 'type', 'scopes'],
-            },
-        },
-    },
-    'additionalProperties': True,
-    'required': ['name', 'version'],
-    'definitions': {
-        'doc': {
-            'type': 'object',
-            'properties': {
-                'title': {'type': 'string'},
-                'content': {'type': 'string'},
-            },
-            'additionalProperties': True,
-            'required': ['title', 'content'],
-        },
-        'module': {
-            'type': 'object',
-            'properties': {
-                'name': {'type': 'string'},
-                'storm': {'type': 'string'},
-                'modconf': {'type': 'object'},
-                'apidefs': {
-                    'type': ['array', 'null'],
-                    'items': {'$ref': '#/definitions/apidef'},
-                },
-                'asroot': {'type': 'boolean'},
-                'asroot:perms': {'type': 'array',
-                    'items': {'type': 'array',
-                        'items': {'type': 'string'}},
-                },
-            },
-            'additionalProperties': True,
-            'required': ['name', 'storm']
-        },
-        'apidef': {
-            'type': 'object',
-            'properties': {
-                'name': {'type': 'string'},
-                'desc': {'type': 'string'},
-                'deprecated': {'$ref': '#/definitions/deprecatedItem'},
-                'type': {
-                    'type': 'object',
-                    'properties': {
-                        'type': {
-                            'type': 'string',
-                            'enum': ['function']
-                        },
-                        'args': {
-                            'type': 'array',
-                            'items': {'$ref': '#/definitions/apiarg'},
-                        },
-                        'returns': {
-                            'type': 'object',
-                            'properties': {
-                                'name': {
-                                    'type': 'string',
-                                    'enum': ['yields'],
-                                },
-                                'desc': {'type': 'string'},
-                                'type': {
-                                    'oneOf': [
-                                        {'$ref': '#/definitions/apitype'},
-                                        {'type': 'array', 'items': {'$ref': '#/definitions/apitype'}},
-                                    ],
-                                },
-                            },
-                            'additionalProperties': False,
-                            'required': ['type', 'desc']
-                        },
-                    },
-                    'additionalProperties': False,
-                    'required': ['type', 'returns'],
-                },
-            },
-            'additionalProperties': False,
-            'required': ['name', 'desc', 'type']
-        },
-        'apiarg': {
-            'type': 'object',
-            'properties': {
-                'name': {'type': 'string'},
-                'desc': {'type': 'string'},
-                'type': {
-                    'oneOf': [
-                        {'$ref': '#/definitions/apitype'},
-                        {'type': 'array', 'items': {'$ref': '#/definitions/apitype'}},
-                    ],
-                },
-                'default': {'type': ['boolean', 'integer', 'string', 'null']},
-            },
-            'additionalProperties': False,
-            'required': ['name', 'desc', 'type']
-        },
-        'deprecatedItem': {
-            'type': 'object',
-            'properties': {
-                'eolvers': {'type': 'string', 'minLength': 1,
-                            'description': "The version which will not longer support the item."},
-                'eoldate': {'type': 'string', 'minLength': 1,
-                            'description': 'Optional string indicating Synapse releases after this date may no longer support the item.'},
-                'mesg': {'type': ['string', 'null'], 'default': None,
-                         'description': 'Optional message to include in the warning text.'}
-            },
-            'oneOf': [
-                {
-                    'required': ['eolvers'],
-                    'not': {'required': ['eoldate']}
-                },
-                {
-                    'required': ['eoldate'],
-                    'not': {'required': ['eolvers']}
-                }
-            ],
-            'additionalProperties': False,
-        },
-        'apitype': {
-            'type': 'string',
-        },
-        'command': {
-            'type': 'object',
-            'properties': {
-                'name': {
-                    'type': 'string',
-                    'pattern': s_grammar.re_scmd
-                },
-                'cmdargs': {
-                    'type': ['array', 'null'],
-                    'items': {'$ref': '#/definitions/cmdarg'},
-                },
-                'cmdinputs': {
-                    'type': ['array', 'null'],
-                    'items': {'$ref': '#/definitions/cmdinput'},
-                },
-                'storm': {'type': 'string'},
-                'forms': {'$ref': '#/definitions/cmdformhints'},
-                'perms': {'type': 'array',
-                    'items': {'type': 'array',
-                        'items': {'type': 'string'}},
-                },
-            },
-            'additionalProperties': True,
-            'required': ['name', 'storm']
-        },
-        'cmdarg': {
-            'type': 'array',
-            'items': [
-                {'type': 'string'},
-                {
-                    'type': 'object',
-                    'properties': {
-                        'help': {'type': 'string'},
-                        'default': {},
-                        'dest': {'type': 'string'},
-                        'required': {'type': 'boolean'},
-                        'action': {'type': 'string'},
-                        'nargs': {'type': ['string', 'integer']},
-                        'choices': {
-                            'type': 'array',
-                            'uniqueItems': True,
-                            'minItems': 1,
-                        },
-                        'type': {
-                            'type': 'string',
-                            'enum': list(s_datamodel.Model().types)
-                        },
-                    },
-                }
-            ],
-            'additionalItems': False,
-        },
-        'cmdinput': {
-            'type': 'object',
-            'properties': {
-                'form': {'type': 'string'},
-                'help': {'type': 'string'},
-            },
-            'additionalProperties': True,
-            'required': ['form'],
-        },
-        'configvartype': {
-            'anyOf': [
-                {'type': 'array', 'items': {'$ref': '#/definitions/configvartype'}},
-                {'type': 'string'},
-            ]
-        },
-        # deprecated
-        'cmdformhints': {
-            'type': 'object',
-            'properties': {
-                'input': {
-                    'type': 'array',
-                    'uniqueItems': True,
-                    'items': {
-                        'type': 'string',
-                    }
-                },
-                'output': {
-                    'type': 'array',
-                    'uniqueItems': True,
-                    'items': {
-                        'type': 'string',
-                    }
-                },
-                'nodedata': {
-                    'type': 'array',
-                    'uniqueItems': True,
-                    'items': {
-                        'type': 'array',
-                        'items': [
-                            {'type': 'string'},
-                            {'type': 'string'},
-                        ],
-                        'additionalItems': False,
-                    },
-                },
-            }
-        },
-        'require': {
-            'type': 'object',
-            'properties': {
-                'name': {'type': 'string'},
-                'version': {'type': 'string'},
-                'desc': {'type': 'string'},
-                'optional': {'type': 'boolean'},
-            },
-            'additionalItems': True,
-            'required': ('name', 'version'),
-        },
-        'conflict': {
-            'type': 'object',
-            'properties': {
-                'name': {'type': 'string'},
-                'version': {'type': 'string'},
-                'desc': {'type': 'string'},
-            },
-            'additionalItems': True,
-            'required': ('name',),
-        },
-    }
-})
-
-reqValidDdef = s_config.getJsValidator({
-    'type': 'object',
-    'properties': {
-        'name': {'type': 'string'},
-        'storm': {'type': 'string'},
-        'view': {'type': 'string', 'pattern': s_config.re_iden},
-        'user': {'type': 'string', 'pattern': s_config.re_iden},
-        'iden': {'type': 'string', 'pattern': s_config.re_iden},
-        'enabled': {'type': 'boolean', 'default': True},
-        'stormopts': {
-            'oneOf': [
-                {'type': 'null'},
-                {'$ref': '#/definitions/stormopts'}
-            ]
-        }
-    },
-    'additionalProperties': True,
-    'required': ['iden', 'user', 'storm'],
-    'definitions': {
-        'stormopts': {
-            'type': 'object',
-            'properties': {
-                'repr': {'type': 'boolean'},
-                'path': {'type': 'string'},
-                'show': {'type': 'array', 'items': {'type': 'string'}}
-            },
-            'additionalProperties': True,
-        },
-    }
-})
-
 stormcmds = (
     {
         'name': 'queue.add',
@@ -2463,11 +2077,15 @@ class Runtime(s_base.Base):
 
 class Parser:
 
-    def __init__(self, prog=None, descr=None, root=None):
+    def __init__(self, prog=None, descr=None, root=None, model=None):
 
         if root is None:
             root = self
 
+        if model is None:
+            model = s_datamodel.Model()
+        self.model = model
+
         self.prog = prog
         self.descr = descr
 
@@ -2495,7 +2113,7 @@ class Parser:
         assert len(names)
 
         argtype = opts.get('type')
-        if argtype is not None and argtype not in s_datamodel.Model().types:
+        if argtype is not None and argtype not in s_schemas.datamodel_basetypes:
             mesg = f'Argument type "{argtype}" is not a valid model type name'
             raise s_exc.BadArg(mesg=mesg, argtype=str(argtype))
 
@@ -2643,7 +2261,7 @@ class Parser:
             valu = todo.popleft()
             if argtype is not None:
                 try:
-                    valu = s_datamodel.Model().type(argtype).norm(valu)[0]
+                    valu = self.model.type(argtype).norm(valu)[0]
                 except Exception:
                     mesg = f'Invalid value for type ({argtype}): {valu}'
                     return self.help(mesg=mesg)
@@ -2666,7 +2284,7 @@ class Parser:
                 valu = todo.popleft()
                 if argtype is not None:
                     try:
-                        valu = s_datamodel.Model().type(argtype).norm(valu)[0]
+                        valu = self.model.type(argtype).norm(valu)[0]
                     except Exception:
                         mesg = f'Invalid value for type ({argtype}): {valu}'
                         return self.help(mesg=mesg)
@@ -2689,7 +2307,7 @@ class Parser:
 
                 if argtype is not None:
                     try:
-                        valu = s_datamodel.Model().type(argtype).norm(valu)[0]
+                        valu = self.model.type(argtype).norm(valu)[0]
                     except Exception:
                         mesg = f'Invalid value for type ({argtype}): {valu}'
                         return self.help(mesg=mesg)
@@ -2718,7 +2336,7 @@ class Parser:
             valu = todo.popleft()
             if argtype is not None:
                 try:
-                    valu = s_datamodel.Model().type(argtype).norm(valu)[0]
+                    valu = self.model.type(argtype).norm(valu)[0]
                 except Exception:
                     mesg = f'Invalid value for type ({argtype}): {valu}'
                     return self.help(mesg=mesg)
@@ -2945,7 +2563,7 @@ class Cmd:
         return self.__class__.__doc__
 
     def getArgParser(self):
-        return Parser(prog=self.getName(), descr=self.getDescr())
+        return Parser(prog=self.getName(), descr=self.getDescr(), model=self.runt.model)
 
     async def setArgv(self, argv):
 

synapse/lib/stormlib/graph.py

@@ -5,67 +5,6 @@ import synapse.lib.msgpack as s_msgpack
 import synapse.lib.schemas as s_schemas
 import synapse.lib.stormtypes as s_stormtypes
 
-gdefSchema = {
-    'type': 'object',
-    'properties': {
-        'iden': {'type': 'string', 'pattern': s_config.re_iden},
-        'name': {'type': 'string', 'minLength': 1},
-        'desc': {'type': 'string', 'default': ''},
-        'scope': {'type': 'string', 'enum': ['user', 'power-up']},
-        'creator': {'type': 'string', 'pattern': s_config.re_iden},
-        'power-up': {'type': 'string', 'minLength': 1},
-        'maxsize': {'type': 'number', 'minimum': 0},
-        'existing': {'type': 'array', 'items': {'type': 'string'}},
-        'created': {'type': 'number'},
-        'updated': {'type': 'number'},
-        'refs': {'type': 'boolean', 'default': False},
-        'edges': {'type': 'boolean', 'default': True},
-        'edgelimit': {'type': 'number', 'default': 3000},
-        'degrees': {'type': ['integer', 'null'], 'minimum': 0},
-        'filterinput': {'type': 'boolean', 'default': True},
-        'yieldfiltered': {'type': 'boolean', 'default': False},
-        'filters': {
-            'type': ['array', 'null'],
-            'items': {'type': 'string'}
-        },
-        'pivots': {
-            'type': ['array', 'null'],
-            'items': {'type': 'string'}
-        },
-        'forms': {
-            'type': 'object',
-            'patternProperties': {
-                '^.*$': {
-                    'type': 'object',
-                    'properties': {
-                        'filters': {
-                            'type': ['array', 'null'],
-                            'items': {'type': 'string'}
-                        },
-                        'pivots': {
-                            'type': ['array', 'null'],
-                            'items': {'type': 'string'}
-                        }
-                    },
-                    'additionalProperties': False,
-                }
-            }
-        },
-        'permissions': s_msgpack.deepcopy(s_schemas.easyPermSchema)
-    },
-    'additionalProperties': False,
-    'required': ['iden', 'name', 'scope'],
-    'allOf': [
-        {
-            'if': {'properties': {'scope': {'const': 'power-up'}}},
-            'then': {'required': ['power-up']},
-            'else': {'required': ['creator']},
-        }
-    ]
-}
-
-reqValidGdef = s_config.getJsValidator(gdefSchema)
-
 USER_EDITABLE = {
     'desc',
     'name',

synapse/lib/stormlib/index.py

@@ -0,0 +1,52 @@
+import synapse.lib.stormtypes as s_stormtypes
+
+stormcmds = (
+    {
+        'name': 'index.count.prop',
+        'descr': '''
+            Display the number of properties or property values in the view.
+
+            Examples:
+
+                // Display the number of file:path:ext properties in the view.
+                index.count.prop file:path:ext
+
+                // Display the number of file:path:ext properties with the value
+                // "exe" in the view.
+                index.count.prop file:path:ext --value exe
+
+        ''',
+        'cmdargs': (
+            ('prop', {'help': 'The name of the form or property to count.'}),
+            ('--value', {'help': 'The specific value to count instances of.',
+                         'default': s_stormtypes.undef}),
+        ),
+        'storm': '''
+
+        $conf = (
+            {"columns": [
+                {"name": "Count", "width": 12},
+                {"name": "Layer Iden", "width": 32},
+                {"name": "Layer Name"},
+            ],
+            "separators": {
+                'data:row': ''
+            },
+        })
+        $printer = $lib.tabular.printer($conf)
+
+        $lib.print($printer.header())
+
+        $total = (0)
+        for $layer in $lib.view.get().layers {
+
+            $count = $layer.getPropCount($cmdopts.prop, valu=$cmdopts.value)
+
+            $total = ($total + $count)
+
+            $lib.print($printer.row(($count, $layer.iden, $layer.name)))
+        }
+        $lib.print(`Total: {$total}`)
+        ''',
+    },
+)

synapse/lib/stormtypes.py

@@ -6839,7 +6839,8 @@ class Layer(Prim):
     Implements the Storm api for a layer instance.
     '''
     _storm_locals = (
-        {'name': 'iden', 'desc': 'The iden of the Layer.', 'type': 'str', },
+        {'name': 'iden', 'desc': 'The iden of the Layer.', 'type': 'str'},
+        {'name': 'name', 'desc': 'The name of the Layer.', 'type': 'str'},
         {'name': 'set', 'desc': 'Set an arbitrary value in the Layer definition.',
          'type': {'type': 'function', '_funcname': '_methLayerSet',
                   'args': (
@@ -7194,6 +7195,7 @@ class Layer(Prim):
 
         self.locls.update(self.getObjLocals())
         self.locls['iden'] = self.valu.get('iden')
+        self.locls['name'] = self.valu.get('name')
 
     def __hash__(self):
         return hash((self._storm_typename, self.locls['iden']))

synapse/lib/task.py

@@ -1,4 +1,3 @@
-import copy
 import asyncio
 import logging
 
@@ -6,6 +5,7 @@ import synapse.exc as s_exc
 import synapse.common as s_common
 
 import synapse.lib.base as s_base
+import synapse.lib.msgpack as s_msgpack
 
 logger = logging.getLogger(__name__)
 
@@ -104,7 +104,7 @@ class Task(s_base.Base):
         pask = {
             'iden': self.iden,
             'name': self.name,
-            'info': copy.deepcopy(self.info),
+            'info': s_msgpack.deepcopy(self.info),
             'tick': self.tick,
             'user': 'root',
             'kids': {i: k.pack() for i, k in self.kids.items()},
@@ -115,6 +115,17 @@ class Task(s_base.Base):
 
         return pask
 
+    def packv2(self):
+        return {
+            'iden': self.iden,
+            'name': self.name,
+            'info': s_msgpack.deepcopy(self.info),
+            'tick': self.tick,
+            'user': self.user.iden,
+            'username': self.user.name,
+            'kids': {i: k.packv2() for i, k in self.kids.items()},
+        }
+
 def loop():
     try:
         return asyncio.get_running_loop()

synapse/lib/urlhelp.py

@@ -13,7 +13,7 @@ def chopurl(url):
     '''
     ret = {}
     if url.find('://') == -1:
-        raise s_exc.BadUrl(':// not found in [{}]!'.format(url))
+        raise s_exc.BadUrl(mesg=':// not found in [{}]!'.format(url))
 
     scheme, remain = url.split('://', 1)
     ret['scheme'] = scheme.lower()

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 196, 0)
+version = (2, 197, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = '8182360d2ebf8e8bdd00cb72f03812061c819cc2'
+commit = 'c8c758131cae47ef21e23ac8242012459ebfce9d'