synapse 2.196.0__py311-none-any.whl → 2.197.0__py311-none-any.whl

synapse/cortex.py

@@ -77,6 +77,7 @@ import synapse.lib.stormlib.yaml as s_stormlib_yaml  # NOQA
 import synapse.lib.stormlib.basex as s_stormlib_basex  # NOQA
 import synapse.lib.stormlib.cache as s_stormlib_cache  # NOQA
 import synapse.lib.stormlib.graph as s_stormlib_graph  # NOQA
+import synapse.lib.stormlib.index as s_stormlib_index  # NOQA
 import synapse.lib.stormlib.iters as s_stormlib_iters  # NOQA
 import synapse.lib.stormlib.macro as s_stormlib_macro
 import synapse.lib.stormlib.model as s_stormlib_model
@@ -1496,7 +1497,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
              'desc': 'Controls the ability to remove a file from the Axon.'},
         ))
         for pdef in self._cortex_permdefs:
-            s_storm.reqValidPermDef(pdef)
+            s_schemas.reqValidPermDef(pdef)
 
     def _getPermDefs(self):
 
@@ -1770,7 +1771,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         gdef['updated'] = now
         gdef['permissions']['users'][user.iden] = s_cell.PERM_ADMIN
 
-        s_stormlib_graph.reqValidGdef(gdef)
+        s_schemas.reqValidGdef(gdef)
 
         await self.reqValidStormGraph(gdef)
 
@@ -1778,7 +1779,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
     @s_nexus.Pusher.onPush('storm:graph:add')
     async def _addStormGraph(self, gdef):
-        s_stormlib_graph.reqValidGdef(gdef)
+        s_schemas.reqValidGdef(gdef)
 
         await self.reqValidStormGraph(gdef)
 
@@ -1857,7 +1858,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         gdef = copy.deepcopy(gdef)
         gdef.update(info)
 
-        s_stormlib_graph.reqValidGdef(gdef)
+        s_schemas.reqValidGdef(gdef)
 
         await self.reqValidStormGraph(gdef)
 
@@ -1879,7 +1880,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         await self._setEasyPerm(gdef, scope, iden, level)
 
-        s_stormlib_graph.reqValidGdef(gdef)
+        s_schemas.reqValidGdef(gdef)
 
         self.graphs.set(gden, gdef)
 
@@ -2876,7 +2877,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                 await self.reqValidStormGraph(gdef)
 
         # Validate package def (post normalization)
-        s_storm.reqValidPkgdef(pkgdef)
+        s_schemas.reqValidPkgdef(pkgdef)
 
         for configvar in pkgdef.get('configvars', ()):
             self._reqStormPkgVarType(pkgname, configvar.get('type'))
@@ -4421,6 +4422,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         for cdef in s_stormlib_vault.stormcmds:
             await self._trySetStormCmd(cdef.get('name'), cdef)
 
+        for cdef in s_stormlib_index.stormcmds:
+            await self._trySetStormCmd(cdef.get('name'), cdef)
+
     async def _initPureStormCmds(self):
         oldcmds = []
         for name, cdef in self.cmddefs.items():
@@ -4451,7 +4455,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         for path, ctor in s_stormtypes.registry.iterLibs():
             # Ensure each ctor's permdefs are valid
             for pdef in ctor._storm_lib_perms:
-                s_storm.reqValidPermDef(pdef)
+                s_schemas.reqValidPermDef(pdef)
             # Skip libbase which is registered as a default ctor in the storm Runtime
             if path:
                 self.addStormLib(path, ctor)
@@ -5672,7 +5676,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     async def runStormDmon(self, iden, ddef):
 
         # validate ddef before firing task
-        s_storm.reqValidDdef(ddef)
+        s_schemas.reqValidDdef(ddef)
 
         dmon = self.stormdmons.getDmon(iden)
         if dmon is not None:

synapse/cryotank.py

@@ -8,7 +8,7 @@ import synapse.common as s_common
 
 import synapse.lib.base as s_base
 import synapse.lib.cell as s_cell
-import synapse.lib.storm as s_storm
+import synapse.lib.schemas as s_schemas
 import synapse.lib.lmdbslab as s_lmdbslab
 import synapse.lib.slabseqn as s_slabseqn
 import synapse.lib.slaboffs as s_slaboffs
@@ -314,7 +314,7 @@ class CryoCell(s_cell.Cell):
         ))
 
         for pdef in self._cryo_permdefs:
-            s_storm.reqValidPermDef(pdef)
+            s_schemas.reqValidPermDef(pdef)
 
     def _getPermDefs(self):
         permdefs = list(s_cell.Cell._getPermDefs(self))

synapse/lib/cell.py

@@ -4478,7 +4478,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         for task in self.boss.ps():
 
-            item = task.pack()
+            item = task.packv2()
             item['service'] = self.ahasvcname
 
             yield item
@@ -4500,7 +4500,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         task = self.boss.get(iden)
         if task is not None:
-            item = task.pack()
+            item = task.packv2()
             item['service'] = self.ahasvcname
             return item
 

synapse/lib/config.py

@@ -32,16 +32,16 @@ def localSchemaRefHandler(uri):
     try:
         parts = urllib.parse.urlparse(uri)
     except ValueError:
-        raise s_exc.BadUrl(f'Malformed URI: {uri}.') from None
+        raise s_exc.BadUrl(mesg=f'Malformed URI: {uri}.') from None
 
     filename = s_data.path('jsonschemas', parts.hostname, *parts.path.split('/'))
 
     # Check for path traversal. Unlikely, but still check
     if not filename.startswith(s_data.path('jsonschemas', parts.hostname)):
-        raise s_exc.BadArg(f'Path traversal in schema URL: {uri}.')
+        raise s_exc.BadArg(mesg=f'Path traversal in schema URL: {uri}.')
 
     if not os.path.exists(filename) or not os.path.isfile(filename):
-        raise s_exc.NoSuchFile(f'Local JSON schema not found for {uri}.')
+        raise s_exc.NoSuchFile(mesg=f'Local JSON schema not found for {uri}.')
 
     with open(filename, 'r') as fp:
         return json.load(fp)

synapse/lib/multislabseqn.py

@@ -150,7 +150,7 @@ class MultiSlabSeqn(s_base.Base):
                     firstidx = firstitem[0]  # might not match the separately stored first index due to culling
 
                     if firstidx < fnstartidx:
-                        raise s_exc.BadCoreStore('Multislab:  filename inconsistent with contents')
+                        raise s_exc.BadCoreStore(mesg='Multislab:  filename inconsistent with contents')
 
                     lastidx = seqn.index() - 1
 
@@ -161,7 +161,7 @@ class MultiSlabSeqn(s_base.Base):
             self.firstindx = lowindx
 
         if self.firstindx > self.indx:
-            raise s_exc.BadCoreStore('Invalid firstindx value')
+            raise s_exc.BadCoreStore(mesg='Invalid firstindx value')
 
         await self._initTailSlab(fnstartidx)
 

synapse/lib/schemas.py

@@ -1,7 +1,8 @@
 import synapse.lib.const as s_const
 import synapse.lib.config as s_config
+import synapse.lib.grammar as s_grammar
 import synapse.lib.msgpack as s_msgpack
-
+import synapse.lib.version as s_version
 
 easyPermSchema = {
     'type': 'object',
@@ -551,3 +552,479 @@ stixIngestBundleSchema = {
     },
 }
 reqValidStixIngestBundle = s_config.getJsValidator(stixIngestBundleSchema)
+
+_reqValidGdefSchema = {
+    'type': 'object',
+    'properties': {
+        'iden': {'type': 'string', 'pattern': s_config.re_iden},
+        'name': {'type': 'string', 'minLength': 1},
+        'desc': {'type': 'string', 'default': ''},
+        'scope': {'type': 'string', 'enum': ['user', 'power-up']},
+        'creator': {'type': 'string', 'pattern': s_config.re_iden},
+        'power-up': {'type': 'string', 'minLength': 1},
+        'maxsize': {'type': 'number', 'minimum': 0},
+        'existing': {'type': 'array', 'items': {'type': 'string'}},
+        'created': {'type': 'number'},
+        'updated': {'type': 'number'},
+        'refs': {'type': 'boolean', 'default': False},
+        'edges': {'type': 'boolean', 'default': True},
+        'edgelimit': {'type': 'number', 'default': 3000},
+        'degrees': {'type': ['integer', 'null'], 'minimum': 0},
+        'filterinput': {'type': 'boolean', 'default': True},
+        'yieldfiltered': {'type': 'boolean', 'default': False},
+        'filters': {
+            'type': ['array', 'null'],
+            'items': {'type': 'string'}
+        },
+        'pivots': {
+            'type': ['array', 'null'],
+            'items': {'type': 'string'}
+        },
+        'forms': {
+            'type': 'object',
+            'patternProperties': {
+                '^.*$': {
+                    'type': 'object',
+                    'properties': {
+                        'filters': {
+                            'type': ['array', 'null'],
+                            'items': {'type': 'string'}
+                        },
+                        'pivots': {
+                            'type': ['array', 'null'],
+                            'items': {'type': 'string'}
+                        }
+                    },
+                    'additionalProperties': False,
+                }
+            }
+        },
+        'permissions': s_msgpack.deepcopy(easyPermSchema)
+    },
+    'additionalProperties': False,
+    'required': ['iden', 'name', 'scope'],
+    'allOf': [
+        {
+            'if': {'properties': {'scope': {'const': 'power-up'}}},
+            'then': {'required': ['power-up']},
+            'else': {'required': ['creator']},
+        }
+    ]
+}
+reqValidGdef = s_config.getJsValidator(_reqValidGdefSchema)
+
+_reqValidPermDefSchema = {
+    'type': 'object',
+    'properties': {
+        'perm': {'type': 'array', 'items': {'type': 'string'}},
+        'desc': {'type': 'string'},
+        'gate': {'type': 'string'},
+        'ex': {'type': 'string'},  # Example string
+        'workflowconfig': {'type': 'boolean'},
+        'default': {'type': 'boolean', 'default': False},
+    },
+    'required': ['perm', 'desc', 'gate'],
+}
+
+reqValidPermDef = s_config.getJsValidator(_reqValidPermDefSchema)
+
+# N.B. This is kept in sync with s_datamodel.Datamodel().types
+# with the DatamodelTest.test_datamodel_schema_basetypes test.
+datamodel_basetypes = [
+    'int',
+    'float',
+    'range',
+    'str',
+    'hex',
+    'bool',
+    'time',
+    'duration',
+    'ival',
+    'guid',
+    'syn:tag:part',
+    'syn:tag',
+    'comp',
+    'loc',
+    'ndef',
+    'array',
+    'edge',
+    'timeedge',
+    'data',
+    'nodeprop',
+    'hugenum',
+    'taxon',
+    'taxonomy',
+    'velocity',
+]
+
+_reqValidPkgdefSchema = {
+    'type': 'object',
+    'properties': {
+        'name': {'type': 'string'},
+        'version': {
+            'type': 'string',
+            'pattern': s_version.semverstr,
+        },
+        'build': {
+            'type' 'object'
+            'properties': {
+                'time': {'type': 'number'},
+            },
+            'required': ['time'],
+        },
+        'codesign': {
+            'type': 'object',
+            'properties': {
+                'sign': {'type': 'string'},
+                'cert': {'type': 'string'},
+            },
+            'required': ['cert', 'sign'],
+        },
+        # TODO: Remove me after Synapse 3.0.0.
+        'synapse_minversion': {
+            'type': ['array', 'null'],
+            'items': {'type': 'number'}
+        },
+        'synapse_version': {
+            'type': 'string',
+        },
+        'modules': {
+            'type': ['array', 'null'],
+            'items': {'$ref': '#/definitions/module'}
+        },
+        'docs': {
+            'type': ['array', 'null'],
+            'items': {'$ref': '#/definitions/doc'},
+        },
+        'logo': {
+            'type': 'object',
+            'properties': {
+                'mime': {'type': 'string'},
+                'file': {'type': 'string'},
+            },
+            'additionalProperties': True,
+            'required': ['mime', 'file'],
+        },
+        'commands': {
+            'type': ['array', 'null'],
+            'items': {'$ref': '#/definitions/command'},
+        },
+        'graphs': {
+            'type': ['array', 'null'],
+            'items': s_msgpack.deepcopy(_reqValidGdefSchema, use_list=True),
+        },
+        'desc': {'type': 'string'},
+        'svciden': {'type': ['string', 'null'], 'pattern': s_config.re_iden},
+        'onload': {'type': 'string'},
+        'author': {
+            'type': 'object',
+            'properties': {
+                'url': {'type': 'string'},
+                'name': {'type': 'string'},
+            },
+            'required': ['name', 'url'],
+        },
+        'depends': {
+            'properties': {
+                'requires': {'type': 'array', 'items': {'$ref': '#/definitions/require'}},
+                'conflicts': {'type': 'array', 'items': {'$ref': '#/definitions/conflict'}},
+            },
+            'additionalProperties': True,
+        },
+        'perms': {
+            'type': 'array',
+            'items': s_msgpack.deepcopy(_reqValidPermDefSchema),
+        },
+        'configvars': {
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                    'name': {'type': 'string'},
+                    'varname': {'type': 'string'},
+                    'desc': {'type': 'string'},
+                    'default': {},
+                    'workflowconfig': {'type': 'boolean'},
+                    'type': {'$ref': '#/definitions/configvartype'},
+                    'scopes': {
+                        'type': 'array',
+                        'items': {
+                            'type': 'string',
+                            'enum': ['global', 'self']
+                        },
+                    },
+                },
+                'required': ['name', 'varname', 'desc', 'type', 'scopes'],
+            },
+        },
+    },
+    'additionalProperties': True,
+    'required': ['name', 'version'],
+    'definitions': {
+        'doc': {
+            'type': 'object',
+            'properties': {
+                'title': {'type': 'string'},
+                'content': {'type': 'string'},
+            },
+            'additionalProperties': True,
+            'required': ['title', 'content'],
+        },
+        'module': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'storm': {'type': 'string'},
+                'modconf': {'type': 'object'},
+                'apidefs': {
+                    'type': ['array', 'null'],
+                    'items': {'$ref': '#/definitions/apidef'},
+                },
+                'asroot': {'type': 'boolean'},
+                'asroot:perms': {'type': 'array',
+                    'items': {'type': 'array',
+                        'items': {'type': 'string'}},
+                },
+            },
+            'additionalProperties': True,
+            'required': ['name', 'storm']
+        },
+        'apidef': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'desc': {'type': 'string'},
+                'deprecated': {'$ref': '#/definitions/deprecatedItem'},
+                'type': {
+                    'type': 'object',
+                    'properties': {
+                        'type': {
+                            'type': 'string',
+                            'enum': ['function']
+                        },
+                        'args': {
+                            'type': 'array',
+                            'items': {'$ref': '#/definitions/apiarg'},
+                        },
+                        'returns': {
+                            'type': 'object',
+                            'properties': {
+                                'name': {
+                                    'type': 'string',
+                                    'enum': ['yields'],
+                                },
+                                'desc': {'type': 'string'},
+                                'type': {
+                                    'oneOf': [
+                                        {'$ref': '#/definitions/apitype'},
+                                        {'type': 'array', 'items': {'$ref': '#/definitions/apitype'}},
+                                    ],
+                                },
+                            },
+                            'additionalProperties': False,
+                            'required': ['type', 'desc']
+                        },
+                    },
+                    'additionalProperties': False,
+                    'required': ['type', 'returns'],
+                },
+            },
+            'additionalProperties': False,
+            'required': ['name', 'desc', 'type']
+        },
+        'apiarg': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'desc': {'type': 'string'},
+                'type': {
+                    'oneOf': [
+                        {'$ref': '#/definitions/apitype'},
+                        {'type': 'array', 'items': {'$ref': '#/definitions/apitype'}},
+                    ],
+                },
+                'default': {'type': ['boolean', 'integer', 'string', 'null']},
+            },
+            'additionalProperties': False,
+            'required': ['name', 'desc', 'type']
+        },
+        'deprecatedItem': {
+            'type': 'object',
+            'properties': {
+                'eolvers': {'type': 'string', 'minLength': 1,
+                            'description': "The version which will not longer support the item."},
+                'eoldate': {'type': 'string', 'minLength': 1,
+                            'description': 'Optional string indicating Synapse releases after this date may no longer support the item.'},
+                'mesg': {'type': ['string', 'null'], 'default': None,
+                         'description': 'Optional message to include in the warning text.'}
+            },
+            'oneOf': [
+                {
+                    'required': ['eolvers'],
+                    'not': {'required': ['eoldate']}
+                },
+                {
+                    'required': ['eoldate'],
+                    'not': {'required': ['eolvers']}
+                }
+            ],
+            'additionalProperties': False,
+        },
+        'apitype': {
+            'type': 'string',
+        },
+        'command': {
+            'type': 'object',
+            'properties': {
+                'name': {
+                    'type': 'string',
+                    'pattern': s_grammar.re_scmd
+                },
+                'cmdargs': {
+                    'type': ['array', 'null'],
+                    'items': {'$ref': '#/definitions/cmdarg'},
+                },
+                'cmdinputs': {
+                    'type': ['array', 'null'],
+                    'items': {'$ref': '#/definitions/cmdinput'},
+                },
+                'storm': {'type': 'string'},
+                'forms': {'$ref': '#/definitions/cmdformhints'},
+                'perms': {'type': 'array',
+                    'items': {'type': 'array',
+                        'items': {'type': 'string'}},
+                },
+            },
+            'additionalProperties': True,
+            'required': ['name', 'storm']
+        },
+        'cmdarg': {
+            'type': 'array',
+            'items': [
+                {'type': 'string'},
+                {
+                    'type': 'object',
+                    'properties': {
+                        'help': {'type': 'string'},
+                        'default': {},
+                        'dest': {'type': 'string'},
+                        'required': {'type': 'boolean'},
+                        'action': {'type': 'string'},
+                        'nargs': {'type': ['string', 'integer']},
+                        'choices': {
+                            'type': 'array',
+                            'uniqueItems': True,
+                            'minItems': 1,
+                        },
+                        'type': {
+                            'type': 'string',
+                            'enum': s_msgpack.deepcopy(datamodel_basetypes),
+                        },
+                    },
+                }
+            ],
+            'additionalItems': False,
+        },
+        'cmdinput': {
+            'type': 'object',
+            'properties': {
+                'form': {'type': 'string'},
+                'help': {'type': 'string'},
+            },
+            'additionalProperties': True,
+            'required': ['form'],
+        },
+        'configvartype': {
+            'anyOf': [
+                {'type': 'array', 'items': {'$ref': '#/definitions/configvartype'}},
+                {'type': 'string'},
+            ]
+        },
+        # deprecated
+        'cmdformhints': {
+            'type': 'object',
+            'properties': {
+                'input': {
+                    'type': 'array',
+                    'uniqueItems': True,
+                    'items': {
+                        'type': 'string',
+                    }
+                },
+                'output': {
+                    'type': 'array',
+                    'uniqueItems': True,
+                    'items': {
+                        'type': 'string',
+                    }
+                },
+                'nodedata': {
+                    'type': 'array',
+                    'uniqueItems': True,
+                    'items': {
+                        'type': 'array',
+                        'items': [
+                            {'type': 'string'},
+                            {'type': 'string'},
+                        ],
+                        'additionalItems': False,
+                    },
+                },
+            }
+        },
+        'require': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'version': {'type': 'string'},
+                'desc': {'type': 'string'},
+                'optional': {'type': 'boolean'},
+            },
+            'additionalItems': True,
+            'required': ('name', 'version'),
+        },
+        'conflict': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'version': {'type': 'string'},
+                'desc': {'type': 'string'},
+            },
+            'additionalItems': True,
+            'required': ('name',),
+        },
+    }
+}
+reqValidPkgdef = s_config.getJsValidator(_reqValidPkgdefSchema)
+
+_reqValidDdefSchema = {
+    'type': 'object',
+    'properties': {
+        'name': {'type': 'string'},
+        'storm': {'type': 'string'},
+        'view': {'type': 'string', 'pattern': s_config.re_iden},
+        'user': {'type': 'string', 'pattern': s_config.re_iden},
+        'iden': {'type': 'string', 'pattern': s_config.re_iden},
+        'enabled': {'type': 'boolean', 'default': True},
+        'stormopts': {
+            'oneOf': [
+                {'type': 'null'},
+                {'$ref': '#/definitions/stormopts'}
+            ]
+        }
+    },
+    'additionalProperties': True,
+    'required': ['iden', 'user', 'storm'],
+    'definitions': {
+        'stormopts': {
+            'type': 'object',
+            'properties': {
+                'repr': {'type': 'boolean'},
+                'path': {'type': 'string'},
+                'show': {'type': 'array', 'items': {'type': 'string'}}
+            },
+            'additionalProperties': True,
+        },
+    }
+}
+reqValidDdef = s_config.getJsValidator(_reqValidDdefSchema)