synapse 2.152.0__py311-none-any.whl → 2.154.0__py311-none-any.whl

synapse/lib/storm.py

@@ -45,6 +45,8 @@ Notes:
         * node:add
         * node:del
         * prop:set
+        * edge:add
+        * edge:del
 
 When condition is tag:add or tag:del, you may optionally provide a form name
 to restrict the trigger to fire only on tags added or deleted from nodes of
@@ -56,6 +58,10 @@ Simple one level tag globbing is supported, only at the end after a period,
 that is aka.* matches aka.foo and aka.bar but not aka.foo.bar. aka* is not
 supported.
 
+When the condition is edge:add or edge:del, you may optionally provide a
+form name or a destination form name to only fire on edges added or deleted
+from nodes of those forms.
+
 Examples:
     # Adds a tag to every inet:ipv4 added
     trigger.add node:add --form inet:ipv4 --query {[ +#mytag ]}
@@ -65,6 +71,13 @@ Examples:
 
     # Adds a tag #todo to every inet:ipv4 as it is tagged #aka
     trigger.add tag:add --form inet:ipv4 --tag aka --query {[ +#todo ]}
+
+    # Adds a tag #todo to the N1 node of every refs edge add
+    trigger.add edge:add --verb refs --query {[ +#todo ]}
+
+    # Adds a tag #todo to the N1 node of every seen edge delete, provided that
+    # both nodes are of form file:bytes
+    trigger.add edge:del --verb seen --form file:bytes --n2form file:bytes --query {[ +#todo ]}
 '''
 
 addcrondescr = '''
@@ -307,6 +320,10 @@ reqValidPkgdef = s_config.getJsValidator({
                 'name': {'type': 'string'},
                 'storm': {'type': 'string'},
                 'modconf': {'type': 'object'},
+                'apidefs': {
+                    'type': ['array', 'null'],
+                    'items': {'$ref': '#/definitions/apidef'},
+                },
                 'asroot': {'type': 'boolean'},
                 'asroot:perms': {'type': 'array',
                     'items': {'type': 'array',
@@ -316,6 +333,67 @@ reqValidPkgdef = s_config.getJsValidator({
             'additionalProperties': True,
             'required': ['name', 'storm']
         },
+        'apidef': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'desc': {'type': 'string'},
+                'type': {
+                    'type': 'object',
+                    'properties': {
+                        'type': {
+                            'type': 'string',
+                            'enum': ['function']
+                        },
+                        'args': {
+                            'type': 'array',
+                            'items': {'$ref': '#/definitions/apiarg'},
+                        },
+                        'returns': {
+                            'type': 'object',
+                            'properties': {
+                                'name': {
+                                    'type': 'string',
+                                    'enum': ['yields'],
+                                },
+                                'desc': {'type': 'string'},
+                                'type': {
+                                    'oneOf': [
+                                        {'$ref': '#/definitions/apitype'},
+                                        {'type': 'array', 'items': {'$ref': '#/definitions/apitype'}},
+                                    ],
+                                },
+                            },
+                            'additionalProperties': False,
+                            'required': ['type', 'desc']
+                        },
+                    },
+                    'additionalProperties': False,
+                    'required': ['type', 'returns'],
+                },
+            },
+            'additionalProperties': False,
+            'required': ['name', 'desc', 'type']
+        },
+        'apiarg': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'desc': {'type': 'string'},
+                'type': {
+                    'oneOf': [
+                        {'$ref': '#/definitions/apitype'},
+                        {'type': 'array', 'items': {'$ref': '#/definitions/apitype'}},
+                    ],
+                },
+                'default': {'type': ['boolean', 'integer', 'string', 'null']},
+            },
+            'additionalProperties': False,
+            'required': ['name', 'desc', 'type']
+        },
+        'apitype': {
+            'type': 'string',
+        },
         'command': {
             'type': 'object',
             'properties': {
@@ -999,6 +1077,8 @@ stormcmds = (
             ('--form', {'help': 'Form to fire on.'}),
             ('--tag', {'help': 'Tag to fire on.'}),
             ('--prop', {'help': 'Property to fire on.'}),
+            ('--verb', {'help': 'Edge verb to fire on.'}),
+            ('--n2form', {'help': 'The form of the n2 node to fire on.'}),
             ('--query', {'help': 'Query for the trigger to execute.', 'required': True,
                          'dest': 'storm', }),
             ('--async', {'default': False, 'action': 'store_true',
@@ -2112,25 +2192,25 @@ class Runtime(s_base.Base):
         try:
             with s_provenance.claim('storm', q=self.query.text, user=self.user.iden):
 
-                nodegenr = self.query.iterNodePaths(self, genr=genr)
-                nodegenr, empty = await s_ast.pullone(nodegenr)
+                async with contextlib.aclosing(self.query.iterNodePaths(self, genr=genr)) as nodegenr:
+                    nodegenr, empty = await s_ast.pullone(nodegenr)
 
-                if empty:
-                    return
+                    if empty:
+                        return
 
-                rules = self.opts.get('graph')
-                if rules not in (False, None):
-                    if rules is True:
-                        rules = {'degrees': None, 'refs': True}
-                    elif isinstance(rules, str):
-                        rules = await self.snap.core.getStormGraph(rules)
+                    rules = self.opts.get('graph')
+                    if rules not in (False, None):
+                        if rules is True:
+                            rules = {'degrees': None, 'refs': True}
+                        elif isinstance(rules, str):
+                            rules = await self.snap.core.getStormGraph(rules)
 
-                    subgraph = s_ast.SubGraph(rules)
-                    nodegenr = subgraph.run(self, nodegenr)
+                        subgraph = s_ast.SubGraph(rules)
+                        nodegenr = subgraph.run(self, nodegenr)
 
-                async for item in nodegenr:
-                    self.tick()
-                    yield item
+                    async for item in nodegenr:
+                        self.tick()
+                        yield item
 
         except RecursionError:
             mesg = 'Maximum Storm pipeline depth exceeded.'
@@ -2147,7 +2227,7 @@ class Runtime(s_base.Base):
 
                 view = snap.core.views.get(viewiden)
                 if view is None:
-                    raise s_exc.NoSuchView(iden=viewiden)
+                    raise s_exc.NoSuchView(mesg=f'No such view iden={viewiden}', iden=viewiden)
 
                 self.user.confirm(('view', 'read'), gateiden=viewiden)
                 snap = await view.snap(self.user)
@@ -2438,8 +2518,7 @@ class Parser:
 
         if nargs == '?':
 
-            # ? will have an implicit default value of None
-            opts.setdefault(dest, None)
+            opts.setdefault(dest, argdef.get('default'))
 
             if todo and not self._is_opt(todo[0]):
 
@@ -3450,7 +3529,7 @@ class CopyToCmd(Cmd):
 
         view = runt.snap.core.getView(iden)
         if view is None:
-            raise s_exc.NoSuchView(mesg=f'No such view: {iden}')
+            raise s_exc.NoSuchView(mesg=f'No such view: {iden=}', iden=iden)
 
         runt.confirm(('view', 'read'), gateiden=view.iden)
 

synapse/lib/storm_format.py

@@ -107,7 +107,7 @@ TerminalPygMap = {
     '_LEFTPIVOT': p_t.Punctuation,
     '_LPARNOSPACE': p_t.Punctuation,
     '_MATCHHASH': p_t.Punctuation,
-    '_MATCHHASHSPACE': p_t.Punctuation,
+    '_MATCHHASHWILD': p_t.Punctuation,
     '_RETURN': p_t.Keyword,
     '_REVERSE': p_t.Keyword,
     '_RIGHTJOIN': p_t.Punctuation,

synapse/lib/stormhttp.py

@@ -250,6 +250,10 @@ class LibHttp(s_stormtypes.Lib):
                   'returns': {'type': 'str', 'desc': 'The reason phrase for the status code.', }}},
     )
     _storm_lib_path = ('inet', 'http')
+    _storm_lib_perms = (
+        {'perm': ('storm', 'lib', 'inet', 'http', 'proxy'), 'gate': 'cortex',
+         'desc': 'Permits a user to specify the proxy used with `$lib.inet.http` APIs.'},
+    )
 
     def getObjLocals(self):
         return {
@@ -270,14 +274,17 @@ class LibHttp(s_stormtypes.Lib):
             return {str(k): str(v) for k, v in item.items()}
         return item
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def urlencode(self, text):
         text = await s_stormtypes.tostr(text)
         return urllib.parse.quote_plus(text)
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def urldecode(self, text):
         text = await s_stormtypes.tostr(text)
         return urllib.parse.unquote_plus(text)
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def codereason(self, code):
         code = await s_stormtypes.toint(code)
         return s_common.httpcodereason(code)
@@ -310,8 +317,8 @@ class LibHttp(s_stormtypes.Lib):
 
         sock = await WebSocket.anit()
 
-        if proxy is not None and not self.runt.isAdmin():
-            raise s_exc.AuthDeny(mesg=s_exc.proxy_admin_mesg)
+        if proxy is not None:
+            self.runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy'))
 
         if proxy is None:
             proxy = await self.runt.snap.core.getConfOpt('http:proxy')
@@ -374,8 +381,8 @@ class LibHttp(s_stormtypes.Lib):
 
         headers = self.strify(headers)
 
-        if proxy is not None and not self.runt.isAdmin():
-            raise s_exc.AuthDeny(mesg=s_exc.proxy_admin_mesg)
+        if proxy is not None:
+            self.runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy'))
 
         if fields:
             if any(['sha256' in field for field in fields]):

synapse/lib/stormlib/auth.py

@@ -63,6 +63,12 @@ stormcmds = (
 
                 // Unlock the user "visi" and set their email to "visi@vertex.link"
                 auth.user.mod visi --locked $lib.false --email visi@vertex.link
+
+                // Grant admin access to user visi for the current view
+                auth.user.mod visi --admin $lib.true --gate $lib.view.get().iden
+
+                // Revoke admin access to user visi for the current view
+                auth.user.mod visi --admin $lib.false --gate $lib.view.get().iden
         ''',
         'cmdargs': (
             ('username', {'type': 'str', 'help': 'The name of the user.'}),
@@ -70,6 +76,7 @@ stormcmds = (
             ('--email', {'type': 'str', 'help': 'The email address to set for the user.'}),
             ('--passwd', {'type': 'str', 'help': 'The new password for the user. This is best passed into the runtime as a variable.'}),
             ('--admin', {'type': 'bool', 'help': 'True to make the user and admin, false to remove their remove their admin status.'}),
+            ('--gate', {'type': 'str', 'help': 'The auth gate iden to grant or revoke admin status on. Use in conjunction with `--admin <bool>`.'}),
             ('--locked', {'type': 'bool', 'help': 'True to lock the user, false to unlock them.'}),
         ),
         'storm': '''
@@ -92,8 +99,15 @@ stormcmds = (
                     $lib.print(`User ({$cmdopts.username}) locked status set to {$cmdopts.locked=1}.`)
                 }
                 if ($cmdopts.admin != $lib.null) {
-                    $user.setAdmin($cmdopts.admin)
-                    $lib.print(`User ({$cmdopts.username}) admin status set to {$cmdopts.admin=1}.`)
+                    $user.setAdmin($cmdopts.admin, gateiden=$cmdopts.gate)
+                    if $cmdopts.gate {
+                        $lib.print(`User ({$cmdopts.username}) admin status set to {$cmdopts.admin=1} for auth gate {$cmdopts.gate}.`)
+                    } else {
+                        $lib.print(`User ({$cmdopts.username}) admin status set to {$cmdopts.admin=1}.`)
+                    }
+                }
+                if ($cmdopts.gate != $lib.null and $cmdopts.admin = $lib.null) {
+                    $lib.exit('Granting/revoking admin status on an auth gate, requires the use of `--admin <true|false>` also.')
                 }
             } else {
                 $lib.warn(`User ({$cmdopts.username}) not found!`)

synapse/lib/stormlib/backup.py

@@ -50,6 +50,7 @@ class BackupLib(s_stormtypes.Lib):
         gatekeys = ((self.runt.user.iden, ('backup', 'run'), None),)
         return await self.dyncall('cortex', todo, gatekeys=gatekeys)
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def _listBackups(self):
         todo = s_common.todo('getBackups')
         gatekeys = ((self.runt.user.iden, ('backup', 'list'), None),)

synapse/lib/stormlib/basex.py

@@ -35,6 +35,7 @@ class BaseXLib(s_stormtypes.Lib):
             'decode': self.decode,
         }
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def encode(self, byts, charset):
         if not isinstance(byts, bytes):
             raise s_exc.BadArg(mesg='$lib.basex.encode() requires a bytes argument.')
@@ -54,6 +55,7 @@ class BaseXLib(s_stormtypes.Lib):
 
         return ''.join(retn[::-1])
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def decode(self, text, charset):
         text = await s_stormtypes.tostr(text)
         charset = await s_stormtypes.tostr(charset)

synapse/lib/stormlib/cell.py

@@ -192,6 +192,7 @@ class CellLib(s_stormtypes.Lib):
 
         return curv
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def _hotFixesCheck(self):
         if not self.runt.isAdmin():
             mesg = '$lib.cell.stormFixesCheck() requires admin privs.'
@@ -210,30 +211,35 @@ class CellLib(s_stormtypes.Lib):
 
         return dowork
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def _getCellInfo(self):
         if not self.runt.isAdmin():
             mesg = '$lib.cell.getCellInfo() requires admin privs.'
             raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
         return await self.runt.snap.core.getCellInfo()
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def _getSystemInfo(self):
         if not self.runt.isAdmin():
             mesg = '$lib.cell.getSystemInfo() requires admin privs.'
             raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
         return await self.runt.snap.core.getSystemInfo()
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def _getBackupInfo(self):
         if not self.runt.isAdmin():
             mesg = '$lib.cell.getBackupInfo() requires admin privs.'
             raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
         return await self.runt.snap.core.getBackupInfo()
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def _getHealthCheck(self):
         if not self.runt.isAdmin():
             mesg = '$lib.cell.getHealthCheck() requires admin privs.'
             raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
         return await self.runt.snap.core.getHealthCheck()
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def _getMirrorUrls(self, name=None):
 
         if not self.runt.isAdmin():
@@ -265,6 +271,7 @@ class CellLib(s_stormtypes.Lib):
 
         return await self.runt.snap.core.trimNexsLog(consumers=consumers, timeout=timeout)
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def _uptime(self, name=None):
 
         name = await s_stormtypes.tostr(name, noneok=True)

synapse/lib/stormlib/compression.py

@@ -47,6 +47,7 @@ class Bzip2Lib(s_stormtypes.Lib):
             'un': self.un,
         }
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def en(self, valu):
         valu = await s_stormtypes.toprim(valu)
         try:
@@ -103,6 +104,7 @@ class GzipLib(s_stormtypes.Lib):
             'un': self.un,
         }
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def en(self, valu):
         valu = await s_stormtypes.toprim(valu)
         try:
@@ -159,6 +161,7 @@ class ZlibLib(s_stormtypes.Lib):
             'un': self.un,
         }
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def en(self, valu):
         valu = await s_stormtypes.toprim(valu)
         try: