synapse 2.152.0__py311-none-any.whl → 2.154.0__py311-none-any.whl

synapse/lib/cache.py

@@ -179,6 +179,14 @@ def getTagGlobRegx(name):
     regq = regexizeTagGlob(name)
     return regex.compile(regq)
 
+def regexizeEdgeGlob(edge):
+    return ReRegex.sub(r'(.+)', regex.escape(edge))
+
+@memoize()
+def getEdgeGlobRegx(name):
+    regq = regexizeEdgeGlob(name)
+    return regex.compile(regq)
+
 class TagGlobs:
     '''
     An object that manages multiple tag globs and values for caching.
@@ -191,7 +199,7 @@ class TagGlobs:
 
         self.cache.clear()
 
-        regx = getTagGlobRegx(name)
+        regx = self._getGlobRegex(name)
 
         glob = (regx, (name, valu))
 
@@ -213,5 +221,12 @@ class TagGlobs:
     def get(self, name):
         return self.cache.get(name)
 
+    def _getGlobRegex(self, name):
+        return getTagGlobRegx(name)
+
     def _getGlobMatches(self, name):
         return [g[1] for g in self.globs if g[0].fullmatch(name) is not None]
+
+class EdgeGlobs(TagGlobs):
+    def _getGlobRegex(self, name):
+        return getEdgeGlobRegx(name)

synapse/lib/cell.py

@@ -633,8 +633,8 @@ class CellApi(s_base.Base):
         return await self.cell.getRoleDefs()
 
     @adminapi()
-    async def isUserAllowed(self, iden, perm, gateiden=None):
-        return await self.cell.isUserAllowed(iden, perm, gateiden=gateiden)
+    async def isUserAllowed(self, iden, perm, gateiden=None, default=False):
+        return await self.cell.isUserAllowed(iden, perm, gateiden=gateiden, default=default)
 
     @adminapi()
     async def isRoleAllowed(self, iden, perm, gateiden=None):
@@ -2232,12 +2232,12 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             if remove:
                 self.backupstreaming = False
 
-    async def isUserAllowed(self, iden, perm, gateiden=None):
-        user = self.auth.user(iden)
+    async def isUserAllowed(self, iden, perm, gateiden=None, default=False):
+        user = self.auth.user(iden)  # type: s_hiveauth.HiveUser
         if user is None:
             return False
 
-        return user.allowed(perm, gateiden=gateiden)
+        return user.allowed(perm, default=default, gateiden=gateiden)
 
     async def isRoleAllowed(self, iden, perm, gateiden=None):
         role = self.auth.role(iden)

synapse/lib/httpapi.py

@@ -326,12 +326,13 @@ class HandlerBase:
         self.web_username = udef.get('name')
         return self.web_useriden
 
-    async def allowed(self, perm, gateiden=None):
+    async def allowed(self, perm, default=False, gateiden=None):
         '''
         Check if the authenticated user has the given permission.
 
         Args:
             perm (tuple): The permission tuple to check.
+            default (boolean): The default value for the permission.
             gateiden (str): The gateiden to check the permission against.
 
         Notes:
@@ -347,11 +348,16 @@ class HandlerBase:
             self.sendAuthRequired()
             return False
 
-        if await authcell.isUserAllowed(useriden, perm, gateiden=gateiden):
+        if await authcell.isUserAllowed(useriden, perm, gateiden=gateiden, default=default):
             return True
 
         self.set_status(403)
+
         mesg = f'User ({self.web_username}) must have permission {".".join(perm)}'
+        if default:
+            mesg = f'User ({self.web_username}) is denied the permission {".".join(perm)}'
+        if gateiden:
+            mesg = f'{mesg} on object {gateiden}'
         self.sendRestErr('AuthDeny', mesg)
         return False
 
@@ -1240,3 +1246,193 @@ class CoreInfoV1(Handler):
 
         resp = await self.cell.getCoreInfoV2()
         return self.sendRestRetn(resp)
+
+class ExtApiHandler(StormHandler):
+    '''
+    /api/ext/.*
+    '''
+
+    storm_prefix = 'init { $request = $lib.cortex.httpapi.response($_http_request_info) }'
+
+    # Disables the etag header from being computed and set. It is too much magic for
+    # a user defined API to utilize.
+    def compute_etag(self):
+        return None
+
+    def set_default_headers(self):
+        self.clear_header('Server')
+
+    async def get(self, path):
+        return await self._runHttpExt('get', path)
+
+    async def head(self, path):
+        return await self._runHttpExt('head', path)
+
+    async def post(self, path):
+        return await self._runHttpExt('post', path)
+
+    async def put(self, path):
+        return await self._runHttpExt('put', path)
+
+    async def delete(self, path):
+        return await self._runHttpExt('delete', path)
+
+    async def patch(self, path):
+        return await self._runHttpExt('patch', path)
+
+    async def options(self, path):
+        return await self._runHttpExt('options', path)
+
+    async def _runHttpExt(self, meth, path):
+        core = self.getCore()
+        adef, args = await core.getHttpExtApiByPath(path)
+        if adef is None:
+            self.set_status(404)
+            self.sendRestErr('NoSuchPath', f'No Extended HTTP API endpoint matches {path}')
+            return await self.finish()
+
+        requester = ''
+        iden = adef.get("iden")
+        useriden = adef.get('owner')
+
+        if adef.get('authenticated'):
+
+            requester = await self.useriden()
+
+            if requester is None:
+                await self.reqAuthUser()
+                return
+
+            for pdef in adef.get('perms'):
+                if not await self.allowed(pdef.get('perm'), default=pdef.get('default')):
+                    return
+
+            if adef.get('runas') == 'user':
+                useriden = requester
+
+        storm = adef['methods'].get(meth)
+        if storm is None:
+            self.set_status(405)
+            meths = [meth.upper() for meth in adef.get('methods')]
+            self.set_header('Allowed', ', '.join(meths))
+            mesg = f'Extended HTTP API {iden} has no method for {meth.upper()}.'
+            if meths:
+                mesg = f'{mesg} Supports {", ".join(meths)}.'
+            self.sendRestErr('NeedConfValu', mesg)
+            return await self.finish()
+
+        # We flatten the request headers and parameters into a flat key/valu map.
+        # The first instance of a given key wins.
+        request_headers = {}
+        for key, valu in self.request.headers.get_all():
+            request_headers.setdefault(key.lower(), valu)
+
+        params = {}
+        for key, valus in self.request.query_arguments.items():
+            for valu in valus:
+                params.setdefault(key, valu.decode())
+
+        info = {
+            'uri': self.request.uri,
+            'body': self.request.body,
+            'iden': iden,
+            'path': path,
+            'user': requester,
+            'method': self.request.method,
+            'params': params,
+            'headers': request_headers,
+            'args': args,
+            'client': self.request.remote_ip,
+        }
+
+        varz = adef.get('vars')
+        varz['_http_request_info'] = info
+
+        opts = {
+            'readonly': adef.get('readonly'),
+            'show': (
+                'http:resp:body',
+                'http:resp:code',
+                'http:resp:headers',
+            ),
+            'user': useriden,
+            'vars': varz,
+            'view': adef.get('view'),
+        }
+
+        query = '\n'.join((self.storm_prefix, storm))
+
+        rcode = False
+        rbody = False
+
+        try:
+            async for mtyp, info in core.storm(query, opts=opts):
+                if mtyp == 'http:resp:code':
+                    if rbody:
+                        # We've already flushed() the stream at this point, so we cannot
+                        # change the status code or the response headers. We just have to
+                        # log the error and move along.
+                        mesg = f'Extended HTTP API {iden} tried to set code after sending body.'
+                        logger.error(mesg)
+                        continue
+
+                    rcode = True
+                    self.set_status(info['code'])
+
+                elif mtyp == 'http:resp:headers':
+                    if rbody:
+                        # We've already flushed() the stream at this point, so we cannot
+                        # change the status code or the response headers. We just have to
+                        # log the error and move along.
+                        mesg = f'Extended HTTP API {iden} tried to set headers after sending body.'
+                        logger.error(mesg)
+                        continue
+                    for hkey, hval in info['headers'].items():
+                        self.set_header(hkey, hval)
+
+                elif mtyp == 'http:resp:body':
+                    if not rcode:
+                        self.clear()
+                        self.set_status(500)
+                        self.sendRestErr('StormRuntimeError',
+                                         f'Extended HTTP API {iden} must set status code before sending body.')
+                        return await self.finish()
+                    rbody = True
+                    body = info['body']
+                    self.write(body)
+                    await self.flush()
+
+                elif mtyp == 'err':
+                    errname, erfo = info
+                    mesg = f'Error executing Extended HTTP API {iden}: {errname} {erfo.get("mesg")}'
+                    logger.error(mesg)
+                    if rbody:
+                        # We've already flushed() the stream at this point, so we cannot
+                        # change the status code or the response headers. We just have to
+                        # log the error and move along.
+                        continue
+
+                    # Since we haven't flushed the body yet, we can clear the handler
+                    # and send the error the user.
+                    self.clear()
+                    self.set_status(500)
+                    self.sendRestErr(errname, erfo.get('mesg'))
+                    rcode = True
+                    rbody = True
+
+        except Exception as e:
+            rcode = True
+            enfo = s_common.err(e)
+            logger.exception(f'Extended HTTP API {iden} encountered fatal error: {enfo[1].get("mesg")}')
+            if rbody is False:
+                self.clear()
+                self.set_status(500)
+                self.sendRestErr(enfo[0],
+                                 f'Extended HTTP API {iden} encountered fatal error: {enfo[1].get("mesg")}')
+
+        if rcode is False:
+            self.clear()
+            self.set_status(500)
+            self.sendRestErr('StormRuntimeError', f'Extended HTTP API {iden} never set status code.')
+
+        await self.finish()

synapse/lib/layer.py

@@ -105,6 +105,8 @@ reqValidLdef = s_config.getJsValidator({
     'required': ['iden', 'creator', 'lockmemory'],
 })
 
+WINDOW_MAXSIZE = 10_000
+
 class LayerApi(s_cell.CellApi):
 
     async def __anit__(self, core, link, user, layr):
@@ -123,6 +125,7 @@ class LayerApi(s_cell.CellApi):
         await self._reqUserAllowed(self.liftperm)
         async for item in self.layr.iterLayerNodeEdits():
             yield item
+            await asyncio.sleep(0)
 
     @s_cell.adminapi()
     async def saveNodeEdits(self, edits, meta):
@@ -509,7 +512,7 @@ class StorType:
 
     async def _liftRegx(self, liftby, valu, reverse=False):
 
-        regx = regex.compile(valu)
+        regx = regex.compile(valu, flags=regex.I)
 
         abrvlen = liftby.abrvlen
         isarray = isinstance(liftby, IndxByPropArray)
@@ -4425,7 +4428,7 @@ class Layer(s_nexus.Pusher):
         if not self.logedits:
             raise s_exc.BadConfValu(mesg='Layer logging must be enabled for getting nodeedits')
 
-        async with await s_queue.Window.anit(maxsize=10000) as wind:
+        async with await s_queue.Window.anit(maxsize=WINDOW_MAXSIZE) as wind:
 
             async def fini():
                 self.windows.remove(wind)

synapse/lib/modelrev.py

@@ -8,7 +8,7 @@ import synapse.lib.layer as s_layer
 
 logger = logging.getLogger(__name__)
 
-maxvers = (0, 2, 21)
+maxvers = (0, 2, 22)
 
 class ModelRev:
 
@@ -35,6 +35,7 @@ class ModelRev:
             ((0, 2, 19), self.revModel_0_2_19),
             ((0, 2, 20), self.revModel_0_2_20),
             ((0, 2, 21), self.revModel_0_2_21),
+            ((0, 2, 22), self.revModel_0_2_22),
         )
 
     async def _uniqSortArray(self, todoprops, layers):
@@ -725,6 +726,9 @@ class ModelRev:
         await self._normPropValu(layers, 'risk:vuln:name')
         await self._propToForm(layers, 'risk:vuln:name', 'risk:vulnname')
 
+    async def revModel_0_2_22(self, layers):
+        await self._normFormSubs(layers, 'inet:ipv4', cmprvalu='100.64.0.0/10')
+
     async def runStorm(self, text, opts=None):
         '''
         Run storm code in a schedcoro and log the output messages.
@@ -880,7 +884,7 @@ class ModelRev:
             if nodeedits:
                 await save()
 
-    async def _normFormSubs(self, layers, formname, liftprop=None):
+    async def _normFormSubs(self, layers, formname, liftprop=None, cmprvalu=s_common.novalu, cmpr='='):
 
         # NOTE: this API may be used to re-normalize subs but *not* to change their storage types
         # and will *not* auto-populate linked forms from subs which are form types.
@@ -898,7 +902,36 @@ class ModelRev:
                 await layr.storNodeEdits(nodeedits, meta)
                 nodeedits.clear()
 
-            async for lkey, buid, sode in layr.liftByProp(form.name, liftprop):
+            if cmprvalu is s_common.novalu:
+                # This is for lifts such as:
+                #   <formname>
+                #   <formname>:<liftprop>
+                # E.g.:
+                #   inet:ipv4
+                #   inet:ipv4:type
+                genr = layr.liftByProp(form.name, liftprop)
+
+            elif liftprop is None:
+                # This is for lifts such as:
+                #   <formname><cmpr><cmprvalu>
+                # E.g.:
+                #   inet:ipv4=1.2.3.4
+
+                # Don't norm cmprvalu first because it may not be normable
+                cmprvals = form.type.getStorCmprs(cmpr, cmprvalu)
+                genr = layr.liftByFormValu(form.name, cmprvals)
+
+            else: # liftprop is not None  # pragma: no cover
+                # This is for lifts such as:
+                #   <formname>:<liftprop><cmpr><cmprvalu>
+                # E.g.:
+                #   inet:ipv4:type=private
+
+                # Don't norm cmprvalu first because it may not be normable
+                cmprvals = form.type.getStorCmprs(cmpr, cmprvalu)
+                genr = layr.liftByPropValu(form.name, liftprop, cmprvals)
+
+            async for _, buid, sode in genr:
 
                 sodevalu = sode.get('valu')
                 if sodevalu is None: # pragma: no cover

synapse/lib/node.py

@@ -485,10 +485,6 @@ class Node:
             raise s_exc.IsRuntForm(mesg='Cannot delete tags from runt nodes.',
                                    form=self.form.full, tag=tag)
 
-        curv = self.tags.get(name, s_common.novalu)
-        if curv is s_common.novalu:
-            return ()
-
         pref = name + '.'
 
         todel = [(len(t), t) for t in self.tags.keys() if t.startswith(pref)]
@@ -529,7 +525,8 @@ class Node:
             edits.append((s_layer.EDIT_TAG_DEL, (subtag, None), ()))
 
         edits.extend(self._getTagPropDel(name))
-        edits.append((s_layer.EDIT_TAG_DEL, (name, None), ()))
+        if self.getTag(name, defval=s_common.novalu) is not s_common.novalu:
+            edits.append((s_layer.EDIT_TAG_DEL, (name, None), ()))
 
         return edits
 

synapse/lib/parser.py

@@ -125,7 +125,7 @@ terminalEnglishMap = {
     '_LEFTPIVOT': '<-',
     '_LPARNOSPACE': '(',
     '_MATCHHASH': '#',
-    '_MATCHHASHSPACE': '#',
+    '_MATCHHASHWILD': '#',
     '_RETURN': 'return',
     '_REVERSE': 'reverse',
     '_RIGHTJOIN': '-+>',

synapse/lib/schemas.py

@@ -0,0 +1,51 @@
+import synapse.lib.config as s_config
+
+
+_HttpExtAPIConfSchema = {
+    'type': 'object',
+    'properties': {
+        'iden': {'type': 'string', 'pattern': s_config.re_iden},
+        'methods': {
+            'type': 'object',
+            'default': {},
+            'properties': {
+                'get': {'type': 'string', 'minLen': 1},
+                'head': {'type': 'string', 'minLen': 1},
+                'post': {'type': 'string', 'minLen': 1},
+                'put': {'type': 'string', 'minLen': 1},
+                'delete': {'type': 'string', 'minLen': 1},
+                'patch': {'type': 'string', 'minLen': 1},
+                'options': {'type': 'string', 'minLen': 1},
+            },
+            'additionalProperties': False,
+        },
+        'authenticated': {'type': 'boolean', 'default': True},
+        'name': {'type': 'string', 'default': ''},
+        'desc': {'type': 'string', 'default': ''},
+        'path': {'type': 'string', 'minlen': 1},
+        'view': {'type': 'string', 'pattern': s_config.re_iden},
+        'runas': {'type': 'string', 'pattern': '^(owner|user)$'},
+        'owner': {'type': 'string', 'pattern': s_config.re_iden},
+        'creator': {'type': 'string', 'pattern': s_config.re_iden},
+        'created': {'type': 'integer', 'minimum': 0},
+        'updated': {'type': 'integer', 'minimum': 0},
+        'readonly': {'type': 'boolean', 'default': False},
+        'perms': {
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                    'perm': {'type': 'array', 'items': {'type': 'string', 'minlen': 1}},
+                    'default': {'type': 'boolean', 'default': False},
+                }
+            },
+            'default': [],
+        },
+        'vars': {'type': 'object', 'default': {}}
+
+    },
+    'additionalProperties': False
+}
+
+
+reqValidHttpExtAPIConf = s_config.getJsValidator(_HttpExtAPIConfSchema)

synapse/lib/snap.py

@@ -1147,6 +1147,16 @@ class Snap(s_base.Base):
                     node.nodedata.pop(name, None)
                     continue
 
+                if etyp == s_layer.EDIT_EDGE_ADD:
+                    verb, n2iden = parms
+                    n2 = await self.getNodeByBuid(s_common.uhex(n2iden))
+                    callbacks.append((self.view.runEdgeAdd, (node, verb, n2), {}))
+
+                if etyp == s_layer.EDIT_EDGE_DEL:
+                    verb, n2iden = parms
+                    n2 = await self.getNodeByBuid(s_common.uhex(n2iden))
+                    callbacks.append((self.view.runEdgeDel, (node, verb, n2), {}))
+
         [await func(*args, **kwargs) for (func, args, kwargs) in callbacks]
 
         if actualedits:

synapse/lib/storm.lark

@@ -241,15 +241,35 @@ _tagsegs:  TAGSEGNOVAR ( "." (TAGSEGNOVAR | "$" varvalue))*
 TAGSEGNOVAR: /[\w]+/
 
 // A tag name with asterisks or $var as segment after the first segment
-tagmatch: (_MATCHHASH | _MATCHHASHSPACE) (_varvalu | _wildtagsegs)
-_MATCHHASH.3: /(?<!\s)#/
-_MATCHHASHSPACE.3: /(?<=\s)#/
+tagmatch: _MATCHHASH (_varvalu | _tagsegs)
+    | _MATCHHASHWILD _wildtagsegs
+
+_MATCHHASH.3: /#/
+
+// A tagmatch with wildcards cannot be followed by a cmpr
+_MATCHHASHWILD.3: /\#
+(?=
+    (?:[\w.$]|"(?:[^"\\]|\\.)*"|'[^']*'(?!'))*       # match tag parts
+    \*                                               # match a *
+    (?!                                              # negative lookahead for cmpr
+        (?:[\w*.$:]|"(?:[^"\\]|\\.)*"|'[^']*'(?!'))* # continue matching tag parts
+        \s*                                          # match any whitespace before cmpr
+        (?!                                          # avoid matching EDGEN2INIT as a cmpr
+            <\(
+            (?=(?>(?<N2MATCHRECUR>\(((?>[^()"'`]+|(?&N2MATCHRECUR)|`(?:[^`\\]|\\.)*`|"(?:[^"\\]|\\.)*"|'''.*?'''|'[^']*'(?!'))*)\))|'''.*?'''|`(?:[^`\\]|\\.)*`|"(?:[^"\\]|\\.)*"|'[^']*'(?!')|[^)])*\)-
+            )
+        )
+        [@?!<>^~=]+                                # match any cmprs (regex fail)
+    )
+)
+/x
+
 _wildtagsegs:  WILDTAGSEGNOVAR ( "." (WILDTAGSEGNOVAR | "$" varvalue))*
 WILDTAGSEGNOVAR: /[\w*]+/
 
 // A comparison operator
 _cmpr: "*" BYNAME | CMPR | CMPROTHER | EQSPACE | EQNOSPACE | TRYSET | SETTAGOPER
-BYNAME.2: /[^=\s]+=/
+BYNAME.2: /\w+[@?!<>^~=]*[<>=]/
 
 _safe_cmpr: BYNAME | CMPR | CMPROTHER | EQSPACE | EQNOSPACE | TRYSET | SETTAGOPER