synapse 2.184.0__py311-none-any.whl → 2.185.0__py311-none-any.whl

synapse/cortex.py

@@ -946,7 +946,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         self._exthttpapicache = s_cache.FixedCache(self._getHttpExtApiByPath, size=1000)
         self._initCortexExtHttpApi()
 
-        self.model = s_datamodel.Model()
+        self.model = s_datamodel.Model(core=self)
 
         await self._bumpCellVers('cortex:extmodel', (
             (1, self._migrateTaxonomyIface),

synapse/datamodel.py

@@ -482,8 +482,9 @@ class Model:
     '''
     The data model used by a Cortex hypergraph.
     '''
-    def __init__(self):
+    def __init__(self, core=None):
 
+        self.core = core
         self.types = {}  # name: Type()
         self.forms = {}  # name: Form()
         self.props = {}  # (form,name): Prop() and full: Prop()
@@ -653,13 +654,16 @@ class Model:
             self.formprefixcache[prefix] = forms
         return forms
 
-    def reqProp(self, name):
+    def reqProp(self, name, extra=None):
         prop = self.prop(name)
         if prop is not None:
             return prop
 
-        mesg = f'No property named {name}.'
-        raise s_exc.NoSuchProp(mesg=mesg, name=name)
+        exc = s_exc.NoSuchProp.init(name)
+        if extra is not None:
+            exc = extra(exc)
+
+        raise exc
 
     def reqUniv(self, name):
         prop = self.univ(name)
@@ -1082,6 +1086,30 @@ class Model:
         self.props[prop.full] = prop
         return prop
 
+    def _prepFormIface(self, form, iface):
+
+        template = iface.get('template', {})
+        template.update(form.type.info.get('template', {}))
+
+        def convert(item):
+
+            if isinstance(item, str):
+
+                if item == '$self':
+                    return form.name
+
+                return item.format(**template)
+
+            if isinstance(item, dict):
+                return {convert(k): convert(v) for (k, v) in item.items()}
+
+            if isinstance(item, (list, tuple)):
+                return tuple([convert(v) for v in item])
+
+            return item
+
+        return convert(iface)
+
     def _addFormIface(self, form, name, subifaces=None):
 
         iface = self.ifaces.get(name)
@@ -1094,9 +1122,14 @@ class Model:
             mesg = f'Form {form.name} depends on deprecated interface {name} which will be removed in 3.0.0'
             logger.warning(mesg)
 
+        iface = self._prepFormIface(form, iface)
+
         for propname, typedef, propinfo in iface.get('props', ()):
-            if typedef[0] == '$self':
-                typedef = (form.name, typedef[1])
+
+            # allow form props to take precedence
+            if form.prop(propname) is not None:
+                continue
+
             prop = self._addFormProp(form, propname, typedef, propinfo)
             self.ifaceprops[f'{name}:{propname}'].append(prop.full)
 
@@ -1123,6 +1156,8 @@ class Model:
         if (iface := self.ifaces.get(name)) is None:
             return
 
+        iface = self._prepFormIface(form, iface)
+
         for propname, typedef, propinfo in iface.get('props', ()):
             fullprop = f'{form.name}:{propname}'
             self.delFormProp(form.name, propname)

synapse/lib/ast.py

@@ -1426,6 +1426,32 @@ class LiftOper(Oper):
         self.astinfo = astinfo
         self.reverse = True
 
+    def getPivNames(self, runt, prop, pivs):
+        pivnames = []
+        typename = prop.type.name
+        for piv in pivs:
+            pivprop = runt.model.reqProp(f'{typename}:{piv}', extra=self.kids[0].addExcInfo)
+            pivnames.append(pivprop.full)
+            typename = pivprop.type.name
+
+        return pivnames
+
+    async def pivlift(self, runt, props, pivnames, genr):
+
+        async def pivvals(prop, pivgenr):
+            async for node in pivgenr:
+                async for pivo in runt.snap.nodesByPropValu(prop, '=', node.ndef[1], reverse=self.reverse):
+                    yield pivo
+
+        for pivname in pivnames[-2::-1]:
+            genr = pivvals(pivname, genr)
+
+        async for node in genr:
+            valu = node.ndef[1]
+            for prop in props:
+                async for node in runt.snap.nodesByPropValu(prop.full, '=', valu, reverse=self.reverse):
+                    yield node
+
     async def run(self, runt, genr):
 
         if self.isRuntSafe(runt):
@@ -1584,30 +1610,52 @@ class LiftByArray(LiftOper):
         cmpr = await self.kids[1].compute(runt, path)
         valu = await s_stormtypes.tostor(await self.kids[2].compute(runt, path))
 
-        prop = runt.model.props.get(name)
-        if prop is not None:
-            async for node in runt.snap.nodesByPropArray(name, cmpr, valu, reverse=self.reverse):
-                yield node
-            return
+        pivs = None
+        if name.find('::') != -1:
+            parts = name.split('::')
+            name, pivs = parts[0], parts[1:]
 
-        proplist = runt.model.ifaceprops.get(name)
-        if proplist is None:
-            raise self.kids[0].addExcInfo(s_exc.NoSuchProp.init(name))
+        if (prop := runt.model.props.get(name)) is not None:
+            props = (prop,)
+        else:
+            proplist = runt.model.ifaceprops.get(name)
+            if proplist is None:
+                raise self.kids[0].addExcInfo(s_exc.NoSuchProp.init(name))
 
-        props = []
-        for propname in proplist:
-            props.append(runt.model.props.get(propname))
+            props = []
+            for propname in proplist:
+                props.append(runt.model.props.get(propname))
 
-        relname = props[0].name
-        def cmprkey(node):
-            return node.props.get(relname)
+        try:
+            if pivs is not None:
+                pivnames = self.getPivNames(runt, props[0], pivs)
 
-        genrs = []
-        for prop in props:
-            genrs.append(runt.snap.nodesByPropArray(prop.full, cmpr, valu, reverse=self.reverse))
+                genr = runt.snap.nodesByPropArray(pivnames[-1], cmpr, valu, reverse=self.reverse)
+                async for node in self.pivlift(runt, props, pivnames, genr):
+                    yield node
+                return
 
-        async for node in s_common.merggenr2(genrs, cmprkey, reverse=self.reverse):
-            yield node
+            if len(props) == 1:
+                async for node in runt.snap.nodesByPropArray(name, cmpr, valu, reverse=self.reverse):
+                    yield node
+                return
+
+            relname = props[0].name
+            def cmprkey(node):
+                return node.props.get(relname)
+
+            genrs = []
+            for prop in props:
+                genrs.append(runt.snap.nodesByPropArray(prop.full, cmpr, valu, reverse=self.reverse))
+
+            async for node in s_common.merggenr2(genrs, cmprkey, reverse=self.reverse):
+                yield node
+
+        except s_exc.BadTypeValu as e:
+            raise self.kids[2].addExcInfo(e)
+
+        except s_exc.SynErr as e:
+            raise self.addExcInfo(e)
 
 class LiftTagProp(LiftOper):
     '''
@@ -1839,6 +1887,11 @@ class LiftPropBy(LiftOper):
         if not isinstance(valu, s_node.Node):
             valu = await s_stormtypes.tostor(valu)
 
+        pivs = None
+        if name.find('::') != -1:
+            parts = name.split('::')
+            name, pivs = parts[0], parts[1:]
+
         prop = runt.model.props.get(name)
         if prop is not None:
             props = (prop,)
@@ -1852,6 +1905,14 @@ class LiftPropBy(LiftOper):
                 props.append(runt.model.props.get(propname))
 
         try:
+            if pivs is not None:
+                pivnames = self.getPivNames(runt, props[0], pivs)
+
+                genr = runt.snap.nodesByPropValu(pivnames[-1], cmpr, valu, reverse=self.reverse)
+                async for node in self.pivlift(runt, props, pivnames, genr):
+                    yield node
+                return
+
             if len(props) == 1:
                 prop = props[0]
                 async for node in runt.snap.nodesByPropValu(prop.full, cmpr, valu, reverse=self.reverse):
@@ -3794,7 +3855,7 @@ class ExprDict(Value):
 
     def prepare(self):
         self.const = None
-        if all(isinstance(k, Const) for k in self.kids):
+        if all(isinstance(k, Const) and not isinstance(k, EmbedQuery) for k in self.kids):
             valu = {}
             for i in range(0, len(self.kids), 2):
                 valu[self.kids[i].value()] = self.kids[i + 1].value()
@@ -3824,7 +3885,7 @@ class ExprList(Value):
 
     def prepare(self):
         self.const = None
-        if all(isinstance(k, Const) for k in self.kids):
+        if all(isinstance(k, Const) and not isinstance(k, EmbedQuery) for k in self.kids):
             self.const = s_msgpack.en([k.value() for k in self.kids])
 
     async def compute(self, runt, path):

synapse/lib/auth.py

@@ -264,6 +264,19 @@ class Auth(s_nexus.Pusher):
     def _getRoleIden(self, name):
         return self.roleidenbyname.get(name)
 
+    # TODO convert getUserByName() and getRoleByName()
+    # back from async? These were plumbed to avoid infecting
+    # type norm/repr functions with async...
+    def _getRoleByName(self, name):
+        roleiden = self.roleidenbynamecache.get(name)
+        if roleiden is not None:
+            return self.role(roleiden)
+
+    def _getUserByName(self, name):
+        useriden = self.useridenbynamecache.get(name)
+        if useriden is not None:
+            return self.user(useriden)
+
     @s_nexus.Pusher.onPushAuto('user:profile:set')
     async def setUserProfileValu(self, iden, name, valu):
         user = await self.reqUser(iden)

synapse/lib/cell.py

@@ -2118,6 +2118,13 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         Hand off leadership to a mirror in a transactional fashion.
         '''
         _dispname = f' ahaname={self.conf.get("aha:name")}' if self.conf.get('aha:name') else ''
+
+        if not self.isactive:
+            mesg = f'HANDOFF: {_dispname} is not the current leader and cannot handoff leadership to' \
+                   f' {s_urlhelp.sanitizeUrl(turl)}.'
+            logger.error(mesg)
+            raise s_exc.BadState(mesg=mesg, turl=turl, cursvc=_dispname)
+
         logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)}{_dispname}.')
         async with await s_telepath.openurl(turl) as cell:
 
@@ -4439,6 +4446,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         Returns:
             Dict: A Dictionary of metadata.
         '''
+
+        mirror = self.conf.get('mirror')
+        if mirror is not None:
+            mirror = s_urlhelp.sanitizeUrl(mirror)
+
         ret = {
             'synapse': {
                 'commit': s_version.commit,
@@ -4458,6 +4470,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 'cellvers': dict(self.cellvers.items()),
                 'nexsindx': await self.getNexsIndx(),
                 'uplink': self.nexsroot.miruplink.is_set(),
+                'mirror': mirror,
                 'aha': {
                     'name': self.conf.get('aha:name'),
                     'leader': self.conf.get('aha:leader'),

synapse/lib/modules.py

@@ -2,6 +2,7 @@
 Module which implements the synapse module API/convention.
 '''
 coremods = (
+    'synapse.models.doc.DocModule',
     'synapse.models.dns.DnsModule',
     'synapse.models.orgs.OuModule',
     'synapse.models.syn.SynModule',

synapse/lib/parser.py

@@ -46,6 +46,7 @@ terminalEnglishMap = {
     'DOT': '.',
     'DOUBLEQUOTEDSTRING': 'double-quoted string',
     'ELIF': 'elif',
+    'EMBEDPROPS': 'absolute property name with embed properties',
     'EQNOSPACE': '=',
     'EQSPACE': '=',
     'EQUAL': '=',

synapse/lib/snap.py

@@ -1692,6 +1692,7 @@ class Snap(s_base.Base):
 
         todo = s_common.todo('runRuntLift', full, valu, cmpr, self.view.iden)
         async for sode in self.core.dyniter('cortex', todo):
+            await asyncio.sleep(0)
 
             node = s_node.Node(self, sode)
             node.isrunt = True

synapse/lib/storm.lark

@@ -222,9 +222,9 @@ liftreverse: _REVERSE "(" (liftformtag | liftpropby | liftprop | liftbyarray | l
 _DEREF.3:  /\*(?=\$)/
 
 liftformtag: ((PROPS | UNIVNAME | WILDPROPS) | _DEREF _varvalu) tagname [_cmpr _valu]
-liftpropby: ((PROPS | UNIVNAME) | _DEREF _varvalu) _cmpr _valu
+liftpropby: ((PROPS | EMBEDPROPS | UNIVNAME) | _DEREF _varvalu) _cmpr _valu
 liftprop: ((PROPS | UNIVNAME | WILDPROPS) | _DEREF _varvalu)
-liftbyarray: ((PROPS | UNIVNAME) | _DEREF _varvalu) "*[" _safe_cmpr _valu "]"
+liftbyarray: ((PROPS | EMBEDPROPS | UNIVNAME) | _DEREF _varvalu) "*[" _safe_cmpr _valu "]"
 lifttagtag: (_HASH | _HASHSPACE) tagname [_cmpr _valu]
 liftbytag: (tagname | tagnamewithspace) [_cmpr _valu]
 liftbytagprop: (tagprop | tagpropwithspace) [_cmpr _valu]
@@ -321,10 +321,10 @@ _valu: _basevalu | NONQUOTEWORD
 evalvalu: _valu
 exprdict: "{" ((_exprvalu | VARTOKN)  (":" | _EXPRCOLONNOSPACE) (_exprvalu | VARTOKN) ("," (_exprvalu | VARTOKN) (":" | _EXPRCOLONNOSPACE) (_exprvalu | VARTOKN))* ","? )? "}"
 exprlist: "[" ((_exprvalu | VARTOKN) ("," (_exprvalu | VARTOKN))* ","? )? "]"
-// Just like _valu, but doesn't allow valu lists or unquoted strings or queries
+// Just like _valu, but doesn't allow valu lists or unquoted strings
 _exprvalu: NUMBER | HEXNUMBER | OCTNUMBER | BOOL | NULL | exprlist | exprdict | _exprvarvalu | exprrelpropvalu
     | exprunivpropvalu | exprtagvalu | exprtagpropvalu | TRIPLEQUOTEDSTRING | DOUBLEQUOTEDSTRING
-    | SINGLEQUOTEDSTRING | formatstring | _innerdollarexprs
+    | SINGLEQUOTEDSTRING | formatstring | _innerdollarexprs | embedquery
 
 // Expr versions of rules to avoid invalid state merges
 _innerdollarexprs: "$"? innerdollaroper
@@ -454,8 +454,14 @@ formatstring: "`" (_formatexpr | FORMATTEXT)* "`"
 
 // Must be kept consistent with same regexes in synapse/lib/grammar.py
 // A full property.  Must contain at least 1 colon
-PROPS.2: /[a-z_][a-z0-9_]*(:[a-z0-9_]+)+([:.][a-z0-9_]+)*(?![:.a-z0-9_\-])/
-WILDPROPS.2: /[a-z_][a-z0-9_]*(:\*|(:[a-z0-9_]+)+([:.][a-z0-9_]+)*:?\*)(?![:.a-z0-9_\-\[])/
+PROPS.3: /[a-z_][a-z0-9_]*(:[a-z0-9_]+)+([:.][a-z0-9_]+)*(?![:.a-z0-9_\-])/
+
+// A full property containing a wildcard
+WILDPROPS.3: /[a-z_][a-z0-9_]*(:\*|(:[a-z0-9_]+)+([:.][a-z0-9_]+)*:?\*)(?![:.a-z0-9_\-\[])/
+
+// A full property with embed properties
+EMBEDPROPS.2: /[a-z_][a-z0-9_]*(:[a-z0-9_]+)+((\:\:|\:|\.)[a-z0-9_]+)*(?![:.a-z0-9_\-])/
+
 // A universal property
 UNIVNAME.2: /(?<=^|[\s\|\{\(\[+=-])\.[a-z_][a-z0-9_]*([:.][a-z0-9_]+)*/
 univprop:  UNIVNAME | "." _varvalu

synapse/lib/storm.py

@@ -1778,8 +1778,10 @@ class StormDmon(s_base.Base):
 
         text = self.ddef.get('storm')
         opts = self.ddef.get('stormopts', {})
-        vars = opts.setdefault('vars', {})
+
+        vars = await s_stormtypes.toprim(opts.get('vars', {}), use_list=True)
         vars.setdefault('auto', {'iden': self.iden, 'type': 'dmon'})
+        opts['vars'] = vars
 
         viewiden = opts.get('view')
 
@@ -2732,15 +2734,28 @@ class Parser:
         self.exited = True
         return False
 
+    def _wrap_text(self, text, width):
+        lines, curline, curlen = [], [], 0
+        for word in text.split():
+            if curlen + len(word) + bool(curline) > width:
+                lines.append(' '.join(curline))
+                curline, curlen = [word], len(word)
+            else:
+                curline.append(word)
+                curlen += len(word) + bool(curline)
+        if curline:
+            lines.append(' '.join(curline))
+        return lines
+
     def _print_optarg(self, names, argdef):
 
         dest = self._get_dest_str(argdef)
         oact = argdef.get('action', 'store')
 
         if oact in ('store_true', 'store_false'):
-            base = f'  {names[0]}'.ljust(30)
+            base = f'  {names[0]}'
         else:
-            base = f'  {names[0]} {dest}'.ljust(30)
+            base = f'  {names[0]} {dest}'
 
         defval = argdef.get('default', s_common.novalu)
         choices = argdef.get('choices')
@@ -2748,12 +2763,14 @@ class Parser:
 
         if defval is not s_common.novalu and oact not in ('store_true', 'store_false'):
             if isinstance(defval, (tuple, list, dict)):
-                defval = pprint.pformat(defval, indent=34, width=100)
-                if '\n' in defval:
-                    defval = '\n' + defval
+                defval_ls = pprint.pformat(defval, width=120).split('\n')
+                defval = '\n'.join(ln.strip() for ln in defval_ls)
 
             if choices is None:
-                helpstr = f'{helpstr} (default: {defval})'
+                if (lambda tst: '\n' in tst if isinstance(tst, str) else False)(defval):
+                    helpstr = f'{helpstr} (default: \n{defval})'
+                else:
+                    helpstr = f'{helpstr} (default: {defval})'
             else:
                 cstr = ', '.join(str(c) for c in choices)
                 helpstr = f'{helpstr} (default: {defval}, choices: {cstr})'
@@ -2762,7 +2779,26 @@ class Parser:
             cstr = ', '.join(str(c) for c in choices)
             helpstr = f'{helpstr} (choices: {cstr})'
 
-        self._printf(f'{base}: {helpstr}')
+        helplst = helpstr.split('\n')
+        if helplst and not helplst[0].strip():
+            helplst = helplst[1:]
+        min_space = min((len(ln) - len(ln.lstrip()) for ln in helplst if ln.strip()), default=0)
+
+        base_w = 32
+        wrap_w = 120 - base_w
+
+        first = helplst[0][min_space:]
+        wrap_first = self._wrap_text(first, wrap_w)
+        self._printf(f'{base:<{base_w-2}}: {wrap_first[0]}')
+
+        for ln in wrap_first[1:]: self._printf(f'{"":<{base_w}}{ln}')
+        for ln in helplst[1:]:
+            lead_s = len(ln) - len(ln.lstrip())
+            rel_ind = lead_s - min_space
+            ind = ' ' * (base_w + rel_ind)
+            wrapped = self._wrap_text(ln.lstrip(), wrap_w - rel_ind)
+            for wl in wrapped:
+                self._printf(f'{ind}{wl}')
 
     def _print_posarg(self, name, argdef):
         dest = self._get_dest_str(argdef)
@@ -5232,7 +5268,7 @@ class BackgroundCmd(Cmd):
         async for item in genr:
             yield item
 
-        runtprims = await s_stormtypes.toprim(self.runt.getScopeVars())
+        runtprims = await s_stormtypes.toprim(self.runt.getScopeVars(), use_list=True)
         runtvars = {k: v for (k, v) in runtprims.items() if s_msgpack.isok(v)}
 
         opts = {

synapse/lib/storm_format.py

@@ -29,6 +29,7 @@ TerminalPygMap = {
     'DOT': p_t.Punctuation,
     'DOUBLEQUOTEDSTRING': p_t.Literal.String,
     'ELIF': p_t.Keyword,
+    'EMBEDPROPS': p_t.Name,
     'EQNOSPACE': p_t.Punctuation,
     'EQSPACE': p_t.Punctuation,
     'EQUAL': p_t.Punctuation,

synapse/lib/stormlib/stix.py

@@ -486,7 +486,10 @@ _DefaultConfig = {
     },
 }
 
-def _validateConfig(core, config):
+perm_maxsize = ('storm', 'lib', 'stix', 'export', 'maxsize')
+def _validateConfig(runt, config):
+
+    core = runt.snap.core
 
     maxsize = config.get('maxsize', 10000)
 
@@ -506,9 +509,10 @@ def _validateConfig(core, config):
         mesg = f'STIX Bundle config maxsize option must be an integer.'
         raise s_exc.BadConfValu(mesg=mesg)
 
-    if maxsize > 10000:
-        mesg = f'STIX Bundle config maxsize option must be <= 10000.'
-        raise s_exc.BadConfValu(mesg=mesg)
+    if maxsize > 10000 and not runt.allowed(perm_maxsize):
+        permstr = '.'.join(perm_maxsize)
+        mesg = f'Setting STIX export maxsize > 10,000 requires permission: {permstr}'
+        raise s_exc.AuthDeny(mesg=mesg, perm=permstr)
 
     formmaps = config.get('forms')
     if formmaps is None:
@@ -1040,6 +1044,11 @@ class LibStixExport(s_stormtypes.Lib):
     '''
     A Storm Library for exporting to STIX version 2.1 CS02.
     '''
+    _storm_lib_perms = (
+        {'perm': ('storm', 'lib', 'stix', 'export', 'maxsize'), 'gate': 'cortex',
+         'desc': 'Controls the ability to specify a STIX export bundle maxsize of greater than 10,000.'},
+    )
+
     _storm_locals = (  # type: ignore
         {
             'name': 'bundle',
@@ -1172,7 +1181,7 @@ class LibStixExport(s_stormtypes.Lib):
             config = _DefaultConfig
 
         config = await s_stormtypes.toprim(config)
-        _validateConfig(self.runt.snap.core, config)
+        _validateConfig(self.runt, config)
 
         return StixBundle(self, self.runt, config)