synapse 3.2.0__tar.gz → 3.3.0__tar.gz

{synapse-3.2.0/synapse.egg-info → synapse-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 3.2.0
+Version: 3.3.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0

{synapse-3.2.0 → synapse-3.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '3.2.0'
+version = '3.3.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]

{synapse-3.2.0 → synapse-3.3.0}/synapse/cortex.py

@@ -6075,7 +6075,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
         except Exception as e:
             return s_common.retnexc(e)
 
-    def _logStormQuery(self, text, user, info=None, meta=None):
+    def _logStormQuery(self, text, user, info=None, meta=None, via=None):
         '''
         Log a storm query.
 
@@ -6084,6 +6084,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             user: The User who ran the query.
             info (dict): Fields to record alongside the query.
             meta (dict): The caller supplied meta opt, if any.
+            via (str): The name of the Storm command or library which dispatched the query.
         '''
         if self.stormlog:
             if info is None:
@@ -6097,8 +6098,12 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             if meta is not None:
                 info['meta'] = meta
 
-            stormlogger.log(self.stormloglvl, 'Executing storm query {%s} as [%s]', text, user.name,
-                            extra=self.getLogExtra(**info))
+            if via is None:
+                stormlogger.log(self.stormloglvl, 'Executing storm query {%s} as [%s]', text, user.name,
+                                extra=self.getLogExtra(**info))
+            else:
+                stormlogger.log(self.stormloglvl, 'Executing storm query via %s {%s} as [%s]', via, text, user.name,
+                                extra=self.getLogExtra(**info))
 
     async def getCoreInfoV2(self):
         return {

{synapse-3.2.0 → synapse-3.3.0}/synapse/data/skills/storm/SKILL.md

@@ -754,6 +754,12 @@ $x = (true)
 $y = (999)
 $dict = ({})
 $list = ([])
+
+// WRONG - redundant parens inside a JSON expression
+$props = ({"ids": ([$iden]), "opts": ({"asof": $asof})})
+
+// CORRECT
+$props = ({"ids": [$iden], "opts": {"asof": $asof}})
 ```
 
 ### 3. Forgetting `|` After Inline Commands

{synapse-3.2.0 → synapse-3.3.0}/synapse/datamodel.py

@@ -677,6 +677,7 @@ class Model:
         self.formprefixcache = s_cache.LruDict(PREFIX_CACHE_SIZE)
 
         self._lookup_hints = None
+        self._virts_by_type = None
 
         self._type_pends = collections.defaultdict(list)
         self._modeldef = {
@@ -748,6 +749,33 @@ class Model:
             return ()
         return list(props.values())
 
+    def getVirtsByType(self, name):
+        '''
+        Return the (formname, virtname) tuples for the indexed virtual properties
+        of a form primary property which reference the given form name. A virt on
+        a secondary property is not a ref because the node for the property value
+        carries the ref, the same as Form.getRefsOut().
+        '''
+        if self._virts_by_type is None:
+
+            virtsbytype = collections.defaultdict(list)
+
+            for formname, form in self.forms.items():
+                for virtname, virtinfo in form.type.virts.items():
+
+                    typename = virtinfo[0].name
+                    if self.forms.get(typename) is None:
+                        continue
+
+                    if form.type.virtindx.get(virtname) is None:
+                        continue
+
+                    virtsbytype[typename].append((formname, virtname))
+
+            self._virts_by_type = virtsbytype
+
+        return self._virts_by_type.get(name, ())
+
     def getProps(self):
         return [pobj for pname, pobj in self.props.items()
                 if not (isinstance(pname, tuple))]
@@ -1790,6 +1818,7 @@ class Model:
         self.formprefixcache.clear()
         self.pivchaincache.clear()
         self._lookup_hints = None
+        self._virts_by_type = None
 
         return form
 
@@ -1904,6 +1933,7 @@ class Model:
         self.formprefixcache.clear()
         self.pivchaincache.clear()
         self._lookup_hints = None
+        self._virts_by_type = None
 
         if parentform:
             self.childforms[parentform.name].remove(formname)

{synapse-3.2.0/synapse/tools/storm → synapse-3.3.0/synapse/lib}/_http.py

@@ -15,6 +15,8 @@ import synapse.lib.const as s_const
 import synapse.lib.httpapi as s_httpapi
 import synapse.lib.msgpack as s_msgpack
 import synapse.lib.urlhelp as s_urlhelp
+import synapse.lib.version as s_version
+import synapse.lib.httpclient as s_httpclient
 import synapse.lib.crypto.passwd as s_passwd
 
 logger = logging.getLogger(__name__)
@@ -34,6 +36,11 @@ EXPORT_TIMEOUT = 3600
 # The number of chunks buffered while streaming bytes into an Axon HTTP upload.
 UPLOAD_QSIZE = 4
 
+# The product token in the default outbound User-Agent. A Cell derives its own token
+# from its cell type ( see Cell._getUserAgentProd ); this client is not a Cell, and it
+# ships with Synapse, so its version is the Synapse version.
+USER_AGENT_PROD = 'Synapse-Client'
+
 apikeymesg = 'A user API key is required to connect to a Cortex over HTTP. ' \
              'Provide it in the URL as https://<apikey>@host:port/.'
 
@@ -49,6 +56,43 @@ def isHttpsUrl(url):
     '''
     return url.lower().startswith('https://')
 
+def addHttpsArgs(pars):
+    '''
+    Add the options which configure the Cortex HTTP API client to an argument parser.
+
+    Note:
+        Shared by every CLI tool which accepts an https:// Cortex URL, so the options
+        and their help text cannot drift apart between them.
+
+    Args:
+        pars (s_cmd.Parser): The argument parser.
+    '''
+    pars.add_argument('--https-proxy', default=None,
+                      help='An aiohttp-socks compatible proxy URL to use for https:// URLs.')
+    pars.add_argument('--https-ca-dir', default=None,
+                      help='A directory of CAs which are added to the TLS CA chain for https:// URLs.')
+    pars.add_argument('--https-noverify', default=False, action='store_true',
+                      help='Ignore SSL certificate validation errors for https:// URLs.')
+
+def reqTeleOpts(opts):
+    '''
+    Require that the https only options are not used with a telepath URL.
+
+    Args:
+        opts (argparse.Namespace): The parsed arguments.
+    '''
+    httponly = (
+        ('--https-proxy', opts.https_proxy),
+        ('--https-ca-dir', opts.https_ca_dir),
+        ('--https-noverify', opts.https_noverify or None),
+    )
+
+    for name, valu in httponly:
+
+        if valu is not None:
+            mesg = f'The {name} option may only be used with an https:// Cortex URL.'
+            raise s_exc.BadArg(mesg=mesg, arg=name)
+
 async def openurl(url, cadir=None, proxy=None, verify=True):
     '''
     Open a Cortex HTTP API client which duck types the Telepath CoreApi.
@@ -190,7 +234,8 @@ class HttpCortex(s_base.Base):
     '''
     A Cortex client which uses the HTTP API rather than Telepath.
 
-    This implements the subset of the Telepath CoreApi which is used by the Storm CLI.
+    This implements the subset of the Telepath CoreApi used by the CLI tools which
+    accept an https:// Cortex URL.
 
     Note:
         Messages are deserialized from JSON, so nested sequences are lists where the
@@ -237,9 +282,14 @@ class HttpCortex(s_base.Base):
         # running Storm queries.
         timeout = aiohttp.ClientTimeout(total=None)
 
+        headers = {
+            'X-API-KEY': apikey,
+            'User-Agent': s_httpclient.getUserAgent(USER_AGENT_PROD, s_version.version),
+        }
+
         sess = aiohttp.ClientSession(connector=connector,
                                      timeout=timeout,
-                                     headers={'X-API-KEY': apikey},
+                                     headers=headers,
                                      max_line_size=s_const.MAX_LINE_SIZE,
                                      max_field_size=s_const.MAX_FIELD_SIZE)
 
@@ -378,6 +428,18 @@ class HttpCortex(s_base.Base):
                 mesg = 'The Cortex HTTP API export stream ended with a partial node.'
                 raise s_exc.BadDataValu(mesg=mesg)
 
+    async def addStormPkg(self, pkgdef, *, verify=False):
+        '''
+        Add a Storm package to the Cortex.
+
+        Note:
+            The HTTP API has no package endpoint, so this runs $lib.pkg.add() which
+            confirms the same pkg.add permission and calls the same Cortex API the
+            Telepath CoreApi method does.
+        '''
+        opts = {'vars': {'pkgdef': pkgdef, 'verify': verify}}
+        return await self.callStorm('return($lib.pkg.add($pkgdef, verify=$verify))', opts=opts)
+
     async def getCoreInfoV2(self):
 
         async with self.sess.get(self._getUrl('/api/v3/core/info'), **self.reqinfo) as resp:

{synapse-3.2.0 → synapse-3.3.0}/synapse/lib/ast.py

@@ -312,9 +312,10 @@ class Lookup(Query):
                 return
 
             hints = runt.model.getLookupHints()
-            if not hints:
-                return
 
+            # the remainder tokens are resolved by the data model lookup hints and
+            # then by the search interface. Nodes from those two steps are
+            # deduplicated against each other.
             async with await s_spooled.Set.anit(dirn=view.core.dirn, cell=view.core) as nidset:
 
                 for tokn in remtokns:
@@ -331,6 +332,22 @@ class Lookup(Query):
                         except (s_exc.BadTypeValu, s_exc.BadCmprValu, s_exc.NoSuchCmpr):
                             continue
 
+                todo = s_common.todo('search', remtokns)
+                async for _, nid in view.mergeStormIface('search', todo):
+
+                    nid = s_common.int64en(nid)
+                    if nid in nidset:
+                        await asyncio.sleep(0)
+                        continue
+
+                    await nidset.add(nid)
+
+                    node = await view.getNodeByNid(nid)
+                    if node is None:
+                        continue
+
+                    yield node, runt.initPath(node)
+
         realgenr = lookgenr()
         if len(self.kids) > 1:
             realgenr = self.kids[1].run(runt, realgenr)
@@ -2714,6 +2731,13 @@ class PivotIn(PivotOper):
 
         name, valu = node.ndef
 
+        for formtype in node.form.formtypes:
+            for formname, virtname in runt.model.getVirtsByType(formtype):
+                link = {'type': 'prop', 'prop': f'.{virtname}', 'reverse': True}
+
+                async for pivo in runt.view.nodesByPropValu(formname, '=', valu, norm=False, virt=virtname):
+                    yield pivo, path.fork(pivo, link)
+
         for formtype in node.form.formtypes:
             for prop in runt.model.getPropsByType(formtype):
                 link = {'type': 'prop', 'prop': prop.name, 'reverse': True}
@@ -4044,12 +4068,13 @@ class TagVirtCond(Cond):
                 valu = await rval.compute(runt, path)
                 virt = await vkid.compute(runt, path)
 
-                (ptyp, getr) = ival.getVirtInfo(virt)
+                (ptyp, getr) = ival.getTagVirtInfo(virt)
 
                 if (cmprctor := ptyp.getCmprCtor(cmpr)) is None:
                     raise self.kids[2].addExcInfo(s_exc.NoSuchCmpr(cmpr=cmpr, name=ptyp.name))
 
-                tval = getr(node.getTag(name))
+                if (tval := node.getTag(name, getr=getr)) is None:
+                    return False
 
                 return await (await cmprctor(valu))(tval)
 
@@ -4061,7 +4086,7 @@ class TagVirtCond(Cond):
             valu = await rval.compute(runt, None)
             virt = await vkid.compute(runt, None)
 
-            (ptyp, getr) = ival.getVirtInfo(virt)
+            (ptyp, getr) = ival.getTagVirtInfo(virt)
 
             if (cmprctor := ptyp.getCmprCtor(cmpr)) is None:
                 raise self.kids[2].addExcInfo(s_exc.NoSuchCmpr(cmpr=cmpr, name=ptyp.name))
@@ -4069,7 +4094,9 @@ class TagVirtCond(Cond):
             cmpr = await cmprctor(valu)
 
             async def cond(node, path):
-                tval = getr(node.getTag(name))
+                if (tval := node.getTag(name, getr=getr)) is None:
+                    return False
+
                 return await cmpr(tval)
 
             return cond
@@ -4079,12 +4106,13 @@ class TagVirtCond(Cond):
             valu = await rval.compute(runt, path)
             virt = await vkid.compute(runt, path)
 
-            (ptyp, getr) = ival.getVirtInfo(virt)
+            (ptyp, getr) = ival.getTagVirtInfo(virt)
 
             if (cmprctor := ptyp.getCmprCtor(cmpr)) is None:
                 raise self.kids[2].addExcInfo(s_exc.NoSuchCmpr(cmpr=cmpr, name=ptyp.name))
 
-            tval = getr(node.getTag(name))
+            if (tval := node.getTag(name, getr=getr)) is None:
+                return False
 
             return await (await cmprctor(valu))(tval)
 
@@ -4433,8 +4461,8 @@ class TagVirtValue(Value):
         name = await self.kids[0].compute(runt, path)
         virt = await self.kids[1].compute(runt, path)
 
-        getr = runt.model.type('ival').getVirtGetr(virt)
-        return getr(path.node.getTag(name))
+        (_, getr) = runt.model.type('ival').getTagVirtInfo(virt)
+        return path.node.getTag(name, getr=getr)
 
 class TagProp(Value):
 
@@ -5862,10 +5890,12 @@ class N1Walk(Oper):
         for destform in destforms:
             prop = runt.model.prop(destform)
             if prop is not None:
+                # a form or prop name also matches the forms which inherit from it
                 if prop.isform:
-                    forms.add(destform)
+                    forms.update(runt.model.getChildForms(destform))
                 else:
-                    formprops[prop.form.name][prop.name] = prop
+                    for cprop in runt.model.getChildProps(prop):
+                        formprops[cprop.form.name][cprop.name] = cprop
                 continue
 
             formlist = runt.model.reqFormsByLook(destform, extra=self.kids[0].addExcInfo)

{synapse-3.2.0 → synapse-3.3.0}/synapse/lib/auth.py

@@ -44,14 +44,7 @@ async def _getEmailType():
 
     return _emailtype
 
-async def normEmail(email):
-    '''
-    Normalize an email address for use as a unique key via the inet:email type.
-
-    Returns the normalized form or None if the input is None or normalizes
-    to an empty string. Raises BadArg if the value is not a string or does
-    not parse as an email address.
-    '''
+async def _normEmail(email):
     if email is None:
         return None
 
@@ -64,11 +57,37 @@ async def normEmail(email):
 
     try:
         etype = await _getEmailType()
-        norm, _info = await etype.norm(valu)
+        return await etype.norm(valu)
     except s_exc.BadTypeValu as e:
         raise s_exc.BadArg(mesg=e.get('mesg'), name='email') from None
 
-    return norm
+async def normEmail(email):
+    '''
+    Normalize an email address for use as a unique key via the inet:email type.
+
+    Returns the normalized form or None if the input is None or normalizes
+    to an empty string. Raises BadArg if the value is not a string or does
+    not parse as an email address.
+    '''
+    retn = await _normEmail(email)
+    if retn is None:
+        return None
+
+    return retn[0]
+
+async def normEmailFqdn(email):
+    '''
+    Extract the normalized FQDN from an email address via the inet:email type.
+
+    Returns the normalized FQDN or None if the input is None or normalizes
+    to an empty string. Raises BadArg if the value is not a string or does
+    not parse as an email address.
+    '''
+    retn = await _normEmail(email)
+    if retn is None:
+        return None
+
+    return retn[1]['subs']['fqdn'][1]
 
 @dataclasses.dataclass(slots=True)
 class _allowedReason:

{synapse-3.2.0 → synapse-3.3.0}/synapse/lib/crypto/passwd.py

@@ -21,6 +21,9 @@ PBKDF2_ITERATIONS = 310_000  # OWASP recommended value 20221004
 
 DEFAULT_PTYP = PBKDF2
 
+APIKEY_PREFIX = 'syn-'
+APIKEY_PREFIX_SIZE = len(APIKEY_PREFIX)
+
 def _getPbkdf2(passwd):
     salt = os.urandom(32)
     func_params = {'salt': salt,
@@ -130,11 +133,11 @@ async def generateApiKey(iden=None):
         if not s_common.isguid(iden):
             raise s_exc.CryptoErr(mesg=f'Invalid iden provided: {iden}, must be a guid.')
     secv = s_common.guid()
-    key = base64.b64encode(s_common.uhex(iden) + s_common.uhex(secv), altchars=b'-_').decode('utf-8')
+    key = APIKEY_PREFIX + base64.b64encode(s_common.uhex(iden) + s_common.uhex(secv), altchars=b'-_').decode('utf-8')
     shadow = await getShadowV2(secv)
     return iden, key, shadow
 
-def parseApiKey(valu):
+def _parseApiKey(valu):
     if '+' in valu or '/' in valu:
         return False, 'Invalid character in API key.'
     try:
@@ -144,3 +147,15 @@ def parseApiKey(valu):
     if len(buf) != 32:
         return False, f'Incorrect length, got {len(valu)}'
     return True, (s_common.ehex(buf[:16]), s_common.ehex(buf[16:]))
+
+def parseApiKey(valu):
+    # keys issued before APIKEY_PREFIX was added carry no prefix and must still parse.
+    # '-_' is the altchars alphabet, so a legacy key can itself begin with the prefix
+    # ('s', 'y', 'n', '-' are all valid altchars); fall back to the unprefixed parse
+    # rather than stripping unconditionally.
+    if valu.startswith(APIKEY_PREFIX):
+        isok, retn = _parseApiKey(valu[APIKEY_PREFIX_SIZE:])
+        if isok:
+            return True, retn
+
+    return _parseApiKey(valu)