synapse 3.3.0__tar.gz → 3.4.0__tar.gz

{synapse-3.3.0/synapse.egg-info → synapse-3.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 3.3.0
+Version: 3.4.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0

{synapse-3.3.0 → synapse-3.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '3.3.0'
+version = '3.4.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]

{synapse-3.3.0 → synapse-3.4.0}/synapse/cortex.py

@@ -742,6 +742,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
         self.viewsbylayer = collections.defaultdict(list)
 
         self.stormcmds = {}
+        self._stormdocs = None
 
         self.maxnodes = self.conf.get('max:nodes')
         self.nodecount = 0
@@ -2103,9 +2104,11 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
 
         name = cdef.get('name')
         self.stormcmds[name] = ctor
+        self._stormdocs = None
 
     def _popStormCmd(self, name):
         self.stormcmds.pop(name, None)
+        self._stormdocs = None
 
     async def delStormCmd(self, name):
         '''
@@ -2135,6 +2138,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
         if not self.readonly:
             self.cmddefs.pop(name)
         self.stormcmds.pop(name, None)
+        self._stormdocs = None
 
     async def addStormPkg(self, pkgdef, verify=False):
         '''
@@ -2332,13 +2336,13 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
         '''
         Resolve a dependency/conflict name to a currently loaded package def.
 
-        The reserved names in self.PKG_PROVIDES refer to the running Synapse version rather
-        than a loaded Storm package.
+        The reserved names in self.PKG_PROVIDES refer to self.VERSION rather than to a
+        loaded Storm package.
         '''
         if pkgname in self.PKG_PROVIDES:
             return {
                 'name': pkgname,
-                'version': s_version.version
+                'version': self.VERSION
             }
 
         return await self.getStormPkg(pkgname)
@@ -5344,6 +5348,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             raise s_exc.BadCmdName(name=ctor.name)
 
         self.stormcmds[ctor.name] = ctor
+        self._stormdocs = None
 
     async def addStormDmon(self, ddef):
         '''
@@ -6116,10 +6121,18 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
         '''
         Get a struct containing the Storm Types documentation.
 
+        Notes:
+            The return value is a cached structure shared across callers and
+            must not be mutated.
+
         Returns:
             dict: A Dictionary of storm documentation information.
         '''
 
+        stormdocs = self._stormdocs
+        if stormdocs is not None:
+            return stormdocs
+
         cmds = []
 
         for name, cmd in self.stormcmds.items():
@@ -6142,6 +6155,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             'commands': cmds,
             # 'packages': ...  # TODO - Support inline information for packages?
         }
+        self._stormdocs = ret
         return ret
 
     async def reqFeedDataAllowed(self, items, user, viewiden=None):

{synapse-3.3.0 → synapse-3.4.0}/synapse/datamodel.py

@@ -679,6 +679,8 @@ class Model:
         self._lookup_hints = None
         self._virts_by_type = None
 
+        self._modeldict = None
+
         self._type_pends = collections.defaultdict(list)
         self._modeldef = {
             'types': [],
@@ -1210,6 +1212,20 @@ class Model:
         return resolved
 
     def getModelDict(self):
+        '''
+        Get a dictionary which describes the data model.
+
+        Notes:
+            The return value is a cached structure shared across callers and
+            must not be mutated.
+
+        Returns:
+            dict: A dictionary representation of the data model.
+        '''
+        modeldict = self._modeldict
+        if modeldict is not None:
+            return modeldict
+
         retn = {
             'metas': (
                 ('created', ('time', {}), {
@@ -1240,6 +1256,7 @@ class Model:
         for eobj in self.edges.values():
             retn['edges'].append(eobj.pack())
 
+        self._modeldict = retn
         return retn
 
     def convertPolyinfo(self, propdef):
@@ -1589,6 +1606,12 @@ class Model:
             self.metatypes['created'] = self.getTypeClone(('time', {'ismin': True}))
             self.metatypes['updated'] = self.getTypeClone(('time', {}))
 
+        # addType()/addForm()/addEdge()/addTagProp()/addIface() above each
+        # already clear the cache, but the ctor-based type pass above (which
+        # writes self.types / self.ifacepolys directly) does not, so clear
+        # once more to be certain.
+        self._modeldict = None
+
     def _getFormsMaybeIface(self, name):
 
         form = self.forms.get(name)
@@ -1636,6 +1659,8 @@ class Model:
             for n2form in n2forms:
                 self._valid_edges[(n1form, verb, n2form)] = edge
 
+        self._modeldict = None
+
     def delEdge(self, edgetype):
 
         edge = self.edges.get(edgetype)
@@ -1662,6 +1687,8 @@ class Model:
             for n2form in n2forms:
                 self._valid_edges.pop((n1form, verb, n2form), None)
 
+        self._modeldict = None
+
     def _reqFormName(self, name):
         form = self.forms.get(name)
         if form is None:
@@ -1694,6 +1721,8 @@ class Model:
         if not skipinit:
             self._modeldef['types'].append(newtype.getTypeDef())
 
+        self._modeldict = None
+
     def reqVirtTypes(self, virts):
 
         for (name, tdef, info) in virts:
@@ -1819,6 +1848,7 @@ class Model:
         self.pivchaincache.clear()
         self._lookup_hints = None
         self._virts_by_type = None
+        self._modeldict = None
 
         return form
 
@@ -1934,6 +1964,7 @@ class Model:
         self.pivchaincache.clear()
         self._lookup_hints = None
         self._virts_by_type = None
+        self._modeldict = None
 
         if parentform:
             self.childforms[parentform.name].remove(formname)
@@ -1947,6 +1978,7 @@ class Model:
             raise s_exc.DupName(mesg=f'Interface name conflicts with existing interface: {name}')
 
         self.ifaces[name] = info
+        self._modeldict = None
 
     def reqTypeNotInUse(self, typename):
         if self.propsbytype.get(typename):
@@ -1978,6 +2010,7 @@ class Model:
         self.propsbytype.pop(typename, None)
         self.arraysbytype.pop(typename, None)
         self.tagpropsbytype.pop(typename, None)
+        self._modeldict = None
 
     def addFormProp(self, formname, propname, tdef, info):
         form = self.forms.get(formname)
@@ -2090,6 +2123,7 @@ class Model:
         self.childpropcache.clear()
         self.pivchaincache.clear()
         self._lookup_hints = None
+        self._modeldict = None
 
         return prop
 
@@ -2238,6 +2272,7 @@ class Model:
     def delTagProp(self, name):
         if (prop := self.tagprops.pop(name, None)) is not None:
             self.tagpropsbytype[prop.type.name].pop(name, None)
+            self._modeldict = None
         return prop
 
     def addTagProp(self, name, tdef, info):
@@ -2252,6 +2287,8 @@ class Model:
                    f' be removed in 4.0.0'
             logger.warning(mesg)
 
+        self._modeldict = None
+
         return prop
 
     def getTagProp(self, name):
@@ -2302,6 +2339,7 @@ class Model:
         self.childpropcache.clear()
         self.pivchaincache.clear()
         self._lookup_hints = None
+        self._modeldict = None
 
     def type(self, name):
         '''

{synapse-3.3.0 → synapse-3.4.0}/synapse/lib/cmd.py

@@ -36,7 +36,7 @@ class Parser(argparse.ArgumentParser):
         if message is not None:
             self.outp.printf(message)
 
-        raise s_exc.ParserExit(mesg=message, status=status)
+        raise s_exc.ParserExit(mesg=message, exitcode=status)
 
     def _print_message(self, text, fd=None):
         '''
@@ -54,8 +54,8 @@ async def wrapmain(func, logconf=None): # pragma: no cover
     try:
         return await func(sys.argv[1:])
 
-    except s_exc.ParserExit:
-        return 1
+    except s_exc.ParserExit as e:
+        return e.get('exitcode', 1)
 
     except Exception as e:
         print(f'ERROR: {s_exc.reprexc(e)}')

{synapse-3.3.0 → synapse-3.4.0}/synapse/lib/node.py

@@ -363,16 +363,33 @@ class NodeBase:
     def valuvirts(self, defv=None):
         return defv
 
+    def _nidNormValu(self, ptyp, valu):
+        # fold a normalized value of type ptyp the same way the index and nid
+        # deconfliction do (e.g. case folding a text value)
+        if valu is None:
+            return None
+
+        if ptyp.isarray:
+            # a whole array value is a tuple of typed values
+            return self.view.wlyr.arraytype.nidNorm(valu)
+
+        typename, tvalu = valu
+        if (ntyp := self.view.core.model.type(typename)) is None:
+            return valu
+
+        return (typename, self.view.wlyr.stortypes[ntyp.stortype].nidNorm(tvalu))
+
     def hasPropAltsValu(self, prop, valu):
         # valu must be normalized in advance
+        valu = self._nidNormValu(prop.type, valu)
         prophash = prop.type.typehash
         for prop in prop.getAlts():
             if prop.type.isarray and prop.type.arraytype.typehash is prophash:
                 arryvalu = self.get(prop.name)
-                if arryvalu is not None and valu in arryvalu:
+                if arryvalu is not None and any(self._nidNormValu(prop.type.arraytype, v) == valu for v in arryvalu):
                     return True
             else:
-                if self.get(prop.name) == valu:
+                if self._nidNormValu(prop.type, self.get(prop.name)) == valu:
                     return True
 
         return False

{synapse-3.3.0 → synapse-3.4.0}/synapse/lib/parser.py

@@ -1,8 +1,12 @@
+import os
 import ast
+import logging
 import collections
 
 import lark  # type: ignore
 import regex  # type: ignore
+import lark.lexer  # type: ignore
+import lark.grammar  # type: ignore
 
 import synapse.exc as s_exc
 import synapse.data as s_data
@@ -10,8 +14,11 @@ import synapse.common as s_common
 
 import synapse.lib.ast as s_ast
 import synapse.lib.cache as s_cache
+import synapse.lib.msgpack as s_msgpack
 import synapse.lib.processpool as s_processpool
 
+logger = logging.getLogger(__name__)
+
 # TL;DR:  *rules* are the internal nodes of an abstract syntax tree (AST), *terminals* are the leaves
 
 # Note: this file is coupled strongly to synapse/data/lark/storm.lark.  Any changes to that file will probably require
@@ -543,8 +550,61 @@ class AstConverter(lark.Transformer):
         return s_ast.TryCatch(astinfo, kids)
 
 _grammar = s_data.getLark('storm')
-LarkParser = lark.Lark(_grammar, regex=True, start=['query', 'lookup', 'cmdargs', 'evalvalu'],
-                       maybe_placeholders=False, propagate_positions=True, parser='lalr')
+_larkopts = {
+    'regex': True,
+    'start': ['query', 'lookup', 'cmdargs', 'evalvalu'],
+    'maybe_placeholders': False,
+    'propagate_positions': True,
+    'parser': 'lalr',
+}
+
+LARK_CACHE_PATH = s_data.path('lark', 'storm.lark.mpk')
+
+def _getLarkIden():
+    return s_common.guid((lark.__version__, _grammar, sorted(_larkopts.items())))
+
+def loadLarkParser(path=LARK_CACHE_PATH):
+    '''
+    Load the Storm grammar parser from a cache written by saveLarkParser(), or build it.
+
+    Notes:
+        Building the LALR tables takes seconds per process. The cache is only used when
+        it was written from the same grammar, options, and lark version.
+
+    Returns:
+        (lark.Lark, bool): The parser and True if it was loaded from the cache.
+    '''
+    if os.path.isfile(path):
+        try:
+            with open(path, 'rb') as fd:
+                info = s_msgpack.un(fd.read(), use_list=True)
+
+            if info.get('iden') == _getLarkIden():
+                return lark.Lark.load({'data': info['data'], 'memo': info['memo']}), True
+
+        except Exception as e:
+            logger.warning(f'Failed to load the Storm grammar cache at {path}: {e}')
+
+    return lark.Lark(_grammar, **_larkopts), False
+
+def saveLarkParser(path=LARK_CACHE_PATH):
+    '''
+    Write the Storm grammar parser to a cache which loadLarkParser() reads.
+
+    Notes:
+        The cache is msgpack rather than lark.Lark.save() so that loading it does not
+        unpickle anything.
+    '''
+    parser = lark.Lark(_grammar, **_larkopts)
+    data, memo = parser.memo_serialize([lark.lexer.TerminalDef, lark.grammar.Rule])
+
+    with open(path, 'wb') as fd:
+        fd.write(s_msgpack.en({'iden': _getLarkIden(), 'data': data, 'memo': memo}))
+
+    if not loadLarkParser(path=path)[1]:
+        raise s_exc.BadState(mesg=f'The Storm grammar cache at {path} could not be loaded.')
+
+LarkParser, larkcached = loadLarkParser()
 
 class Parser:
     '''

{synapse-3.3.0 → synapse-3.4.0}/synapse/lib/stormlib/pkg.py

@@ -1,9 +1,14 @@
 import asyncio
 
 import synapse.exc as s_exc
+import synapse.common as s_common
 
+import synapse.lib.json as s_json
+import synapse.lib.const as s_const
 import synapse.lib.stormtypes as s_stormtypes
 
+MAX_DOC_SIZE = 25 * s_const.mebibyte
+
 stormcmds = [
     {
         'name': 'pkg.list',
@@ -127,26 +132,98 @@ stormcmds = [
         'name': 'pkg.docs',
         'desc': 'Display documentation included in a storm package.',
         'cmdargs': (
-            ('name', {'help': 'The name (or name prefix) of the package.'}),
+            ('name', {'help': 'The name of the package.'}),
+            ('doc', {'nargs': '?', 'default': None,
+                'help': 'The path or title of a single document to display.'}),
         ),
         'storm': '''
-            $pdef = (null)
-            for $pkg in $lib.pkg.list() {
-                if $pkg.name.startswith($cmdopts.name) {
-                    $pdef = $pkg
-                    break
+            function printDocs(docs) {
+                $conf = ({
+                    "columns": [
+                        {"name": "Path", "width": 30},
+                        {"name": "Title"},
+                    ],
+                })
+                $printer = $lib.tabular.printer($conf)
+
+                $lib.print($printer.header())
+
+                for $doc in $docs {
+                    $title = $doc.title
+                    if (not $title) { $title = '' }
+                    $lib.print($printer.row(($doc.path, $title)))
                 }
+
+                return()
             }
 
+            $pdef = $lib.pkg.get($cmdopts.name)
+
             if (not $pdef) {
                 $lib.warn(`Package ({$cmdopts.name}) not found!`)
+                $lib.exit()
+            }
+
+            $docs = $lib.pkg.docs.list($pdef.name)
+
+            if ($lib.len($docs) = 0) {
+                $lib.print(`Package ({$pdef.name}) contains no documentation.`)
+                $lib.exit()
+            }
+
+            if (not $cmdopts.doc) {
+
+                $lib.print(`Documentation for package ({$pdef.name}):`)
+                $lib.print('')
+
+                $printDocs($docs)
+
+                $lib.print('')
+                $lib.print(`Use: pkg.docs {$pdef.name} <doc>`)
+
             } else {
-                if $pdef.docs {
-                    for $doc in $pdef.docs {
-                        $lib.print($doc.content)
+
+                $needle = $cmdopts.doc.lower()
+                $matches = ([])
+
+                for $doc in $docs {
+
+                    $relpath = $doc.path
+                    $stem = $relpath
+                    if $stem.endswith('.md') { $stem = $stem.slice(0, -3) }
+                    $title = $doc.title
+
+                    $fullpath = `docs/{$relpath}`
+                    $cands = ([$fullpath.lower(), $relpath.lower(), $stem.lower()])
+                    if $title { $cands.append($title.lower()) }
+
+                    if ($needle in $cands) {
+                        $matches.append($relpath)
+                    }
+                }
+
+                if ($lib.len($matches) = 0) {
+
+                    $lib.warn(`No document matches "{$cmdopts.doc}" in package ({$pdef.name}).`)
+                    $lib.print('')
+                    $printDocs($docs)
+
+                } elif ($lib.len($matches) > 1) {
+
+                    $lib.warn(`Multiple documents match "{$cmdopts.doc}" in package ({$pdef.name}):`)
+                    for $relpath in $matches {
+                        $lib.print($relpath)
                     }
+
                 } else {
-                    $lib.print(`Package ({$cmdopts.name}) contains no documentation.`)
+
+                    $relpath = $matches.index(0)
+                    try {
+                        $doctext = $lib.pkg.docs.get($pdef.name, doc=$relpath)
+                        $lib.print($doctext.$relpath)
+                    } catch NoSuchFile as err {
+                        $lib.warn($err.mesg)
+                    }
                 }
             }
         '''
@@ -336,6 +413,147 @@ class LibPkg(s_stormtypes.Lib):
         name = await s_stormtypes.tostr(name)
         return PkgState(self.runt, name)
 
+@s_stormtypes.registry.registerLib
+class LibPkgDocs(s_stormtypes.Lib):
+    '''
+    A Storm Library for reading a Storm Package's documentation.
+    '''
+    _storm_locals = (
+        {'name': 'list', 'desc': "List a Storm Package's documentation pages.",
+         'type': {'type': 'function', '_funcname': '_libPkgDocsList',
+                  'args': (
+                      {'name': 'name', 'type': 'str', 'desc': 'A Storm Package name.', },
+                  ),
+                  'returns': {'type': 'list', 'desc': 'A list of `{"path": ..., "title": ...}` dicts, or null if '
+                              'the package does not exist.', }}},
+        {'name': 'get', 'desc': "Get a Storm Package's documentation.",
+         'type': {'type': 'function', '_funcname': '_libPkgDocsGet',
+                  'args': (
+                      {'name': 'name', 'type': 'str', 'desc': 'A Storm Package name.', },
+                      {'name': 'doc', 'type': 'str', 'default': None,
+                       'desc': 'The path (relative to docs/) of a single document to retrieve.', },
+                  ),
+                  'returns': {'type': 'dict',
+                              'desc': 'A dict of {path: text} -- every document, or just the named one '
+                              'if doc was specified -- or null if the package or document does not exist.', }}},
+    )
+    _storm_lib_path = ('pkg', 'docs')
+
+    def getObjLocals(self):
+        return {
+            'list': self._libPkgDocsList,
+            'get': self._libPkgDocsGet,
+        }
+
+    async def _getPkgDocs(self, name):
+
+        pkgdef = await self.runt.view.core.getStormPkg(name)
+        if pkgdef is None:
+            return None
+
+        # a package's documentation ships as ordinary files under docs/
+        # rather than inline pkgdef content -- a doc page is a files:
+        # entry whose path starts with docs/ and ends in .md, which
+        # excludes docs/metadata.json (not .md) and any images.
+        docs = {}
+        for docpath, filedef in (pkgdef.get('files') or {}).items():
+            if docpath.startswith('docs/') and docpath.endswith('.md'):
+                docs[docpath[5:]] = filedef.get('sha256')
+
+        return pkgdef, docs
+
+    async def _readPkgFile(self, axon, relpath, sha256):
+
+        sha256b = s_common.uhex(sha256)
+
+        size = await axon.size(sha256b)
+        if size is None:
+            mesg = f'Document ({relpath}) is missing from the Axon.'
+            raise s_exc.NoSuchFile(mesg=mesg)
+
+        if size > MAX_DOC_SIZE:
+            mesg = f'Document ({relpath}) is larger than the {MAX_DOC_SIZE} byte limit.'
+            raise s_exc.StormRuntimeError(mesg=mesg)
+
+        byts = b''
+        async for chunk in axon.get(sha256b):
+            byts += chunk
+
+        return byts
+
+    async def _getDocTitles(self, axon, pkgdef):
+
+        titles = {}
+
+        metafiledef = (pkgdef.get('files') or {}).get('docs/metadata.json')
+        if metafiledef is None:
+            return titles
+
+        metasha256 = metafiledef['sha256']
+
+        try:
+            byts = await self._readPkgFile(axon, 'metadata.json', metasha256)
+            meta = s_json.loads(byts)
+        except (s_exc.NoSuchFile, s_exc.StormRuntimeError, s_exc.BadJsonText):
+            return titles
+
+        queue = list(meta.get('toc') or ())
+        while queue:
+            entry = queue.pop(0)
+            href = entry.get('href')
+            if href:
+                titles[href] = entry.get('title')
+            queue.extend(entry.get('children') or ())
+
+        return titles
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _libPkgDocsList(self, name):
+
+        name = await s_stormtypes.tostr(name)
+
+        retn = await self._getPkgDocs(name)
+        if retn is None:
+            return None
+
+        pkgdef, docs = retn
+
+        if not docs:
+            return []
+
+        axon = await self.runt.view.core.getAxon()
+        titles = await self._getDocTitles(axon, pkgdef)
+
+        return [{'path': relpath, 'title': titles.get(relpath)} for relpath in sorted(docs.keys())]
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _libPkgDocsGet(self, name, doc=None):
+
+        name = await s_stormtypes.tostr(name)
+        doc = await s_stormtypes.tostr(doc, noneok=True)
+
+        retn = await self._getPkgDocs(name)
+        if retn is None:
+            return None
+
+        _, docs = retn
+
+        axon = await self.runt.view.core.getAxon()
+
+        if doc is not None:
+            sha256 = docs.get(doc)
+            if sha256 is None:
+                return None
+            byts = await self._readPkgFile(axon, doc, sha256)
+            return {doc: byts.decode('utf8', errors='ignore')}
+
+        retn = {}
+        for relpath, sha256 in docs.items():
+            byts = await self._readPkgFile(axon, relpath, sha256)
+            retn[relpath] = byts.decode('utf8', errors='ignore')
+
+        return retn
+
 @s_stormtypes.registry.registerType
 class PkgVars(s_stormtypes.Prim):
     '''

{synapse-3.3.0 → synapse-3.4.0}/synapse/lib/stormlib/stix.py

@@ -15,8 +15,6 @@ import synapse.lib.stormctrl as s_stormctrl
 import synapse.lib.stormtypes as s_stormtypes
 import synapse.lib.processpool as s_processpool
 
-import stix2validator
-
 logger = logging.getLogger(__name__)
 
 def uuid5(valu=None):
@@ -594,6 +592,10 @@ def _validateConfig(runt, config):
                             raise s_exc.BadConfValu(mesg=mesg)
 
 def validateStix(bundle, version='2.1'):
+
+    # stix2validator is slow to import, so it only loads in the process which validates a bundle.
+    import stix2validator
+
     ret = {
         'ok': False,
         'mesg': '',