synapse 2.175.0__py311-none-any.whl → 2.177.0__py311-none-any.whl

synapse/lib/stormlib/model.py

@@ -919,8 +919,63 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
                        'desc': 'Do not copy nodedata to the risk:vulnerable node.'},
                  ),
                  'returns': {'type': 'list', 'desc': 'A list of idens for the risk:vulnerable nodes.'}}},
+        {'name': 'inetSslCertToTlsServerCert', 'desc': '''
+            Create a inet:tls:servercert node from the provided inet:ssl:cert node.
+
+            Edits will be made to the inet:tls:servercert node in the current write layer.
+
+            Tags, tag properties, edges, and node data will be copied
+            to the inet:tls:servercert node. However, existing tag properties and
+            node data will not be overwritten.
+        ''',
+        'type': {'type': 'function', '_funcname': '_storm_query',
+                 'args': (
+                      {'name': 'n', 'type': 'node', 'desc': 'The inet:ssl:cert node to migrate.'},
+                      {'name': 'nodata', 'type': 'bool', 'default': False,
+                       'desc': 'Do not copy nodedata to the inet:tls:servercert node.'},
+                 ),
+                 'returns': {'type': 'node', 'desc': 'The newly created inet:tls:servercert node.'}}},
+
     )
     _storm_lib_path = ('model', 'migration', 's')
+    _storm_query = '''
+        function inetSslCertToTlsServerCert(n, nodata=$lib.false) {
+            $form = $n.form()
+            if ($form != 'inet:ssl:cert') {
+                $mesg = `$lib.model.migration.s.inetSslCertToTlsServerCert() only accepts inet:ssl:cert nodes, not {$form}`
+                $lib.raise(BadArg, $mesg)
+            }
+
+            $server = $n.props.server
+            $sha256 = { yield $n -> file:bytes -> hash:sha256 }
+
+            if $sha256 {
+
+                yield $lib.gen.inetTlsServerCertByServerAndSha256($server, $sha256)
+
+            } else {
+
+                // File doesn't have a :sha256, try to lift/create a crypto:x509:node based on the file link
+                $crypto = { yield $n -> file:bytes -> crypto:x509:cert:file }
+                if (not $crypto) {
+                    $crypto = {[ crypto:x509:cert=($n.props.file,) :file=$n.props.file ]}
+                }
+
+                [ inet:tls:servercert=($server, $crypto) ]
+
+            }
+
+            [ .seen ?= $n.props.".seen" ]
+
+            $lib.model.migration.copyTags($n, $node, overwrite=$lib.false)
+            $lib.model.migration.copyEdges($n, $node)
+            if (not $nodata) {
+                $lib.model.migration.copyData($n, $node, overwrite=$lib.false)
+            }
+
+            return($node)
+        }
+    '''
 
     def getObjLocals(self):
         return {

synapse/lib/stormlib/modelext.py

@@ -77,6 +77,27 @@ class LibModelExt(s_stormtypes.Lib):
                       {'name': 'model', 'type': 'dict', 'desc': 'A model dictionary from getExtModel().', },
                   ),
                   'returns': {'type': 'boolean'}}},
+        {'name': 'addEdge', 'desc': 'Add an extended edge definition to the data model.',
+         'type': {'type': 'function', '_funcname': 'addEdge',
+                  'args': (
+                      {'name': 'n1form', 'type': 'str',
+                       'desc': 'The form of the n1 node. May be "*" or null to specify "any".'},
+                      {'name': 'verb', 'type': 'str', 'desc': 'The edge verb, which must begin with "_".'},
+                      {'name': 'n2form', 'type': 'str',
+                       'desc': 'The form of the n2 node. May be "*" or null to specify "any".'},
+                      {'name': 'edgeinfo', 'type': 'dict', 'desc': 'A Synapse edge info dictionary.'},
+                  ),
+                  'returns': {'type': 'null'}}},
+        {'name': 'delEdge', 'desc': 'Remove an extended edge definition from the data model.',
+         'type': {'type': 'function', '_funcname': 'delEdge',
+                  'args': (
+                      {'name': 'n1form', 'type': 'str',
+                       'desc': 'The form of the n1 node. May be "*" or null to specify "any".'},
+                      {'name': 'verb', 'type': 'str', 'desc': 'The edge verb, which must begin with "_".'},
+                      {'name': 'n2form', 'type': 'str',
+                       'desc': 'The form of the n2 node. May be "*" or null to specify "any".'},
+                  ),
+                  'returns': {'type': 'null'}}},
     )
     _storm_lib_path = ('model', 'ext')
 
@@ -92,6 +113,8 @@ class LibModelExt(s_stormtypes.Lib):
             'delTagProp': self.delTagProp,
             'getExtModel': self.getExtModel,
             'addExtModel': self.addExtModel,
+            'addEdge': self.addEdge,
+            'delEdge': self.delEdge,
         }
 
     # TODO type docs in the new convention
@@ -163,3 +186,40 @@ class LibModelExt(s_stormtypes.Lib):
     async def addExtModel(self, model):
         model = await s_stormtypes.toprim(model)
         return await self.runt.snap.core.addExtModel(model)
+
+    async def addEdge(self, n1form, verb, n2form, edgeinfo):
+        verb = await s_stormtypes.tostr(verb)
+        n1form = await s_stormtypes.tostr(n1form, noneok=True)
+        n2form = await s_stormtypes.tostr(n2form, noneok=True)
+        edgeinfo = await s_stormtypes.toprim(edgeinfo)
+
+        if not (s_grammar.isEdgeVerb(verb) and verb.islower()):
+            mesg = f'Invalid edge verb {verb}'
+            raise s_exc.BadEdgeDef(mesg=mesg, n1form=n1form, verb=verb, n2form=n2form)
+
+        if n1form == '*':
+            n1form = None
+
+        if n2form == '*':
+            n2form = None
+
+        s_stormtypes.confirm(('model', 'edge', 'add'))
+        await self.runt.snap.core.addEdge((n1form, verb, n2form), edgeinfo)
+
+    async def delEdge(self, n1form, verb, n2form):
+        verb = await s_stormtypes.tostr(verb)
+        n1form = await s_stormtypes.tostr(n1form, noneok=True)
+        n2form = await s_stormtypes.tostr(n2form, noneok=True)
+
+        if not (s_grammar.isEdgeVerb(verb) and verb.islower()):
+            mesg = f'Invalid edge verb {verb}'
+            raise s_exc.BadEdgeDef(mesg=mesg, n1form=n1form, verb=verb, n2form=n2form)
+
+        if n1form == '*':
+            n1form = None
+
+        if n2form == '*':
+            n2form = None
+
+        s_stormtypes.confirm(('model', 'edge', 'del'))
+        await self.runt.snap.core.delEdge((n1form, verb, n2form))

synapse/lib/stormlib/storm.py

@@ -1,18 +1,102 @@
 import logging
 
 import synapse.exc as s_exc
+import synapse.common as s_common
 
+import synapse.lib.storm as s_storm
 import synapse.lib.stormtypes as s_stormtypes
 
 evaldesc = '''\
-Evaluate a storm runtime value and optionally cast/coerce it.
+Evaluate a Storm runtime value and optionally cast/coerce it.
 
-NOTE: If storm logging is enabled, the expression being evaluated will be logged
-separately.
+Note:
+    If Storm logging is enabled, the expression being evaluated will be logged
+    separately.
+'''
+
+rundesc = '''
+Run a Storm query and yield the messages output by the Storm interpreter.
+
+Note:
+    If Storm logging is enabled, the query being run will be logged separately.
 '''
 
 stormlogger = logging.getLogger('synapse.storm')
 
+class StormExecCmd(s_storm.Cmd):
+    '''
+    Execute text or an embedded query object as Storm in the current pipeline.
+
+    NOTE: It is recommended to avoid using this where possible to avoid potential
+    query injection risks. If you must use this, take care to ensure any values
+    being executed have been properly sanitized.
+
+    Examples:
+
+        // Add nodes using text in a variable
+        $query = '[ inet:fqdn=foo.com inet:fqdn=bar.net ]'
+        storm.exec $query
+
+        // Filter nodes in the pipeline using text in a variable
+        $filter = '-:asn=10'
+        inet:ipv4:asn
+        storm.exec $filter
+
+        // Pivot using an embedded query
+        $pivot = ${ -> inet:asn }
+        inet:ipv4:asn
+        storm.exec $pivot
+    '''
+    name = 'storm.exec'
+    def getArgParser(self):
+        pars = s_storm.Cmd.getArgParser(self)
+        pars.add_argument('query', help='The Storm to execute.')
+        return pars
+
+    async def execStormCmd(self, runt, genr):
+
+        if self.runtsafe:
+
+            text = await s_stormtypes.tostr(self.opts.query)
+            query = await runt.getStormQuery(text)
+
+            extra = await self.runt.snap.core.getLogExtra(text=text, view=self.runt.snap.view.iden)
+            stormlogger.info(f'Executing storm query via storm.exec {{{text}}} as [{self.runt.user.name}]', extra=extra)
+
+            async with runt.getSubRuntime(query) as subr:
+                async for subp in subr.execute(genr=genr):
+                    yield subp
+
+        else:
+
+            item = None
+            async for item in genr:
+                break
+
+            text = await s_stormtypes.tostr(self.opts.query)
+            query = await runt.getStormQuery(text)
+
+            extra = await self.runt.snap.core.getLogExtra(text=text, view=self.runt.snap.view.iden)
+            stormlogger.info(f'Executing storm query via storm.exec {{{text}}} as [{self.runt.user.name}]', extra=extra)
+
+            async with runt.getSubRuntime(query) as subr:
+                async for subp in subr.execute(genr=s_common.agen(item)):
+                    yield subp
+
+                async for item in genr:
+                    text = await s_stormtypes.tostr(self.opts.query)
+                    query = await runt.getStormQuery(text)
+
+                    subr.runtvars.clear()
+                    subr.query = query
+                    subr._initRuntVars(query)
+
+                    extra = await self.runt.snap.core.getLogExtra(text=text, view=self.runt.snap.view.iden)
+                    stormlogger.info(f'Executing storm query via storm.exec {{{text}}} as [{self.runt.user.name}]', extra=extra)
+
+                    async for subp in subr.execute(genr=s_common.agen(item)):
+                        yield subp
+
 @s_stormtypes.registry.registerLib
 class LibStorm(s_stormtypes.Lib):
     '''
@@ -25,15 +109,43 @@ class LibStorm(s_stormtypes.Lib):
                       {'name': 'text', 'type': 'str', 'desc': 'A storm expression string.'},
                       {'name': 'cast', 'type': 'str', 'desc': 'A type to cast the result to.', 'default': None},
                   ),
-                  'returns': {'type': 'any', 'desc': 'The value of the expression and optional cast.', }}},
+                  'returns': {'type': 'any', 'desc': 'The value of the expression and optional cast.'}}},
+        {'name': 'run', 'desc': rundesc,
+         'type': {'type': 'function', '_funcname': '_runStorm',
+                  'args': (
+                      {'name': 'query', 'type': 'str', 'desc': 'A Storm query string.'},
+                      {'name': 'opts', 'type': 'dict', 'desc': 'Storm options dictionary.', 'default': None},
+                  ),
+                  'returns': {'name': 'yields', 'type': 'list', 'desc': 'The output messages from the Storm runtime.'}}},
     )
     _storm_lib_path = ('storm',)
 
     def getObjLocals(self):
         return {
+            'run': self._runStorm,
             'eval': self._evalStorm,
         }
 
+    async def _runStorm(self, query, opts=None):
+
+        opts = await s_stormtypes.toprim(opts)
+        query = await s_stormtypes.tostr(query)
+
+        if opts is None:
+            opts = {}
+
+        user = opts.get('user')
+        if user is None:
+            user = opts['user'] = self.runt.user.iden
+
+        if user != self.runt.user.iden:
+            self.runt.confirm(('impersonate',))
+
+        opts.setdefault('view', self.runt.snap.view.iden)
+
+        async for mesg in self.runt.snap.view.core.storm(query, opts=opts):
+            yield mesg
+
     @s_stormtypes.stormfunc(readonly=True)
     async def _evalStorm(self, text, cast=None):
 
@@ -41,7 +153,7 @@ class LibStorm(s_stormtypes.Lib):
         cast = await s_stormtypes.tostr(cast, noneok=True)
 
         if self.runt.snap.core.stormlog:
-            extra = await self.runt.snap.core.getLogExtra(text=text)
+            extra = await self.runt.snap.core.getLogExtra(text=text, view=self.runt.snap.view.iden)
             stormlogger.info(f'Executing storm query via $lib.storm.eval() {{{text}}} as [{self.runt.user.name}]', extra=extra)
 
         casttype = None

synapse/lib/stormlib/tabular.py

@@ -0,0 +1,212 @@
+import copy
+import textwrap
+import itertools
+
+import synapse.exc as s_exc
+import synapse.common as s_common
+import synapse.lib.schemas as s_schemas
+import synapse.lib.stormtypes as s_stormtypes
+
+justers = {
+    'left': str.ljust,
+    'center': str.center,
+    'right': str.rjust,
+}
+
+@s_stormtypes.registry.registerLib
+class LibTabular(s_stormtypes.Lib):
+    '''
+    A Storm Library for creating printable tables.
+    '''
+    _storm_locals = (
+        {'name': 'printer', 'desc': '''
+        Construct a new printer.
+
+        Examples:
+            Create a simple table using the default separators::
+
+                $conf = ({
+                    "columns": [
+                        {"name": "Year", "width": 4},
+                        {"name": "Author", "width": 20},
+                        {"name": "Title", "width": 12},
+                    ]
+                })
+
+                $printer = $lib.tabular.printer($conf)
+
+                $lib.print($printer.header())
+
+                for ($year, $author, $title, $publisher) in $data {
+                    $lib.print($printer.row(($year, $author, $title))
+                }
+
+            Create a configuration with custom separators and column options::
+
+                $conf = ({
+                    "separators": {
+                        "row:outline": true,
+                        "column:outline": true,
+                        "header:row": "#",
+                        "data:row": "*",
+                        "column": "+",
+                    },
+                    "columns": [
+                        {"name": "Year", "width": 4, "justify": "right"},
+                        {"name": "Author", "width": 20, "justify": "center"},
+                        {"name": "Title", "width": 12, "overflow": "wrap"},
+                    ]
+                })
+
+                $printer = $lib.tabular.printer($conf)
+        ''',
+         'type': {'type': 'function', '_funcname': '_methPrinter',
+                  'args': (
+                      {'name': 'conf', 'type': 'dict',
+                       'desc': 'The table configuration dictionary.'},
+                  ),
+                  'returns': {'type': 'tabular:printer',
+                              'desc': 'The newly constructed tabular:printer.'}}},
+        {'name': 'schema', 'desc': '''
+        Get a copy of the table configuration schema.
+
+        Examples:
+            Print a human-readable version of the schema::
+
+                $schema = $lib.tabular.schema()
+                $lib.print($lib.yaml.save($schema))
+        ''',
+         'type': {'type': 'function', '_funcname': '_methSchema',
+                  'returns': {'type': 'dict',
+                              'desc': 'The table configuration schema.'}}},
+    )
+    _storm_lib_path = ('tabular',)
+
+    def getObjLocals(self):
+        return {
+            'printer': self._methPrinter,
+            'schema': self._methSchema,
+        }
+
+    async def _methSchema(self):
+        return copy.deepcopy(s_schemas.tabularConfSchema)
+
+    async def _methPrinter(self, conf):
+        conf = await s_stormtypes.toprim(conf)
+        conf.setdefault('separators', {})
+        conf = s_schemas.reqValidTabularConf(conf)
+        return TabularPrinter(self.runt, conf)
+
+@s_stormtypes.registry.registerType
+class TabularPrinter(s_stormtypes.StormType):
+    '''
+    A Storm object for printing tabular data using a defined configuration.
+    '''
+    _storm_typename = 'tabular:printer'
+    _storm_locals = (
+        {'name': 'row', 'desc': 'Create a new row string from a data list.',
+         'type': {'type': 'function', '_funcname': 'row',
+                  'args': (
+                      {'name': 'data', 'type': 'list',
+                       'desc': 'The data to create the row from; length must match the number of configured columns.'},
+                  ),
+                  'returns': {'type': 'str', 'desc': 'The row string.'}}},
+        {'name': 'header',
+         'desc': 'Create a header row string.',
+         'type': {'type': 'function', '_funcname': 'header',
+                  'returns': {'type': 'str', 'desc': 'The header row string.'}}},
+    )
+
+    def __init__(self, runt, conf):
+        s_stormtypes.StormType.__init__(self, None)
+        self.runt = runt
+        self.conf = conf
+
+        self.firstrow = True
+
+        self.seprconf = conf['separators']
+
+        self.colconf = conf['columns']
+        self.colcnt = len(self.colconf)
+        self.colwidths = [coldef.get('width') or len(coldef['name']) for coldef in self.colconf]
+
+        self.locls.update({
+            'row': self.row,
+            'header': self.header,
+        })
+
+    def _formatRowLine(self, lineitems, pad=' '):
+        sepr = self.seprconf['column']
+        endstr = sepr if self.seprconf['column:outline'] else ''
+        return f'{endstr}{pad}{f"{pad}{sepr}{pad}".join(lineitems)}{pad}{endstr}'
+
+    def _makeSeparatorRowStr(self, rowsepr):
+        if rowsepr:
+            lineitems = (width * rowsepr for width in self.colwidths)
+            return self._formatRowLine(lineitems, pad=rowsepr)
+        return None
+
+    def _makeDataRowStrs(self, values):
+
+        items = []
+        for coldef, valu in zip(self.colconf, values):
+            width = coldef.get('width')
+            valu = str(valu) if valu is not None else ''
+
+            if width is None:
+                if coldef['newlines'] == 'split':
+                    items.append(valu.split('\n'))
+                else:
+                    items.append([valu.replace('\n', ' ')])
+                continue
+
+            valu = valu.replace('\n', ' ')
+
+            if len(valu) <= width:
+                items.append([justers[coldef['justify']](valu, width)])
+                continue
+
+            if coldef['overflow'] == 'trim':
+                items.append([s_common.trimText(valu, n=width)])
+                continue
+
+            items.append([justers[coldef['justify']](item, width) for item in textwrap.wrap(valu, width)])
+
+        rowstrs = []
+        for lineitems in itertools.zip_longest(*items, fillvalue=None):
+            lineitems = ([item or self.colwidths[i] * ' ' for i, item in enumerate(lineitems)])
+            rowstrs.append(self._formatRowLine(lineitems))
+
+        return rowstrs
+
+    async def header(self):
+        rowstrs = self._makeDataRowStrs([col['name'] for col in self.colconf])
+
+        if seprstr := self._makeSeparatorRowStr(self.seprconf['header:row']):
+            rowstrs.append(seprstr)
+            if self.seprconf['row:outline']:
+                rowstrs.insert(0, seprstr)
+
+        return '\n'.join(rowstrs)
+
+    async def row(self, data):
+        data = await s_stormtypes.toprim(data)
+
+        if not isinstance(data, (list, tuple)):
+            raise s_exc.BadArg(mesg='tabular:printer row() requires a list argument')
+
+        if len(data) != self.colcnt:
+            mesg = 'tabular:printer row() requires data length to equal column count'
+            raise s_exc.BadArg(mesg=mesg, length=len(data), column_length=self.colcnt)
+
+        rowstrs = self._makeDataRowStrs(data)
+
+        if seprstr := self._makeSeparatorRowStr(self.seprconf['data:row']):
+            if self.seprconf['row:outline']:
+                rowstrs.append(seprstr)
+            elif self.firstrow:
+                self.firstrow = False
+            else:
+                rowstrs.insert(0, seprstr)
+
+        return '\n'.join(rowstrs)

synapse/lib/stormtypes.py

@@ -4138,7 +4138,7 @@ class LibBase64(Lib):
             if urlsafe:
                 return base64.urlsafe_b64decode(valu)
             return base64.b64decode(valu)
-        except binascii.Error as e:
+        except (binascii.Error, TypeError) as e:
             mesg = f'Error during base64 decoding - {str(e)}: {s_common.trimText(repr(valu))}'
             raise s_exc.StormRuntimeError(mesg=mesg, urlsafe=urlsafe) from None
 
@@ -7532,6 +7532,9 @@ class View(Prim):
         {'name': 'wipeLayer', 'desc': 'Delete all nodes and nodedata from the write layer. Triggers will be run.',
          'type': {'type': 'function', '_funcname': '_methWipeLayer',
                   'returns': {'type': 'null', }}},
+        {'name': 'swapLayer', 'desc': 'Swaps the top layer for a fresh one and deletes the old layer.',
+         'type': {'type': 'function', '_funcname': '_methSwapLayer',
+                  'returns': {'type': 'null', }}},
         {'name': 'addNode', 'desc': '''Transactionally add a single node and all it's properties. If any validation fails, no changes are made.''',
          'type': {'type': 'function', '_funcname': 'addNode',
                   'args': (
@@ -7717,6 +7720,7 @@ class View(Prim):
             'addNode': self.addNode,
             'getEdges': self._methGetEdges,
             'wipeLayer': self._methWipeLayer,
+            'swapLayer': self._methSwapLayer,
             'addNodeEdits': self._methAddNodeEdits,
             'getEdgeVerbs': self._methGetEdgeVerbs,
             'getFormCounts': self._methGetFormcount,
@@ -8017,6 +8021,15 @@ class View(Prim):
         view = self.runt.snap.core.getView(viewiden)
         await view.wipeLayer(useriden=useriden)
 
+    async def _methSwapLayer(self):
+
+        view = self._reqView()
+
+        self.runt.reqAdmin(gateiden=view.iden)
+        self.runt.confirm(('layer', 'del'), gateiden=view.layers[0].iden)
+
+        await view.swapLayer()
+
     async def getMerges(self):
         view = self._reqView()
         async for merge in view.getMerges():

synapse/lib/trigger.py

@@ -494,7 +494,7 @@ class Trigger:
             await self.view.core.getStormQuery(valu)
 
         self.tdef[name] = valu
-        await self.view.trigdict.set(self.iden, self.tdef)
+        self.view.trigdict.set(self.iden, self.tdef)
 
     def get(self, name):
         return self.tdef.get(name)

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 175, 0)
+version = (2, 177, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'c331c50ef1e7b2234be997c4b791fb8b8bed7cf2'
+commit = 'b7da795aeb690020e38e99dc5aa1f505cdc2d659'