synapse 2.152.0__py311-none-any.whl → 2.154.0__py311-none-any.whl

synapse/lib/stormlib/stats.py

@@ -0,0 +1,223 @@
+import collections
+
+import synapse.exc as s_exc
+import synapse.common as s_common
+
+import synapse.lib.storm as s_storm
+import synapse.lib.stormtypes as s_stormtypes
+
+class StatsCountByCmd(s_storm.Cmd):
+    '''
+    Tally occurrences of values and display a bar chart of the results.
+
+    Examples:
+
+        // Show counts of geo:name values referenced by media:news nodes.
+        media:news -(refs)> geo:name | stats.countby
+
+        // Show counts of ASN values in a set of IPs.
+        inet:ipv4#myips | stats.countby :asn
+
+        // Show counts of attacker names for risk:compromise nodes.
+        risk:compromise | stats.countby :attacker::name
+    '''
+
+    name = 'stats.countby'
+    readonly = True
+
+    def getArgParser(self):
+        pars = s_storm.Cmd.getArgParser(self)
+        pars.add_argument('valu', nargs='?', default=s_common.novalu,
+                          help='A relative property or variable to tally.')
+        pars.add_argument('--reverse', default=False, action='store_true',
+                          help='Display results in ascending instead of descending order.')
+        pars.add_argument('--size', type='int', default=None,
+                          help='Maximum number of bars to display.')
+        pars.add_argument('--char', type='str', default='#',
+                          help='Character to use for bars.')
+        pars.add_argument('--bar-width', type='int', default=50,
+                          help='Width of the bars to display.')
+        pars.add_argument('--label-max-width', type='int', default=None,
+                          help='Maximum width of the labels to display.')
+        pars.add_argument('--yield', default=False, action='store_true',
+                          dest='yieldnodes', help='Yield inbound nodes.')
+        return pars
+
+    async def execStormCmd(self, runt, genr):
+
+        labelwidth = await s_stormtypes.toint(self.opts.label_max_width, noneok=True)
+        if labelwidth is not None and labelwidth < 0:
+            mesg = f'Value for --label-max-width must be >= 0, got: {labelwidth}'
+            raise s_exc.BadArg(mesg=mesg)
+
+        barwidth = await s_stormtypes.toint(self.opts.bar_width)
+        if barwidth < 0:
+            mesg = f'Value for --bar-width must be >= 0, got: {barwidth}'
+            raise s_exc.BadArg(mesg=mesg)
+
+        counts = collections.defaultdict(int)
+
+        usenode = self.opts.valu is s_common.novalu
+
+        async for node, path in genr:
+            if self.opts.yieldnodes:
+                yield node, path
+
+            if usenode:
+                valu = node.repr()
+            else:
+                valu = self.opts.valu
+                if s_stormtypes.ismutable(valu):
+                    raise s_exc.BadArg(mesg='Mutable values cannot be used for counting.')
+
+                valu = await s_stormtypes.tostr(await s_stormtypes.toprim(valu))
+
+            counts[valu] += 1
+
+        if len(counts) == 0:
+            await runt.printf('No values to display!')
+            return
+
+        values = list(sorted(counts.items(), key=lambda x: x[1]))
+
+        maxv = values[-1][1]
+        size = await s_stormtypes.toint(self.opts.size, noneok=True)
+        char = (await s_stormtypes.tostr(self.opts.char))[0]
+        reverse = self.opts.reverse
+
+        if reverse:
+            order = 1
+            if size:
+                values = values[:size]
+        else:
+            order = -1
+            if size:
+                values = values[len(values) - size:]
+
+        namewidth = 0
+        countwidth = 0
+        for (name, count) in values:
+            if (namelen := len(str(name))) > namewidth:
+                namewidth = namelen
+
+            if (countlen := len(str(count))) > countwidth:
+                countwidth = countlen
+
+        if labelwidth is not None:
+            namewidth = min(labelwidth, namewidth)
+
+        for (name, count) in values[::order]:
+
+            barsize = int((count / maxv) * barwidth)
+            bar = ''.ljust(barsize, char)
+            line = f'{name[0:namewidth].rjust(namewidth)} | {count:>{countwidth}} | {bar}'
+
+            await runt.printf(line)
+
+@s_stormtypes.registry.registerLib
+class LibStats(s_stormtypes.Lib):
+    '''
+    A Storm Library for statistics related functionality.
+    '''
+    _storm_locals = (
+        {'name': 'tally', 'desc': 'Get a Tally object.',
+         'type': {'type': 'function', '_funcname': 'tally',
+                  'returns': {'type': 'stat:tally', 'desc': 'A new tally object.', }}},
+    )
+    _storm_lib_path = ('stats',)
+
+    def getObjLocals(self):
+        return {
+            'tally': self.tally,
+        }
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def tally(self):
+        return StatTally(path=self.path)
+
+@s_stormtypes.registry.registerType
+class StatTally(s_stormtypes.Prim):
+    '''
+    A tally object.
+
+    An example of using it::
+
+        $tally = $lib.stats.tally()
+
+        $tally.inc(foo)
+
+        for $name, $total in $tally {
+            $doStuff($name, $total)
+        }
+
+    '''
+    _storm_typename = 'stat:tally'
+    _storm_locals = (
+        {'name': 'inc', 'desc': 'Increment a given counter.',
+         'type': {'type': 'function', '_funcname': 'inc',
+                  'args': (
+                      {'name': 'name', 'desc': 'The name of the counter to increment.', 'type': 'str', },
+                      {'name': 'valu', 'desc': 'The value to increment the counter by.', 'type': 'int', 'default': 1, },
+                  ),
+                  'returns': {'type': 'null', }}},
+        {'name': 'get', 'desc': 'Get the value of a given counter.',
+         'type': {'type': 'function', '_funcname': 'get',
+                  'args': (
+                      {'name': 'name', 'type': 'str', 'desc': 'The name of the counter to get.', },
+                  ),
+                  'returns': {'type': 'int',
+                              'desc': 'The value of the counter, or 0 if the counter does not exist.', }}},
+        {'name': 'sorted', 'desc': 'Get a list of (counter, value) tuples in sorted order.',
+         'type': {'type': 'function', '_funcname': 'sorted',
+                  'args': (
+                      {'name': 'byname', 'desc': 'Sort by counter name instead of value.',
+                       'type': 'bool', 'default': False},
+                      {'name': 'reverse', 'desc': 'Sort in descending order instead of ascending order.',
+                       'type': 'bool', 'default': False},
+                  ),
+                  'returns': {'type': 'list',
+                              'desc': 'List of (counter, value) tuples in sorted order.'}}},
+    )
+    _ismutable = True
+
+    def __init__(self, path=None):
+        s_stormtypes.Prim.__init__(self, {}, path=path)
+        self.counters = collections.defaultdict(int)
+        self.locls.update(self.getObjLocals())
+
+    def getObjLocals(self):
+        return {
+            'inc': self.inc,
+            'get': self.get,
+            'sorted': self.sorted,
+        }
+
+    async def __aiter__(self):
+        for name, valu in self.counters.items():
+            yield name, valu
+
+    def __len__(self):
+        return len(self.counters)
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def inc(self, name, valu=1):
+        valu = await s_stormtypes.toint(valu)
+        self.counters[name] += valu
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def get(self, name):
+        return self.counters.get(name, 0)
+
+    def value(self):
+        return dict(self.counters)
+
+    async def iter(self):
+        for item in tuple(self.counters.items()):
+            yield item
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def sorted(self, byname=False, reverse=False):
+        if byname:
+            return list(sorted(self.counters.items(), reverse=reverse))
+        else:
+            return list(sorted(self.counters.items(), key=lambda x: x[1], reverse=reverse))

synapse/lib/stormlib/stix.py

@@ -653,10 +653,12 @@ class LibStix(s_stormtypes.Lib):
             'validate': self.validateBundle,
         }
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def validateBundle(self, bundle):
         bundle = await s_stormtypes.toprim(bundle)
         return await s_coro.semafork(validateStix, bundle)
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def liftBundle(self, bundle):
         bundle = await s_stormtypes.toprim(bundle)
 
@@ -861,6 +863,7 @@ class LibStixImport(s_stormtypes.Lib):
             'ingest': self.ingest,
         }
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def config(self):
         return s_msgpack.deepcopy(stixingest, use_list=True)
 
@@ -1129,10 +1132,12 @@ class LibStixExport(s_stormtypes.Lib):
             'timestamp': self.timestamp,
         }
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def config(self):
         # make a new mutable config
         return json.loads(json.dumps(_DefaultConfig))
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def bundle(self, config=None):
 
         if config is None:
@@ -1265,6 +1270,7 @@ class StixBundle(s_stormtypes.Prim):
     # async def addPropMap(self, formname, stixtype, propname, stormtext):
     # async def addRelsMap(self, formname, stixtype, relname, targtype,  stormtext):
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def add(self, node, stixtype=None):
 
         if len(self.objs) >= self.maxsize:
@@ -1380,6 +1386,7 @@ class StixBundle(s_stormtypes.Prim):
         }
         return ret
 
+    @s_stormtypes.stormfunc(readonly=True)
     def pack(self):
         objects = list(self.objs.values())
         if self.synextension:
@@ -1391,6 +1398,7 @@ class StixBundle(s_stormtypes.Prim):
         }
         return bundle
 
+    @s_stormtypes.stormfunc(readonly=True)
     def size(self):
         return len(self.objs)
 

synapse/lib/stormlib/storm.py

@@ -23,6 +23,7 @@ class LibStorm(s_stormtypes.Lib):
             'eval': self._evalStorm,
         }
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def _evalStorm(self, text, cast=None):
 
         text = await s_stormtypes.tostr(text)

synapse/lib/stormlib/version.py

@@ -45,12 +45,15 @@ class VersionLib(s_stormtypes.Lib):
             'synapse': self._getSynVersion,
         }
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def _getSynVersion(self):
         return s_version.version
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def _getSynCommit(self):
         return s_version.commit
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def matches(self, vertup, reqstr):
         reqstr = await s_stormtypes.tostr(reqstr)
         vertup = tuple(await s_stormtypes.toprim(vertup))

synapse/lib/stormlib/xml.py

@@ -49,6 +49,7 @@ class XmlElement(s_stormtypes.Prim):
         for elem in self.elem:
             yield XmlElement(self.runt, elem)
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def find(self, name, nested=True):
         name = await s_stormtypes.tostr(name)
         nested = await s_stormtypes.tobool(nested)
@@ -60,6 +61,7 @@ class XmlElement(s_stormtypes.Prim):
             for elem in self.elem.findall(name):
                 yield XmlElement(self.runt, elem)
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def get(self, name):
         name = await s_stormtypes.tostr(name)
         elem = self.elem.find(name)
@@ -87,6 +89,7 @@ class LibXml(s_stormtypes.Lib):
             'parse': self.parse,
         }
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def parse(self, valu):
         valu = await s_stormtypes.tostr(valu)
         try:

synapse/lib/stormlib/yaml.py

@@ -34,11 +34,13 @@ class LibYaml(s_stormtypes.Lib):
             'load': self.load,
         }
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def save(self, valu, sort_keys=True):
         valu = await s_stormtypes.toprim(valu)
         sort_keys = await s_stormtypes.tobool(sort_keys)
         return yaml.dump(valu, sort_keys=sort_keys, Dumper=s_common.Dumper)
 
+    @s_stormtypes.stormfunc(readonly=True)
     async def load(self, valu):
         valu = await s_stormtypes.tostr(valu)
         try: