synapse 2.180.1__py311-none-any.whl → 2.182.0__py311-none-any.whl

synapse/lib/stormtypes.py

@@ -1337,7 +1337,7 @@ class LibBase(Lib):
                 Check if the runtime is in debug and print a message::
 
                     if $lib.debug {
-                        $lib.print('Doing stuff!")
+                        $lib.print('Doing stuff!')
                     }
 
                 Update the current runtime to enable debugging::
@@ -1996,7 +1996,7 @@ class LibAxon(Lib):
                        'desc': 'Set to False to disable SSL/TLS certificate verification.', 'default': True},
                       {'name': 'timeout', 'type': 'int', 'desc': 'Timeout for the download operation.',
                        'default': None},
-                      {'name': 'proxy', 'type': ['bool', 'null', 'str'],
+                      {'name': 'proxy', 'type': ['boolean', 'null', 'str'],
                        'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
                       {'name': 'ssl_opts', 'type': 'dict',
                        'desc': 'Optional SSL/TLS options. See $lib.axon help for additional details.',
@@ -2019,7 +2019,7 @@ class LibAxon(Lib):
                        'desc': 'Set to False to disable SSL/TLS certificate verification.', 'default': True},
                       {'name': 'timeout', 'type': 'int', 'desc': 'Timeout for the download operation.',
                        'default': None},
-                      {'name': 'proxy', 'type': ['bool', 'null', 'str'],
+                      {'name': 'proxy', 'type': ['boolean', 'null', 'str'],
                        'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None},
                       {'name': 'ssl_opts', 'type': 'dict',
                        'desc': 'Optional SSL/TLS options. See $lib.axon help for additional details.',
@@ -4994,7 +4994,7 @@ class List(Prim):
         {'name': 'sort', 'desc': 'Sort the list in place.',
          'type': {'type': 'function', '_funcname': '_methListSort',
                   'args': (
-                      {'name': 'reverse', 'type': 'bool', 'desc': 'Sort the list in reverse order.',
+                      {'name': 'reverse', 'type': 'boolean', 'desc': 'Sort the list in reverse order.',
                        'default': False},
                   ),
                   'returns': {'type': 'null', }}},
@@ -5061,6 +5061,14 @@ class List(Prim):
         {'name': 'unique', 'desc': 'Get a copy of the list containing unique items.',
          'type': {'type': 'function', '_funcname': '_methListUnique',
                   'returns': {'type': 'list'}}},
+        {'name': 'rem', 'desc': 'Remove a specific item from anywhere in the list.',
+         'type': {'type': 'function', '_funcname': '_methListRemove',
+                  'args': (
+                      {'name': 'item', 'type': 'any', 'desc': 'An item in the list.'},
+                      {'name': 'all', 'type': 'boolean', 'default': False,
+                       'desc': 'Remove all instances of item from the list.'},
+                  ),
+                  'returns': {'type': 'boolean', 'desc': 'Boolean indicating if the item was removed from the list.'}}},
     )
     _storm_typename = 'list'
     _ismutable = True
@@ -5082,6 +5090,7 @@ class List(Prim):
             'slice': self._methListSlice,
             'extend': self._methListExtend,
             'unique': self._methListUnique,
+            'rem': self._methListRemove,
         }
 
     @stormfunc(readonly=True)
@@ -5176,7 +5185,6 @@ class List(Prim):
         end = await toint(end)
         return self.valu[start:end]
 
-    @stormfunc(readonly=True)
     async def _methListExtend(self, valu):
         async for item in toiter(valu):
             self.valu.append(item)
@@ -5204,6 +5212,21 @@ class List(Prim):
             ret.append(val)
         return ret
 
+    async def _methListRemove(self, item, all=False):
+        item = await toprim(item)
+        all = await tobool(all)
+
+        if item not in self.valu:
+            return False
+
+        while item in self.valu:
+            self.valu.remove(item)
+
+            if not all:
+                break
+
+        return True
+
     async def stormrepr(self):
         reprs = [await torepr(k) for k in self.valu]
         rval = ', '.join(reprs)
@@ -6061,7 +6084,7 @@ class Node(Prim):
                       {'name': 'glob', 'type': 'str', 'default': None,
                        'desc': 'A tag glob expression. If this is provided, only tags which match the expression '
                                'are returned.'},
-                      {'name': 'leaf', 'type': 'bool', 'default': False,
+                      {'name': 'leaf', 'type': 'boolean', 'default': False,
                        'desc': 'If true, only leaf tags are included in the returned tags.'},
                   ),
                   'returns': {'type': 'list',
@@ -6721,6 +6744,13 @@ class Layer(Prim):
          'type': {'type': 'function', '_funcname': '_methGetFormcount',
                   'returns': {'type': 'dict',
                               'desc': 'Dictionary containing form names and the count of the nodes in the Layer.', }}},
+        {'name': 'getPropValues',
+         'desc': 'Yield unique property values in the layer for the given form or property name.',
+         'type': {'type': 'function', '_funcname': '_methGetPropValues',
+                  'args': (
+                      {'name': 'propname', 'type': 'str', 'desc': 'The property or form name to look up.', },
+                  ),
+                  'returns': {'name': 'yields', 'type': 'any', 'desc': 'Unique property values.', }}},
         {'name': 'getStorNodes', 'desc': '''
             Get buid, sode tuples representing the data stored in the layer.
 
@@ -6921,6 +6951,7 @@ class Layer(Prim):
             'liftByProp': self.liftByProp,
             'getTagCount': self._methGetTagCount,
             'getPropCount': self._methGetPropCount,
+            'getPropValues': self._methGetPropValues,
             'getTagPropCount': self._methGetTagPropCount,
             'getPropArrayCount': self._methGetPropArrayCount,
             'getFormCounts': self._methGetFormcount,
@@ -7190,6 +7221,29 @@ class Layer(Prim):
 
         return layr.getTagPropValuCount(form, tag, prop.name, prop.type.stortype, norm)
 
+    @stormfunc(readonly=True)
+    async def _methGetPropValues(self, propname):
+        propname = await tostr(propname)
+
+        prop = self.runt.snap.core.model.reqProp(propname)
+
+        layriden = self.valu.get('iden')
+        await self.runt.reqUserCanReadLayer(layriden)
+        layr = self.runt.snap.core.getLayer(layriden)
+
+        formname = None
+        propname = None
+
+        if prop.isform:
+            formname = prop.name
+        else:
+            propname = prop.name
+            if not prop.isuniv:
+                formname = prop.form.name
+
+        async for indx, valu in layr.iterPropValues(formname, propname, prop.type.stortype):
+            yield valu
+
     @stormfunc(readonly=True)
     async def _methLayerEdits(self, offs=0, wait=True, size=None, reverse=False):
         offs = await toint(offs)
@@ -7367,7 +7421,7 @@ class LibView(Lib):
         {'name': 'list', 'desc': 'List the Views in the Cortex.',
          'type': {'type': 'function', '_funcname': '_methViewList',
                   'args': (
-                      {'name': 'deporder', 'type': 'bool', 'default': False,
+                      {'name': 'deporder', 'type': 'boolean', 'default': False,
                         'desc': 'Return the lists in bottom-up dependency order.', },
                   ),
                   'returns': {'type': 'list', 'desc': 'List of ``view`` objects.', }}},
@@ -7622,6 +7676,14 @@ class View(Prim):
                   ),
                   'returns': {'type': 'int', 'desc': 'The count of nodes.', }}},
 
+        {'name': 'getPropValues',
+         'desc': 'Yield unique property values in the view for the given form or property name.',
+         'type': {'type': 'function', '_funcname': '_methGetPropValues',
+                  'args': (
+                      {'name': 'propname', 'type': 'str', 'desc': 'The property or form name to look up.', },
+                  ),
+                  'returns': {'name': 'yields', 'type': 'any', 'desc': 'Unique property values.', }}},
+
         {'name': 'detach', 'desc': 'Detach the view from its parent. WARNING: This cannot be reversed.',
          'type': {'type': 'function', '_funcname': 'detach',
                   'args': (),
@@ -7725,6 +7787,7 @@ class View(Prim):
             'getEdgeVerbs': self._methGetEdgeVerbs,
             'getFormCounts': self._methGetFormcount,
             'getPropCount': self._methGetPropCount,
+            'getPropValues': self._methGetPropValues,
             'getTagPropCount': self._methGetTagPropCount,
             'getPropArrayCount': self._methGetPropArrayCount,
 
@@ -7841,6 +7904,17 @@ class View(Prim):
 
         return await view.getPropArrayCount(propname, valu=valu)
 
+    @stormfunc(readonly=True)
+    async def _methGetPropValues(self, propname):
+        propname = await tostr(propname)
+
+        viewiden = self.valu.get('iden')
+        self.runt.confirm(('view', 'read'), gateiden=viewiden)
+        view = self.runt.snap.core.getView(viewiden)
+
+        async for valu in view.iterPropValues(propname):
+            yield valu
+
     @stormfunc(readonly=True)
     async def _methGetEdges(self, verb=None):
         verb = await toprim(verb)

synapse/lib/types.py

@@ -368,6 +368,13 @@ class Type:
         topt.update(opts)
         return self.__class__(self.modl, self.name, self.info, topt)
 
+    def __eq__(self, othr):
+        if self.name != othr.name:
+            return False
+        if self.opts != othr.opts:
+            return False
+        return True
+
 class Bool(Type):
 
     stortype = s_layer.STOR_TYPE_U8

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 180, 1)
+version = (2, 182, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = '1f1f8364fe330b8d92350436a6a10b99c5cc4b69'
+commit = '72e535c8c22073fd1b7428121849de4b9fb034ae'

synapse/lib/view.py

@@ -824,6 +824,53 @@ class View(s_nexus.Pusher):  # type: ignore
 
         return count
 
+    async def iterPropValues(self, propname):
+        prop = self.core.model.reqProp(propname)
+
+        formname = None
+        propname = None
+
+        if prop.isform:
+            formname = prop.name
+        else:
+            propname = prop.name
+            if not prop.isuniv:
+                formname = prop.form.name
+
+        async def wrapgenr(lidx, genr):
+            async for indx, valu in genr:
+                yield indx, valu, lidx
+
+        genrs = []
+        for lidx, layr in enumerate(self.layers):
+            genr = layr.iterPropValues(formname, propname, prop.type.stortype)
+            genrs.append(wrapgenr(lidx, genr))
+
+        lastvalu = None
+        async for indx, valu, lidx in s_common.merggenr2(genrs):
+            if valu == lastvalu:
+                continue
+
+            if lidx == 0 or propname is None:
+                lastvalu = valu
+                yield valu
+            else:
+                valid = False
+                async for buid in self.layers[lidx].iterPropIndxBuids(formname, propname, indx):
+                    for layr in self.layers[0:lidx]:
+                        if (sode := layr._getStorNode(buid)) is None:
+                            continue
+
+                        if sode['props'].get(propname) is not None:
+                            break
+                    else:
+                        valid = True
+
+                    if valid:
+                        lastvalu = valu
+                        yield valu
+                        break
+
     async def getEdgeVerbs(self):
 
         async with await s_spooled.Set.anit(dirn=self.core.dirn, cell=self.core) as vset:

synapse/models/inet.py

@@ -2,8 +2,6 @@ import socket
 import asyncio
 import hashlib
 import logging
-import ipaddress
-import email.utils
 import urllib.parse
 
 import idna
@@ -20,7 +18,12 @@ import synapse.lib.scrape as s_scrape
 import synapse.lib.module as s_module
 import synapse.lookup.iana as s_l_iana
 
+import synapse.vendor.cpython.lib.email.utils as s_v_email_utils
+
 logger = logging.getLogger(__name__)
+
+ipaddress = s_common.ipaddress
+
 drivre = regex.compile(r'^\w[:|]')
 fqdnre = regex.compile(r'^[\w._-]+$', regex.U)
 srv6re = regex.compile(r'^\[([a-f0-9\.:]+)\](?::(\d+))?$', regex.IGNORECASE)
@@ -811,13 +814,17 @@ class Rfc2822Addr(s_types.Str):
         valu = ' '.join(valu.split())
 
         try:
-            name, addr = email.utils.parseaddr(valu)
+            name, addr = s_v_email_utils.parseaddr(valu, strict=True)
         except Exception as e:  # pragma: no cover
             # not sure we can ever really trigger this with a string as input
             mesg = f'email.utils.parsaddr failed: {str(e)}'
             raise s_exc.BadTypeValu(valu=valu, name=self.name,
                                     mesg=mesg) from None
 
+        if not name and not addr:
+            raise s_exc.BadTypeValu(valu=valu, name=self.name,
+                                    mesg=f'No name or email parsed from {valu}')
+
         subs = {}
         if name:
             subs['name'] = name

synapse/models/infotech.py

@@ -199,7 +199,6 @@ URI_PERCENT_CHARS = [
 ]
 
 def uri_quote(text):
-    ret = ''
     for (pct, char) in URI_PERCENT_CHARS:
         text = text.replace(char, pct)
     return text
@@ -1070,6 +1069,8 @@ class ItModule(s_module.CoreModule):
                     'doc': 'The snort rule is intended for use in detecting the target node.'}),
                 (('it:app:yara:rule', 'detects', None), {
                     'doc': 'The YARA rule is intended for use in detecting the target node.'}),
+                (('it:dev:repo', 'has', 'inet:url'), {
+                    'doc': 'The repo has content hosted at the URL.'}),
             ),
             'forms': (
                 ('it:hostname', {}, ()),

synapse/models/language.py

@@ -12,6 +12,9 @@ class LangModule(s_module.CoreModule):
                     'deprecated': True,
                     'doc': 'Deprecated. Please use lang:translation.'}),
 
+                ('lang:phrase', ('str', {'lower': True, 'onespace': True}), {
+                    'doc': 'A small group of words which stand together as a concept.'}),
+
                 ('lang:trans', ('str', {}), {
                     'deprecated': True,
                     'doc': 'Deprecated. Please use lang:translation.'}),
@@ -31,6 +34,7 @@ class LangModule(s_module.CoreModule):
             ),
             'forms': (
 
+                ('lang:phrase', {}, ()),
                 ('lang:idiom', {}, (
 
                     ('url', ('inet:url', {}), {

synapse/models/math.py

@@ -0,0 +1,50 @@
+import synapse.lib.module as s_module
+
+class MathModule(s_module.CoreModule):
+
+    def getModelDefs(self):
+        return (('math', {
+            'types': (
+
+                ('math:algorithm', ('guid', {}), {
+                    'doc': 'A mathematical algorithm.'}),
+
+                ('math:algorithm:type:taxonomy', ('taxonomy', {}), {
+                    'interfaces': ('meta:taxonomy',),
+                    'doc': 'A hierarchical taxonomy of algorithm types.'}),
+            ),
+            'edges': (
+
+                (('risk:tool:software', 'uses', 'math:algorithm'), {
+                    'doc': 'The tool uses the algorithm.'}),
+
+                (('it:prod:softver', 'uses', 'math:algorithm'), {
+                    'doc': 'The software uses the algorithm.'}),
+
+                (('file:bytes', 'uses', 'math:algorithm'), {
+                    'doc': 'The file uses the algorithm.'}),
+
+                (('math:algorithm', 'generates', None), {
+                    'doc': 'The target node was generated by the algorithm.'}),
+            ),
+            'forms': (
+
+                ('math:algorithm:type:taxonomy', {}, ()),
+
+                ('math:algorithm', {}, (
+
+                    ('name', ('str', {'lower': True, 'onespace': True}), {
+                        'doc': 'The name of the algorithm.'}),
+
+                    ('type', ('math:algorithm:type:taxonomy', {}), {
+                        'doc': 'The type of algorithm.'}),
+
+                    ('desc', ('str', {}), {
+                        'disp': {'hint': 'text'},
+                        'doc': 'A description of the algorithm.'}),
+
+                    ('created', ('time', {}), {
+                        'doc': 'The time that the algorithm was authored.'}),
+                )),
+            ),
+        }),)

synapse/models/orgs.py

@@ -480,6 +480,9 @@ class OuModule(s_module.CoreModule):
                     ('goal', ('ou:goal', {}), {
                         'doc': 'The assessed primary goal of the campaign.'}),
 
+                    ('slogan', ('lang:phrase', {}), {
+                        'doc': 'The slogan used by the campaign.'}),
+
                     ('actors', ('array', {'type': 'ps:contact', 'split': ',', 'uniq': True, 'sorted': True}), {
                         'doc': 'Actors who participated in the campaign.'}),
 
@@ -633,12 +636,16 @@ class OuModule(s_module.CoreModule):
                         'doc': 'Location for an organization.'
                     }),
                     ('name', ('ou:name', {}), {
+                        'alts': ('names',),
                         'doc': 'The localized name of an organization.',
                     }),
                     ('type', ('str', {'lower': True, 'strip': True}), {
                         'deprecated': True,
                         'doc': 'The type of organization.',
                     }),
+                    ('motto', ('lang:phrase', {}), {
+                        'doc': 'The motto used by the organization.'}),
+
                     ('orgtype', ('ou:orgtype', {}), {
                         'doc': 'The type of organization.',
                         'disp': {'hint': 'taxonomy'},
@@ -775,6 +782,7 @@ class OuModule(s_module.CoreModule):
                         'deprecated': True,
                         'doc': 'A list of types that apply to the contract.'}),
                 )),
+                ('ou:industry:type:taxonomy', {}, ()),
                 ('ou:industry', {}, (
 
                     ('name', ('ou:industryname', {}), {

synapse/models/risk.py

@@ -648,12 +648,21 @@ class RiskModule(s_module.CoreModule):
                 )),
 
                 ('risk:vulnerable', {}, (
+
                     ('vuln', ('risk:vuln', {}), {
                         'doc': 'The vulnerability that the node is susceptible to.'}),
+
                     ('period', ('ival', {}), {
                         'doc': 'The time window where the node was vulnerable.'}),
+
                     ('node', ('ndef', {}), {
                         'doc': 'The node which is vulnerable.'}),
+
+                    ('mitigated', ('bool', {}), {
+                        'doc': 'Set to true if the vulnerable node has been mitigated.'}),
+
+                    ('mitigations', ('array', {'type': 'risk:mitigation', 'sorted': True, 'uniq': True}), {
+                        'doc': 'The mitigations which were used to address the vulnerable node.'}),
                 )),
 
                 ('risk:alert:taxonomy', {}, {}),

synapse/tests/files/stormcov/pragma-nocov.storm

@@ -0,0 +1,18 @@
+function foo() {
+    if $lib.true {
+        stop
+    }
+
+    if $lib.false { $lib.print("no cover") } // pragma: no cover
+
+    if $lib.false { // pragma: no cover start
+        $lib.print("multi-line no cover")
+    } // pragma: no cover stop
+
+    if $lib.false { $lib.print("stop before start") } // pragma: no cover stop
+
+    if $lib.false { // pragma: no cover start
+        $lib.print("start in a block") // pragma: no cover start
+    } // pragma: no cover stop
+}
+yield $foo()

synapse/tests/test_assets.py

@@ -0,0 +1,25 @@
+import os
+import synapse.assets as s_assets
+
+import synapse.tests.utils as s_t_utils
+
+class TestAssets(s_t_utils.SynTest):
+
+    def test_assets_path(self):
+
+        fp = s_assets.getAssetPath('storm', 'migrations', 'model-0.2.28.storm')
+        self.true(os.path.isfile(fp))
+
+        with self.raises(ValueError) as cm:
+            s_assets.getAssetPath('../../../../../../../etc/passwd')
+        self.isin('Path escaping', str(cm.exception))
+
+        with self.raises(ValueError) as cm:
+            s_assets.getAssetPath('newp', 'does', 'not', 'exit')
+        self.isin('Asset does not exist', str(cm.exception))
+
+    def test_assets_storm(self):
+
+        text = s_assets.getStorm('migrations', 'model-0.2.28.storm')
+        self.isinstance(text, str)
+        self.gt(len(text), 0)