synapse 2.185.0__py311-none-any.whl → 2.187.0__py311-none-any.whl

synapse/lib/stormtypes.py

@@ -3816,13 +3816,13 @@ class Queue(StormType):
                   'args': (
                       {'name': 'item', 'type': 'prim', 'desc': 'The item being put into the queue.', },
                   ),
-                  'returns': {'type': 'null', }}},
+                  'returns': {'type': 'int', 'desc': 'The queue offset of the item.'}}},
         {'name': 'puts', 'desc': 'Put multiple items into the Queue.',
          'type': {'type': 'function', '_funcname': '_methQueuePuts',
                   'args': (
                       {'name': 'items', 'type': 'list', 'desc': 'The items to put into the Queue.', },
                   ),
-                  'returns': {'type': 'null', }}},
+                  'returns': {'type': 'int', 'desc': 'The queue offset of the first item.'}}},
         {'name': 'gets', 'desc': 'Get multiple items from the Queue as a iterator.',
          'type': {'type': 'function', '_funcname': '_methQueueGets',
                   'args': (
@@ -6130,14 +6130,14 @@ class Node(Prim):
          'type': {'type': 'function', '_funcname': '_methNodeAddEdge',
                   'args': (
                       {'name': 'verb', 'type': 'str', 'desc': 'The edge verb to add.'},
-                      {'name': 'iden', 'type': 'str', 'desc': 'The node id of the destination node.'},
+                      {'name': 'iden', 'type': 'str', 'desc': 'The node iden of the destination node.'},
                   ),
                   'returns': {'type': 'null', }}},
         {'name': 'delEdge', 'desc': 'Remove a light-weight edge.',
          'type': {'type': 'function', '_funcname': '_methNodeDelEdge',
                   'args': (
                       {'name': 'verb', 'type': 'str', 'desc': 'The edge verb to remove.'},
-                      {'name': 'iden', 'type': 'str', 'desc': 'The node id of the destination node to remove.'},
+                      {'name': 'iden', 'type': 'str', 'desc': 'The node iden of the destination node to remove.'},
                   ),
                   'returns': {'type': 'null', }}},
         {'name': 'globtags', 'desc': 'Get a list of the tag components from a Node which match a tag glob expression.',
@@ -6827,11 +6827,11 @@ class Layer(Prim):
                   'returns': {'name': 'Yields', 'type': 'list',
                               'desc': 'Yields messages describing any index inconsistencies.', }}},
         {'name': 'getStorNode', 'desc': '''
-            Retrieve the raw storage node for the specified node id.
+            Retrieve the raw storage node for the specified node iden.
             ''',
          'type': {'type': 'function', '_funcname': 'getStorNode',
                   'args': (
-                      {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node id.'},
+                      {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node iden.'},
                   ),
                   'returns': {'type': 'dict', 'desc': 'The storage node dictionary.', }}},
         {'name': 'liftByProp', 'desc': '''
@@ -6897,7 +6897,7 @@ class Layer(Prim):
                               'desc': 'Yields (<n1iden>, <verb>, <n2iden>) tuples', }}},
 
         {'name': 'getEdgesByN1', 'desc': '''
-            Yield (verb, n2iden) tuples for any light edges in the layer for the source node id.
+            Yield (verb, n2iden) tuples for any light edges in the layer for the source node iden.
 
             Example:
                 Iterate the N1 edges for ``$node``::
@@ -6909,13 +6909,13 @@ class Layer(Prim):
             ''',
          'type': {'type': 'function', '_funcname': 'getEdgesByN1',
                   'args': (
-                      {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node id.'},
+                      {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node iden.'},
                   ),
                   'returns': {'name': 'Yields', 'type': 'list',
                               'desc': 'Yields (<verb>, <n2iden>) tuples', }}},
 
         {'name': 'getEdgesByN2', 'desc': '''
-            Yield (verb, n1iden) tuples for any light edges in the layer for the target node id.
+            Yield (verb, n1iden) tuples for any light edges in the layer for the target node iden.
 
             Example:
                 Iterate the N2 edges for ``$node``::
@@ -6926,10 +6926,26 @@ class Layer(Prim):
             ''',
          'type': {'type': 'function', '_funcname': 'getEdgesByN2',
                   'args': (
-                      {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node id.'},
+                      {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node iden.'},
                   ),
                   'returns': {'name': 'Yields', 'type': 'list',
                               'desc': 'Yields (<verb>, <n1iden>) tuples', }}},
+        {'name': 'getNodeData', 'desc': '''
+            Yield (name, valu) tuples for any node data in the layer for the target node iden.
+
+            Example:
+                Iterate the node data for ``$node``::
+
+                    for ($name, $valu) in $layer.getNodeData($node.iden()) {
+                        $lib.print(`{$name} = {$valu}`)
+                    }
+            ''',
+         'type': {'type': 'function', '_funcname': 'getNodeData',
+                  'args': (
+                      {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node iden.'},
+                  ),
+                  'returns': {'name': 'Yields', 'type': 'list',
+                              'desc': 'Yields (<name>, <valu>) tuples', }}},
     )
     _storm_typename = 'layer'
     _ismutable = False
@@ -6986,6 +7002,7 @@ class Layer(Prim):
             'getStorNodesByForm': self.getStorNodesByForm,
             'getEdgesByN1': self.getEdgesByN1,
             'getEdgesByN2': self.getEdgesByN2,
+            'getNodeData': self.getNodeData,
             'getMirrorStatus': self.getMirrorStatus,
         }
 
@@ -7355,6 +7372,15 @@ class Layer(Prim):
         async for item in layr.iterNodeEdgesN2(s_common.uhex(nodeid)):
             yield item
 
+    @stormfunc(readonly=True)
+    async def getNodeData(self, nodeid):
+        nodeid = await tostr(nodeid)
+        layriden = self.valu.get('iden')
+        await self.runt.reqUserCanReadLayer(layriden)
+        layr = self.runt.snap.core.getLayer(layriden)
+        async for item in layr.iterNodeData(s_common.uhex(nodeid)):
+            yield item
+
     @stormfunc(readonly=True)
     async def _methLayerGet(self, name, defv=None):
         return self.valu.get(name, defv)

synapse/lib/types.py

@@ -1903,6 +1903,12 @@ class Tag(Str):
             mesg = f'Tag does not match tagre: [{s_grammar.tagre.pattern}]'
             raise s_exc.BadTypeValu(valu=norm, name=self.name, mesg=mesg)
 
+        core = self.modl.core
+        if core is not None:
+            (ok, mesg) = core.isTagValid(norm)
+            if not ok:
+                raise s_exc.BadTypeValu(valu=norm, name=self.name, mesg=mesg)
+
         return norm, {'subs': subs, 'toks': toks}
 
     def _normPyStr(self, text):

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 185, 0)
+version = (2, 187, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'c2ef0eded02f8ac5f34704a117947f44095f41f4'
+commit = '5cdabb0eeea74eff019d8bcdc461b48bb5eb6651'

synapse/models/infotech.py

@@ -18,10 +18,10 @@ import synapse.lib.version as s_version
 
 logger = logging.getLogger(__name__)
 
-# This is the regular expression pattern for CPE2.2. It's kind of a hybrid
+# This is the regular expression pattern for CPE 2.2. It's kind of a hybrid
 # between compatible binding and preferred binding. Differences are here:
 # - Use only the list of percent encoded values specified by preferred binding.
-#   This is to ensure it converts properly to CPE2.3.
+#   This is to ensure it converts properly to CPE 2.3.
 # - Add tilde (~) to the UNRESERVED list which removes the need to specify the
 #   PACKED encoding specifically.
 ALPHA = '[A-Za-z]'
@@ -60,6 +60,14 @@ COMPONENT_LIST = f'''
 cpe22_regex = regex.compile(f'cpe:/{COMPONENT_LIST}', regex.VERBOSE | regex.IGNORECASE)
 cpe23_regex = regex.compile(s_scrape._cpe23_regex, regex.VERBOSE | regex.IGNORECASE)
 
+def isValidCpe22(text):
+    rgx = cpe22_regex.fullmatch(text)
+    return rgx is not None
+
+def isValidCpe23(text):
+    rgx = cpe23_regex.fullmatch(text)
+    return rgx is not None
+
 def cpesplit(text):
     part = ''
     parts = []
@@ -251,9 +259,20 @@ class Cpe22Str(s_types.Str):
     def _normPyStr(self, valu):
 
         text = valu.lower()
+
         if text.startswith('cpe:/'):
+
+            if not isValidCpe22(text):
+                mesg = 'CPE 2.2 string appears to be invalid.'
+                raise s_exc.BadTypeValu(mesg=mesg, valu=valu)
+
             parts = chopCpe22(text)
         elif text.startswith('cpe:2.3:'):
+
+            if not isValidCpe23(text):
+                mesg = 'CPE 2.3 string appears to be invalid.'
+                raise s_exc.BadTypeValu(mesg=mesg, valu=valu)
+
             parts = cpesplit(text[8:])
         else:
             mesg = 'CPE 2.2 string is expected to start with "cpe:/"'
@@ -261,8 +280,7 @@ class Cpe22Str(s_types.Str):
 
         v2_2 = zipCpe22(parts)
 
-        rgx = cpe22_regex.match(v2_2)
-        if rgx is None or rgx.group() != v2_2:
+        if not isValidCpe22(v2_2): # pragma: no cover
             mesg = 'CPE 2.2 string appears to be invalid.'
             raise s_exc.BadTypeValu(mesg=mesg, valu=valu)
 
@@ -330,6 +348,12 @@ class Cpe23Str(s_types.Str):
     def _normPyStr(self, valu):
         text = valu.lower()
         if text.startswith('cpe:2.3:'):
+
+            # Validate the CPE 2.3 string immediately
+            if not isValidCpe23(text):
+                mesg = 'CPE 2.3 string appears to be invalid.'
+                raise s_exc.BadTypeValu(mesg=mesg, valu=valu)
+
             parts = cpesplit(text[8:])
             if len(parts) > 11:
                 mesg = f'CPE 2.3 string has {len(parts)} fields, expected up to 11.'
@@ -346,6 +370,10 @@ class Cpe23Str(s_types.Str):
                     v2_2[idx] = ''
                     continue
 
+                if idx in (PART_IDX_PART, PART_IDX_LANG) and part == '-':
+                    v2_2[idx] = ''
+                    continue
+
                 part = fsb_unescape(part)
                 v2_2[idx] = uri_quote(part)
 
@@ -357,12 +385,22 @@ class Cpe23Str(s_types.Str):
                 v2_2[PART_IDX_OTHER]
             )
 
-            v2_2 = v2_2[:7]
+            v2_2 = zipCpe22(v2_2[:7])
+
+            # Now validate the downconvert
+            if not isValidCpe22(v2_2): # pragma: no cover
+                mesg = 'Invalid CPE 2.3 to CPE 2.2 conversion.'
+                raise s_exc.BadTypeValu(mesg=mesg, valu=valu, v2_2=v2_2)
 
             parts = [fsb_unescape(k) for k in parts]
 
         elif text.startswith('cpe:/'):
 
+            # Validate the CPE 2.2 string immediately
+            if not isValidCpe22(text):
+                mesg = 'CPE 2.2 string appears to be invalid.'
+                raise s_exc.BadTypeValu(mesg=mesg, valu=valu)
+
             v2_2 = text
             # automatically normalize CPE 2.2 format to CPE 2.3
             parts = chopCpe22(text)
@@ -408,24 +446,15 @@ class Cpe23Str(s_types.Str):
 
             v2_3 = 'cpe:2.3:' + ':'.join(escaped)
 
+            # Now validate the upconvert
+            if not isValidCpe23(v2_3): # pragma: no cover
+                mesg = 'Invalid CPE 2.2 to CPE 2.3 conversion.'
+                raise s_exc.BadTypeValu(mesg=mesg, valu=valu, v2_3=v2_3)
+
         else:
             mesg = 'CPE 2.3 string is expected to start with "cpe:2.3:"'
             raise s_exc.BadTypeValu(valu=valu, mesg=mesg)
 
-        rgx = cpe23_regex.match(v2_3)
-        if rgx is None or rgx.group() != v2_3:
-            mesg = 'CPE 2.3 string appears to be invalid.'
-            raise s_exc.BadTypeValu(mesg=mesg, valu=valu)
-
-        if isinstance(v2_2, list):
-            cpe22 = zipCpe22(v2_2)
-        else:
-            cpe22 = v2_2
-
-        rgx = cpe22_regex.match(cpe22)
-        if rgx is None or rgx.group() != cpe22:
-            v2_2 = None
-
         subs = {
             'part': parts[PART_IDX_PART],
             'vendor': parts[PART_IDX_VENDOR],
@@ -438,11 +467,9 @@ class Cpe23Str(s_types.Str):
             'target_sw': parts[PART_IDX_TARGET_SW],
             'target_hw': parts[PART_IDX_TARGET_HW],
             'other': parts[PART_IDX_OTHER],
+            'v2_2': v2_2,
         }
 
-        if v2_2 is not None:
-            subs['v2_2'] = v2_2
-
         return v2_3, {'subs': subs}
 
 class SemVer(s_types.Int):

synapse/models/risk.py

@@ -351,6 +351,9 @@ class RiskModule(s_module.CoreModule):
                     ('mitre:attack:software', ('it:mitre:attack:software', {}), {
                         'doc': 'A mapping to a MITRE ATT&CK software if applicable.'}),
 
+                    ('id', ('str', {'strip': True}), {
+                        'doc': 'An ID for the tool.'}),
+
                 )),
                 ('risk:mitigation:type:taxonomy', {}, ()),
                 ('risk:mitigation', {}, (

synapse/tests/test_cortex.py

@@ -7202,16 +7202,16 @@ class CortexBasicTest(s_t_utils.SynTest):
 
             nodes = await core.nodes('[ inet:ipv4=1.2.3.4 +#cno.cve.2021.12345 ]')
 
-            with self.raises(s_exc.BadTag):
+            with self.raises(s_exc.BadTypeValu):
                 await core.nodes('[ inet:ipv4=1.2.3.4 +#cno.cve.foo ]')
 
-            with self.raises(s_exc.BadTag):
+            with self.raises(s_exc.BadTypeValu):
                 await core.nodes('[ inet:ipv4=1.2.3.4 +#cno.cve.2021.hehe ]')
 
-            with self.raises(s_exc.BadTag):
+            with self.raises(s_exc.BadTypeValu):
                 await core.nodes('[ inet:ipv4=1.2.3.4 +#cno.cve.2021.123456 ]')
 
-            with self.raises(s_exc.BadTag):
+            with self.raises(s_exc.BadTypeValu):
                 await core.nodes('[ inet:ipv4=1.2.3.4 +#cno.cve.12345 ]')
 
             nodes = await core.nodes('[ test:str=beep +?#cno.cve.12345 ]')
@@ -7229,9 +7229,14 @@ class CortexBasicTest(s_t_utils.SynTest):
             await core.nodes('[ inet:ipv4=1.2.3.4 +#cno.cve.2021.hehe ]')
 
             await core.setTagModel('cno.cve', 'regex', (None, None, '[0-9]{4}', '[0-9]{5}'))
-            with self.raises(s_exc.BadTag):
+            with self.raises(s_exc.BadTypeValu):
                 await core.nodes('[ inet:ipv4=1.2.3.4 +#cno.cve.2021.haha ]')
 
+            self.eq((False, None), await core.callStorm('return($lib.trycast(syn:tag, cno.cve.2021.haha))'))
+
+            with self.raises(s_exc.BadTypeValu):
+                await core.callStorm('return($lib.cast(syn:tag, cno.cve.2021.haha))')
+
             self.none(await core.callStorm('$lib.model.tags.del(cno.cve)'))
             self.none(await core.callStorm('return($lib.model.tags.get(cno.cve))'))
 

synapse/tests/test_lib_base.py

@@ -378,7 +378,7 @@ class BaseTest(s_t_utils.SynTest):
         proc = ctx.Process(target=block_processing, args=(evt1,))
         proc.start()
 
-        self.true(evt1.wait(timeout=10))
+        self.true(evt1.wait(timeout=30))
         os.kill(proc.pid, signal.SIGTERM)
         proc.join(timeout=10)
         self.eq(proc.exitcode, 137)
@@ -396,7 +396,7 @@ class BaseTest(s_t_utils.SynTest):
         proc = ctx.Process(target=block_processing, args=(evt1,))
         proc.start()
 
-        self.true(evt1.wait(timeout=10))
+        self.true(evt1.wait(timeout=30))
         os.kill(proc.pid, signal.SIGINT)
 
         proc.join(timeout=10)

synapse/tests/test_lib_cell.py

@@ -208,7 +208,12 @@ class CellTest(s_t_utils.SynTest):
 
             # TODO how to handle iden match with additional property mismatch
 
-            await cell.drive.setTypeSchema('woot', testDataSchema_v0)
+            self.true(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=0))
+            self.true(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=1))
+            self.false(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=1))
+
+            with self.raises(s_exc.BadVersion):
+                await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=0)
 
             info = {'name': 'win32k.sys', 'type': 'woot'}
             info = await cell.addDriveItem(info, reldir=rootdir)
@@ -284,6 +289,12 @@ class CellTest(s_t_utils.SynTest):
             versinfo, data = await cell.getDriveData(iden, vers=(1, 1, 0))
             self.eq('woot', data.get('woot'))
 
+            with self.raises(s_exc.NoSuchIden):
+                await cell.reqDriveInfo('d7d6107b200e2c039540fc627bc5537d')
+
+            with self.raises(s_exc.TypeMismatch):
+                await cell.getDriveInfo(iden, typename='newp')
+
             self.nn(await cell.getDriveInfo(iden))
             self.len(2, [vers async for vers in cell.getDriveDataVersions(iden)])
 
@@ -1899,7 +1910,7 @@ class CellTest(s_t_utils.SynTest):
             proc = ctx.Process(target=lock_target, args=(dirn, evt1,))
             proc.start()
 
-            self.true(evt1.wait(timeout=10))
+            self.true(evt1.wait(timeout=30))
 
             with self.raises(s_exc.FatalErr) as cm:
                 async with await s_cell.Cell.anit(dirn) as cell:
@@ -2600,7 +2611,7 @@ class CellTest(s_t_utils.SynTest):
             proc = ctx.Process(target=reload_target, args=(dirn, evt1, evt2))
             proc.start()
 
-            self.true(evt1.wait(timeout=10))
+            self.true(evt1.wait(timeout=30))
 
             async with await s_telepath.openurl(f'cell://{dirn}') as prox:
                 cnfo = await prox.getCellInfo()
@@ -3008,6 +3019,7 @@ class CellTest(s_t_utils.SynTest):
                 self.eq(node.get('._woot'), 5)
                 self.nn(node.getTagProp('test', 'score'), 6)
 
+                self.maxDiff = None
                 roles = s_t_utils.deguidify('[{"type": "role", "iden": "e1ef725990aa62ae3c4b98be8736d89f", "name": "all", "rules": [], "authgates": {"46cfde2c1682566602860f8df7d0cc83": {"rules": [[true, ["layer", "read"]]]}, "4d50eb257549436414643a71e057091a": {"rules": [[true, ["view", "read"]]]}}}]')
                 users = s_t_utils.deguidify('[{"type": "user", "iden": "a357138db50780b62093a6ce0d057fd8", "name": "root", "rules": [], "roles": [], "admin": true, "email": null, "locked": false, "archived": false, "authgates": {"46cfde2c1682566602860f8df7d0cc83": {"admin": true}, "4d50eb257549436414643a71e057091a": {"admin": true}}}, {"type": "user", "iden": "f77ac6744671a845c27e571071877827", "name": "visi", "rules": [[true, ["cron", "add"]], [true, ["dmon", "add"]], [true, ["trigger", "add"]]], "roles": [{"type": "role", "iden": "e1ef725990aa62ae3c4b98be8736d89f", "name": "all", "rules": [], "authgates": {"46cfde2c1682566602860f8df7d0cc83": {"rules": [[true, ["layer", "read"]]]}, "4d50eb257549436414643a71e057091a": {"rules": [[true, ["view", "read"]]]}}}], "admin": false, "email": null, "locked": false, "archived": false, "authgates": {"f21b7ae79c2dacb89484929a8409e5d8": {"admin": true}, "d7d0380dd4e743e35af31a20d014ed48": {"admin": true}}}]')
                 gates = s_t_utils.deguidify('[{"iden": "46cfde2c1682566602860f8df7d0cc83", "type": "layer", "users": [{"iden": "a357138db50780b62093a6ce0d057fd8", "rules": [], "admin": true}], "roles": [{"iden": "e1ef725990aa62ae3c4b98be8736d89f", "rules": [[true, ["layer", "read"]]], "admin": false}]}, {"iden": "d7d0380dd4e743e35af31a20d014ed48", "type": "trigger", "users": [{"iden": "f77ac6744671a845c27e571071877827", "rules": [], "admin": true}], "roles": []}, {"iden": "f21b7ae79c2dacb89484929a8409e5d8", "type": "cronjob", "users": [{"iden": "f77ac6744671a845c27e571071877827", "rules": [], "admin": true}], "roles": []}, {"iden": "4d50eb257549436414643a71e057091a", "type": "view", "users": [{"iden": "a357138db50780b62093a6ce0d057fd8", "rules": [], "admin": true}], "roles": [{"iden": "e1ef725990aa62ae3c4b98be8736d89f", "rules": [[true, ["view", "read"]]], "admin": false}]}, {"iden": "cortex", "type": "cortex", "users": [], "roles": []}]')
@@ -3016,7 +3028,7 @@ class CellTest(s_t_utils.SynTest):
                 self.eq(users, s_t_utils.deguidify(json.dumps(await core.callStorm('return($lib.auth.users.list())'))))
                 self.eq(gates, s_t_utils.deguidify(json.dumps(await core.callStorm('return($lib.auth.gates.list())'))))
 
-                with self.raises(s_exc.BadTag):
+                with self.raises(s_exc.BadTypeValu):
                     await core.nodes('[ it:dev:str=foo +#test.newp ]')
 
         stream.seek(0)