synapse 2.186.0__py311-none-any.whl → 2.187.1__py311-none-any.whl

synapse/tests/test_lib_spooled.py

@@ -28,11 +28,34 @@ class SpooledTest(s_test.SynTest):
 
             await sset.add(10)
 
+            newset = await sset.copy()
+            self.len(1, newset)
+            self.true(10 in newset)
+            self.false(newset.fallback)
+
             # Trigger fallback
             await sset.add(20)
             await sset.add(30)
             await sset.add(None)
 
+            newset = await sset.copy()
+            self.true(10 in newset)
+            self.true(20 in newset)
+            self.true(30 in newset)
+            self.true(None in newset)
+            self.len(4, newset)
+
+            await newset.clear()
+            self.false(10 in newset)
+            self.false(20 in newset)
+            self.false(30 in newset)
+            self.false(None in newset)
+            self.len(0, newset)
+
+            self.true(os.path.isdir(newset.slab.path))
+            await newset.fini()
+            self.false(os.path.isdir(newset.slab.path))
+
             self.len(4, sset)
 
             await sset.add(20)
@@ -68,6 +91,17 @@ class SpooledTest(s_test.SynTest):
                 self.true(os.path.isdir(sset.slab.path))
                 self.true(os.path.abspath(sset.slab.path).startswith(dirn))
 
+                newset = await sset.copy()
+                self.true(os.path.isdir(newset.slab.path))
+                self.true(os.path.abspath(newset.slab.path).startswith(dirn))
+
+            # Slabs should get removed on fini
+            self.false(os.path.isdir(sset.slab.path))
+
+            self.true(os.path.isdir(newset.slab.path))
+            await newset.fini()
+            self.false(os.path.isdir(newset.slab.path))
+
     async def test_spooled_dict(self):
 
         async def runtest(x):
@@ -91,6 +125,8 @@ class SpooledTest(s_test.SynTest):
 
         async with await s_spooled.Dict.anit(size=2) as sd0:
             await runtest(sd0)
+            self.true(sd0.fallback)
 
         async with await s_spooled.Dict.anit(size=1000) as sd1:
             await runtest(sd1)
+            self.false(sd1.fallback)

synapse/tests/test_lib_stormlib_model.py

@@ -403,24 +403,7 @@ class StormlibModelTest(s_test.SynTest):
             q = 'test:str=src $n=$node -> { test:str=deny $lib.model.migration.copyTags($n, $node) }'
             await self.asyncraises(s_exc.AuthDeny, core.nodes(q, opts=aslow))
 
-            with self.raises(s_exc.NoSuchProp) as exc:
-                await core.callStorm('$lib.model.migration.liftByPropValuNoNorm(formname, propname, valu)')
-            self.eq(exc.exception.get('mesg'), 'Could not find prop: formname:propname')
-
-            with self.raises(s_exc.AuthDeny) as exc:
-                await core.callStorm('$lib.model.migration.liftByPropValuNoNorm(it:prod:soft, cpe, valu)')
-            self.eq(exc.exception.get('mesg'), '$lib.model.migration.liftByPropValuNoNorm() is restricted to model migrations only.')
-
-            with self.raises(s_exc.BadArg) as exc:
-                await core.callStorm('$lib.model.migration.setNodePropValuNoNorm(notanode, propname, valu)')
-            self.eq(exc.exception.get('mesg'), '$lib.model.migration.setNodePropValuNoNorm() argument must be a node.')
-
-            with self.raises(s_exc.AuthDeny) as exc:
-                await core.callStorm('test:str $lib.model.migration.setNodePropValuNoNorm($node, propname, valu)')
-            self.eq(exc.exception.get('mesg'), '$lib.model.migration.setNodePropValuNoNorm() is restricted to model migrations only.')
-
             # copy extended properties
-
             await self.asyncraises(s_exc.BadArg, core.nodes('test:str=src $lib.model.migration.copyExtProps($node, newp)'))
             await self.asyncraises(s_exc.BadArg, core.nodes('test:str=dst $lib.model.migration.copyExtProps(newp, $node)'))
 
@@ -442,259 +425,6 @@ class StormlibModelTest(s_test.SynTest):
             self.len(1, nodes)
             self.eq(nodes[0].get('_foo'), 'foobarbaz')
 
-    async def test_model_migration_s_itSecCpe_2_170_0(self):
-
-        async with self.getRegrCore('itSecCpe_2_170_0', maxvers=(0, 2, 27)) as core:
-            # Migrate it:sec:cpe nodes with a valid CPE2.3, valid CPE2.2
-            q = 'it:sec:cpe +#test.cpe.23valid +#test.cpe.22valid'
-            nodes = await core.nodes(q)
-            self.len(2, nodes)
-            self.eq(
-                [
-                    ('it:sec:cpe', 'cpe:2.3:a:abine:donottrackme_-_mobile_privacy:1.1.8:*:*:*:*:android:*:*'),
-                    ('it:sec:cpe', 'cpe:2.3:a:01generator:pireospay:-:*:*:*:*:prestashop:*:*')
-                ],
-                [node.ndef for node in nodes]
-            )
-
-            q = '''
-            it:sec:cpe +#test.cpe.23valid +#test.cpe.22valid
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node)
-            $node.data.load(migration.s.itSecCpe_2_170_0)
-            '''
-            nodes = await core.nodes(q)
-
-            data = nodes[0].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.none(data.get('reason'))
-
-            data = nodes[1].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.none(data.get('reason'))
-
-        async with self.getRegrCore('itSecCpe_2_170_0', maxvers=(0, 2, 27)) as core:
-            # Migrate it:sec:cpe nodes with a valid CPE2.3, invalid CPE2.2
-            q = '''
-            it:sec:cpe +#test.cpe.23valid +#test.cpe.22invalid
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node)
-            '''
-            nodes = await core.nodes(q)
-            self.len(3, nodes)
-            self.eq(
-                [
-                    ('it:sec:cpe', 'cpe:2.3:a:1c:1c\\:enterprise:-:*:*:*:*:*:*:*'),
-                    ('it:sec:cpe', 'cpe:2.3:o:zyxel:nas542_firmware:5.21\\%28aazf.15\\%29co:*:*:*:*:*:*:*'),
-                    ('it:sec:cpe', 'cpe:2.3:a:abinitio:control\\>center:-:*:*:*:*:*:*:*'),
-                ],
-                [node.ndef for node in nodes]
-            )
-
-            q = '''
-            it:sec:cpe +#test.cpe.23valid +#test.cpe.22invalid
-            $node.data.load(migration.s.itSecCpe_2_170_0)
-            '''
-            nodes = await core.nodes(q)
-            self.len(3, nodes)
-
-            data = nodes[0].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['v2_2', 'product'])
-            self.eq(nodes[0].get('v2_2'), 'cpe:/a:1c:1c%3aenterprise:-')
-            self.eq(nodes[0].get('product'), '1c:enterprise')
-
-            data = nodes[1].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.none(data.get('valu'))
-            self.eq(data['updated'], ['v2_2', 'version'])
-            self.eq(nodes[1].get('v2_2'), 'cpe:/o:zyxel:nas542_firmware:5.21%2528aazf.15%2529co')
-            self.eq(nodes[1].get('version'), '5.21%28aazf.15%29co')
-
-            data = nodes[2].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['v2_2', 'product'])
-            self.eq(nodes[2].get('v2_2'), 'cpe:/a:abinitio:control%3ecenter:-')
-            self.eq(nodes[2].get('product'), 'control>center')
-
-            # The migration of this node was not correct because the CPE2.3 string (primary property) is valid but was
-            # not created correctly due to a bad CPE2.2 input value. Now we update :v2_2 to be correct, and re-run the
-            # migration. This time, we specify `prefer_v22=True` and `force=True` so the migration will use the updated
-            # :v2_2 prop for reparsing the strings. Force will cause the migration to continue past the check where both
-            # the primary property and :v2_2 are valid.
-            q = '''
-            it:sec:cpe:product=nas542_firmware [ :v2_2="cpe:/o:zyxel:nas542_firmware:5.21%28aazf.15%29co" ]
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node, prefer_v22=$lib.true, force=$lib.true)
-            '''
-            nodes = await core.nodes(q)
-            self.len(1, nodes)
-
-            # Lift the updated node and check the migration did what was expected.
-            q = '''
-            it:sec:cpe:product=nas542_firmware
-            $node.data.load(migration.s.itSecCpe_2_170_0)
-            '''
-            nodes = await core.nodes(q)
-            self.len(1, nodes)
-
-            data = nodes[0].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['version'])
-            self.eq(data['valu'], 'cpe:2.3:o:zyxel:nas542_firmware:5.21\\(aazf.15\\)co:*:*:*:*:*:*:*')
-            self.eq(nodes[0].get('v2_2'), 'cpe:/o:zyxel:nas542_firmware:5.21%28aazf.15%29co')
-            self.eq(nodes[0].get('version'), '5.21(aazf.15)co')
-
-        async with self.getRegrCore('itSecCpe_2_170_0', maxvers=(0, 2, 27)) as core:
-            # Migrate it:sec:cpe nodes with a invalid CPE2.3, valid CPE2.2
-            q = '''
-            it:sec:cpe +#test.cpe.23invalid +#test.cpe.22valid
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node)
-            '''
-            nodes = await core.nodes(q)
-            self.len(4, nodes)
-            self.eq(
-                [
-                    ('it:sec:cpe', 'cpe:2.3:h:d\\-link:dir\\-850l:*:*:*:*:*:*:*:*'),
-                    ('it:sec:cpe', 'cpe:2.3:a:acurax:under_construction_%2f_maintenance_mode:-::~~~wordpress~~:*:*:*:*:*'),
-                    ('it:sec:cpe', 'cpe:2.3:a:10web:social_feed_for_instagram:1.0.0::~~premium~wordpress~~:*:*:*:*:*'),
-                    ('it:sec:cpe', 'cpe:2.3:o:zyxel:nas326_firmware:5.21%28aazf.14%29c0:*:*:*:*:*:*:*'),
-                ],
-                [node.ndef for node in nodes]
-            )
-
-            q = '''
-            it:sec:cpe +#test.cpe.23invalid +#test.cpe.22valid
-            $node.data.load(migration.s.itSecCpe_2_170_0)
-            '''
-            nodes = await core.nodes(q)
-            self.len(4, nodes)
-
-            data = nodes[0].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['vendor', 'product'])
-            self.eq(data['valu'], 'cpe:2.3:h:d-link:dir-850l:*:*:*:*:*:*:*:*')
-            self.eq(nodes[0].get('vendor'), 'd-link')
-            self.eq(nodes[0].get('product'), 'dir-850l')
-
-            data = nodes[1].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['product', 'update', 'edition', 'target_sw'])
-            self.eq(data['valu'], 'cpe:2.3:a:acurax:under_construction_\\/_maintenance_mode:-:*:*:*:*:wordpress:*:*')
-            self.eq(nodes[1].get('product'), 'under_construction_/_maintenance_mode')
-            self.eq(nodes[1].get('update'), '*')
-            self.eq(nodes[1].get('edition'), '*')
-            self.eq(nodes[1].get('target_sw'), 'wordpress')
-
-            data = nodes[2].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['update', 'edition', 'sw_edition', 'target_sw'])
-            self.eq(data['valu'], 'cpe:2.3:a:10web:social_feed_for_instagram:1.0.0:*:*:*:premium:wordpress:*:*')
-            self.eq(nodes[2].get('update'), '*')
-            self.eq(nodes[2].get('edition'), '*')
-            self.eq(nodes[2].get('sw_edition'), 'premium')
-            self.eq(nodes[2].get('target_sw'), 'wordpress')
-
-            data = nodes[3].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['version'])
-            self.eq(data['valu'], 'cpe:2.3:o:zyxel:nas326_firmware:5.21\\(aazf.14\\)c0:*:*:*:*:*:*:*')
-            self.eq(nodes[3].get('version'), '5.21(aazf.14)c0')
-
-        async with self.getRegrCore('itSecCpe_2_170_0', maxvers=(0, 2, 27)) as core:
-            # Migrate it:sec:cpe nodes with a invalid CPE2.3, invalid CPE2.2
-            q = '''
-            it:sec:cpe +#test.cpe.23invalid +#test.cpe.22invalid
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node)
-            '''
-            msgs = await core.stormlist(q)
-            mesg = 'itSecCpe_2_170_0(it:sec:cpe=cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*): '
-            mesg += 'Unable to migrate due to invalid data. Primary property and :v2_2 are both invalid.'
-            self.stormIsInWarn(mesg, msgs)
-
-            ndefs = [m[1][0] for m in msgs if m[0] == 'node']
-            self.eq(
-                [
-                    ('it:sec:cpe', 'cpe:2.3:a:openbsd:openssh:7.4\r\n:*:*:*:*:*:*:*'),
-                    ('it:sec:cpe', 'cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*')
-                ],
-                ndefs
-            )
-
-            q = '''
-            it:sec:cpe +#test.cpe.23invalid +#test.cpe.22invalid
-            $node.data.load(migration.s.itSecCpe_2_170_0)
-            '''
-            nodes = await core.nodes(q)
-            self.len(2, nodes)
-
-            for node in nodes:
-                data = node.nodedata['migration.s.itSecCpe_2_170_0']
-                self.eq(data, {
-                    'status': 'failed',
-                    'reason': 'Unable to migrate due to invalid data. Primary property and :v2_2 are both invalid.',
-                })
-
-            # Now update the :v2_2 on one of the nodes and migrate again
-            q = '''
-            it:sec:cpe:version^=8.2p1 [ :v2_2="cpe:/a:openbsd:openssh:8.2p1_ubuntu-4ubuntu0.2" ]
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node)
-            '''
-            msgs = await core.stormlist(q)
-            self.stormHasNoWarnErr(msgs)
-
-            q = '''
-            it:sec:cpe:version^=8.2p1
-            $node.data.load(migration.s.itSecCpe_2_170_0)
-            '''
-            nodes = await core.nodes(q)
-            self.len(1, nodes)
-
-            data = nodes[0].nodedata['migration.s.itSecCpe_2_170_0']
-            self.nn(data)
-            self.eq(data['status'], 'success')
-            self.eq(data['updated'], ['version'])
-            self.eq(data['valu'], 'cpe:2.3:a:openbsd:openssh:8.2p1_ubuntu-4ubuntu0.2:*:*:*:*:*:*:*')
-            self.eq(nodes[0].get('version'), '8.2p1_ubuntu-4ubuntu0.2')
-
-            # Run the migration again to make sure we identify already migrated
-            # nodes correctly and bail early.
-            q = '''
-            it:sec:cpe:version^=8.2p1
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node)
-            '''
-            msgs = await core.stormlist(q)
-            self.stormIsInPrint('DEBUG: itSecCpe_2_170_0(it:sec:cpe=cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*): Node already migrated.', msgs)
-
-            q = '''
-            it:sec:cpe:version^=8.2p1
-            $lib.debug=$lib.true
-            $lib.model.migration.s.itSecCpe_2_170_0($node, force=$lib.true)
-            '''
-            msgs = await core.stormlist(q)
-            self.stormIsInPrint('DEBUG: itSecCpe_2_170_0(it:sec:cpe=cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*): No property updates required.', msgs)
-
-        async with self.getTestCore() as core:
-            with self.raises(s_exc.BadArg):
-                await core.callStorm('$lib.model.migration.s.itSecCpe_2_170_0(newp)')
-
-            with self.raises(s_exc.BadArg):
-                await core.callStorm('[ inet:fqdn=vertex.link ] $lib.model.migration.s.itSecCpe_2_170_0($node)')
-
     async def test_stormlib_model_migrations_risk_hasvuln_vulnerable(self):
 
         async with self.getTestCore() as core:

synapse/tests/test_lib_stormtypes.py

@@ -907,6 +907,17 @@ class StormTypesTest(s_test.SynTest):
                        'refs',
                        '20153b758f9d5eaaa38e4f4a65c36da797c3e59e549620fa7c4895e1a920991f'), edges)
 
+            data = await core.callStorm('''
+                $data = ({})
+                inet:user=visi
+                for ($name, $valu) in $lib.layer.get().getNodeData($node.iden()) { $data.$name = $valu }
+                return($data)
+            ''')
+            foo = data.get('foo')
+            self.nn(foo)
+            self.nn(foo.get('asof'))
+            self.eq('bar', foo.get('data'))
+
             msgs = await core.stormlist('$lib.print($lib.null)')
             self.stormIsInPrint('$lib.null', msgs)
             self.stormNotInPrint('None', msgs)

synapse/tests/test_model_infotech.py

@@ -1778,8 +1778,17 @@ class InfotechModelTest(s_t_utils.SynTest):
             with self.raises(s_exc.BadTypeValu):
                 nodes = await core.nodes('[it:sec:cpe=cpe:2.3:1:2:3:4:5:6:7:8:9:10:11:12]')
 
-            nodes = await core.nodes('[ it:sec:cpe=cpe:2.3:a:vertex:synapse ]')
-            self.eq(nodes[0].ndef, ('it:sec:cpe', 'cpe:2.3:a:vertex:synapse:*:*:*:*:*:*:*:*'))
+            with self.raises(s_exc.BadTypeValu):
+                await core.nodes('[ it:sec:cpe=cpe:2.3:a:vertex:synapse ]')
+
+            with self.raises(s_exc.BadTypeValu):
+                await core.callStorm(r'$lib.cast(it:sec:cpe, "cpe:2.3:a:openbsd:openssh:7.4\r\n:*:*:*:*:*:*:*")')
+
+            with self.raises(s_exc.BadTypeValu):
+                await core.callStorm(r'$lib.cast(it:sec:cpe:v2_2, "cpe:/a:01generator:pireospay\r\n:-::~~~prestashop~~")')
+
+            with self.raises(s_exc.BadTypeValu):
+                await core.callStorm('$lib.cast(it:sec:cpe:v2_2, "cpe:2.3:*")')
 
             nodes = await core.nodes('''[
                 it:sec:cpe=cpe:2.3:a:microsoft:internet_explorer:8.0.6001:beta:*:*:*:*:*:*
@@ -1792,14 +1801,8 @@ class InfotechModelTest(s_t_utils.SynTest):
             self.eq(nodes[0].get('version'), '8.0.6001')
             self.eq(nodes[0].get('update'), 'beta')
 
-            nodes = await core.nodes("[ it:sec:cpe='cpe:2.3:a:openbsd:openssh:7.4\r\n:*:*:*:*:*:*:*' ]")
-            self.len(1, nodes)
-            self.eq(nodes[0].ndef, ('it:sec:cpe', 'cpe:2.3:a:openbsd:openssh:7.4:*:*:*:*:*:*:*'))
-            self.eq(nodes[0].get('part'), 'a')
-            self.eq(nodes[0].get('product'), 'openssh')
-            self.eq(nodes[0].get('vendor'), 'openbsd')
-            self.eq(nodes[0].get('version'), '7.4')
-            self.eq(nodes[0].get('v2_2'), 'cpe:/a:openbsd:openssh:7.4')
+            with self.raises(s_exc.BadTypeValu):
+                await core.nodes("[ it:sec:cpe='cpe:2.3:a:openbsd:openssh:7.4\r\n:*:*:*:*:*:*:*' ]")
 
             nodes = await core.nodes(r'[ it:sec:cpe="cpe:2.3:o:cisco:ios:12.1\(22\)ea1a:*:*:*:*:*:*:*" ]')
             self.len(1, nodes)
@@ -1923,7 +1926,7 @@ class InfotechModelTest(s_t_utils.SynTest):
                 nodes = await core.nodes(q, opts={'vars': {'valu': valu}})
                 self.len(1, nodes)
                 node = nodes[0]
-                self.eq(node.ndef[1], valu.lower())
+                self.eq(node.ndef[1], valu.lower(), msg=valu.lower())
 
     async def test_infotech_c2config(self):
         async with self.getTestCore() as core:

synapse/tests/test_utils_getrefs.py

@@ -1,10 +1,13 @@
-import os.path
+import os
 import pathlib
 
 import vcr
 
+from unittest import mock
+
 import synapse.exc as s_exc
 import synapse.data as s_data
+import synapse.common as s_common
 
 import synapse.lib.config as s_config
 
@@ -24,7 +27,7 @@ class TestUtilsGetrefs(s_utils.SynTest):
         cm = myvcr.use_cassette(fp)
         return cm
 
-    async def test_basics(self):
+    def test_basics(self):
 
         args = s_getrefs.parse_args([
             s_data.path('attack-flow', 'attack-flow-schema-2.0.0.json')
@@ -45,30 +48,34 @@ class TestUtilsGetrefs(s_utils.SynTest):
         with self.raises(s_exc.BadArg):
             s_getrefs.download_refs_handler('http://raw.githubusercontent.com/../../attack-flow/attack-flow-schema-2.0.0.json')
 
-        filename = pathlib.Path(s_data.path(
-            'jsonschemas',
-            'raw.githubusercontent.com',
-            'oasis-open',
-            'cti-stix2-json-schemas',
-            'stix2.1',
-            'schemas',
-            'common',
-            'core.json'
-        ))
-
-        self.true(filename.exists())
-        filename.unlink()
-
-        # Clear the cached validator funcs so the ref handlers in s_getrefs get called
-        s_config._JsValidators = {}
-
-        with self.getLoggerStream('synapse.utils.getrefs') as stream:
-            with self.getVcr() as cass:
-                s_getrefs.main(args)
-
-        stream.seek(0)
-        mesgs = stream.read()
-        mesg = f'Downloading schema from {CORE_URL}.'
-        self.true(filename.exists())
-        self.isin(mesg, mesgs)
-        self.notin(f'Schema {CORE_URL} already exists in local cache, skipping.', mesgs)
+        with self.getTestDir(copyfrom=s_getrefs.BASEDIR) as dirn:
+
+            filename = pathlib.Path(s_common.genpath(
+                dirn,
+                'raw.githubusercontent.com',
+                'oasis-open',
+                'cti-stix2-json-schemas',
+                'stix2.1',
+                'schemas',
+                'common',
+                'core.json'
+            ))
+
+            self.true(filename.exists())
+            filename.unlink()
+            self.false(filename.exists())
+
+            # Clear the cached validator funcs so the ref handlers in s_getrefs get called
+            s_config._JsValidators = {}
+
+            with self.getLoggerStream('synapse.utils.getrefs') as stream:
+                with mock.patch('synapse.utils.getrefs.BASEDIR', dirn):
+                    with self.getVcr():
+                        s_getrefs.main(args)
+
+            stream.seek(0)
+            mesgs = stream.read()
+            mesg = f'Downloading schema from {CORE_URL}.'
+            self.true(filename.exists())
+            self.isin(mesg, mesgs)
+            self.notin(f'Schema {CORE_URL} already exists in local cache, skipping.', mesgs)

synapse/tools/changelog.py

@@ -883,9 +883,17 @@ async def format(opts: argparse.Namespace,
             text = text + f'\n{header}\n{"-" * len(header)}'
             dataz.sort(key=lambda x: x.get('prs'))
             for data in dataz:
-                desc = data.get('desc')
-                for line in textwrap.wrap(desc, initial_indent='- ', subsequent_indent='  ', width=opts.width):
-                    text = f'{text}\n{line}'
+                desc = data.get('desc')  # type: str
+                desc_lines = desc.splitlines()
+                for i, chunk in enumerate(desc_lines):
+                    if i == 0:
+                        for line in textwrap.wrap(chunk, initial_indent='- ', subsequent_indent='  ', width=opts.width):
+                            text = f'{text}\n{line}'
+                    else:
+                        text = text + '\n'
+                        for line in textwrap.wrap(chunk, initial_indent='  ', subsequent_indent='  ', width=opts.width):
+                            text = f'{text}\n{line}'
+
                 if not opts.hide_prs:
                     for pr in data.get('prs'):
                         text = f'{text}\n  (`#{pr} <https://github.com/vertexproject/synapse/pull/{pr}>`_)'

synapse/utils/getrefs.py

@@ -1,11 +1,12 @@
 import sys
 import json
 import urllib
+import asyncio
 import logging
 import pathlib
 import argparse
 
-import requests
+import aiohttp
 
 import synapse.exc as s_exc
 import synapse.data as s_data
@@ -15,23 +16,31 @@ import synapse.lib.config as s_config
 
 logger = logging.getLogger(__name__)
 
+BASEDIR = s_data.path('jsonschemas')
+
 def download_refs_handler(uri):
     '''
     This function downloads the JSON schema at the given URI, parses the given
     URI to get the path component, and then saves the referenced schema to the
     'jsonschemas' directory of synapse.data.
+
+    This function runs its own asyncio loop for each URI being requested.
     '''
+    ret = asyncio.run(_download_refs_handler(uri))
+    return ret
+
+async def _download_refs_handler(uri):
 
     try:
         parts = urllib.parse.urlparse(uri)
     except ValueError:
         raise s_exc.BadUrl(mesg=f'Malformed URI: {uri}.') from None
 
-    filename = s_data.path('jsonschemas', parts.hostname, *parts.path.split('/'))
+    filename = s_data.path(BASEDIR, parts.hostname, *parts.path.split('/'))
     filepath = pathlib.Path(filename)
 
     # Check for path traversal. Unlikely, but still check
-    if not str(filepath.absolute()).startswith(s_data.path('jsonschemas')):
+    if not str(filepath.absolute()).startswith(BASEDIR):
         raise s_exc.BadArg(mesg=f'Path traversal in schema URL: {uri} ?')
 
     # If we already have the file, return it
@@ -45,8 +54,12 @@ def download_refs_handler(uri):
 
     # Get the data from the interwebs
     logger.info(f'Downloading schema from {uri}.')
-    resp = requests.get(uri)
-    data = resp.json()
+    async with aiohttp.ClientSession() as session:
+        async with session.get(uri) as resp:
+            resp.raise_for_status()
+            buf = await resp.read()
+
+    data = json.loads(buf.decode())
 
     # Save the json schema to disk
     with filepath.open('w') as fp:

{synapse-2.186.0.dist-info → synapse-2.187.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: synapse
-Version: 2.186.0
+Version: 2.187.1
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License: Apache License 2.0
@@ -19,7 +19,7 @@ Classifier: Operating System :: POSIX :: Linux
 Requires-Python: >=3.11
 Description-Content-Type: text/x-rst
 License-File: LICENSE
-Requires-Dist: pyOpenSSL<25.0.0,>=24.0.0
+Requires-Dist: pyOpenSSL<24.3.0,>=24.0.0
 Requires-Dist: cryptography<44.0.0,>=43.0.1
 Requires-Dist: msgpack<1.1.0,>=1.0.5
 Requires-Dist: xxhash<3.6.0,>=1.4.4
@@ -63,6 +63,7 @@ Requires-Dist: sphinx<9.0.0,>=8.0.0; extra == "docs"
 Requires-Dist: sphinx-rtd-theme<4.0.0,>=3.0.0; extra == "docs"
 Requires-Dist: sphinx-notfound-page<2.0.0,>=1.0.4; extra == "docs"
 Requires-Dist: jinja2<4.0.0,>=3.1.4; extra == "docs"
+Dynamic: license-file
 
 Synapse
 =======