synapse 2.201.0__py311-none-any.whl → 2.203.0__py311-none-any.whl

synapse/telepath.py

@@ -31,6 +31,8 @@ televers = (3, 0)
 
 aha_clients = {}
 
+LINK_CULL_INTERVAL = 10
+
 async def addAhaUrl(url):
     '''
     Add (incref) an aha registry URL.
@@ -392,7 +394,7 @@ class Share(s_base.Base):
             This should never be used by synapse core code.  This is for sync client code convenience only.
         '''
         if s_threads.iden() == self.tid:
-            raise s_exc.SynErr('Use of synchronous context manager in async code')
+            raise s_exc.SynErr(mesg='Use of synchronous context manager in async code')
 
         self._ctxobj = self.schedCoroSafePend(self.__aenter__())
         return self
@@ -590,6 +592,10 @@ class Proxy(s_base.Base):
         valu = proxy.getFooValu(x, y)
 
     '''
+    _link_task = None
+    _link_event = None
+    _all_proxies = set()
+
     async def __anit__(self, link, name):
 
         await s_base.Base.__anit__(self)
@@ -608,11 +614,15 @@ class Proxy(s_base.Base):
         self.methinfo = {}
 
         self.sess = None
+
         self.links = collections.deque()
-        self._link_poolsize = 4
+        self.alllinks = collections.deque()
+
+        self._link_add = 0      # counter for pending links being connected
+        self._links_min = 4     # low water mark for the link pool
+        self._links_max = 12    # high water mark for the link pool
 
         self.synack = None
-        self.syndone = asyncio.Event()
 
         self.handlers = {
             'task:fini': self._onTaskFini,
@@ -626,19 +636,54 @@ class Proxy(s_base.Base):
                 await item.fini()
 
             mesg = ('task:fini', {'retn': (False, ('IsFini', {}))})
-            for name, task in list(self.tasks.items()):
+            for iden, task in list(self.tasks.items()):  # pragma: no cover
                 task.reply(mesg)
-                del self.tasks[name]
+                del self.tasks[iden]
 
-            for link in self.links:
-                await link.fini()
+            # fini all the links from a different task to prevent
+            # delaying the proxy shutdown...
+            s_coro.create_task(self._finiAllLinks())
 
-            del self.syndone
-            await self.link.fini()
+            if self in self._all_proxies:
+                self._all_proxies.remove(self)
+
+            if not Proxy._all_proxies and Proxy._link_task is not None:
+                Proxy._link_task.cancel()
+                Proxy._link_task = None
+                Proxy._link_event = None
+
+        Proxy._all_proxies.add(self)
 
         self.onfini(fini)
         self.link.onfini(self.fini)
 
+        if Proxy._link_task is None:
+            Proxy._link_event = asyncio.Event()
+            Proxy._link_task = s_coro.create_task(Proxy._linkLoopTask())
+
+    @classmethod
+    async def _linkLoopTask(clas):
+        while True:
+            try:
+                await s_coro.event_wait(Proxy._link_event, timeout=LINK_CULL_INTERVAL)
+
+                for proxy in list(Proxy._all_proxies):
+
+                    if proxy.isfini:
+                        continue
+
+                    # close one link per proxy per period if the number of
+                    # available links is greater than _links_max...
+                    if len(proxy.links) > proxy._links_max:
+                        link = proxy.links.popleft()
+                        await link.fini()
+                        await proxy.fire('pool:link:fini', link=link)
+
+                Proxy._link_event.clear()
+
+            except Exception:  # pragma: no cover
+                logger.exception('synapse.telepath.Proxy.linkLoopTask()')
+
     def _hasTeleFeat(self, name, vers=1):
         return self._features.get(name, 0) >= vers
 
@@ -692,6 +737,12 @@ class Proxy(s_base.Base):
 
             link = self.links.popleft()
 
+            # fire a task to replace the link if we are
+            # below the low-water mark for link count.
+            if len(self.links) + self._link_add < self._links_min:
+                self._link_add += 1
+                self.schedCoro(self._addPoolLink())
+
             if link.isfini:
                 continue
 
@@ -700,6 +751,11 @@ class Proxy(s_base.Base):
         # we need a new one...
         return await self._initPoolLink()
 
+    async def _addPoolLink(self):
+        link = await self._initPoolLink()
+        self.links.append(link)
+        self._link_add -= 1
+
     async def getPipeline(self, genr, name=None):
         '''
         Construct a proxy API call pipeline in order to make
@@ -724,8 +780,6 @@ class Proxy(s_base.Base):
 
     async def _initPoolLink(self):
 
-        # TODO loop / backoff
-
         if self.link.get('unix'):
 
             path = self.link.get('path')
@@ -739,19 +793,26 @@ class Proxy(s_base.Base):
 
             link = await s_link.connect(host, port, ssl=ssl)
 
-        self.onfini(link)
+        self.alllinks.append(link)
+        async def fini():
+            self.alllinks.remove(link)
+            if link in self.links:
+                self.links.remove(link)
+
+        link.onfini(fini)
 
         return link
 
+    async def _finiAllLinks(self):
+        for link in list(self.alllinks):
+            await link.fini()
+        await self.link.fini()
+
     async def _putPoolLink(self, link):
 
         if link.isfini:
             return
 
-        # If we've exceeded our poolsize, discard the current link.
-        if len(self.links) >= self._link_poolsize:
-            return await link.fini()
-
         self.links.append(link)
 
     def __enter__(self):
@@ -762,7 +823,7 @@ class Proxy(s_base.Base):
             This must not be used from async code, and it should never be used in core synapse code.
         '''
         if s_threads.iden() == self.tid:
-            raise s_exc.SynErr('Use of synchronous context manager in async code')
+            raise s_exc.SynErr(mesg='Use of synchronous context manager in async code')
         self._ctxobj = self.schedCoroSafePend(self.__aenter__())
         return self
 
@@ -862,6 +923,7 @@ class Proxy(s_base.Base):
 
                 except GeneratorExit:
                     # if they bail early on the genr, fini the link
+                    # TODO: devise a tx/rx strategy to recover these links...
                     await link.fini()
 
             return s_coro.GenrHelp(genrloop())
@@ -1321,7 +1383,6 @@ class Client(s_base.Base):
         info.update(self._t_opts)
         self._t_proxy = await openinfo(info)
         self._t_methinfo = self._t_proxy.methinfo
-        self._t_proxy._link_poolsize = self._t_conf.get('link_poolsize', 4)
 
         async def fini():
             if self._t_named_meths:

synapse/tests/files/__init__.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # Stdlib
 import os
-import json
 import logging
 
 log = logging.getLogger(__name__)

synapse/tests/test_axon.py

@@ -103,7 +103,7 @@ class HttpPushFile(s_httpapi.StreamHandler):
         assert item['filename'] == 'bytes'
 
         assert args.get('zip_password') == [b'test']
-        assert args.get('dict') == [b'{"foo": "bar"}']
+        assert args.get('dict') == [b'{"foo":"bar"}']
         self.sendRestRetn(self.gotsize)
 
 class AxonTest(s_t_utils.SynTest):

synapse/tests/test_cmds_cortex.py

@@ -4,6 +4,7 @@ import asyncio
 import synapse.common as s_common
 
 import synapse.lib.cmdr as s_cmdr
+import synapse.lib.json as s_json
 import synapse.lib.encoding as s_encoding
 import synapse.lib.lmdbslab as s_lmdbslab
 
@@ -352,7 +353,7 @@ class CmdCoreTest(s_t_utils.SynTest):
             await cmdr.runCmdLine(f'storm --save-nodes {path} test:int')
             outp.expect('2 nodes')
 
-            jsdata = [item for item in s_common.jslines(path)]
+            jsdata = [item for item in s_json.jslines(path)]
             self.len(2, jsdata)
             self.eq({tuple(n[0]) for n in jsdata},
                     {('test:int', 20), ('test:int', 30)})
@@ -371,7 +372,7 @@ class CmdCoreTest(s_t_utils.SynTest):
             with s_common.genfile(stormfile) as fd:
                 fd.write(b'[ inet:fqdn=$hehe ]')
 
-            s_common.jssave(test_opts, optsfile)
+            s_json.jssave(test_opts, optsfile)
             s_common.yamlsave(test_opts, optsfile_yaml)
 
             outp = self.getTestOutp()

synapse/tests/test_cmds_hive.py

@@ -5,10 +5,10 @@ import synapse.lib.cmdr as s_cmdr
 import synapse.tests.utils as s_t_utils
 
 _json_output = '''[
-    1,
-    2,
-    3,
-    4
+  1,
+  2,
+  3,
+  4
 ]'''
 
 class CmdHiveTest(s_t_utils.SynTest):

synapse/tests/test_common.py

@@ -1,12 +1,17 @@
 import os
+import http
 import asyncio
 import logging
 import subprocess
 
 import yaml
+import aiohttp
 
 import synapse.exc as s_exc
 import synapse.common as s_common
+
+import synapse.lib.httpapi as s_httpapi
+
 import synapse.tests.utils as s_t_utils
 
 logger = logging.getLogger(__name__)
@@ -426,25 +431,6 @@ class CommonTest(s_t_utils.SynTest):
         retn = s_common.merggenr2([asyncl(lt) for lt in (l3, l2, l1)], reverse=True)
         self.eq((9, 8, 7, 6, 5, 4, 3, 2, 1), await alist(retn))
 
-    def test_jsonsafe(self):
-        items = (
-            (None, None),
-            (1234, None),
-            ('1234', None),
-            ({'asdf': 'haha'}, None),
-            ({'a': (1,), 'b': [{'': 4}, 56, None, {'t': True, 'f': False}, 'oh my']}, None),
-            (b'1234', s_exc.BadArg),
-            ({'a': 'a', 2: 2}, s_exc.BadArg),
-            ({'a', 'b', 'c'}, s_exc.BadArg),
-            (s_common.novalu, s_exc.BadArg),
-        )
-        for (item, eret) in items:
-            if eret is None:
-                self.none(s_common.reqJsonSafeStrict(item))
-            else:
-                with self.raises(eret):
-                    s_common.reqJsonSafeStrict(item)
-
     def test_sslctx(self):
         with self.getTestDir(mirror='certdir') as dirn:
             cadir = s_common.genpath(dirn, 'cas')
@@ -481,3 +467,27 @@ class CommonTest(s_t_utils.SynTest):
             v = s_common.trimText(iv, n=n)
             self.le(len(v), n)
             self.eq(v, ev)
+
+    async def test_tornado_monkeypatch(self):
+        class JsonHandler(s_httpapi.Handler):
+            async def get(self):
+                resp = {
+                    'foo': 'bar',
+                    'html': '<html></html>'
+                }
+                self.write(resp)
+
+        async with self.getTestCore() as core:
+            core.addHttpApi('/api/v1/test_tornado/', JsonHandler, {'cell': core})
+            _, port = await core.addHttpsPort(0)
+            url = f'https://127.0.0.1:{port}/api/v1/test_tornado/'
+
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url, ssl=False) as resp:
+                    self.eq(resp.status, http.HTTPStatus.OK)
+
+                    text = await resp.text()
+                    self.eq(text, '{"foo":"bar","html":"<html><\\/html>"}')
+
+                    json = await resp.json()
+                    self.eq(json, {'foo': 'bar', 'html': '<html></html>'})

synapse/tests/test_cortex.py

@@ -1559,8 +1559,8 @@ class CortexTest(s_t_utils.SynTest):
             self.eq(refs.get('fqdn'), ('inet:fqdn', 'woot.com'))
             self.eq(refs.get('ipv4'), ('inet:ipv4', 0x01020304))
 
-            self.len(1, await core.nodes('[meta:seen=($sorc, $valu)]',
-                                         opts={'vars': {'sorc': sorc, 'valu': node.ndef}}))
+            self.len(1, await core.nodes('[test:str=testndef :somestr=$somestr :bar=$valu]',
+                                         opts={'vars': {'somestr': sorc, 'valu': node.ndef}}))
 
             # test un-populated properties
             nodes = await core.nodes('[ps:contact="*"]')
@@ -1592,13 +1592,13 @@ class CortexTest(s_t_utils.SynTest):
             self.eq(ints, (('test:int', 1), ('test:int', 2), ('test:int', 3)))
 
             opts = {'vars': {'sorc': sorc}}
-            nodes = await core.nodes('meta:seen:source=$sorc -> *', opts=opts)
+            nodes = await core.nodes('test:str:somestr=$sorc -> *', opts=opts)
 
             self.len(2, nodes)
             self.isin('inet:dns:a', {n.ndef[0] for n in nodes})
 
             opts = {'vars': {'sorc': sorc}}
-            nodes = await core.nodes('meta:seen:source=$sorc :node -> *', opts=opts)
+            nodes = await core.nodes('test:str:somestr=$sorc :bar -> *', opts=opts)
 
             self.len(1, nodes)
             self.eq('inet:dns:a', nodes[0].ndef[0])
@@ -3955,7 +3955,7 @@ class CortexBasicTest(s_t_utils.SynTest):
                 'refs': True,
                 'edges': False,
                 'forms': {},
-                'pivots': ['<- meta:seen'],
+                'pivots': ['<(seen)- meta:source'],
                 'degrees': 3,
                 'filters': ['+#nope'],
                 'filterinput': False,
@@ -8504,6 +8504,26 @@ class CortexBasicTest(s_t_utils.SynTest):
                     self.notin('Timeout waiting for pool mirror', data)
                     self.notin('Timeout waiting for query mirror', data)
 
+                    orig = s_telepath.ClientV2.proxy
+                    async def finidproxy(self, timeout=None):
+                        prox = await orig(self, timeout=timeout)
+                        await prox.fini()
+                        return prox
+
+                    with patch('synapse.telepath.ClientV2.proxy', finidproxy):
+                        with self.getLoggerStream('synapse') as stream:
+                            msgs = await alist(core00.storm('inet:asn=0'))
+                            self.len(1, [m for m in msgs if m[0] == 'node'])
+
+                    stream.seek(0)
+                    data = stream.read()
+                    self.isin('Proxy for pool mirror [01.core.synapse] was shutdown. Skipping.', data)
+
+                    msgs = await core00.stormlist('cortex.storm.pool.set --connection-timeout 1 --sync-timeout 1 aha://pool00...')
+                    self.stormHasNoWarnErr(msgs)
+                    self.stormIsInPrint('Storm pool configuration set.', msgs)
+                    await core00.stormpool.waitready(timeout=12)
+
                     core01.nexsroot.nexslog.indx = 0
 
                     with patch('synapse.cortex.MAX_NEXUS_DELTA', 1):
@@ -8516,8 +8536,7 @@ class CortexBasicTest(s_t_utils.SynTest):
 
                         stream.seek(0)
                         data = stream.read()
-                        explog = (f'Pool mirror [01.core.synapse] Nexus offset delta too large '
-                                  f'({nexsoffs} > 1), running query locally')
+                        explog = ('Pool mirror [01.core.synapse] is too far out of sync. Skipping.')
                         self.isin(explog, data)
                         self.notin('Offloading Storm query', data)
 
@@ -8575,7 +8594,8 @@ class CortexBasicTest(s_t_utils.SynTest):
 
                     stream.seek(0)
                     data = stream.read()
-                    self.isin('Timeout waiting for pool mirror, running query locally', data)
+                    self.isin('Timeout waiting for pool mirror proxy.', data)
+                    self.isin('Pool members exhausted. Running query locally.', data)
 
                     await core01.fini()
 

synapse/tests/test_lib_ast.py

@@ -1,4 +1,3 @@
-import json
 import math
 import asyncio
 
@@ -9,6 +8,7 @@ import synapse.common as s_common
 import synapse.datamodel as s_datamodel
 
 import synapse.lib.ast as s_ast
+import synapse.lib.json as s_json
 import synapse.lib.snap as s_snap
 
 import synapse.tests.utils as s_test
@@ -1523,7 +1523,7 @@ class AstTest(s_test.SynTest):
             self.stormIsInPrint('stormpkg', msgs)
 
             # Make sure a JSON package loads
-            jsonpkg = json.loads(json.dumps(jsonpkg))
+            jsonpkg = s_json.loads(s_json.dumps(jsonpkg))
             await core.stormlist('$lib.pkg.add($pkg)',
                                  opts={'vars': {'pkg': jsonpkg}})
             msgs = await core.stormlist('pkg.list')
@@ -1731,7 +1731,7 @@ class AstTest(s_test.SynTest):
             prints = list(filter(lambda m: m[0] == 'print', msgs))
             self.eq(len(prints), 3)
 
-            jmsgs = list(map(lambda m: json.loads(m[1]['mesg']), prints))
+            jmsgs = list(map(lambda m: s_json.loads(m[1]['mesg']), prints))
             omsgs = sorted(jmsgs, key=lambda m: m[0])
             self.eq(omsgs[0][1], 'this should be first')
             self.eq(omsgs[1][1], 'toreturn called')

synapse/tests/test_lib_autodoc.py

@@ -56,7 +56,7 @@ shave'''
             )
         }
         callsig = s_autodoc.genCallsig(rtype)
-        self.eq(callsig, '(foo, bar=$lib.null, **kwargs)')
+        self.eq(callsig, '(foo, bar=(null), **kwargs)')
         self.eq(s_autodoc.genCallsig({}), '()')
         self.eq(s_autodoc.genCallsig({'args': ({'name': 'beep'},)}), '(beep)')
 
@@ -176,8 +176,8 @@ Returns:
 
 .. _test-lib-test-someargs:
 
-someargs(valu, bar=$lib.true, faz=$lib.null)
-============================================
+someargs(valu, bar=(true), faz=(null))
+======================================
 
 .. warning::
    ``$lib.test.someargs`` has been deprecated and will be removed in version v3.0.0.
@@ -252,8 +252,8 @@ Returns:
 
 .. _test-lib-test-someargs:
 
-$lib.test.someargs(valu, bar=$lib.true, faz=$lib.null)
-======================================================
+$lib.test.someargs(valu, bar=(true), faz=(null))
+================================================
 
 .. warning::
    ``$lib.test.someargs`` has been deprecated and will be removed in version v3.0.0.

synapse/tests/test_lib_base.py

@@ -483,7 +483,7 @@ class BaseTest(s_t_utils.SynTest):
         async def badgenr():
             yield 'foo'
             await asyncio.sleep(0)
-            raise s_exc.SynErr('rando')
+            raise s_exc.SynErr(mesg='rando')
 
         async def slowgenr():
             yield 'foo'

synapse/tests/test_lib_cell.py

@@ -1,7 +1,6 @@
 import os
 import ssl
 import sys
-import json
 import time
 import base64
 import signal
@@ -26,6 +25,7 @@ import synapse.lib.auth as s_auth
 import synapse.lib.base as s_base
 import synapse.lib.cell as s_cell
 import synapse.lib.coro as s_coro
+import synapse.lib.json as s_json
 import synapse.lib.link as s_link
 import synapse.lib.drive as s_drive
 import synapse.lib.nexus as s_nexus
@@ -1817,8 +1817,19 @@ class CellTest(s_t_utils.SynTest):
 
                     with open(bkuppath3, 'wb') as bkup3:
                         async for msg in proxy.iterNewBackupArchive('bkup3', remove=True):
+                            self.true(core.backupstreaming)
                             bkup3.write(msg)
 
+                    async def streamdone():
+                        while core.backupstreaming:
+                            await asyncio.sleep(0.1)
+
+                    task = core.schedCoro(streamdone())
+                    try:
+                        await asyncio.wait_for(task, 5)
+                    except TimeoutError:
+                        raise TimeoutError('Timeout waiting for streaming backup cleanup of bkup3 to complete.')
+
                     self.eq(('bkup', 'bkup2'), sorted(await proxy.getBackups()))
                     self.false(os.path.isdir(os.path.join(backdirn, 'bkup3')))
 
@@ -1830,6 +1841,12 @@ class CellTest(s_t_utils.SynTest):
                         async for msg in proxy.iterNewBackupArchive(remove=True):
                             bkup4.write(msg)
 
+                    task = core.schedCoro(streamdone())
+                    try:
+                        await asyncio.wait_for(task, 5)
+                    except TimeoutError:
+                        raise TimeoutError('Timeout waiting for streaming backup cleanup of bkup4 to complete.')
+
                     self.eq(('bkup', 'bkup2'), sorted(await proxy.getBackups()))
 
                     # Start another backup while one is already running
@@ -3117,13 +3134,13 @@ class CellTest(s_t_utils.SynTest):
                 self.nn(node.getTagProp('test', 'score'), 6)
 
                 self.maxDiff = None
-                roles = s_t_utils.deguidify('[{"type": "role", "iden": "e1ef725990aa62ae3c4b98be8736d89f", "name": "all", "rules": [], "authgates": {"46cfde2c1682566602860f8df7d0cc83": {"rules": [[true, ["layer", "read"]]]}, "4d50eb257549436414643a71e057091a": {"rules": [[true, ["view", "read"]]]}}}]')
-                users = s_t_utils.deguidify('[{"type": "user", "iden": "a357138db50780b62093a6ce0d057fd8", "name": "root", "rules": [], "roles": [], "admin": true, "email": null, "locked": false, "archived": false, "authgates": {"46cfde2c1682566602860f8df7d0cc83": {"admin": true}, "4d50eb257549436414643a71e057091a": {"admin": true}}}, {"type": "user", "iden": "f77ac6744671a845c27e571071877827", "name": "visi", "rules": [[true, ["cron", "add"]], [true, ["dmon", "add"]], [true, ["trigger", "add"]]], "roles": [{"type": "role", "iden": "e1ef725990aa62ae3c4b98be8736d89f", "name": "all", "rules": [], "authgates": {"46cfde2c1682566602860f8df7d0cc83": {"rules": [[true, ["layer", "read"]]]}, "4d50eb257549436414643a71e057091a": {"rules": [[true, ["view", "read"]]]}}}], "admin": false, "email": null, "locked": false, "archived": false, "authgates": {"f21b7ae79c2dacb89484929a8409e5d8": {"admin": true}, "d7d0380dd4e743e35af31a20d014ed48": {"admin": true}}}]')
-                gates = s_t_utils.deguidify('[{"iden": "46cfde2c1682566602860f8df7d0cc83", "type": "layer", "users": [{"iden": "a357138db50780b62093a6ce0d057fd8", "rules": [], "admin": true}], "roles": [{"iden": "e1ef725990aa62ae3c4b98be8736d89f", "rules": [[true, ["layer", "read"]]], "admin": false}]}, {"iden": "d7d0380dd4e743e35af31a20d014ed48", "type": "trigger", "users": [{"iden": "f77ac6744671a845c27e571071877827", "rules": [], "admin": true}], "roles": []}, {"iden": "f21b7ae79c2dacb89484929a8409e5d8", "type": "cronjob", "users": [{"iden": "f77ac6744671a845c27e571071877827", "rules": [], "admin": true}], "roles": []}, {"iden": "4d50eb257549436414643a71e057091a", "type": "view", "users": [{"iden": "a357138db50780b62093a6ce0d057fd8", "rules": [], "admin": true}], "roles": [{"iden": "e1ef725990aa62ae3c4b98be8736d89f", "rules": [[true, ["view", "read"]]], "admin": false}]}, {"iden": "cortex", "type": "cortex", "users": [], "roles": []}]')
+                roles = s_t_utils.deguidify('[{"type":"role","iden":"e1ef725990aa62ae3c4b98be8736d89f","name":"all","rules":[],"authgates":{"46cfde2c1682566602860f8df7d0cc83":{"rules":[[true,["layer","read"]]]},"4d50eb257549436414643a71e057091a":{"rules":[[true,["view","read"]]]}}}]')
+                users = s_t_utils.deguidify('[{"type":"user","iden":"a357138db50780b62093a6ce0d057fd8","name":"root","rules":[],"roles":[],"admin":true,"email":null,"locked":false,"archived":false,"authgates":{"46cfde2c1682566602860f8df7d0cc83":{"admin":true},"4d50eb257549436414643a71e057091a":{"admin":true}}},{"type":"user","iden":"f77ac6744671a845c27e571071877827","name":"visi","rules":[[true,["cron","add"]],[true,["dmon","add"]],[true,["trigger","add"]]],"roles":[{"type":"role","iden":"e1ef725990aa62ae3c4b98be8736d89f","name":"all","rules":[],"authgates":{"46cfde2c1682566602860f8df7d0cc83":{"rules":[[true,["layer","read"]]]},"4d50eb257549436414643a71e057091a":{"rules":[[true,["view","read"]]]}}}],"admin":false,"email":null,"locked":false,"archived":false,"authgates":{"f21b7ae79c2dacb89484929a8409e5d8":{"admin":true},"d7d0380dd4e743e35af31a20d014ed48":{"admin":true}}}]')
+                gates = s_t_utils.deguidify('[{"iden":"46cfde2c1682566602860f8df7d0cc83","type":"layer","users":[{"iden":"a357138db50780b62093a6ce0d057fd8","rules":[],"admin":true}],"roles":[{"iden":"e1ef725990aa62ae3c4b98be8736d89f","rules":[[true,["layer","read"]]],"admin":false}]},{"iden":"d7d0380dd4e743e35af31a20d014ed48","type":"trigger","users":[{"iden":"f77ac6744671a845c27e571071877827","rules":[],"admin":true}],"roles":[]},{"iden":"f21b7ae79c2dacb89484929a8409e5d8","type":"cronjob","users":[{"iden":"f77ac6744671a845c27e571071877827","rules":[],"admin":true}],"roles":[]},{"iden":"4d50eb257549436414643a71e057091a","type":"view","users":[{"iden":"a357138db50780b62093a6ce0d057fd8","rules":[],"admin":true}],"roles":[{"iden":"e1ef725990aa62ae3c4b98be8736d89f","rules":[[true,["view","read"]]],"admin":false}]},{"iden":"cortex","type":"cortex","users":[],"roles":[]}]')
 
-                self.eq(roles, s_t_utils.deguidify(json.dumps(await core.callStorm('return($lib.auth.roles.list())'))))
-                self.eq(users, s_t_utils.deguidify(json.dumps(await core.callStorm('return($lib.auth.users.list())'))))
-                self.eq(gates, s_t_utils.deguidify(json.dumps(await core.callStorm('return($lib.auth.gates.list())'))))
+                self.eq(roles, s_t_utils.deguidify(s_json.dumps(await core.callStorm('return($lib.auth.roles.list())')).decode()))
+                self.eq(users, s_t_utils.deguidify(s_json.dumps(await core.callStorm('return($lib.auth.users.list())')).decode()))
+                self.eq(gates, s_t_utils.deguidify(s_json.dumps(await core.callStorm('return($lib.auth.gates.list())')).decode()))
 
                 with self.raises(s_exc.BadTypeValu):
                     await core.nodes('[ it:dev:str=foo +#test.newp ]')

synapse/tests/test_lib_config.py

@@ -1,5 +1,4 @@
 import copy
-import json
 import regex
 import pathlib
 import argparse
@@ -11,6 +10,7 @@ import synapse.data as s_data
 import synapse.common as s_common
 
 import synapse.lib.cell as s_cell
+import synapse.lib.json as s_json
 import synapse.lib.config as s_config
 
 import synapse.tests.utils as s_test
@@ -396,7 +396,7 @@ class ConfTest(s_test.SynTest):
         self.true(filename.exists())
 
         with filename.open() as fp:
-            schema = json.load(fp)
+            schema = s_json.load(fp)
 
         self.eq(schema, s_config.localSchemaRefHandler(
             "http://raw.githubusercontent.com/oasis-open/cti-stix2-json-schemas/stix2.1/schemas/common/core.json"

synapse/tests/test_lib_encoding.py

@@ -1,8 +1,8 @@
 import io
-import json
 
 import synapse.common as s_common
 
+import synapse.lib.json as s_json
 import synapse.lib.msgpack as s_msgpack
 import synapse.lib.encoding as s_encoding
 
@@ -190,7 +190,7 @@ class EncTest(s_t_utils.SynTest):
 
         }
 
-        buf = io.BytesIO(json.dumps(data).encode())
+        buf = io.BytesIO(s_json.dumps(data))
 
         lines = list(s_encoding.iterdata(buf, format='json'))
         self.len(1, lines)