synapse 2.201.0__py311-none-any.whl → 2.203.0__py311-none-any.whl

synapse/axon.py

@@ -1,5 +1,4 @@
 import csv
-import json
 import struct
 import asyncio
 import hashlib
@@ -16,6 +15,7 @@ import synapse.common as s_common
 import synapse.lib.cell as s_cell
 import synapse.lib.coro as s_coro
 import synapse.lib.base as s_base
+import synapse.lib.json as s_json
 import synapse.lib.link as s_link
 import synapse.lib.const as s_const
 import synapse.lib.nexus as s_nexus
@@ -1509,8 +1509,8 @@ class Axon(s_cell.Cell):
                 continue
 
             try:
-                yield json.loads(line)
-            except json.JSONDecodeError as e:
+                yield s_json.loads(line)
+            except s_exc.BadJsonText as e:
                 logger.exception(f'Bad json line encountered for {sha256}')
                 raise s_exc.BadJsonText(mesg=f'Bad json line encountered while processing {sha256}, ({e})',
                                         sha256=sha256) from None
@@ -1643,7 +1643,7 @@ class Axon(s_cell.Cell):
                     else:
                         valu = field.get('value')
                         if not isinstance(valu, (bytes, str)):
-                            valu = json.dumps(valu)
+                            valu = s_json.dumps(valu).decode()
 
                     data.add_field(name,
                                    valu,

synapse/cmds/cortex.py

@@ -1,5 +1,4 @@
 import os
-import json
 import queue
 import shlex
 import pprint
@@ -10,6 +9,7 @@ import synapse.common as s_common
 
 import synapse.lib.cli as s_cli
 import synapse.lib.cmd as s_cmd
+import synapse.lib.json as s_json
 import synapse.lib.node as s_node
 import synapse.lib.time as s_time
 import synapse.lib.msgpack as s_msgpack
@@ -118,9 +118,7 @@ Examples:
 
         fmt = self.locs.get('log:fmt')
         if fmt == 'jsonl':
-            s = json.dumps(mesg, sort_keys=True) + '\n'
-            buf = s.encode()
-            return buf
+            return s_json.dumps(mesg, sort_keys=True, newline=True)
 
         elif fmt == 'mpk':
             buf = s_msgpack.en(mesg)
@@ -430,8 +428,8 @@ class StormCmd(s_cli.Cmd):
                 if mesg[0] == 'node':
 
                     if nodesfd is not None:
-                        byts = json.dumps(mesg[1]).encode()
-                        nodesfd.write(byts + b'\n')
+                        byts = s_json.dumps(mesg[1], newline=True)
+                        nodesfd.write(byts)
 
                 try:
                     func = self.cmdmeths[mesg[0]]

synapse/cmds/hive.py

@@ -1,5 +1,4 @@
 import os
-import json
 import shlex
 import pprint
 import asyncio
@@ -12,6 +11,7 @@ import synapse.common as s_common
 
 import synapse.lib.cmd as s_cmd
 import synapse.lib.cli as s_cli
+import synapse.lib.json as s_json
 
 ListHelp = '''
 Lists all the keys underneath a particular key in the hive.
@@ -140,8 +140,8 @@ A Hive is a hierarchy persistent storage mechanism typically used for configurat
             return
 
         if opts.json:
-            prend = json.dumps(valu, indent=4, sort_keys=True)
-            rend = prend.encode()
+            rend = s_json.dumps(valu, indent=True, sort_keys=True)
+            prend = rend.decode()
         elif isinstance(valu, str):
             rend = valu.encode()
             prend = valu
@@ -149,7 +149,7 @@ A Hive is a hierarchy persistent storage mechanism typically used for configurat
             rend = valu
             prend = pprint.pformat(valu)
         else:
-            rend = json.dumps(valu, indent=4, sort_keys=True).encode()
+            rend = s_json.dumps(valu, indent=True, sort_keys=True)
             prend = pprint.pformat(valu)
 
         if opts.file:
@@ -172,7 +172,7 @@ A Hive is a hierarchy persistent storage mechanism typically used for configurat
             if opts.value[0] not in '([{"':
                 data = opts.value
             else:
-                data = json.loads(opts.value)
+                data = s_json.loads(opts.value)
             await core.setHiveKey(path, data)
             return
         elif opts.file is not None:
@@ -181,7 +181,7 @@ A Hive is a hierarchy persistent storage mechanism typically used for configurat
                 if len(s) == 0:
                     self.printf('Empty file.  Not writing key.')
                     return
-                data = s if opts.string else json.loads(s)
+                data = s if opts.string else s_json.loads(s)
                 await core.setHiveKey(path, data)
                 return
 
@@ -201,8 +201,8 @@ A Hive is a hierarchy persistent storage mechanism typically used for configurat
                         data = old_valu
                     else:
                         try:
-                            data = json.dumps(old_valu, indent=4, sort_keys=True)
-                        except (ValueError, TypeError):
+                            data = s_json.dumps(old_valu, indent=True, sort_keys=True).decode()
+                        except s_exc.MustBeJsonSafe:
                             self.printf('Value is not JSON-encodable, therefore not editable.')
                             return
                     fh.write(data)
@@ -218,8 +218,8 @@ A Hive is a hierarchy persistent storage mechanism typically used for configurat
                         self.printf('Empty file.  Not writing key.')
                         return
                     try:
-                        valu = rawval if opts.string else json.loads(rawval)
-                    except json.JSONDecodeError as e:  # pragma: no cover
+                        valu = rawval if opts.string else s_json.loads(rawval)
+                    except s_exc.BadJsonText as e:  # pragma: no cover
                         self.printf(f'JSON decode failure: [{e}].  Reopening.')
                         await asyncio.sleep(1)
                         continue

synapse/common.py

@@ -3,7 +3,6 @@ import os
 import ssl
 import sys
 import enum
-import json
 import http
 import stat
 import time
@@ -30,6 +29,7 @@ import contextlib
 import collections
 
 import http.cookies
+import tornado.escape
 
 import yaml
 import regex
@@ -511,24 +511,6 @@ def getDirSize(*paths):
 
     return realsum, apprsum
 
-def jsload(*paths):
-    with genfile(*paths) as fd:
-        byts = fd.read()
-        if not byts:
-            return None
-
-        return json.loads(byts.decode('utf8'))
-
-def jslines(*paths):
-    with genfile(*paths) as fd:
-        for line in fd:
-            yield json.loads(line)
-
-def jssave(js, *paths):
-    path = genpath(*paths)
-    with io.open(path, 'wb') as fd:
-        fd.write(json.dumps(js, sort_keys=True, indent=2).encode('utf8'))
-
 def yamlloads(data):
     return yaml.load(data, Loader)
 
@@ -959,16 +941,6 @@ def deprdate(name, date):  # pragma: no cover
     mesg = f'{name} is deprecated and will be removed on {date}.'
     warnings.warn(mesg, DeprecationWarning)
 
-def reqjsonsafe(item):
-    '''
-    Returns None if item is json serializable, otherwise raises an exception.
-    Uses default type coercion from built-in json.dumps.
-    '''
-    try:
-        json.dumps(item)
-    except TypeError as e:
-        raise s_exc.MustBeJsonSafe(mesg=str(e)) from None
-
 def jsonsafe_nodeedits(nodeedits):
     '''
     Hexlify the buid of each node:edits
@@ -998,35 +970,6 @@ def reprauthrule(rule):
         text = '!' + text
     return text
 
-def reqJsonSafeStrict(item):
-    '''
-    Require the item to be safe to serialize to JSON without type coercion issues.
-
-    Args:
-        item: The python primitive to check.
-
-    Returns:
-        None
-
-    Raise:
-        s_exc.BadArg: If the item contains invalid data.
-    '''
-    if item is None:
-        return
-    if isinstance(item, (str, int,)):
-        return
-    if isinstance(item, (list, tuple)):
-        for valu in item:
-            reqJsonSafeStrict(valu)
-        return
-    if isinstance(item, dict):
-        for key, valu in item.items():
-            if not isinstance(key, str):
-                raise s_exc.BadArg(mesg='Non-string keys are not valid json', key=key)
-            reqJsonSafeStrict(valu)
-        return
-    raise s_exc.BadArg(mesg=f'Invalid item type encountered: {item.__class__.__name__}')
-
 async def merggenr(genrs, cmprkey):
     '''
     Iterate multiple sorted async generators and yield their results in order.
@@ -1234,7 +1177,23 @@ def _patch_http_cookies():
         return
     http.cookies._unquote = v_cookies._unquote
 
+def _patch_tornado_json():
+    import synapse.lib.json as s_json
+
+    if hasattr(tornado.escape, 'json_encode'):
+        # This exists for a specific reason. See the following URL for explanation:
+        # https://github.com/tornadoweb/tornado/blob/d5ac65c1f1453c2aeddd089d8e68c159645c13e1/tornado/escape.py#L83-L96
+        # https://github.com/tornadoweb/tornado/pull/706
+        def _tornado_json_encode(value):
+            return s_json.dumps(value).replace(b'</', br'<\/').decode()
+
+        tornado.escape.json_encode = _tornado_json_encode
+
+    if hasattr(tornado.escape, 'json_decode'):
+        tornado.escape.json_decode = s_json.loads
+
 _patch_http_cookies()
+_patch_tornado_json()
 
 # TODO:  Switch back to using asyncio.wait_for when we are using py 3.12+
 # This is a workaround for a race where asyncio.wait_for can end up

synapse/cortex.py

@@ -1265,7 +1265,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
             byts = self.slab.get(newname.encode(), db=self.macrodb)
             if byts is not None:
-                raise s_exc.DupName('A macro named {newname} already exists!')
+                raise s_exc.DupName(mesg=f'A macro named {newname} already exists!', name=newname)
 
             self.slab.put(newname.encode(), s_msgpack.en(mdef), db=self.macrodb)
             self.slab.pop(name.encode(), db=self.macrodb)
@@ -5874,38 +5874,45 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         if self.stormpool is None:  # pragma: no cover
             return None
 
-        if self.stormpool.size() == 0:
+        size = self.stormpool.size()
+        if size == 0:
             logger.warning('Storm query mirror pool is empty, running query locally.')
             return None
 
-        proxy = None
+        for _ in range(size):
 
-        try:
-            timeout = self.stormpoolopts.get('timeout:connection')
-            proxy = await self.stormpool.proxy(timeout=timeout)
-            proxyname = proxy._ahainfo.get('name')
-            if proxyname is not None and proxyname == self.ahasvcname:
-                # we are part of the pool and were selected. Convert to local use.
-                return None
+            try:
+                timeout = self.stormpoolopts.get('timeout:connection')
+                proxy = await self.stormpool.proxy(timeout=timeout)
+                proxyname = proxy._ahainfo.get('name')
+                if proxyname is not None and proxyname == self.ahasvcname:
+                    # we are part of the pool and were selected. Convert to local use.
+                    return None
+
+            except TimeoutError:
+                logger.warning('Timeout waiting for pool mirror proxy.')
+                continue
+
+            try:
+
+                curoffs = opts.setdefault('nexsoffs', await self.getNexsIndx() - 1)
+                miroffs = await s_common.wait_for(proxy.getNexsIndx(), timeout) - 1
+                if (delta := curoffs - miroffs) <= MAX_NEXUS_DELTA:
+                    return proxy
 
-            curoffs = opts.setdefault('nexsoffs', await self.getNexsIndx() - 1)
-            miroffs = await s_common.wait_for(proxy.getNexsIndx(), timeout) - 1
-            if (delta := curoffs - miroffs) > MAX_NEXUS_DELTA:
-                mesg = (f'Pool mirror [{proxyname}] Nexus offset delta too large '
-                        f'({delta} > {MAX_NEXUS_DELTA}), running query locally.')
+                mesg = f'Pool mirror [{proxyname}] is too far out of sync. Skipping.'
                 logger.warning(mesg, extra=await self.getLogExtra(delta=delta, mirror=proxyname, mirror_offset=miroffs))
-                return None
 
-            return proxy
+            except s_exc.IsFini:
+                mesg = f'Proxy for pool mirror [{proxyname}] was shutdown. Skipping.'
+                logger.warning(mesg, extra=await self.getLogExtra(mirror=proxyname))
 
-        except (TimeoutError, s_exc.IsFini):
-            if proxy is None:
-                logger.warning('Timeout waiting for pool mirror, running query locally.')
-            else:
-                mesg = f'Timeout waiting for pool mirror [{proxyname}] Nexus offset, running query locally.'
+            except TimeoutError:
+                mesg = f'Timeout waiting for pool mirror [{proxyname}] Nexus offset.'
                 logger.warning(mesg, extra=await self.getLogExtra(mirror=proxyname))
-                await proxy.fini()
-            return None
+
+        logger.warning('Pool members exhausted. Running query locally.', extra=await self.getLogExtra())
+        return None
 
     async def storm(self, text, opts=None):
 
@@ -6528,7 +6535,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
             cdef['reqs'] = reqs
         except KeyError:
-            raise s_exc.BadConfValu('Unrecognized time unit')
+            raise s_exc.BadConfValu(mesg='Unrecognized time unit')
 
         if not cdef.get('iden'):
             cdef['iden'] = s_common.guid()
@@ -6985,7 +6992,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         vault = self.getVault(iden)
         if vault is None:
-            raise s_exc.NoSuchIden(mesg=f'Vault not found for iden: {iden}.')
+            raise s_exc.NoSuchIden(mesg=f'Vault not found for iden: {iden}.', iden=iden)
 
         return vault
 
@@ -7338,7 +7345,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             scope = 'roles'
             ident = self.auth.role(iden)
             if ident is None:
-                raise s_exc.NoSuchIden(mesg=f'Iden {iden} is not a valid user or role.')
+                raise s_exc.NoSuchIden(mesg=f'Iden {iden} is not a valid user or role.', iden=iden)
 
         await self._setEasyPerm(vault, scope, ident.iden, level)
         permissions = vault.get('permissions')
@@ -7359,14 +7366,14 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         Returns: Updated vault.
         '''
         if self.getVaultByName(name) is not None:
-            raise s_exc.DupName(mesg=f'Vault with name {name} already exists.')
+            raise s_exc.DupName(mesg=f'Vault with name {name} already exists.', name=name)
 
         return await self._push(('vault:set'), iden, 'name', name)
 
     @s_nexus.Pusher.onPush('vault:set')
     async def _setVault(self, iden, key, valu):
         if key not in ('name', 'permissions'):  # pragma: no cover
-            raise s_exc.BadArg('Only vault names and permissions can be changed.')
+            raise s_exc.BadArg(mesg='Only vault names and permissions can be changed.')
 
         vault = self.reqVault(iden)
         oldv = vault.get(key)

synapse/data/__init__.py

@@ -1,8 +1,9 @@
 import os
-import json
 import logging
 
 import synapse.common as s_common
+
+import synapse.lib.json as s_json
 import synapse.lib.datfile as s_datfile
 import synapse.lib.msgpack as s_msgpack
 
@@ -26,7 +27,7 @@ def get(name, defval=None):
 
 def getJSON(name):
     with s_datfile.openDatFile(f'synapse.data/{name}.json') as fd:
-        return json.loads(fd.read())
+        return s_json.load(fd)
 
 def path(*names):
     return s_common.genpath(dirname, *names)

synapse/data/iana.uris.mpk

@@ -0,0 +1 @@
+�
��aaa�aaas�about�acap�acct�acd�acr�adiumxtra�adt�afp�afs�aim�amss�android�appdata�apt�ar�ari�ark�at�attachment�aw�barion�bb�beshare�bitcoin�bitcoincash�blob�bluetooth�bolo�brid�browserext�cabal�calculator�callto�cap�cast�casts�chrome�chrome-extension�cid�coap�coap+tcp�coap+ws�coaps�coaps+tcp�coaps+ws�com-eventbrite-attendee�content�content-type�crid�cstr�cvs�dab�dat�data�dav�dhttp�diaspora�dict�did�dis�dlna-playcontainer�dlna-playsingle�dns�dntp�doi�dpp�drm�drop�dtmi�dtn�dvb�dvx�dweb�ed2k�eid�elsi�embedded�ens�ethereum�example�facetime�fax�feed�feedready�fido�file�filesystem�finger�first-run-pen-experience�fish�fm�ftp�fuchsia-pkg�geo�gg�git�gitoid�gizmoproject�go�gopher�graph�grd�gtalk�h323�ham�hcap�hcp�hs20�http�https�hxxp�hxxps�hydrazone�hyper�iax�icap�icon�ilstring�im�imap�info�iotdisco�ipfs�ipn�ipns�ipp�ipps�irc�irc6�ircs�iris�iris.beep�iris.lwz�iris.xpc�iris.xpcs�isostore�itms�jabber�jar�jms�keyparc�lastfm�lbry�ldap�ldaps�leaptofrogans�lid�lorawan�lpa�lvlt�#machineProvisioningProgressReporter�magnet�mailserver�mailto�maps�market�matrix�message�microsoft.windows.camera�$microsoft.windows.camera.multipicker�microsoft.windows.camera.picker�mid�mms�modem�mongodb�moz�ms-access�ms-appinstaller�ms-browser-extension�ms-calculator�ms-drive-to�ms-enrollment�ms-excel�ms-eyecontrolspeech�ms-gamebarservices�ms-gamingoverlay�ms-getoffice�ms-help�ms-infopath�ms-inputapp�ms-launchremotedesktop�ms-lockscreencomponent-config�ms-media-stream-id�ms-meetnow�ms-mixedrealitycapture�ms-mobileplans�ms-newsandinterests�ms-officeapp�ms-people�ms-personacard�ms-project�ms-powerpoint�ms-publisher�ms-recall�ms-remotedesktop�ms-remotedesktop-launch�ms-restoretabcompanion�ms-screenclip�ms-screensketch�ms-search�ms-search-repair�ms-secondary-screen-controller�ms-secondary-screen-setup�ms-settings�ms-settings-airplanemode�ms-settings-bluetooth�ms-settings-camera�ms-settings-cellular�ms-settings-cloudstorage�ms-settings-connectabledevices�ms-settings-displays-topology�ms-settings-emailandaccounts�ms-settings-language�ms-settings-location�ms-settings-lock�ms-settings-nfctransactions�ms-settings-notifications�ms-settings-power�ms-settings-privacy�ms-settings-proximity�ms-settings-screenrotation�ms-settings-wifi�ms-settings-workplace�ms-spd�ms-stickers�ms-sttoverlay�ms-transit-to�ms-useractivityset�ms-virtualtouchpad�ms-visio�ms-walk-to�ms-whiteboard�ms-whiteboard-cmd�ms-widgetboard�ms-widgets�ms-word�msnim�msrp�msrps�mss�mt�mtqp�mumble�mupdate�mvn�mvrp�mvrps�news�nfs�ni�nih�nntp�notes�num�ocf�oid�onenote�onenote-cmd�opaquelocktoken�openid�openpgp4fpr�otpauth�p1�pack�palm�paparazzi�payment�payto�pkcs11�platform�pop�pres�prospero�proxy�pwid�psyc�pttp�qb�query�quic-transport�redis�rediss�reload�res�resource�rmi�rsync�rtmfp�rtmp�rtsp�rtsps�rtspu�sarif�secondlife�secret-token�service�session�sftp�sgn�shc�shelter�sieve�simpleledger�simplex�sip�sips�skype�smb�smp�sms�smtp�snews�snmp�soap.beep�soap.beeps�soldat�spiffe�spotify�ssb�ssh�starknet�steam�stun�stuns�submit�svn�swh�swid�swidpath�tag�taler�teamspeak�teapot�teapots�tel�teliaeid�telnet�tftp�things�thismessage�thzp�tip�tn3270�tool�turn�turns�tv�udp�unreal�upt�urn�ut2004�uuid-in-package�v-event�vemmi�ventrilo�ves�videotex�vnc�view-source�vscode�vscode-insiders�vsls�w3�wais�web3�wcr�webcal�web+ap�wifi�wpid�ws�wss�wtai�wyciwyg�xcon�xcon-userid�xfire�xmlrpc.beep�xmlrpc.beeps�xmpp�xftp�xrcp�xri�ymsgr�z39.50�z39.50r�z39.50s

synapse/lib/autodoc.py

@@ -286,11 +286,11 @@ def genCallsig(rtype):
         if defv is s_common.novalu:
             item = name
         elif defv is None:
-            item = f'{name}=$lib.null'
+            item = f'{name}=(null)'
         elif defv is True:
-            item = f'{name}=$lib.true'
+            item = f'{name}=(true)'
         elif defv is False:
-            item = f'{name}=$lib.false'
+            item = f'{name}=(false)'
         elif isinstance(defv, str):
             item = f'{name}={defv}'
         elif isinstance(defv, int):

synapse/lib/base.py

@@ -128,7 +128,7 @@ class Base:
 
         self.isfini = False
         self.anitted = True  # For assertion purposes
-        self.finievt = None
+        self.finievt = asyncio.Event()
         self.entered = False
 
         # hold a weak ref to other bases we should fini if they
@@ -431,10 +431,7 @@ class Base:
         self._syn_funcs.clear()
         self._fini_funcs.clear()
 
-        fevt = self.finievt
-
-        if fevt is not None:
-            fevt.set()
+        self.finievt.set()
 
         return 0
 
@@ -468,13 +465,6 @@ class Base:
             base.waitfini(timeout=30)
 
         '''
-
-        if self.isfini:
-            return True
-
-        if self.finievt is None:
-            self.finievt = asyncio.Event()
-
         return await s_coro.event_wait(self.finievt, timeout)
 
     def schedCoro(self, coro):

synapse/lib/cell.py

@@ -2711,6 +2711,12 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             raise s_exc.BadArg(mesg=mesg, arg='path', valu=path)
         await self._streamBackupArchive(path, user, name)
 
+    async def _removeStreamingBackup(self, path):
+        logger.debug(f'Removing {path}')
+        await s_coro.executor(shutil.rmtree, path, ignore_errors=True)
+        logger.debug(f'Removed {path}')
+        self.backupstreaming = False
+
     async def iterNewBackupArchive(self, user, name=None, remove=False):
 
         if self.backupstreaming:
@@ -2728,23 +2734,13 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 mesg = 'Backup with name already exists'
                 raise s_exc.BadArg(mesg=mesg)
 
-            try:
-                await self.runBackup(name)
-            except Exception:
-                if remove:
-                    logger.debug(f'Removing {path}')
-                    await s_coro.executor(shutil.rmtree, path, ignore_errors=True)
-                    logger.debug(f'Removed {path}')
-                raise
-
+            await self.runBackup(name)
             await self._streamBackupArchive(path, user, name)
 
         finally:
             if remove:
-                logger.debug(f'Removing {path}')
-                await s_coro.executor(shutil.rmtree, path, ignore_errors=True)
-                logger.debug(f'Removed {path}')
-                self.backupstreaming = False
+                self.removetask = asyncio.create_task(self._removeStreamingBackup(path))
+                await asyncio.shield(self.removetask)
 
     async def isUserAllowed(self, iden, perm, gateiden=None, default=False):
         user = self.auth.user(iden)  # type: s_auth.User

synapse/lib/cli.py

@@ -1,5 +1,4 @@
 import os
-import json
 import signal
 import asyncio
 import logging
@@ -18,6 +17,7 @@ import synapse.common as s_common
 import synapse.telepath as s_telepath
 
 import synapse.lib.base as s_base
+import synapse.lib.json as s_json
 import synapse.lib.output as s_output
 import synapse.lib.parser as s_parser
 import synapse.lib.grammar as s_grammar
@@ -542,5 +542,5 @@ class CmdLocals(Cmd):
                 ret[k] = v
             else:
                 ret[k] = repr(v)
-        mesg = json.dumps(ret, indent=2, sort_keys=True)
+        mesg = s_json.dumps(ret, indent=True, sort_keys=True).decode()
         self.printf(mesg)

synapse/lib/config.py

@@ -1,6 +1,5 @@
 import os
 import copy
-import json
 import urllib
 import logging
 import argparse
@@ -12,6 +11,7 @@ import synapse.exc as s_exc
 import synapse.data as s_data
 import synapse.common as s_common
 
+import synapse.lib.json as s_json
 import synapse.lib.hashitem as s_hashitem
 
 from fastjsonschema.exceptions import JsonSchemaValueException
@@ -44,7 +44,7 @@ def localSchemaRefHandler(uri):
         raise s_exc.NoSuchFile(mesg=f'Local JSON schema not found for {uri}.')
 
     with open(filename, 'r') as fp:
-        return json.load(fp)
+        return s_json.load(fp)
 
 # This handlers dictionary is used by the jsonschema validator to attempt to
 # resolve schema '$ref' values locally from disk and will raise an exception

synapse/lib/encoding.py

@@ -1,11 +1,12 @@
 import csv
-import json
 import regex
 import base64
 import codecs
 import xml.etree.ElementTree as x_etree
 
 import synapse.exc as s_exc
+
+import synapse.lib.json as s_json
 import synapse.lib.msgpack as s_msgpack
 
 def _de_base64(item, **opts):
@@ -176,11 +177,11 @@ def _fmt_lines(fd, gest):
         yield line
 
 def _fmt_json(fd, info):
-    yield json.loads(fd.read())
+    yield s_json.load(fd)
 
 def _fmt_jsonl(fd, info):
     for line in fd:
-        yield json.loads(line)
+        yield s_json.loads(line)
 
 def _fmt_mpk(fd, info):
     yield from s_msgpack.iterfd(fd)

synapse/lib/httpapi.py

@@ -1,4 +1,3 @@
-import json
 import base64
 import asyncio
 import logging
@@ -12,6 +11,7 @@ import synapse.exc as s_exc
 import synapse.common as s_common
 
 import synapse.lib.base as s_base
+import synapse.lib.json as s_json
 import synapse.lib.msgpack as s_msgpack
 
 logger = logging.getLogger(__name__)
@@ -132,7 +132,7 @@ class HandlerBase:
 
     def loadJsonMesg(self, byts, validator=None):
         try:
-            item = json.loads(byts)
+            item = s_json.loads(byts)
             if validator is not None:
                 validator(item)
             return item
@@ -418,7 +418,7 @@ class HandlerBase:
 class WebSocket(HandlerBase, t_websocket.WebSocketHandler):
 
     async def xmit(self, name, **info):
-        await self.write_message(json.dumps({'type': name, 'data': info}))
+        await self.write_message(s_json.dumps({'type': name, 'data': info}))
 
     async def _reqUserAllow(self, perm):
 
@@ -513,9 +513,7 @@ class StormNodesV1(StormHandler):
         await self.cell.boss.promote('storm', user=user, info=taskinfo)
 
         async for pode in view.iterStormPodes(query, opts=opts):
-            self.write(json.dumps(pode))
-            if jsonlines:
-                self.write("\n")
+            self.write(s_json.dumps(pode, newline=jsonlines))
             await self.flush()
 
 class StormV1(StormHandler):
@@ -545,9 +543,7 @@ class StormV1(StormHandler):
         opts.setdefault('editformat', 'nodeedits')
 
         async for mesg in self.getCore().storm(query, opts=opts):
-            self.write(json.dumps(mesg))
-            if jsonlines:
-                self.write("\n")
+            self.write(s_json.dumps(mesg, newline=jsonlines))
             await self.flush()
 
 class StormCallV1(StormHandler):
@@ -640,9 +636,9 @@ class BeholdSockV1(WebSocket):
 
     async def onInitMessage(self, byts):
         try:
-            mesg = json.loads(byts)
+            mesg = s_json.loads(byts)
             if mesg.get('type') != 'call:init':
-                raise s_exc.BadMesgFormat('Invalid initial message')
+                raise s_exc.BadMesgFormat(mesg='Invalid initial message')
 
             admin = await self.isUserAdmin()
             if not admin: