synapse 2.201.0__py311-none-any.whl → 2.203.0__py311-none-any.whl

synapse/tests/utils.py

@@ -21,13 +21,11 @@ import io
 import os
 import sys
 import copy
-import json
 import math
 import types
 import shutil
 import typing
 import asyncio
-import hashlib
 import inspect
 import logging
 import tempfile
@@ -59,6 +57,7 @@ import synapse.lib.cell as s_cell
 import synapse.lib.coro as s_coro
 import synapse.lib.cmdr as s_cmdr
 import synapse.lib.hive as s_hive
+import synapse.lib.json as s_json
 import synapse.lib.task as s_task
 import synapse.lib.const as s_const
 import synapse.lib.layer as s_layer
@@ -101,7 +100,7 @@ def deguidify(x):
 
 def jsonlines(text: str):
     lines = [k for k in text.split('\n') if k]
-    return [json.loads(line) for line in lines]
+    return [s_json.loads(line) for line in lines]
 
 async def waitForBehold(core, events):
     async for mesg in core.behold():
@@ -457,6 +456,7 @@ testmodel = {
             ('tick', ('test:time', {}), {}),
             ('hehe', ('str', {}), {}),
             ('ndefs', ('array', {'type': 'ndef'}), {}),
+            ('somestr', ('test:str', {}), {}),
         )),
 
         ('test:migr', {}, (

synapse/tools/autodoc.py

@@ -1,6 +1,5 @@
 import sys
 import copy
-import json
 import asyncio
 import logging
 import argparse
@@ -13,6 +12,7 @@ import synapse.common as s_common
 import synapse.cortex as s_cortex
 import synapse.telepath as s_telepath
 
+import synapse.lib.json as s_json
 import synapse.lib.storm as s_storm
 import synapse.lib.config as s_config
 import synapse.lib.output as s_output
@@ -262,7 +262,7 @@ def processTypes(rst, dochelp, types):
                         rst.addLines(f' * {key}: ``{valu}``')
                         continue
                     lines = [f' * {key}:\n', '  ::\n\n']
-                    json_lines = json.dumps(valu, indent=1, sort_keys=True)
+                    json_lines = s_json.dumps(valu, indent=True, sort_keys=True).decode()
                     json_lines = ['   ' + line for line in json_lines.split('\n')]
                     lines.extend(json_lines)
                     lines.append('\n')
@@ -810,7 +810,7 @@ async def docConfdefs(ctor):
                 data = {k: v for k, v in conf.items() if k not in (
                     'description', 'default', 'type', 'hideconf', 'hidecmdl',
                 )}
-                parts = json.dumps(data, sort_keys=True, indent=2).split('\n')
+                parts = s_json.dumps(data, sort_keys=True, indent=True).decode().split('\n')
                 lines.append('    ::')
                 lines.append('\n')
                 lines.extend([f'      {p}' for p in parts])

synapse/tools/changelog.py

@@ -476,7 +476,7 @@ async def gen(opts: argparse.Namespace,
     if opts.verbose:
         outp.printf('Validating data against schema')
 
-    s_schemas._reqChanglogSchema(data)
+    s_schemas._reqChangelogSchema(data)
 
     if opts.verbose:
         outp.printf('Saving the following information:')
@@ -835,7 +835,7 @@ async def format(opts: argparse.Namespace,
 
         files_processed.append(fp)
 
-        s_schemas._reqChanglogSchema(data)
+        s_schemas._reqChangelogSchema(data)
 
         data.setdefault('prs', [])
         prs = data.get('prs')

synapse/tools/cryo/cat.py

@@ -1,5 +1,4 @@
 import sys
-import json
 import pprint
 import asyncio
 import argparse
@@ -7,6 +6,7 @@ import logging
 
 import synapse.telepath as s_telepath
 
+import synapse.lib.json as s_json
 import synapse.lib.output as s_output
 import synapse.lib.msgpack as s_msgpack
 
@@ -50,14 +50,14 @@ async def main(argv, outp=s_output.stdout):
                     await tank.puts(items)
                     return 0
 
-                items = [json.loads(line) for line in sys.stdin]
+                items = [s_json.loads(line) for line in sys.stdin]
                 await tank.puts(items)
                 return 0
 
             async for item in tank.slice(opts.offset, opts.size):
 
                 if opts.jsonl:
-                    outp.printf(json.dumps(item[1], sort_keys=True))
+                    outp.printf(s_json.dumps(item[1], sort_keys=True).decode())
 
                 elif opts.msgpack:
                     sys.stdout.buffer.write(s_msgpack.en(item[1]))

synapse/tools/csvtool.py

@@ -1,6 +1,5 @@
 import csv
 import sys
-import json
 import asyncio
 import contextlib
 
@@ -12,6 +11,7 @@ import synapse.telepath as s_telepath
 import synapse.lib.cmd as s_cmd
 import synapse.lib.base as s_base
 import synapse.lib.cmdr as s_cmdr
+import synapse.lib.json as s_json
 import synapse.lib.output as s_output
 import synapse.lib.version as s_version
 
@@ -114,8 +114,7 @@ async def runCsvImport(opts, outp, text, stormopts):
                     outp.printf(repr(mesg))
 
                 if logfd is not None:
-                    byts = json.dumps(mesg).encode('utf8')
-                    logfd.write(byts + b'\n')
+                    logfd.write(s_json.dumps(mesg, newline=True))
 
         if opts.cli:
             await s_cmdr.runItemCmdr(core, outp, True)

synapse/tools/docker/validate.py

@@ -1,7 +1,6 @@
 import os
 import re
 import sys
-import json
 import base64
 import pprint
 import argparse
@@ -11,6 +10,7 @@ import synapse.exc as s_exc
 import synapse.data as s_data
 import synapse.common as s_common
 
+import synapse.lib.json as s_json
 import synapse.lib.output as s_outp
 import synapse.lib.certdir as s_certdir
 
@@ -50,8 +50,8 @@ def getCosignSignature(outp, image):
 
     blob = proc.stdout
     try:
-        sigd = json.loads(blob)
-    except json.JSONDecodeError as e:
+        sigd = s_json.loads(blob)
+    except s_exc.BadJsonText as e:
         outp.printf(f'Error decoding blob: {blob}: {e}')
         return None
     if not isinstance(sigd, dict):
@@ -104,8 +104,8 @@ def checkCosignSignature(outp, pubk_byts, image_to_verify):
         except subprocess.CalledProcessError as e:  # pragma: no cover
             outp.printf(f'Error calling {" ".join(args)}: {e}')
             return None
-        blob = json.loads(proc.stdout.decode())
-        outp.printf(f'Cosign output:')
+        blob = s_json.loads(proc.stdout)
+        outp.printf('Cosign output:')
         outp.printf(pprint.pformat(blob))
         return True
 

synapse/tools/feed.py

@@ -11,6 +11,7 @@ import synapse.cortex as s_cortex
 import synapse.telepath as s_telepath
 
 import synapse.lib.cmdr as s_cmdr
+import synapse.lib.json as s_json
 import synapse.lib.output as s_output
 import synapse.lib.msgpack as s_msgpack
 import synapse.lib.version as s_version
@@ -24,7 +25,7 @@ def getItems(*paths):
     items = []
     for path in paths:
         if path.endswith('.json'):
-            item = s_common.jsload(path)
+            item = s_json.jsload(path)
             if not isinstance(item, list):
                 item = [item]
             items.append((path, item))

synapse/tools/genpkg.py

@@ -12,6 +12,7 @@ import synapse.exc as s_exc
 import synapse.common as s_common
 import synapse.telepath as s_telepath
 
+import synapse.lib.json as s_json
 import synapse.lib.output as s_output
 import synapse.lib.certdir as s_certdir
 import synapse.lib.dyndeps as s_dyndeps
@@ -224,7 +225,7 @@ def loadPkgProto(path, opticdir=None, no_docs=False, readonly=False):
     s_schemas.reqValidPkgdef(pkgdef)
 
     # Ensure the package is json safe and tuplify it.
-    s_common.reqJsonSafeStrict(pkgdef)
+    s_json.reqjsonsafe(pkgdef, strict=True)
     pkgdef = s_common.tuplify(pkgdef)
     return pkgdef
 
@@ -286,7 +287,7 @@ async def main(argv, outp=s_output.stdout):
         return 1
 
     if opts.save:
-        s_common.jssave(pkgdef, opts.save)
+        s_json.jssave(pkgdef, opts.save)
 
     if opts.push:
 

synapse/tools/healthcheck.py

@@ -1,5 +1,4 @@
 import sys
-import json
 import socket
 import asyncio
 
@@ -8,13 +7,13 @@ import synapse.common as s_common
 import synapse.telepath as s_telepath
 
 import synapse.lib.cmd as s_cmd
+import synapse.lib.json as s_json
 import synapse.lib.output as s_output
 import synapse.lib.health as s_health
 import synapse.lib.urlhelp as s_urlhelp
 
 def serialize(ret):
-    s = json.dumps(ret, separators=(',', ':'))
-    return s
+    return s_json.dumps(ret).decode()
 
 def format_component(e, mesg: str) -> dict:
     d = {

synapse/tools/json2mpk.py

@@ -1,8 +1,8 @@
 import os
 import sys
-import json
 import argparse
 
+import synapse.lib.json as s_json
 import synapse.lib.output as s_output
 import synapse.lib.msgpack as s_msgpack
 
@@ -37,7 +37,7 @@ def main(argv, outp=None):
         with open(path, 'r', encoding='utf8') as fd:
             with open(newp, 'wb') as pk:
                 for line in fd:
-                    item = json.loads(line)
+                    item = s_json.loads(line)
                     pk.write(s_msgpack.en(item))
 
         if opts.rm:

synapse/utils/getrefs.py

@@ -1,5 +1,4 @@
 import sys
-import json
 import urllib
 import asyncio
 import logging
@@ -12,10 +11,13 @@ import synapse.exc as s_exc
 import synapse.data as s_data
 import synapse.common as s_common
 
+import synapse.lib.json as s_json
 import synapse.lib.config as s_config
 
 logger = logging.getLogger(__name__)
 
+BASEDIR = s_data.path('jsonschemas')
+
 def download_refs_handler(uri):
     '''
     This function downloads the JSON schema at the given URI, parses the given
@@ -34,18 +36,18 @@ async def _download_refs_handler(uri):
     except ValueError:
         raise s_exc.BadUrl(mesg=f'Malformed URI: {uri}.') from None
 
-    filename = s_data.path('jsonschemas', parts.hostname, *parts.path.split('/'))
+    filename = s_common.genpath(BASEDIR, parts.hostname, *parts.path.split('/'))
     filepath = pathlib.Path(filename)
 
     # Check for path traversal. Unlikely, but still check
-    if not str(filepath.absolute()).startswith(s_data.path('jsonschemas')):
+    if not str(filepath.absolute()).startswith(BASEDIR):
         raise s_exc.BadArg(mesg=f'Path traversal in schema URL: {uri} ?')
 
     # If we already have the file, return it
     if filepath.exists():
         logger.info(f'Schema {uri} already exists in local cache, skipping.')
         with filepath.open() as fp:
-            return json.load(fp)
+            return s_json.load(fp)
 
     # Create parent directory structure if it doesn't already exist
     filepath.parent.mkdir(parents=True, exist_ok=True)
@@ -57,11 +59,11 @@ async def _download_refs_handler(uri):
             resp.raise_for_status()
             buf = await resp.read()
 
-    data = json.loads(buf.decode())
+    data = s_json.loads(buf)
 
     # Save the json schema to disk
-    with filepath.open('w') as fp:
-        json.dump(data, fp, indent=2)
+    with filepath.open('wb') as fp:
+        s_json.dump(data, fp, indent=True)
 
     # Return the schema to satisfy fastjsonschema
     return data
@@ -76,7 +78,7 @@ def download_refs(schema):
 
 def main(argv):
     with argv.schema.open() as fp:
-        schema = json.load(fp)
+        schema = s_json.load(fp)
 
     download_refs(schema)
 

synapse/vendor/cpython/lib/json.py

@@ -0,0 +1,35 @@
+##############################################################################
+# Taken from the cpython 3.11 source branch after the 3.11.10 release.
+##############################################################################
+import codecs
+
+# vendored from:
+# https://github.com/python/cpython/blob/v3.11.10/Lib/json/__init__.py#L244-L271
+def detect_encoding(b):
+    bstartswith = b.startswith
+    if bstartswith((codecs.BOM_UTF32_BE, codecs.BOM_UTF32_LE)):
+        return 'utf-32'
+    if bstartswith((codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE)):
+        return 'utf-16'
+    if bstartswith(codecs.BOM_UTF8):
+        return 'utf-8-sig'
+
+    if len(b) >= 4:
+        if not b[0]:
+            # 00 00 -- -- - utf-32-be
+            # 00 XX -- -- - utf-16-be
+            return 'utf-16-be' if b[1] else 'utf-32-be'
+        if not b[1]:
+            # XX 00 00 00 - utf-32-le
+            # XX 00 00 XX - utf-16-le
+            # XX 00 XX -- - utf-16-le
+            return 'utf-16-le' if b[2] or b[3] else 'utf-32-le'
+    elif len(b) == 2:
+        if not b[0]:
+            # 00 XX - utf-16-be
+            return 'utf-16-be'
+        if not b[1]:
+            # XX 00 - utf-16-le
+            return 'utf-16-le'
+    # default
+    return 'utf-8'

synapse/vendor/cpython/lib/test/test_json.py

@@ -0,0 +1,22 @@
+# From the python source, tests/test_json/test_unicode.py does not have a
+# direct test for detect_encoding. It instead tests the various encoding
+# schemes in test_bytes_decode, which uses the detection in json.loads().
+# Since there is not a standalone test to vendor, we have written a simple
+# test on its own.
+
+import synapse.vendor.cpython.lib.json as v_json
+
+import synapse.vendor.utils as s_v_utils
+
+class JsonVendorTest(s_v_utils.VendorTest):
+    def test_json_detect_encoding(self):
+
+        ENCODINGS = (
+            'utf-8', 'utf-8-sig',
+            'utf-16', 'utf-16-le', 'utf-16-be',
+            'utf-32', 'utf-32-le', 'utf-32-be',
+        )
+
+        for encoding in ENCODINGS:
+            self.assertEqual(encoding, v_json.detect_encoding('a'.encode(encoding)))
+            self.assertEqual(encoding, v_json.detect_encoding('ab'.encode(encoding)))

{synapse-2.201.0.dist-info → synapse-2.203.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: synapse
-Version: 2.201.0
+Version: 2.203.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License: Apache License 2.0
@@ -50,6 +50,7 @@ Requires-Dist: idna<3.8,>=3.6
 Requires-Dist: python-dateutil<3.0,>=2.8
 Requires-Dist: pytz<2024.1,>=2023.3
 Requires-Dist: beautifulsoup4[html5lib]<5.0,>=4.11.1
+Requires-Dist: orjson<4.0,>=3.10.15
 Provides-Extra: dev
 Requires-Dist: pytest<8.0.0,>=7.2.0; extra == "dev"
 Requires-Dist: autopep8<3.0.0,>=2.0.4; extra == "dev"