synapse 2.202.0__py311-none-any.whl → 2.204.0__py311-none-any.whl

synapse/telepath.py

@@ -394,7 +394,7 @@ class Share(s_base.Base):
             This should never be used by synapse core code.  This is for sync client code convenience only.
         '''
         if s_threads.iden() == self.tid:
-            raise s_exc.SynErr('Use of synchronous context manager in async code')
+            raise s_exc.SynErr(mesg='Use of synchronous context manager in async code')
 
         self._ctxobj = self.schedCoroSafePend(self.__aenter__())
         return self
@@ -593,7 +593,7 @@ class Proxy(s_base.Base):
 
     '''
     _link_task = None
-    _link_event = asyncio.Event()
+    _link_event = None
     _all_proxies = set()
 
     async def __anit__(self, link, name):
@@ -650,6 +650,7 @@ class Proxy(s_base.Base):
             if not Proxy._all_proxies and Proxy._link_task is not None:
                 Proxy._link_task.cancel()
                 Proxy._link_task = None
+                Proxy._link_event = None
 
         Proxy._all_proxies.add(self)
 
@@ -657,6 +658,7 @@ class Proxy(s_base.Base):
         self.link.onfini(self.fini)
 
         if Proxy._link_task is None:
+            Proxy._link_event = asyncio.Event()
             Proxy._link_task = s_coro.create_task(Proxy._linkLoopTask())
 
     @classmethod
@@ -821,7 +823,7 @@ class Proxy(s_base.Base):
             This must not be used from async code, and it should never be used in core synapse code.
         '''
         if s_threads.iden() == self.tid:
-            raise s_exc.SynErr('Use of synchronous context manager in async code')
+            raise s_exc.SynErr(mesg='Use of synchronous context manager in async code')
         self._ctxobj = self.schedCoroSafePend(self.__aenter__())
         return self
 

synapse/tests/files/__init__.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # Stdlib
 import os
-import json
 import logging
 
 log = logging.getLogger(__name__)

synapse/tests/test_axon.py

@@ -103,7 +103,7 @@ class HttpPushFile(s_httpapi.StreamHandler):
         assert item['filename'] == 'bytes'
 
         assert args.get('zip_password') == [b'test']
-        assert args.get('dict') == [b'{"foo": "bar"}']
+        assert args.get('dict') == [b'{"foo":"bar"}']
         self.sendRestRetn(self.gotsize)
 
 class AxonTest(s_t_utils.SynTest):

synapse/tests/test_cmds_cortex.py

@@ -4,6 +4,7 @@ import asyncio
 import synapse.common as s_common
 
 import synapse.lib.cmdr as s_cmdr
+import synapse.lib.json as s_json
 import synapse.lib.encoding as s_encoding
 import synapse.lib.lmdbslab as s_lmdbslab
 
@@ -352,7 +353,7 @@ class CmdCoreTest(s_t_utils.SynTest):
             await cmdr.runCmdLine(f'storm --save-nodes {path} test:int')
             outp.expect('2 nodes')
 
-            jsdata = [item for item in s_common.jslines(path)]
+            jsdata = [item for item in s_json.jslines(path)]
             self.len(2, jsdata)
             self.eq({tuple(n[0]) for n in jsdata},
                     {('test:int', 20), ('test:int', 30)})
@@ -371,7 +372,7 @@ class CmdCoreTest(s_t_utils.SynTest):
             with s_common.genfile(stormfile) as fd:
                 fd.write(b'[ inet:fqdn=$hehe ]')
 
-            s_common.jssave(test_opts, optsfile)
+            s_json.jssave(test_opts, optsfile)
             s_common.yamlsave(test_opts, optsfile_yaml)
 
             outp = self.getTestOutp()

synapse/tests/test_cmds_hive.py

@@ -5,10 +5,10 @@ import synapse.lib.cmdr as s_cmdr
 import synapse.tests.utils as s_t_utils
 
 _json_output = '''[
-    1,
-    2,
-    3,
-    4
+  1,
+  2,
+  3,
+  4
 ]'''
 
 class CmdHiveTest(s_t_utils.SynTest):

synapse/tests/test_common.py

@@ -1,12 +1,17 @@
 import os
+import http
 import asyncio
 import logging
 import subprocess
 
 import yaml
+import aiohttp
 
 import synapse.exc as s_exc
 import synapse.common as s_common
+
+import synapse.lib.httpapi as s_httpapi
+
 import synapse.tests.utils as s_t_utils
 
 logger = logging.getLogger(__name__)
@@ -426,25 +431,6 @@ class CommonTest(s_t_utils.SynTest):
         retn = s_common.merggenr2([asyncl(lt) for lt in (l3, l2, l1)], reverse=True)
         self.eq((9, 8, 7, 6, 5, 4, 3, 2, 1), await alist(retn))
 
-    def test_jsonsafe(self):
-        items = (
-            (None, None),
-            (1234, None),
-            ('1234', None),
-            ({'asdf': 'haha'}, None),
-            ({'a': (1,), 'b': [{'': 4}, 56, None, {'t': True, 'f': False}, 'oh my']}, None),
-            (b'1234', s_exc.BadArg),
-            ({'a': 'a', 2: 2}, s_exc.BadArg),
-            ({'a', 'b', 'c'}, s_exc.BadArg),
-            (s_common.novalu, s_exc.BadArg),
-        )
-        for (item, eret) in items:
-            if eret is None:
-                self.none(s_common.reqJsonSafeStrict(item))
-            else:
-                with self.raises(eret):
-                    s_common.reqJsonSafeStrict(item)
-
     def test_sslctx(self):
         with self.getTestDir(mirror='certdir') as dirn:
             cadir = s_common.genpath(dirn, 'cas')
@@ -481,3 +467,27 @@ class CommonTest(s_t_utils.SynTest):
             v = s_common.trimText(iv, n=n)
             self.le(len(v), n)
             self.eq(v, ev)
+
+    async def test_tornado_monkeypatch(self):
+        class JsonHandler(s_httpapi.Handler):
+            async def get(self):
+                resp = {
+                    'foo': 'bar',
+                    'html': '<html></html>'
+                }
+                self.write(resp)
+
+        async with self.getTestCore() as core:
+            core.addHttpApi('/api/v1/test_tornado/', JsonHandler, {'cell': core})
+            _, port = await core.addHttpsPort(0)
+            url = f'https://127.0.0.1:{port}/api/v1/test_tornado/'
+
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url, ssl=False) as resp:
+                    self.eq(resp.status, http.HTTPStatus.OK)
+
+                    text = await resp.text()
+                    self.eq(text, '{"foo":"bar","html":"<html><\\/html>"}')
+
+                    json = await resp.json()
+                    self.eq(json, {'foo': 'bar', 'html': '<html></html>'})

synapse/tests/test_cortex.py

@@ -1559,8 +1559,8 @@ class CortexTest(s_t_utils.SynTest):
             self.eq(refs.get('fqdn'), ('inet:fqdn', 'woot.com'))
             self.eq(refs.get('ipv4'), ('inet:ipv4', 0x01020304))
 
-            self.len(1, await core.nodes('[meta:seen=($sorc, $valu)]',
-                                         opts={'vars': {'sorc': sorc, 'valu': node.ndef}}))
+            self.len(1, await core.nodes('[test:str=testndef :somestr=$somestr :bar=$valu]',
+                                         opts={'vars': {'somestr': sorc, 'valu': node.ndef}}))
 
             # test un-populated properties
             nodes = await core.nodes('[ps:contact="*"]')
@@ -1592,13 +1592,13 @@ class CortexTest(s_t_utils.SynTest):
             self.eq(ints, (('test:int', 1), ('test:int', 2), ('test:int', 3)))
 
             opts = {'vars': {'sorc': sorc}}
-            nodes = await core.nodes('meta:seen:source=$sorc -> *', opts=opts)
+            nodes = await core.nodes('test:str:somestr=$sorc -> *', opts=opts)
 
             self.len(2, nodes)
             self.isin('inet:dns:a', {n.ndef[0] for n in nodes})
 
             opts = {'vars': {'sorc': sorc}}
-            nodes = await core.nodes('meta:seen:source=$sorc :node -> *', opts=opts)
+            nodes = await core.nodes('test:str:somestr=$sorc :bar -> *', opts=opts)
 
             self.len(1, nodes)
             self.eq('inet:dns:a', nodes[0].ndef[0])
@@ -3955,7 +3955,7 @@ class CortexBasicTest(s_t_utils.SynTest):
                 'refs': True,
                 'edges': False,
                 'forms': {},
-                'pivots': ['<- meta:seen'],
+                'pivots': ['<(seen)- meta:source'],
                 'degrees': 3,
                 'filters': ['+#nope'],
                 'filterinput': False,

synapse/tests/test_lib_ast.py

@@ -1,4 +1,3 @@
-import json
 import math
 import asyncio
 
@@ -9,6 +8,7 @@ import synapse.common as s_common
 import synapse.datamodel as s_datamodel
 
 import synapse.lib.ast as s_ast
+import synapse.lib.json as s_json
 import synapse.lib.snap as s_snap
 
 import synapse.tests.utils as s_test
@@ -1523,7 +1523,7 @@ class AstTest(s_test.SynTest):
             self.stormIsInPrint('stormpkg', msgs)
 
             # Make sure a JSON package loads
-            jsonpkg = json.loads(json.dumps(jsonpkg))
+            jsonpkg = s_json.loads(s_json.dumps(jsonpkg))
             await core.stormlist('$lib.pkg.add($pkg)',
                                  opts={'vars': {'pkg': jsonpkg}})
             msgs = await core.stormlist('pkg.list')
@@ -1731,7 +1731,7 @@ class AstTest(s_test.SynTest):
             prints = list(filter(lambda m: m[0] == 'print', msgs))
             self.eq(len(prints), 3)
 
-            jmsgs = list(map(lambda m: json.loads(m[1]['mesg']), prints))
+            jmsgs = list(map(lambda m: s_json.loads(m[1]['mesg']), prints))
             omsgs = sorted(jmsgs, key=lambda m: m[0])
             self.eq(omsgs[0][1], 'this should be first')
             self.eq(omsgs[1][1], 'toreturn called')

synapse/tests/test_lib_autodoc.py

@@ -56,7 +56,7 @@ shave'''
             )
         }
         callsig = s_autodoc.genCallsig(rtype)
-        self.eq(callsig, '(foo, bar=$lib.null, **kwargs)')
+        self.eq(callsig, '(foo, bar=(null), **kwargs)')
         self.eq(s_autodoc.genCallsig({}), '()')
         self.eq(s_autodoc.genCallsig({'args': ({'name': 'beep'},)}), '(beep)')
 
@@ -176,8 +176,8 @@ Returns:
 
 .. _test-lib-test-someargs:
 
-someargs(valu, bar=$lib.true, faz=$lib.null)
-============================================
+someargs(valu, bar=(true), faz=(null))
+======================================
 
 .. warning::
    ``$lib.test.someargs`` has been deprecated and will be removed in version v3.0.0.
@@ -252,8 +252,8 @@ Returns:
 
 .. _test-lib-test-someargs:
 
-$lib.test.someargs(valu, bar=$lib.true, faz=$lib.null)
-======================================================
+$lib.test.someargs(valu, bar=(true), faz=(null))
+================================================
 
 .. warning::
    ``$lib.test.someargs`` has been deprecated and will be removed in version v3.0.0.

synapse/tests/test_lib_base.py

@@ -483,7 +483,7 @@ class BaseTest(s_t_utils.SynTest):
         async def badgenr():
             yield 'foo'
             await asyncio.sleep(0)
-            raise s_exc.SynErr('rando')
+            raise s_exc.SynErr(mesg='rando')
 
         async def slowgenr():
             yield 'foo'

synapse/tests/test_lib_cell.py

@@ -1,7 +1,6 @@
 import os
 import ssl
 import sys
-import json
 import time
 import base64
 import signal
@@ -26,6 +25,7 @@ import synapse.lib.auth as s_auth
 import synapse.lib.base as s_base
 import synapse.lib.cell as s_cell
 import synapse.lib.coro as s_coro
+import synapse.lib.json as s_json
 import synapse.lib.link as s_link
 import synapse.lib.drive as s_drive
 import synapse.lib.nexus as s_nexus
@@ -1822,10 +1822,13 @@ class CellTest(s_t_utils.SynTest):
 
                     async def streamdone():
                         while core.backupstreaming:
-                            await asyncio.sleep(0)
+                            await asyncio.sleep(0.1)
 
                     task = core.schedCoro(streamdone())
-                    await asyncio.wait_for(task, 15)
+                    try:
+                        await asyncio.wait_for(task, 5)
+                    except TimeoutError:
+                        raise TimeoutError('Timeout waiting for streaming backup cleanup of bkup3 to complete.')
 
                     self.eq(('bkup', 'bkup2'), sorted(await proxy.getBackups()))
                     self.false(os.path.isdir(os.path.join(backdirn, 'bkup3')))
@@ -1839,7 +1842,10 @@ class CellTest(s_t_utils.SynTest):
                             bkup4.write(msg)
 
                     task = core.schedCoro(streamdone())
-                    await asyncio.wait_for(task, 15)
+                    try:
+                        await asyncio.wait_for(task, 5)
+                    except TimeoutError:
+                        raise TimeoutError('Timeout waiting for streaming backup cleanup of bkup4 to complete.')
 
                     self.eq(('bkup', 'bkup2'), sorted(await proxy.getBackups()))
 
@@ -3128,13 +3134,13 @@ class CellTest(s_t_utils.SynTest):
                 self.nn(node.getTagProp('test', 'score'), 6)
 
                 self.maxDiff = None
-                roles = s_t_utils.deguidify('[{"type": "role", "iden": "e1ef725990aa62ae3c4b98be8736d89f", "name": "all", "rules": [], "authgates": {"46cfde2c1682566602860f8df7d0cc83": {"rules": [[true, ["layer", "read"]]]}, "4d50eb257549436414643a71e057091a": {"rules": [[true, ["view", "read"]]]}}}]')
-                users = s_t_utils.deguidify('[{"type": "user", "iden": "a357138db50780b62093a6ce0d057fd8", "name": "root", "rules": [], "roles": [], "admin": true, "email": null, "locked": false, "archived": false, "authgates": {"46cfde2c1682566602860f8df7d0cc83": {"admin": true}, "4d50eb257549436414643a71e057091a": {"admin": true}}}, {"type": "user", "iden": "f77ac6744671a845c27e571071877827", "name": "visi", "rules": [[true, ["cron", "add"]], [true, ["dmon", "add"]], [true, ["trigger", "add"]]], "roles": [{"type": "role", "iden": "e1ef725990aa62ae3c4b98be8736d89f", "name": "all", "rules": [], "authgates": {"46cfde2c1682566602860f8df7d0cc83": {"rules": [[true, ["layer", "read"]]]}, "4d50eb257549436414643a71e057091a": {"rules": [[true, ["view", "read"]]]}}}], "admin": false, "email": null, "locked": false, "archived": false, "authgates": {"f21b7ae79c2dacb89484929a8409e5d8": {"admin": true}, "d7d0380dd4e743e35af31a20d014ed48": {"admin": true}}}]')
-                gates = s_t_utils.deguidify('[{"iden": "46cfde2c1682566602860f8df7d0cc83", "type": "layer", "users": [{"iden": "a357138db50780b62093a6ce0d057fd8", "rules": [], "admin": true}], "roles": [{"iden": "e1ef725990aa62ae3c4b98be8736d89f", "rules": [[true, ["layer", "read"]]], "admin": false}]}, {"iden": "d7d0380dd4e743e35af31a20d014ed48", "type": "trigger", "users": [{"iden": "f77ac6744671a845c27e571071877827", "rules": [], "admin": true}], "roles": []}, {"iden": "f21b7ae79c2dacb89484929a8409e5d8", "type": "cronjob", "users": [{"iden": "f77ac6744671a845c27e571071877827", "rules": [], "admin": true}], "roles": []}, {"iden": "4d50eb257549436414643a71e057091a", "type": "view", "users": [{"iden": "a357138db50780b62093a6ce0d057fd8", "rules": [], "admin": true}], "roles": [{"iden": "e1ef725990aa62ae3c4b98be8736d89f", "rules": [[true, ["view", "read"]]], "admin": false}]}, {"iden": "cortex", "type": "cortex", "users": [], "roles": []}]')
+                roles = s_t_utils.deguidify('[{"type":"role","iden":"e1ef725990aa62ae3c4b98be8736d89f","name":"all","rules":[],"authgates":{"46cfde2c1682566602860f8df7d0cc83":{"rules":[[true,["layer","read"]]]},"4d50eb257549436414643a71e057091a":{"rules":[[true,["view","read"]]]}}}]')
+                users = s_t_utils.deguidify('[{"type":"user","iden":"a357138db50780b62093a6ce0d057fd8","name":"root","rules":[],"roles":[],"admin":true,"email":null,"locked":false,"archived":false,"authgates":{"46cfde2c1682566602860f8df7d0cc83":{"admin":true},"4d50eb257549436414643a71e057091a":{"admin":true}}},{"type":"user","iden":"f77ac6744671a845c27e571071877827","name":"visi","rules":[[true,["cron","add"]],[true,["dmon","add"]],[true,["trigger","add"]]],"roles":[{"type":"role","iden":"e1ef725990aa62ae3c4b98be8736d89f","name":"all","rules":[],"authgates":{"46cfde2c1682566602860f8df7d0cc83":{"rules":[[true,["layer","read"]]]},"4d50eb257549436414643a71e057091a":{"rules":[[true,["view","read"]]]}}}],"admin":false,"email":null,"locked":false,"archived":false,"authgates":{"f21b7ae79c2dacb89484929a8409e5d8":{"admin":true},"d7d0380dd4e743e35af31a20d014ed48":{"admin":true}}}]')
+                gates = s_t_utils.deguidify('[{"iden":"46cfde2c1682566602860f8df7d0cc83","type":"layer","users":[{"iden":"a357138db50780b62093a6ce0d057fd8","rules":[],"admin":true}],"roles":[{"iden":"e1ef725990aa62ae3c4b98be8736d89f","rules":[[true,["layer","read"]]],"admin":false}]},{"iden":"d7d0380dd4e743e35af31a20d014ed48","type":"trigger","users":[{"iden":"f77ac6744671a845c27e571071877827","rules":[],"admin":true}],"roles":[]},{"iden":"f21b7ae79c2dacb89484929a8409e5d8","type":"cronjob","users":[{"iden":"f77ac6744671a845c27e571071877827","rules":[],"admin":true}],"roles":[]},{"iden":"4d50eb257549436414643a71e057091a","type":"view","users":[{"iden":"a357138db50780b62093a6ce0d057fd8","rules":[],"admin":true}],"roles":[{"iden":"e1ef725990aa62ae3c4b98be8736d89f","rules":[[true,["view","read"]]],"admin":false}]},{"iden":"cortex","type":"cortex","users":[],"roles":[]}]')
 
-                self.eq(roles, s_t_utils.deguidify(json.dumps(await core.callStorm('return($lib.auth.roles.list())'))))
-                self.eq(users, s_t_utils.deguidify(json.dumps(await core.callStorm('return($lib.auth.users.list())'))))
-                self.eq(gates, s_t_utils.deguidify(json.dumps(await core.callStorm('return($lib.auth.gates.list())'))))
+                self.eq(roles, s_t_utils.deguidify(s_json.dumps(await core.callStorm('return($lib.auth.roles.list())')).decode()))
+                self.eq(users, s_t_utils.deguidify(s_json.dumps(await core.callStorm('return($lib.auth.users.list())')).decode()))
+                self.eq(gates, s_t_utils.deguidify(s_json.dumps(await core.callStorm('return($lib.auth.gates.list())')).decode()))
 
                 with self.raises(s_exc.BadTypeValu):
                     await core.nodes('[ it:dev:str=foo +#test.newp ]')

synapse/tests/test_lib_config.py

@@ -1,5 +1,4 @@
 import copy
-import json
 import regex
 import pathlib
 import argparse
@@ -11,6 +10,7 @@ import synapse.data as s_data
 import synapse.common as s_common
 
 import synapse.lib.cell as s_cell
+import synapse.lib.json as s_json
 import synapse.lib.config as s_config
 
 import synapse.tests.utils as s_test
@@ -396,7 +396,7 @@ class ConfTest(s_test.SynTest):
         self.true(filename.exists())
 
         with filename.open() as fp:
-            schema = json.load(fp)
+            schema = s_json.load(fp)
 
         self.eq(schema, s_config.localSchemaRefHandler(
             "http://raw.githubusercontent.com/oasis-open/cti-stix2-json-schemas/stix2.1/schemas/common/core.json"

synapse/tests/test_lib_encoding.py

@@ -1,8 +1,8 @@
 import io
-import json
 
 import synapse.common as s_common
 
+import synapse.lib.json as s_json
 import synapse.lib.msgpack as s_msgpack
 import synapse.lib.encoding as s_encoding
 
@@ -190,7 +190,7 @@ class EncTest(s_t_utils.SynTest):
 
         }
 
-        buf = io.BytesIO(json.dumps(data).encode())
+        buf = io.BytesIO(s_json.dumps(data))
 
         lines = list(s_encoding.iterdata(buf, format='json'))
         self.len(1, lines)