synapse 2.201.0__py311-none-any.whl → 2.203.0__py311-none-any.whl

synapse/tests/test_lib_node.py

@@ -1,9 +1,9 @@
-import json
 import collections
 
 import synapse.exc as s_exc
 import synapse.common as s_common
 
+import synapse.lib.json as s_json
 import synapse.lib.node as s_node
 
 import synapse.tests.utils as s_t_utils
@@ -220,7 +220,7 @@ class NodeTest(s_t_utils.SynTest):
                     async for byts, x in resp.content.iter_chunks():
                         if not byts:
                             break
-                        mesg = json.loads(byts)
+                        mesg = s_json.loads(byts)
                         if mesg[0] == 'node':
                             https_nodes.append(mesg[1])
 

synapse/tests/test_lib_scrape.py

@@ -1187,3 +1187,53 @@ class ScrapeTest(s_t_utils.SynTest):
         nodes.remove(('it:sec:cpe', 'cpe:2.3:*:*why*:*:*:*:*:*:*:*:*:*'))
 
         self.len(0, nodes)
+
+    def test_scrape_uris(self):
+
+        nodes = list(s_scrape.scrape('http://vertex.link https://woot.com'))
+        self.len(4, nodes)
+        self.isin(('inet:url', 'http://vertex.link'), nodes)
+        self.isin(('inet:url', 'https://woot.com'), nodes)
+        self.isin(('inet:fqdn', 'vertex.link'), nodes)
+        self.isin(('inet:fqdn', 'woot.com'), nodes)
+
+        nodes = list(s_scrape.scrape('ftps://files.vertex.link tcp://1.2.3.4:8080'))
+        self.len(5, nodes)
+        self.isin(('inet:url', 'ftps://files.vertex.link'), nodes)
+        self.isin(('inet:url', 'tcp://1.2.3.4:8080'), nodes)
+        self.isin(('inet:fqdn', 'files.vertex.link'), nodes)
+        self.isin(('inet:ipv4', '1.2.3.4'), nodes)
+        self.isin(('inet:server', '1.2.3.4:8080'), nodes)
+
+        nodes = list(s_scrape.scrape('invalidscheme://vertex.link newp://woot.com'))
+        self.len(2, nodes)
+        self.isin(('inet:fqdn', 'vertex.link'), nodes)
+        self.isin(('inet:fqdn', 'woot.com'), nodes)
+
+        nodes = list(s_scrape.scrape('[https://vertex.link] (http://woot.com)'))
+        self.len(4, nodes)
+        self.isin(('inet:url', 'https://vertex.link'), nodes)
+        self.isin(('inet:url', 'http://woot.com'), nodes)
+        self.isin(('inet:fqdn', 'vertex.link'), nodes)
+        self.isin(('inet:fqdn', 'woot.com'), nodes)
+
+        nodes = list(s_scrape.scrape('HTTP://vertex.link HTTPS://woot.com'))
+        self.len(4, nodes)
+        self.isin(('inet:url', 'HTTP://vertex.link'), nodes)
+        self.isin(('inet:url', 'HTTPS://woot.com'), nodes)
+        self.isin(('inet:fqdn', 'vertex.link'), nodes)
+        self.isin(('inet:fqdn', 'woot.com'), nodes)
+
+        nodes = list(s_scrape.scrape('hxxps://vertex[.]link hXXp://woot[.]com'))
+        self.len(4, nodes)
+        self.isin(('inet:url', 'https://vertex.link'), nodes)
+        self.isin(('inet:url', 'http://woot.com'), nodes)
+        self.isin(('inet:fqdn', 'vertex.link'), nodes)
+        self.isin(('inet:fqdn', 'woot.com'), nodes)
+
+        nodes = list(s_scrape.scrape('hxxp[:]//vertex(.)link hXXps[://]woot[.]com'))
+        self.len(4, nodes)
+        self.isin(('inet:url', 'http://vertex.link'), nodes)
+        self.isin(('inet:url', 'https://woot.com'), nodes)
+        self.isin(('inet:fqdn', 'vertex.link'), nodes)
+        self.isin(('inet:fqdn', 'woot.com'), nodes)

synapse/tests/test_lib_storm.py

@@ -1,5 +1,4 @@
 import copy
-import json
 import asyncio
 import datetime
 import itertools
@@ -13,6 +12,7 @@ import synapse.datamodel as s_datamodel
 
 import synapse.lib.base as s_base
 import synapse.lib.coro as s_coro
+import synapse.lib.json as s_json
 import synapse.lib.storm as s_storm
 import synapse.lib.httpapi as s_httpapi
 import synapse.lib.msgpack as s_msgpack
@@ -167,6 +167,12 @@ class StormTest(s_t_utils.SynTest):
             self.len(1, await core.nodes('ou:org=({"name": "origname"}) [ :name=newname ]'))
             self.len(0, await core.nodes('ou:org=({"name": "origname"})'))
 
+            nodes = await core.nodes('[ it:exec:proc=(notime,) ]')
+            self.len(1, nodes)
+
+            nodes = await core.nodes('[ it:exec:proc=(nulltime,) ]')
+            self.len(1, nodes)
+
     async def test_lib_storm_jsonexpr(self):
         async with self.getTestCore() as core:
 
@@ -2382,7 +2388,7 @@ class StormTest(s_t_utils.SynTest):
             async for bytz in core.axon.get(s_common.uhex(sha)):
                 buf += bytz
 
-            resp = json.loads(buf.decode('utf8'))
+            resp = s_json.loads(buf)
             return resp
 
         async with self.getTestCore() as core:
@@ -4217,15 +4223,15 @@ class StormTest(s_t_utils.SynTest):
             self.stormIsInPrint('$lib.auth                     : A Storm Library for interacting with Auth in the '
                                 'Cortex.',
                                 msgs)
-            self.stormIsInPrint('$lib.import(name, debug=$lib.false, reqvers=$lib.null)\nImport a Storm module.',
+            self.stormIsInPrint('$lib.import(name, debug=(false), reqvers=(null))\nImport a Storm module.',
                                 msgs)
             self.stormIsInPrint('$lib.debug\nTrue if the current runtime has debugging enabled.', msgs)
             self.stormNotInPrint('Examples', msgs)
 
             msgs = await core.stormlist('help -v $lib')
 
-            self.stormIsInPrint('$lib.import(name, debug=$lib.false, reqvers=$lib.null)\n'
-                                '======================================================\n'
+            self.stormIsInPrint('$lib.import(name, debug=(false), reqvers=(null))\n'
+                                '================================================\n'
                                 'Import a Storm module.', msgs)
 
             msgs = await core.stormlist('help $lib.macro')
@@ -4250,7 +4256,7 @@ class StormTest(s_t_utils.SynTest):
                                 'Regex flag to indicate that multiline matches are allowed.', msgs)
 
             msgs = await core.stormlist('help $lib.inet.http.get')
-            self.stormIsInPrint('$lib.inet.http.get(url, headers=$lib.null', msgs)
+            self.stormIsInPrint('$lib.inet.http.get(url, headers=(null)', msgs)
             self.stormIsInPrint('Get the contents of a given URL.', msgs)
 
             msgs = await core.stormlist('$str=hehe help $str.split')

synapse/tests/test_lib_stormhttp.py

@@ -1,11 +1,11 @@
 import os
 import ssl
-import json
 import shutil
 
 import synapse.exc as s_exc
 import synapse.common as s_common
 
+import synapse.lib.json as s_json
 import synapse.lib.certdir as s_certdir
 import synapse.lib.httpapi as s_httpapi
 import synapse.lib.stormctrl as s_stormctrl
@@ -32,11 +32,11 @@ class TstWebSock(s_httpapi.WebSocket):
         await self.sendJsonMesg(resp)
 
     async def on_message(self, byts):
-        mesg = json.loads(byts)
+        mesg = s_json.loads(byts)
         await self.sendJsonMesg(('echo', mesg), binary=True)
 
     async def sendJsonMesg(self, item, binary=False):
-        byts = json.dumps(item)
+        byts = s_json.dumps(item)
         await self.write_message(byts, binary=binary)
 
 class HttpNotJson(s_httpapi.Handler):
@@ -539,7 +539,7 @@ class StormHttpTest(s_test.SynTest):
             return($lib.inet.http.post($url, ssl_verify=$lib.false, fields=$fields))
             '''
             resp = await core.callStorm(q, opts=opts)
-            request = json.loads(resp.get('body'))
+            request = s_json.loads(resp.get('body'))
             request_headers = request.get('result').get('headers')
             self.isin('multipart/form-data; boundary=', request_headers.get('Content-Type', ''))
 

synapse/tests/test_lib_stormlib_auth.py

@@ -1,9 +1,10 @@
-import json
 import asyncio
 
 import synapse.exc as s_exc
 import synapse.common as s_common
 
+import synapse.lib.json as s_json
+
 import synapse.tests.utils as s_test
 
 visishow = '''
@@ -961,7 +962,7 @@ class StormLibAuthTest(s_test.SynTest):
             '''
             retn = await core.callStorm(q)
             self.eq(retn[0], retn[1])
-            self.eq(json.loads(retn[0]), {'hehe': 'haha', 'd': {'foo': 'bar'}})
+            self.eq(s_json.loads(retn[0]), {'hehe': 'haha', 'd': {'foo': 'bar'}})
 
             q = '''$u=$lib.auth.users.byname(puser) $profile=$u.profile
             for $valu in $profile {

synapse/tests/test_lib_stormlib_cortex.py

@@ -1,11 +1,9 @@
-import json
-
 import unittest.mock as mock
 
-import aiohttp
-
-import synapse.common as s_common
 import synapse.exc as s_exc
+import synapse.common as s_common
+
+import synapse.lib.json as s_json
 
 import synapse.tests.utils as s_test
 
@@ -141,7 +139,7 @@ $request.reply(206, headers=$headers, body=({"no":"body"}))
                 resp = await sess.head(f'https://localhost:{hport}/api/ext/testpath00')
                 self.eq(resp.status, 206)
                 self.eq(resp.headers.get('Secret-Header'), 'Head')
-                self.eq(resp.headers.get('Content-Length'), '14')
+                self.eq(resp.headers.get('Content-Length'), '13')
                 # HEAD had no body in its response
                 self.eq(await resp.read(), b'')
 
@@ -804,12 +802,12 @@ $request.reply(200, headers=$headers, body=$body)
                 resp = await sess.get(f'https://localhost:{hport}/api/ext/raw')
                 self.eq(resp.status, 200)
                 self.eq(resp.headers.get('Content-Type'), 'application/json')
-                self.eq(resp.headers.get('Content-Length'), '12')
+                self.eq(resp.headers.get('Content-Length'), '11')
                 self.eq(resp.headers.get('Secret-Header'), 'OhBoy!')
 
                 buf = await resp.read()
-                self.len(12, buf)
-                self.eq(json.loads(buf), {'oh': 'my'})
+                self.len(11, buf)
+                self.eq(s_json.loads(buf), {'oh': 'my'})
 
     async def test_libcortex_httpapi_jsonlines(self):
         async with self.getTestCore() as core:
@@ -851,8 +849,8 @@ for $i in $values {
                             break
 
                         try:
-                            mesg = json.loads(byts)
-                        except json.JSONDecodeError:
+                            mesg = s_json.loads(byts)
+                        except s_exc.BadJsonText:
                             bufr = jstr
                             break
                         else:
@@ -1357,7 +1355,7 @@ for $i in $values {
                     self.eq(resp.status, 500)
                     data = await resp.json()
                     self.eq(data.get('code'), 'StormRuntimeError')
-                    self.isin('Failed to decode request body as JSON: Expecting value', data.get('mesg'))
+                    self.isin('Failed to decode request body as JSON', data.get('mesg'))
 
     async def test_cortex_httpapi_dynamic(self):
 

synapse/tests/test_lib_stormlib_infosec.py

@@ -1,7 +1,6 @@
-import json
-
 import synapse.exc as s_exc
 
+import synapse.lib.json as s_json
 import synapse.lib.time as s_time
 import synapse.lib.msgpack as s_msgpack
 
@@ -505,7 +504,7 @@ class InfoSecTest(s_test.SynTest):
 
     async def test_stormlib_infosec_attack_flow(self):
 
-        flow = json.loads(s_test_files.getAssetStr('attack_flow/CISA AA22-138B VMWare Workspace (Alt).json'))
+        flow = s_json.loads(s_test_files.getAssetStr('attack_flow/CISA AA22-138B VMWare Workspace (Alt).json'))
         async with self.getTestCore() as core:
             opts = {'vars': {'flow': flow}}
             msgs = await core.stormlist('$lib.infosec.mitre.attack.flow.ingest($flow)', opts=opts)

synapse/tests/test_lib_stormlib_json.py

@@ -1,3 +1,5 @@
+import textwrap
+
 import synapse.exc as s_exc
 
 import synapse.lib.stormlib.json as s_json
@@ -11,27 +13,22 @@ class JsonTest(s_test.SynTest):
         async with self.getTestCore() as core:
 
             self.eq(((1, 2, 3)), await core.callStorm('return($lib.json.load("[1, 2, 3]"))'))
-            self.eq(('["foo", "bar", "baz"]'), await core.callStorm('return($lib.json.save((foo, bar, baz)))'))
-            self.eq(('{"foo": 1, "bar": {"baz": "hello"}}'), await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}})))'))
-            self.eq(('{"foo": 1, "bar": {"baz": "hello"}}'), await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), (null)))'))
-            self.eq((
-'''{
-    "foo": 1,
-    "bar": {
-        "baz": "hello"
-    }
-}'''), await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=(4)))'))
-
-            self.eq((
-'''{
-  "foo": 1,
-  "bar": {
-    "baz": "hello"
-  }
-}'''), await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=2))'))
-
-            with self.raises(s_exc.BadCast):
-                await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=x))')
+            self.eq(('["foo","bar","baz"]'), await core.callStorm('return($lib.json.save((foo, bar, baz)))'))
+            self.eq(('{"foo":1,"bar":{"baz":"hello"}}'), await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}})))'))
+            self.eq(('{"foo":1,"bar":{"baz":"hello"}}'), await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), (null)))'))
+
+            expected = textwrap.dedent('''\
+                {
+                  "foo": 1,
+                  "bar": {
+                    "baz": "hello"
+                  }
+                }'''
+            )
+            self.eq(expected, await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=(1)))'))
+            self.eq(expected, await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=(2)))'))
+            self.eq(expected, await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=(4)))'))
+            self.eq(expected, await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=(true)))'))
 
             with self.raises(s_exc.BadJsonText):
                 await core.callStorm('return($lib.json.load(foo))')

synapse/tests/test_lib_stormlib_log.py

@@ -43,7 +43,7 @@ class LogTest(s_test.SynTest):
                 await core.callStorm('$lib.log.debug("debug message", extra=(foo, bar, baz))')
 
             # structlog test
-            with self.getStructuredAsyncLoggerStream(logname, '"key": "valu"') as stream:
+            with self.getStructuredAsyncLoggerStream(logname, '"key":"valu"') as stream:
                 await core.callStorm('$lib.log.debug("struct1 message")')
                 await core.callStorm('$lib.log.debug("struct2 message", extra=({"key": "valu"}))')
                 self.true(await stream.wait(6))