synapse 2.202.0__py311-none-any.whl → 2.203.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/axon.py +4 -4
- synapse/cmds/cortex.py +4 -6
- synapse/cmds/hive.py +10 -10
- synapse/common.py +17 -58
- synapse/cortex.py +6 -6
- synapse/data/__init__.py +3 -2
- synapse/data/iana.uris.mpk +1 -0
- synapse/lib/autodoc.py +3 -3
- synapse/lib/cli.py +2 -2
- synapse/lib/config.py +2 -2
- synapse/lib/encoding.py +4 -3
- synapse/lib/httpapi.py +7 -11
- synapse/lib/json.py +224 -0
- synapse/lib/lmdbslab.py +1 -1
- synapse/lib/oauth.py +176 -54
- synapse/lib/rstorm.py +18 -14
- synapse/lib/schemas.py +87 -1
- synapse/lib/scrape.py +35 -13
- synapse/lib/snap.py +2 -1
- synapse/lib/storm.py +2 -2
- synapse/lib/stormhttp.py +11 -13
- synapse/lib/stormlib/aha.py +4 -4
- synapse/lib/stormlib/auth.py +1 -1
- synapse/lib/stormlib/cache.py +2 -2
- synapse/lib/stormlib/cortex.py +5 -5
- synapse/lib/stormlib/graph.py +1 -1
- synapse/lib/stormlib/imap.py +1 -1
- synapse/lib/stormlib/json.py +8 -11
- synapse/lib/stormlib/model.py +1 -1
- synapse/lib/stormlib/notifications.py +2 -2
- synapse/lib/stormlib/oauth.py +105 -2
- synapse/lib/stormlib/stats.py +4 -0
- synapse/lib/stormlib/stix.py +3 -4
- synapse/lib/stormlib/vault.py +6 -6
- synapse/lib/stormlib/xml.py +2 -2
- synapse/lib/stormtypes.py +19 -28
- synapse/lib/structlog.py +3 -3
- synapse/lib/types.py +2 -1
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +7 -3
- synapse/models/base.py +51 -2
- synapse/telepath.py +5 -3
- synapse/tests/files/__init__.py +0 -1
- synapse/tests/test_axon.py +1 -1
- synapse/tests/test_cmds_cortex.py +3 -2
- synapse/tests/test_cmds_hive.py +4 -4
- synapse/tests/test_common.py +29 -19
- synapse/tests/test_cortex.py +5 -5
- synapse/tests/test_lib_ast.py +3 -3
- synapse/tests/test_lib_autodoc.py +5 -5
- synapse/tests/test_lib_base.py +1 -1
- synapse/tests/test_lib_cell.py +16 -10
- synapse/tests/test_lib_config.py +2 -2
- synapse/tests/test_lib_encoding.py +2 -2
- synapse/tests/test_lib_grammar.py +64 -64
- synapse/tests/test_lib_httpapi.py +13 -13
- synapse/tests/test_lib_json.py +219 -0
- synapse/tests/test_lib_multislabseqn.py +2 -1
- synapse/tests/test_lib_node.py +2 -2
- synapse/tests/test_lib_scrape.py +50 -0
- synapse/tests/test_lib_storm.py +6 -6
- synapse/tests/test_lib_stormhttp.py +4 -4
- synapse/tests/test_lib_stormlib_auth.py +3 -2
- synapse/tests/test_lib_stormlib_cortex.py +10 -12
- synapse/tests/test_lib_stormlib_infosec.py +2 -3
- synapse/tests/test_lib_stormlib_json.py +18 -21
- synapse/tests/test_lib_stormlib_log.py +1 -1
- synapse/tests/test_lib_stormlib_oauth.py +603 -1
- synapse/tests/test_lib_stormlib_stats.py +13 -3
- synapse/tests/test_lib_stormlib_stix.py +5 -5
- synapse/tests/test_lib_stormtypes.py +4 -4
- synapse/tests/test_lib_structlog.py +5 -6
- synapse/tests/test_lib_view.py +8 -0
- synapse/tests/test_model_base.py +32 -0
- synapse/tests/test_model_infotech.py +2 -2
- synapse/tests/test_telepath.py +0 -1
- synapse/tests/test_tools_cryo_cat.py +4 -3
- synapse/tests/test_tools_docker_validate.py +4 -2
- synapse/tests/test_tools_feed.py +30 -2
- synapse/tests/test_tools_genpkg.py +1 -1
- synapse/tests/test_tools_healthcheck.py +8 -7
- synapse/tests/test_utils.py +2 -2
- synapse/tests/utils.py +3 -3
- synapse/tools/autodoc.py +3 -3
- synapse/tools/changelog.py +2 -2
- synapse/tools/cryo/cat.py +3 -3
- synapse/tools/csvtool.py +2 -3
- synapse/tools/docker/validate.py +5 -5
- synapse/tools/feed.py +2 -1
- synapse/tools/genpkg.py +3 -2
- synapse/tools/healthcheck.py +2 -3
- synapse/tools/json2mpk.py +2 -2
- synapse/utils/getrefs.py +6 -6
- synapse/vendor/cpython/lib/json.py +35 -0
- synapse/vendor/cpython/lib/test/test_json.py +22 -0
- {synapse-2.202.0.dist-info → synapse-2.203.0.dist-info}/METADATA +2 -1
- {synapse-2.202.0.dist-info → synapse-2.203.0.dist-info}/RECORD +100 -95
- {synapse-2.202.0.dist-info → synapse-2.203.0.dist-info}/WHEEL +1 -1
- {synapse-2.202.0.dist-info → synapse-2.203.0.dist-info}/LICENSE +0 -0
- {synapse-2.202.0.dist-info → synapse-2.203.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import io
|
|
2
|
+
import json
|
|
3
|
+
|
|
4
|
+
import orjson
|
|
5
|
+
|
|
6
|
+
import synapse.exc as s_exc
|
|
7
|
+
import synapse.common as s_common
|
|
8
|
+
|
|
9
|
+
import synapse.lib.json as s_json
|
|
10
|
+
|
|
11
|
+
import synapse.tests.utils as s_test
|
|
12
|
+
|
|
13
|
+
class JsonTest(s_test.SynTest):
|
|
14
|
+
|
|
15
|
+
async def test_lib_json_loads(self):
|
|
16
|
+
self.eq({'a': 'b'}, s_json.loads('{"a": "b"}'))
|
|
17
|
+
|
|
18
|
+
with self.raises(s_exc.BadJsonText) as exc:
|
|
19
|
+
s_json.loads('newp')
|
|
20
|
+
self.eq(exc.exception.get('mesg'), 'Expecting value: line 1 column 1 (char 0)')
|
|
21
|
+
|
|
22
|
+
async def test_lib_json_load(self):
|
|
23
|
+
with self.getTestDir() as dirn:
|
|
24
|
+
|
|
25
|
+
with s_common.genfile(dirn, 'file00') as file00:
|
|
26
|
+
file00.write(b'{"a": "b"}')
|
|
27
|
+
|
|
28
|
+
with s_common.genfile(dirn, 'file00') as file00:
|
|
29
|
+
self.eq({'a': 'b'}, s_json.load(file00))
|
|
30
|
+
|
|
31
|
+
with s_common.genfile(dirn, 'empty') as empty:
|
|
32
|
+
with self.raises(s_exc.BadJsonText) as exc:
|
|
33
|
+
s_json.load(empty)
|
|
34
|
+
self.eq(exc.exception.get('mesg'), 'Expecting value: line 1 column 1 (char 0)')
|
|
35
|
+
|
|
36
|
+
buf = io.BytesIO(b'{"a": "b"}')
|
|
37
|
+
self.eq({'a': 'b'}, s_json.load(buf))
|
|
38
|
+
|
|
39
|
+
buf = io.StringIO('{"a": "b"}')
|
|
40
|
+
self.eq({'a': 'b'}, s_json.load(buf))
|
|
41
|
+
|
|
42
|
+
async def test_lib_json_load_surrogates(self):
|
|
43
|
+
|
|
44
|
+
inval = '{"a": "😀\ud83d\ude47"}'
|
|
45
|
+
outval = {'a': '😀\ud83d\ude47'}
|
|
46
|
+
|
|
47
|
+
# orjson.dumps fails because of the surrogate pairs
|
|
48
|
+
with self.raises(orjson.JSONDecodeError):
|
|
49
|
+
orjson.loads(inval)
|
|
50
|
+
|
|
51
|
+
# stdlib json.loads passes because of voodoo magic
|
|
52
|
+
self.eq(outval, json.loads(inval))
|
|
53
|
+
|
|
54
|
+
self.eq(outval, s_json.loads(inval))
|
|
55
|
+
|
|
56
|
+
buf = io.StringIO(inval)
|
|
57
|
+
self.eq(outval, s_json.load(buf))
|
|
58
|
+
|
|
59
|
+
buf = io.BytesIO(inval.encode('utf8', errors='surrogatepass'))
|
|
60
|
+
self.eq(outval, s_json.load(buf))
|
|
61
|
+
|
|
62
|
+
async def test_lib_json_dump_surrogates(self):
|
|
63
|
+
inval = {'a': '😀\ud83d\ude47'}
|
|
64
|
+
outval = b'{"a": "\\ud83d\\ude00\\ud83d\\ude47"}'
|
|
65
|
+
|
|
66
|
+
# orjson.dumps fails because of the surrogate pairs
|
|
67
|
+
with self.raises(TypeError):
|
|
68
|
+
orjson.dumps(inval)
|
|
69
|
+
|
|
70
|
+
# stdlib json.dumps passes because of voodoo magic
|
|
71
|
+
self.eq(outval.decode(), json.dumps(inval))
|
|
72
|
+
|
|
73
|
+
self.eq(outval, s_json.dumps(inval))
|
|
74
|
+
self.eq(outval + b'\n', s_json.dumps(inval, newline=True))
|
|
75
|
+
|
|
76
|
+
buf = io.BytesIO()
|
|
77
|
+
s_json.dump(inval, buf)
|
|
78
|
+
self.eq(outval, buf.getvalue())
|
|
79
|
+
|
|
80
|
+
async def test_lib_json_dumps(self):
|
|
81
|
+
self.eq(b'{"c":"d","a":"b"}', s_json.dumps({'c': 'd', 'a': 'b'}))
|
|
82
|
+
self.eq(b'{"a":"b","c":"d"}', s_json.dumps({'c': 'd', 'a': 'b'}, sort_keys=True))
|
|
83
|
+
self.eq(b'{\n "c": "d",\n "a": "b"\n}', s_json.dumps({'c': 'd', 'a': 'b'}, indent=True))
|
|
84
|
+
self.eq(b'{"c":"d","a":"b"}\n', s_json.dumps({'c': 'd', 'a': 'b'}, newline=True))
|
|
85
|
+
|
|
86
|
+
with self.raises(s_exc.MustBeJsonSafe) as exc:
|
|
87
|
+
s_json.dumps({}.items())
|
|
88
|
+
self.eq(exc.exception.get('mesg'), 'Type is not JSON serializable: dict_items')
|
|
89
|
+
|
|
90
|
+
with self.raises(s_exc.MustBeJsonSafe) as exc:
|
|
91
|
+
s_json.dumps({1: 'foo'})
|
|
92
|
+
self.eq(exc.exception.get('mesg'), 'Dict key must be str')
|
|
93
|
+
|
|
94
|
+
with self.raises(s_exc.MustBeJsonSafe) as exc:
|
|
95
|
+
s_json.dumps({'\ud83d\ude47': {}.items()})
|
|
96
|
+
self.eq(exc.exception.get('mesg'), 'Object of type dict_items is not JSON serializable')
|
|
97
|
+
|
|
98
|
+
self.eq(b'"dict_items([])"', s_json.dumps({}.items(), default=str))
|
|
99
|
+
|
|
100
|
+
async def test_lib_json_dump(self):
|
|
101
|
+
with self.getTestDir() as dirn:
|
|
102
|
+
binfn = s_common.genpath(dirn, 'bin.json')
|
|
103
|
+
|
|
104
|
+
with open(binfn, 'wb') as binfp:
|
|
105
|
+
s_json.dump({'c': 'd', 'a': 'b'}, binfp)
|
|
106
|
+
|
|
107
|
+
with open(binfn, 'rb') as binfp:
|
|
108
|
+
self.eq(b'{"c":"d","a":"b"}', binfp.read())
|
|
109
|
+
|
|
110
|
+
buf = io.BytesIO()
|
|
111
|
+
s_json.dump({'c': 'd', 'a': 'b'}, buf)
|
|
112
|
+
self.eq(b'{"c":"d","a":"b"}', buf.getvalue())
|
|
113
|
+
|
|
114
|
+
async def test_jsload(self):
|
|
115
|
+
with self.getTestDir() as dirn:
|
|
116
|
+
with s_common.genfile(dirn, 'jsload.json') as fp:
|
|
117
|
+
fp.write(b'{"a":"b"}')
|
|
118
|
+
|
|
119
|
+
obj = s_json.jsload(dirn, 'jsload.json')
|
|
120
|
+
self.eq({'a': 'b'}, obj)
|
|
121
|
+
|
|
122
|
+
s_common.genfile(dirn, 'empty.json').close()
|
|
123
|
+
self.none(s_json.jsload(dirn, 'empty.json'))
|
|
124
|
+
|
|
125
|
+
async def test_jslines(self):
|
|
126
|
+
with self.getTestDir() as dirn:
|
|
127
|
+
with s_common.genfile(dirn, 'jslines.json') as fp:
|
|
128
|
+
fp.write(b'{"a":"b"}\n{"c":"d"}')
|
|
129
|
+
|
|
130
|
+
objs = [k for k in s_json.jslines(dirn, 'jslines.json')]
|
|
131
|
+
self.len(2, objs)
|
|
132
|
+
self.eq([{'a': 'b'}, {'c': 'd'}], objs)
|
|
133
|
+
|
|
134
|
+
async def test_jssave(self):
|
|
135
|
+
with self.getTestDir() as dirn:
|
|
136
|
+
s_json.jssave({'a': 'b'}, dirn, 'jssave.json')
|
|
137
|
+
|
|
138
|
+
with s_common.genfile(dirn, 'jssave.json') as fd:
|
|
139
|
+
data = fd.read()
|
|
140
|
+
|
|
141
|
+
self.eq(data, b'{\n "a": "b"\n}')
|
|
142
|
+
|
|
143
|
+
async def test_lib_json_reqjsonsafe(self):
|
|
144
|
+
self.none(s_json.reqjsonsafe('foo'))
|
|
145
|
+
self.none(s_json.reqjsonsafe({'foo': 'bar'}))
|
|
146
|
+
self.none(s_json.reqjsonsafe(['foo', 'bar']))
|
|
147
|
+
|
|
148
|
+
buf = io.BytesIO()
|
|
149
|
+
|
|
150
|
+
with self.raises(s_exc.MustBeJsonSafe) as exc:
|
|
151
|
+
s_json.reqjsonsafe(buf)
|
|
152
|
+
self.isin('Type is not JSON serializable: _io.BytesIO', exc.exception.get('mesg'))
|
|
153
|
+
|
|
154
|
+
with self.raises(s_exc.MustBeJsonSafe) as exc:
|
|
155
|
+
s_json.reqjsonsafe({'foo': buf})
|
|
156
|
+
self.isin('Type is not JSON serializable: _io.BytesIO', exc.exception.get('mesg'))
|
|
157
|
+
|
|
158
|
+
with self.raises(s_exc.MustBeJsonSafe) as exc:
|
|
159
|
+
s_json.reqjsonsafe(['foo', buf])
|
|
160
|
+
self.isin('Type is not JSON serializable: _io.BytesIO', exc.exception.get('mesg'))
|
|
161
|
+
|
|
162
|
+
items = (
|
|
163
|
+
(None, None),
|
|
164
|
+
(1234, None),
|
|
165
|
+
('1234', None),
|
|
166
|
+
({'asdf': 'haha'}, None),
|
|
167
|
+
({'a': (1,), 'b': [{'': 4}, 56, None, {'t': True, 'f': False}, 'oh my']}, None),
|
|
168
|
+
(b'1234', s_exc.MustBeJsonSafe),
|
|
169
|
+
({'a': 'a', 2: 2}, s_exc.MustBeJsonSafe),
|
|
170
|
+
({'a', 'b', 'c'}, s_exc.MustBeJsonSafe),
|
|
171
|
+
(s_common.novalu, s_exc.MustBeJsonSafe),
|
|
172
|
+
)
|
|
173
|
+
for (item, eret) in items:
|
|
174
|
+
if eret is None:
|
|
175
|
+
self.none(s_json.reqjsonsafe(item))
|
|
176
|
+
else:
|
|
177
|
+
with self.raises(eret):
|
|
178
|
+
s_json.reqjsonsafe(item)
|
|
179
|
+
|
|
180
|
+
text = '😀\ud83d\ude47'
|
|
181
|
+
s_json.reqjsonsafe(text)
|
|
182
|
+
with self.raises(s_exc.MustBeJsonSafe) as exc:
|
|
183
|
+
s_json.reqjsonsafe(text, strict=True)
|
|
184
|
+
self.eq(exc.exception.get('mesg'), 'str is not valid UTF-8: surrogates not allowed')
|
|
185
|
+
|
|
186
|
+
async def test_lib_json_data_at_rest(self):
|
|
187
|
+
async with self.getRegrCore('json-data') as core:
|
|
188
|
+
badjson = {
|
|
189
|
+
1: 'foo',
|
|
190
|
+
'foo': '😀\ud83d\ude47',
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
goodjson = {
|
|
194
|
+
'1': 'foo',
|
|
195
|
+
'foo': '😀',
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
# We can lift nodes with bad :data
|
|
199
|
+
nodes = await core.nodes('it:log:event')
|
|
200
|
+
self.len(1, nodes)
|
|
201
|
+
self.eq(nodes[0].get('data'), badjson)
|
|
202
|
+
|
|
203
|
+
iden = nodes[0].iden()
|
|
204
|
+
|
|
205
|
+
# We can't lift nodes with bad data by querying the prop directly
|
|
206
|
+
opts = {'vars': {'data': badjson}}
|
|
207
|
+
with self.raises(s_exc.BadTypeValu):
|
|
208
|
+
await core.callStorm('it:log:event:data=$data', opts=opts)
|
|
209
|
+
|
|
210
|
+
# We can't set nodes with bad data
|
|
211
|
+
with self.raises(s_exc.BadTypeValu):
|
|
212
|
+
await core.callStorm('[ it:log:event=* :data=$data ]', opts=opts)
|
|
213
|
+
|
|
214
|
+
# We can overwrite bad :data props
|
|
215
|
+
opts = {'vars': {'data': goodjson}}
|
|
216
|
+
nodes = await core.nodes('it:log:event:data [ :data=$data ]', opts=opts)
|
|
217
|
+
self.len(1, nodes)
|
|
218
|
+
self.eq(nodes[0].iden(), iden)
|
|
219
|
+
self.eq(nodes[0].get('data'), goodjson)
|
|
@@ -5,6 +5,7 @@ import synapse.exc as s_exc
|
|
|
5
5
|
import synapse.common as s_common
|
|
6
6
|
|
|
7
7
|
import synapse.lib.coro as s_coro
|
|
8
|
+
import synapse.lib.json as s_json
|
|
8
9
|
import synapse.lib.lmdbslab as s_lmdbslab
|
|
9
10
|
import synapse.lib.multislabseqn as s_multislabseqn
|
|
10
11
|
|
|
@@ -314,7 +315,7 @@ class MultiSlabSeqn(s_t_utils.SynTest):
|
|
|
314
315
|
# Make a slab a non-dir
|
|
315
316
|
slab0dirn = s_common.genpath(baddirn, f'seqn{"0" * 16}.lmdb')
|
|
316
317
|
shutil.rmtree(slab0dirn)
|
|
317
|
-
|
|
318
|
+
s_json.jssave('{}', slab0dirn)
|
|
318
319
|
|
|
319
320
|
with self.getAsyncLoggerStream('synapse.lib.multislabseqn', 'non-directory') as stream:
|
|
320
321
|
async with await s_multislabseqn.MultiSlabSeqn.anit(baddirn) as msqn:
|
synapse/tests/test_lib_node.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import json
|
|
2
1
|
import collections
|
|
3
2
|
|
|
4
3
|
import synapse.exc as s_exc
|
|
5
4
|
import synapse.common as s_common
|
|
6
5
|
|
|
6
|
+
import synapse.lib.json as s_json
|
|
7
7
|
import synapse.lib.node as s_node
|
|
8
8
|
|
|
9
9
|
import synapse.tests.utils as s_t_utils
|
|
@@ -220,7 +220,7 @@ class NodeTest(s_t_utils.SynTest):
|
|
|
220
220
|
async for byts, x in resp.content.iter_chunks():
|
|
221
221
|
if not byts:
|
|
222
222
|
break
|
|
223
|
-
mesg =
|
|
223
|
+
mesg = s_json.loads(byts)
|
|
224
224
|
if mesg[0] == 'node':
|
|
225
225
|
https_nodes.append(mesg[1])
|
|
226
226
|
|
synapse/tests/test_lib_scrape.py
CHANGED
|
@@ -1187,3 +1187,53 @@ class ScrapeTest(s_t_utils.SynTest):
|
|
|
1187
1187
|
nodes.remove(('it:sec:cpe', 'cpe:2.3:*:*why*:*:*:*:*:*:*:*:*:*'))
|
|
1188
1188
|
|
|
1189
1189
|
self.len(0, nodes)
|
|
1190
|
+
|
|
1191
|
+
def test_scrape_uris(self):
|
|
1192
|
+
|
|
1193
|
+
nodes = list(s_scrape.scrape('http://vertex.link https://woot.com'))
|
|
1194
|
+
self.len(4, nodes)
|
|
1195
|
+
self.isin(('inet:url', 'http://vertex.link'), nodes)
|
|
1196
|
+
self.isin(('inet:url', 'https://woot.com'), nodes)
|
|
1197
|
+
self.isin(('inet:fqdn', 'vertex.link'), nodes)
|
|
1198
|
+
self.isin(('inet:fqdn', 'woot.com'), nodes)
|
|
1199
|
+
|
|
1200
|
+
nodes = list(s_scrape.scrape('ftps://files.vertex.link tcp://1.2.3.4:8080'))
|
|
1201
|
+
self.len(5, nodes)
|
|
1202
|
+
self.isin(('inet:url', 'ftps://files.vertex.link'), nodes)
|
|
1203
|
+
self.isin(('inet:url', 'tcp://1.2.3.4:8080'), nodes)
|
|
1204
|
+
self.isin(('inet:fqdn', 'files.vertex.link'), nodes)
|
|
1205
|
+
self.isin(('inet:ipv4', '1.2.3.4'), nodes)
|
|
1206
|
+
self.isin(('inet:server', '1.2.3.4:8080'), nodes)
|
|
1207
|
+
|
|
1208
|
+
nodes = list(s_scrape.scrape('invalidscheme://vertex.link newp://woot.com'))
|
|
1209
|
+
self.len(2, nodes)
|
|
1210
|
+
self.isin(('inet:fqdn', 'vertex.link'), nodes)
|
|
1211
|
+
self.isin(('inet:fqdn', 'woot.com'), nodes)
|
|
1212
|
+
|
|
1213
|
+
nodes = list(s_scrape.scrape('[https://vertex.link] (http://woot.com)'))
|
|
1214
|
+
self.len(4, nodes)
|
|
1215
|
+
self.isin(('inet:url', 'https://vertex.link'), nodes)
|
|
1216
|
+
self.isin(('inet:url', 'http://woot.com'), nodes)
|
|
1217
|
+
self.isin(('inet:fqdn', 'vertex.link'), nodes)
|
|
1218
|
+
self.isin(('inet:fqdn', 'woot.com'), nodes)
|
|
1219
|
+
|
|
1220
|
+
nodes = list(s_scrape.scrape('HTTP://vertex.link HTTPS://woot.com'))
|
|
1221
|
+
self.len(4, nodes)
|
|
1222
|
+
self.isin(('inet:url', 'HTTP://vertex.link'), nodes)
|
|
1223
|
+
self.isin(('inet:url', 'HTTPS://woot.com'), nodes)
|
|
1224
|
+
self.isin(('inet:fqdn', 'vertex.link'), nodes)
|
|
1225
|
+
self.isin(('inet:fqdn', 'woot.com'), nodes)
|
|
1226
|
+
|
|
1227
|
+
nodes = list(s_scrape.scrape('hxxps://vertex[.]link hXXp://woot[.]com'))
|
|
1228
|
+
self.len(4, nodes)
|
|
1229
|
+
self.isin(('inet:url', 'https://vertex.link'), nodes)
|
|
1230
|
+
self.isin(('inet:url', 'http://woot.com'), nodes)
|
|
1231
|
+
self.isin(('inet:fqdn', 'vertex.link'), nodes)
|
|
1232
|
+
self.isin(('inet:fqdn', 'woot.com'), nodes)
|
|
1233
|
+
|
|
1234
|
+
nodes = list(s_scrape.scrape('hxxp[:]//vertex(.)link hXXps[://]woot[.]com'))
|
|
1235
|
+
self.len(4, nodes)
|
|
1236
|
+
self.isin(('inet:url', 'http://vertex.link'), nodes)
|
|
1237
|
+
self.isin(('inet:url', 'https://woot.com'), nodes)
|
|
1238
|
+
self.isin(('inet:fqdn', 'vertex.link'), nodes)
|
|
1239
|
+
self.isin(('inet:fqdn', 'woot.com'), nodes)
|
synapse/tests/test_lib_storm.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import copy
|
|
2
|
-
import json
|
|
3
2
|
import asyncio
|
|
4
3
|
import datetime
|
|
5
4
|
import itertools
|
|
@@ -13,6 +12,7 @@ import synapse.datamodel as s_datamodel
|
|
|
13
12
|
|
|
14
13
|
import synapse.lib.base as s_base
|
|
15
14
|
import synapse.lib.coro as s_coro
|
|
15
|
+
import synapse.lib.json as s_json
|
|
16
16
|
import synapse.lib.storm as s_storm
|
|
17
17
|
import synapse.lib.httpapi as s_httpapi
|
|
18
18
|
import synapse.lib.msgpack as s_msgpack
|
|
@@ -2388,7 +2388,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
2388
2388
|
async for bytz in core.axon.get(s_common.uhex(sha)):
|
|
2389
2389
|
buf += bytz
|
|
2390
2390
|
|
|
2391
|
-
resp =
|
|
2391
|
+
resp = s_json.loads(buf)
|
|
2392
2392
|
return resp
|
|
2393
2393
|
|
|
2394
2394
|
async with self.getTestCore() as core:
|
|
@@ -4223,15 +4223,15 @@ class StormTest(s_t_utils.SynTest):
|
|
|
4223
4223
|
self.stormIsInPrint('$lib.auth : A Storm Library for interacting with Auth in the '
|
|
4224
4224
|
'Cortex.',
|
|
4225
4225
|
msgs)
|
|
4226
|
-
self.stormIsInPrint('$lib.import(name, debug
|
|
4226
|
+
self.stormIsInPrint('$lib.import(name, debug=(false), reqvers=(null))\nImport a Storm module.',
|
|
4227
4227
|
msgs)
|
|
4228
4228
|
self.stormIsInPrint('$lib.debug\nTrue if the current runtime has debugging enabled.', msgs)
|
|
4229
4229
|
self.stormNotInPrint('Examples', msgs)
|
|
4230
4230
|
|
|
4231
4231
|
msgs = await core.stormlist('help -v $lib')
|
|
4232
4232
|
|
|
4233
|
-
self.stormIsInPrint('$lib.import(name, debug
|
|
4234
|
-
'
|
|
4233
|
+
self.stormIsInPrint('$lib.import(name, debug=(false), reqvers=(null))\n'
|
|
4234
|
+
'================================================\n'
|
|
4235
4235
|
'Import a Storm module.', msgs)
|
|
4236
4236
|
|
|
4237
4237
|
msgs = await core.stormlist('help $lib.macro')
|
|
@@ -4256,7 +4256,7 @@ class StormTest(s_t_utils.SynTest):
|
|
|
4256
4256
|
'Regex flag to indicate that multiline matches are allowed.', msgs)
|
|
4257
4257
|
|
|
4258
4258
|
msgs = await core.stormlist('help $lib.inet.http.get')
|
|
4259
|
-
self.stormIsInPrint('$lib.inet.http.get(url, headers
|
|
4259
|
+
self.stormIsInPrint('$lib.inet.http.get(url, headers=(null)', msgs)
|
|
4260
4260
|
self.stormIsInPrint('Get the contents of a given URL.', msgs)
|
|
4261
4261
|
|
|
4262
4262
|
msgs = await core.stormlist('$str=hehe help $str.split')
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import ssl
|
|
3
|
-
import json
|
|
4
3
|
import shutil
|
|
5
4
|
|
|
6
5
|
import synapse.exc as s_exc
|
|
7
6
|
import synapse.common as s_common
|
|
8
7
|
|
|
8
|
+
import synapse.lib.json as s_json
|
|
9
9
|
import synapse.lib.certdir as s_certdir
|
|
10
10
|
import synapse.lib.httpapi as s_httpapi
|
|
11
11
|
import synapse.lib.stormctrl as s_stormctrl
|
|
@@ -32,11 +32,11 @@ class TstWebSock(s_httpapi.WebSocket):
|
|
|
32
32
|
await self.sendJsonMesg(resp)
|
|
33
33
|
|
|
34
34
|
async def on_message(self, byts):
|
|
35
|
-
mesg =
|
|
35
|
+
mesg = s_json.loads(byts)
|
|
36
36
|
await self.sendJsonMesg(('echo', mesg), binary=True)
|
|
37
37
|
|
|
38
38
|
async def sendJsonMesg(self, item, binary=False):
|
|
39
|
-
byts =
|
|
39
|
+
byts = s_json.dumps(item)
|
|
40
40
|
await self.write_message(byts, binary=binary)
|
|
41
41
|
|
|
42
42
|
class HttpNotJson(s_httpapi.Handler):
|
|
@@ -539,7 +539,7 @@ class StormHttpTest(s_test.SynTest):
|
|
|
539
539
|
return($lib.inet.http.post($url, ssl_verify=$lib.false, fields=$fields))
|
|
540
540
|
'''
|
|
541
541
|
resp = await core.callStorm(q, opts=opts)
|
|
542
|
-
request =
|
|
542
|
+
request = s_json.loads(resp.get('body'))
|
|
543
543
|
request_headers = request.get('result').get('headers')
|
|
544
544
|
self.isin('multipart/form-data; boundary=', request_headers.get('Content-Type', ''))
|
|
545
545
|
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import json
|
|
2
1
|
import asyncio
|
|
3
2
|
|
|
4
3
|
import synapse.exc as s_exc
|
|
5
4
|
import synapse.common as s_common
|
|
6
5
|
|
|
6
|
+
import synapse.lib.json as s_json
|
|
7
|
+
|
|
7
8
|
import synapse.tests.utils as s_test
|
|
8
9
|
|
|
9
10
|
visishow = '''
|
|
@@ -961,7 +962,7 @@ class StormLibAuthTest(s_test.SynTest):
|
|
|
961
962
|
'''
|
|
962
963
|
retn = await core.callStorm(q)
|
|
963
964
|
self.eq(retn[0], retn[1])
|
|
964
|
-
self.eq(
|
|
965
|
+
self.eq(s_json.loads(retn[0]), {'hehe': 'haha', 'd': {'foo': 'bar'}})
|
|
965
966
|
|
|
966
967
|
q = '''$u=$lib.auth.users.byname(puser) $profile=$u.profile
|
|
967
968
|
for $valu in $profile {
|
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import json
|
|
2
|
-
|
|
3
1
|
import unittest.mock as mock
|
|
4
2
|
|
|
5
|
-
import aiohttp
|
|
6
|
-
|
|
7
|
-
import synapse.common as s_common
|
|
8
3
|
import synapse.exc as s_exc
|
|
4
|
+
import synapse.common as s_common
|
|
5
|
+
|
|
6
|
+
import synapse.lib.json as s_json
|
|
9
7
|
|
|
10
8
|
import synapse.tests.utils as s_test
|
|
11
9
|
|
|
@@ -141,7 +139,7 @@ $request.reply(206, headers=$headers, body=({"no":"body"}))
|
|
|
141
139
|
resp = await sess.head(f'https://localhost:{hport}/api/ext/testpath00')
|
|
142
140
|
self.eq(resp.status, 206)
|
|
143
141
|
self.eq(resp.headers.get('Secret-Header'), 'Head')
|
|
144
|
-
self.eq(resp.headers.get('Content-Length'), '
|
|
142
|
+
self.eq(resp.headers.get('Content-Length'), '13')
|
|
145
143
|
# HEAD had no body in its response
|
|
146
144
|
self.eq(await resp.read(), b'')
|
|
147
145
|
|
|
@@ -804,12 +802,12 @@ $request.reply(200, headers=$headers, body=$body)
|
|
|
804
802
|
resp = await sess.get(f'https://localhost:{hport}/api/ext/raw')
|
|
805
803
|
self.eq(resp.status, 200)
|
|
806
804
|
self.eq(resp.headers.get('Content-Type'), 'application/json')
|
|
807
|
-
self.eq(resp.headers.get('Content-Length'), '
|
|
805
|
+
self.eq(resp.headers.get('Content-Length'), '11')
|
|
808
806
|
self.eq(resp.headers.get('Secret-Header'), 'OhBoy!')
|
|
809
807
|
|
|
810
808
|
buf = await resp.read()
|
|
811
|
-
self.len(
|
|
812
|
-
self.eq(
|
|
809
|
+
self.len(11, buf)
|
|
810
|
+
self.eq(s_json.loads(buf), {'oh': 'my'})
|
|
813
811
|
|
|
814
812
|
async def test_libcortex_httpapi_jsonlines(self):
|
|
815
813
|
async with self.getTestCore() as core:
|
|
@@ -851,8 +849,8 @@ for $i in $values {
|
|
|
851
849
|
break
|
|
852
850
|
|
|
853
851
|
try:
|
|
854
|
-
mesg =
|
|
855
|
-
except
|
|
852
|
+
mesg = s_json.loads(byts)
|
|
853
|
+
except s_exc.BadJsonText:
|
|
856
854
|
bufr = jstr
|
|
857
855
|
break
|
|
858
856
|
else:
|
|
@@ -1357,7 +1355,7 @@ for $i in $values {
|
|
|
1357
1355
|
self.eq(resp.status, 500)
|
|
1358
1356
|
data = await resp.json()
|
|
1359
1357
|
self.eq(data.get('code'), 'StormRuntimeError')
|
|
1360
|
-
self.isin('Failed to decode request body as JSON
|
|
1358
|
+
self.isin('Failed to decode request body as JSON', data.get('mesg'))
|
|
1361
1359
|
|
|
1362
1360
|
async def test_cortex_httpapi_dynamic(self):
|
|
1363
1361
|
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import json
|
|
2
|
-
|
|
3
1
|
import synapse.exc as s_exc
|
|
4
2
|
|
|
3
|
+
import synapse.lib.json as s_json
|
|
5
4
|
import synapse.lib.time as s_time
|
|
6
5
|
import synapse.lib.msgpack as s_msgpack
|
|
7
6
|
|
|
@@ -505,7 +504,7 @@ class InfoSecTest(s_test.SynTest):
|
|
|
505
504
|
|
|
506
505
|
async def test_stormlib_infosec_attack_flow(self):
|
|
507
506
|
|
|
508
|
-
flow =
|
|
507
|
+
flow = s_json.loads(s_test_files.getAssetStr('attack_flow/CISA AA22-138B VMWare Workspace (Alt).json'))
|
|
509
508
|
async with self.getTestCore() as core:
|
|
510
509
|
opts = {'vars': {'flow': flow}}
|
|
511
510
|
msgs = await core.stormlist('$lib.infosec.mitre.attack.flow.ingest($flow)', opts=opts)
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import textwrap
|
|
2
|
+
|
|
1
3
|
import synapse.exc as s_exc
|
|
2
4
|
|
|
3
5
|
import synapse.lib.stormlib.json as s_json
|
|
@@ -11,27 +13,22 @@ class JsonTest(s_test.SynTest):
|
|
|
11
13
|
async with self.getTestCore() as core:
|
|
12
14
|
|
|
13
15
|
self.eq(((1, 2, 3)), await core.callStorm('return($lib.json.load("[1, 2, 3]"))'))
|
|
14
|
-
self.eq(('["foo",
|
|
15
|
-
self.eq(('{"foo":
|
|
16
|
-
self.eq(('{"foo":
|
|
17
|
-
|
|
18
|
-
'''
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
''
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
}
|
|
31
|
-
}'''), await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=2))'))
|
|
32
|
-
|
|
33
|
-
with self.raises(s_exc.BadCast):
|
|
34
|
-
await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=x))')
|
|
16
|
+
self.eq(('["foo","bar","baz"]'), await core.callStorm('return($lib.json.save((foo, bar, baz)))'))
|
|
17
|
+
self.eq(('{"foo":1,"bar":{"baz":"hello"}}'), await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}})))'))
|
|
18
|
+
self.eq(('{"foo":1,"bar":{"baz":"hello"}}'), await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), (null)))'))
|
|
19
|
+
|
|
20
|
+
expected = textwrap.dedent('''\
|
|
21
|
+
{
|
|
22
|
+
"foo": 1,
|
|
23
|
+
"bar": {
|
|
24
|
+
"baz": "hello"
|
|
25
|
+
}
|
|
26
|
+
}'''
|
|
27
|
+
)
|
|
28
|
+
self.eq(expected, await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=(1)))'))
|
|
29
|
+
self.eq(expected, await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=(2)))'))
|
|
30
|
+
self.eq(expected, await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=(4)))'))
|
|
31
|
+
self.eq(expected, await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=(true)))'))
|
|
35
32
|
|
|
36
33
|
with self.raises(s_exc.BadJsonText):
|
|
37
34
|
await core.callStorm('return($lib.json.load(foo))')
|
|
@@ -43,7 +43,7 @@ class LogTest(s_test.SynTest):
|
|
|
43
43
|
await core.callStorm('$lib.log.debug("debug message", extra=(foo, bar, baz))')
|
|
44
44
|
|
|
45
45
|
# structlog test
|
|
46
|
-
with self.getStructuredAsyncLoggerStream(logname, '"key":
|
|
46
|
+
with self.getStructuredAsyncLoggerStream(logname, '"key":"valu"') as stream:
|
|
47
47
|
await core.callStorm('$lib.log.debug("struct1 message")')
|
|
48
48
|
await core.callStorm('$lib.log.debug("struct2 message", extra=({"key": "valu"}))')
|
|
49
49
|
self.true(await stream.wait(6))
|