synapse 2.165.0__py311-none-any.whl → 2.167.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/axon.py +4 -10
- synapse/cmds/cortex.py +1 -6
- synapse/common.py +6 -0
- synapse/cortex.py +104 -57
- synapse/datamodel.py +32 -0
- synapse/exc.py +1 -0
- synapse/lib/agenda.py +81 -51
- synapse/lib/aha.py +2 -0
- synapse/lib/ast.py +21 -23
- synapse/lib/base.py +11 -10
- synapse/lib/cell.py +24 -24
- synapse/lib/hive.py +11 -0
- synapse/lib/httpapi.py +1 -0
- synapse/lib/nexus.py +3 -2
- synapse/lib/node.py +4 -2
- synapse/lib/schemas.py +3 -1
- synapse/lib/snap.py +50 -0
- synapse/lib/storm.py +19 -17
- synapse/lib/stormlib/aha.py +370 -17
- synapse/lib/stormlib/auth.py +11 -4
- synapse/lib/stormlib/cache.py +202 -0
- synapse/lib/stormlib/cortex.py +69 -7
- synapse/lib/stormlib/macro.py +11 -18
- synapse/lib/stormlib/spooled.py +109 -0
- synapse/lib/stormlib/stix.py +1 -1
- synapse/lib/stormtypes.py +61 -17
- synapse/lib/trigger.py +10 -12
- synapse/lib/types.py +3 -1
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +16 -3
- synapse/models/base.py +8 -0
- synapse/models/files.py +3 -0
- synapse/models/inet.py +74 -2
- synapse/models/orgs.py +52 -8
- synapse/models/person.py +30 -11
- synapse/models/risk.py +44 -3
- synapse/telepath.py +115 -32
- synapse/tests/files/stormpkg/dotstorm/dotstorm.yaml +3 -0
- synapse/tests/test_cortex.py +79 -8
- synapse/tests/test_datamodel.py +22 -0
- synapse/tests/test_lib_agenda.py +8 -1
- synapse/tests/test_lib_aha.py +19 -6
- synapse/tests/test_lib_cell.py +6 -2
- synapse/tests/test_lib_grammar.py +62 -64
- synapse/tests/test_lib_httpapi.py +1 -1
- synapse/tests/test_lib_rstorm.py +4 -4
- synapse/tests/test_lib_storm.py +98 -7
- synapse/tests/test_lib_stormlib_aha.py +196 -0
- synapse/tests/test_lib_stormlib_cache.py +272 -0
- synapse/tests/test_lib_stormlib_compression.py +12 -12
- synapse/tests/test_lib_stormlib_cortex.py +71 -0
- synapse/tests/test_lib_stormlib_macro.py +94 -0
- synapse/tests/test_lib_stormlib_spooled.py +190 -0
- synapse/tests/test_lib_stormtypes.py +71 -37
- synapse/tests/test_lib_view.py +50 -3
- synapse/tests/test_model_files.py +3 -0
- synapse/tests/test_model_inet.py +67 -0
- synapse/tests/test_model_risk.py +6 -0
- synapse/tests/test_telepath.py +30 -7
- synapse/tests/test_tools_genpkg.py +26 -0
- synapse/tests/test_tools_hiveload.py +1 -0
- synapse/tests/test_tools_hivesave.py +1 -0
- synapse/tests/test_tools_modrole.py +81 -0
- synapse/tests/test_tools_moduser.py +105 -0
- synapse/tests/utils.py +22 -3
- synapse/tools/autodoc.py +1 -1
- synapse/tools/hive/load.py +3 -0
- synapse/tools/hive/save.py +3 -0
- synapse/tools/modrole.py +59 -7
- synapse/tools/moduser.py +78 -10
- {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/METADATA +3 -3
- {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/RECORD +75 -72
- synapse/lib/provenance.py +0 -111
- synapse/tests/test_lib_provenance.py +0 -37
- {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/LICENSE +0 -0
- {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/WHEEL +0 -0
- {synapse-2.165.0.dist-info → synapse-2.167.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import synapse.exc as s_exc
|
|
2
|
+
import synapse.common as s_common
|
|
3
|
+
import synapse.cortex as s_cortex
|
|
4
|
+
|
|
5
|
+
import synapse.lib.cell as s_cell
|
|
6
|
+
|
|
7
|
+
import synapse.tests.utils as s_test
|
|
8
|
+
|
|
9
|
+
class AhaLibTest(s_test.SynTest):
|
|
10
|
+
|
|
11
|
+
async def test_stormlib_aha_basics(self):
|
|
12
|
+
|
|
13
|
+
async with self.getTestAhaProv() as aha:
|
|
14
|
+
|
|
15
|
+
with self.getTestDir() as dirn:
|
|
16
|
+
|
|
17
|
+
dirn00 = s_common.genpath(dirn, 'cell00')
|
|
18
|
+
dirn01 = s_common.genpath(dirn, 'cell01')
|
|
19
|
+
dirn02 = s_common.genpath(dirn, 'cell02')
|
|
20
|
+
dirn03 = s_common.genpath(dirn, 'cell03')
|
|
21
|
+
|
|
22
|
+
replay = s_common.envbool('SYNDEV_NEXUS_REPLAY')
|
|
23
|
+
nevents = 10 if replay else 5
|
|
24
|
+
|
|
25
|
+
waiter = aha.waiter(nevents, 'aha:svcadd')
|
|
26
|
+
|
|
27
|
+
cell00 = await aha.enter_context(self.addSvcToAha(aha, '00.cell', s_cell.Cell, dirn=dirn00))
|
|
28
|
+
cell01 = await aha.enter_context(self.addSvcToAha(aha, '01.cell', s_cell.Cell, dirn=dirn01,
|
|
29
|
+
provinfo={'mirror': 'cell'}))
|
|
30
|
+
cell02 = await aha.enter_context(self.addSvcToAha(aha, 'mysvc', s_cell.Cell, dirn=dirn02))
|
|
31
|
+
core00 = await aha.enter_context(self.addSvcToAha(aha, 'core', s_cortex.Cortex, dirn=dirn03))
|
|
32
|
+
|
|
33
|
+
self.len(nevents, await waiter.wait(timeout=12))
|
|
34
|
+
|
|
35
|
+
svcs = await core00.callStorm('$l=() for $i in $lib.aha.list() { $l.append($i) } fini { return ($l) }')
|
|
36
|
+
self.len(5, svcs)
|
|
37
|
+
|
|
38
|
+
svc = await core00.callStorm('return( $lib.aha.get(core...) )')
|
|
39
|
+
self.eq('core.loop.vertex.link', svc.get('name'))
|
|
40
|
+
svc = await core00.callStorm('return( $lib.aha.get(core.loop.vertex.link))')
|
|
41
|
+
self.eq('core.loop.vertex.link', svc.get('name'))
|
|
42
|
+
svc = await core00.callStorm('return( $lib.aha.get(00.cell...))')
|
|
43
|
+
self.eq('00.cell.loop.vertex.link', svc.get('name'))
|
|
44
|
+
svc = await core00.callStorm('return( $lib.aha.get(cell...))')
|
|
45
|
+
self.eq('cell.loop.vertex.link', svc.get('name'))
|
|
46
|
+
svc = await core00.callStorm('$f=({"mirror": (true)}) return( $lib.aha.get(cell..., filters=$f))')
|
|
47
|
+
self.eq('01.cell.loop.vertex.link', svc.get('name'))
|
|
48
|
+
|
|
49
|
+
# List the aha services available
|
|
50
|
+
msgs = await core00.stormlist('aha.svc.list --nexus')
|
|
51
|
+
self.stormIsInPrint('Nexus', msgs)
|
|
52
|
+
self.stormIsInPrint('00.cell.loop.vertex.link true true true', msgs)
|
|
53
|
+
self.stormIsInPrint('01.cell.loop.vertex.link false true true', msgs)
|
|
54
|
+
self.stormIsInPrint('cell.loop.vertex.link true true true', msgs)
|
|
55
|
+
self.stormIsInPrint('core.loop.vertex.link null true true', msgs)
|
|
56
|
+
self.stormIsInPrint('mysvc.loop.vertex.link null true true', msgs)
|
|
57
|
+
|
|
58
|
+
msgs = await core00.stormlist('aha.svc.list')
|
|
59
|
+
self.stormNotInPrint('Nexus', msgs)
|
|
60
|
+
msgs = await core00.stormlist('aha.svc.stat cell...')
|
|
61
|
+
|
|
62
|
+
# The connections information includes runtime information such as port/host info.
|
|
63
|
+
# Omit checking part of that.
|
|
64
|
+
emsg = '''Resolved cell... to an AHA Service.
|
|
65
|
+
|
|
66
|
+
Name: cell.loop.vertex.link
|
|
67
|
+
Online: true
|
|
68
|
+
Ready: true
|
|
69
|
+
Run iden: ********************************
|
|
70
|
+
Cell iden: ********************************
|
|
71
|
+
Leader: cell
|
|
72
|
+
Connection information:
|
|
73
|
+
ca: loop.vertex.link
|
|
74
|
+
'''
|
|
75
|
+
self.stormIsInPrint(emsg, msgs, deguid=True)
|
|
76
|
+
self.stormIsInPrint(' hostname: 00.cell.loop.vertex.link', msgs)
|
|
77
|
+
self.stormIsInPrint(' scheme: ssl', msgs)
|
|
78
|
+
self.stormIsInPrint(' user: root', msgs)
|
|
79
|
+
|
|
80
|
+
msgs = await core00.stormlist('aha.svc.stat --nexus cell...')
|
|
81
|
+
emsg = '''Resolved cell... to an AHA Service.
|
|
82
|
+
|
|
83
|
+
Name: cell.loop.vertex.link
|
|
84
|
+
Online: true
|
|
85
|
+
Ready: true
|
|
86
|
+
Run iden: ********************************
|
|
87
|
+
Cell iden: ********************************
|
|
88
|
+
Leader: cell
|
|
89
|
+
Nexus: 1
|
|
90
|
+
Connection information:
|
|
91
|
+
ca: loop.vertex.link
|
|
92
|
+
'''
|
|
93
|
+
self.stormIsInPrint(emsg, msgs, deguid=True)
|
|
94
|
+
|
|
95
|
+
msgs = await core00.stormlist('aha.svc.stat --nexus 01.cell...')
|
|
96
|
+
emsg = '''Resolved 01.cell... to an AHA Service.
|
|
97
|
+
|
|
98
|
+
Name: 01.cell.loop.vertex.link
|
|
99
|
+
Online: true
|
|
100
|
+
Ready: true
|
|
101
|
+
Run iden: ********************************
|
|
102
|
+
Cell iden: ********************************
|
|
103
|
+
Leader: cell
|
|
104
|
+
Nexus: 1
|
|
105
|
+
Connection information:
|
|
106
|
+
ca: loop.vertex.link
|
|
107
|
+
'''
|
|
108
|
+
self.stormIsInPrint(emsg, msgs, deguid=True)
|
|
109
|
+
|
|
110
|
+
# Full name works
|
|
111
|
+
msgs = await core00.stormlist('aha.svc.stat 01.cell.loop.vertex.link')
|
|
112
|
+
emsg = '''Resolved 01.cell.loop.vertex.link to an AHA Service.
|
|
113
|
+
|
|
114
|
+
Name: 01.cell.loop.vertex.link
|
|
115
|
+
'''
|
|
116
|
+
self.stormIsInPrint(emsg, msgs, deguid=True)
|
|
117
|
+
|
|
118
|
+
# No item for pool00 yet..
|
|
119
|
+
msgs = await core00.stormlist('aha.svc.stat pool00...')
|
|
120
|
+
self.stormIsInPrint('No service found for: "pool00..."', msgs)
|
|
121
|
+
|
|
122
|
+
msgs = await core00.stormlist('aha.pool.add pool00...')
|
|
123
|
+
self.stormHasNoWarnErr(msgs)
|
|
124
|
+
msgs = await core00.stormlist('aha.pool.svc.add pool00... 00.cell...')
|
|
125
|
+
self.stormHasNoWarnErr(msgs)
|
|
126
|
+
|
|
127
|
+
msgs = await core00.stormlist('aha.svc.stat --nexus pool00...')
|
|
128
|
+
emsg = '''Resolved pool00... to an AHA Pool.
|
|
129
|
+
|
|
130
|
+
The pool currently has 1 members.
|
|
131
|
+
AHA Pool: pool00.loop.vertex.link
|
|
132
|
+
Member: 00.cell.loop.vertex.link'''
|
|
133
|
+
self.stormIsInPrint(emsg, msgs)
|
|
134
|
+
|
|
135
|
+
# Shut down a service
|
|
136
|
+
nevents = 2 if replay else 1
|
|
137
|
+
waiter = aha.waiter(nevents, 'aha:svcdown')
|
|
138
|
+
await cell01.fini()
|
|
139
|
+
self.len(nevents, await waiter.wait(timeout=12))
|
|
140
|
+
|
|
141
|
+
msgs = await core00.stormlist('aha.svc.list')
|
|
142
|
+
self.stormIsInPrint('01.cell.loop.vertex.link false false true', msgs)
|
|
143
|
+
|
|
144
|
+
# Fake a record
|
|
145
|
+
await aha.addAhaSvc('00.newp', info={'urlinfo': {'scheme': 'tcp', 'host': '0.0.0.0', 'port': '3030'}},
|
|
146
|
+
network='loop.vertex.link')
|
|
147
|
+
|
|
148
|
+
msgs = await core00.stormlist('aha.svc.list --nexus')
|
|
149
|
+
emsg = '00.newp.loop.vertex.link null false null 0.0.0.0 3030 ' \
|
|
150
|
+
'Service is not online. Will not attempt to retrieve its nexus offset.'
|
|
151
|
+
self.stormIsInPrint(emsg, msgs)
|
|
152
|
+
|
|
153
|
+
self.none(await core00.callStorm('return($lib.aha.del(00.newp...))'))
|
|
154
|
+
msgs = await core00.stormlist('aha.svc.list')
|
|
155
|
+
self.stormNotInPrint('00.newp', msgs)
|
|
156
|
+
|
|
157
|
+
# Fake a online record
|
|
158
|
+
guid = s_common.guid()
|
|
159
|
+
await aha.addAhaSvc('00.newp', info={'urlinfo': {'scheme': 'tcp',
|
|
160
|
+
'host': '0.0.0.0',
|
|
161
|
+
'port': '3030'},
|
|
162
|
+
'online': guid,
|
|
163
|
+
},
|
|
164
|
+
network='loop.vertex.link')
|
|
165
|
+
msgs = await core00.stormlist('aha.svc.list --nexus')
|
|
166
|
+
emsg = '00.newp.loop.vertex.link null true null 0.0.0.0 3030 ' \
|
|
167
|
+
'Failed to connect to Telepath service: "aha://00.newp.loop.vertex.link/" error:'
|
|
168
|
+
self.stormIsInPrint(emsg, msgs)
|
|
169
|
+
|
|
170
|
+
msgs = await core00.stormlist('aha.svc.stat --nexus 00.newp...')
|
|
171
|
+
emsg = '''Resolved 00.newp... to an AHA Service.
|
|
172
|
+
|
|
173
|
+
Name: 00.newp.loop.vertex.link
|
|
174
|
+
Online: true
|
|
175
|
+
Ready: null
|
|
176
|
+
Run iden: $lib.null
|
|
177
|
+
Cell iden: $lib.null
|
|
178
|
+
Leader: Service did not register itself with a leader name.
|
|
179
|
+
Nexus: Failed to connect to Telepath service: "aha://00.newp.loop.vertex.link/" error: [Errno 111] Connect call failed ('0.0.0.0', 3030)
|
|
180
|
+
Connection information:
|
|
181
|
+
host: 0.0.0.0
|
|
182
|
+
port: 3030
|
|
183
|
+
scheme: tcp
|
|
184
|
+
user: root'''
|
|
185
|
+
self.stormIsInPrint(emsg, msgs)
|
|
186
|
+
|
|
187
|
+
# Delete the fake service with its full service name
|
|
188
|
+
self.none(await core00.callStorm('return($lib.aha.del(00.newp.loop.vertex.link))'))
|
|
189
|
+
self.none(await core00.callStorm('return($lib.aha.get(00.newp...))'))
|
|
190
|
+
|
|
191
|
+
# Coverage for sad paths
|
|
192
|
+
with self.raises(s_exc.BadArg):
|
|
193
|
+
await core00.callStorm('$lib.aha.del(pool00...)')
|
|
194
|
+
|
|
195
|
+
with self.raises(s_exc.NoSuchName):
|
|
196
|
+
await core00.callStorm('$lib.aha.del(axon...)')
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
import synapse.exc as s_exc
|
|
2
|
+
|
|
3
|
+
import synapse.tests.utils as s_test
|
|
4
|
+
|
|
5
|
+
class StormlibCacheTest(s_test.SynTest):
|
|
6
|
+
|
|
7
|
+
async def test_storm_lib_cache_fixed(self):
|
|
8
|
+
|
|
9
|
+
async with self.getTestCore() as core:
|
|
10
|
+
|
|
11
|
+
# basics
|
|
12
|
+
|
|
13
|
+
rets = await core.callStorm('''
|
|
14
|
+
$rets = ([])
|
|
15
|
+
$cache = $lib.cache.fixed("return(`{$cache_key}-ret`)")
|
|
16
|
+
|
|
17
|
+
$cache.clear()
|
|
18
|
+
|
|
19
|
+
$rets.append($lib.len($cache))
|
|
20
|
+
|
|
21
|
+
$rets.append($cache.get(key))
|
|
22
|
+
$rets.append($lib.len($cache))
|
|
23
|
+
|
|
24
|
+
$cache.put(key, key-put)
|
|
25
|
+
$rets.append($cache.get(key))
|
|
26
|
+
|
|
27
|
+
$cache.clear()
|
|
28
|
+
$rets.append($cache.get(key))
|
|
29
|
+
|
|
30
|
+
$cache.put(key, key-put)
|
|
31
|
+
$rets.append($cache.get(key))
|
|
32
|
+
$rets.append($cache.pop(key))
|
|
33
|
+
$rets.append($cache.get(key))
|
|
34
|
+
|
|
35
|
+
$rets.append($cache.pop(newp))
|
|
36
|
+
|
|
37
|
+
$rets.append($cache.query)
|
|
38
|
+
|
|
39
|
+
return($rets)
|
|
40
|
+
''')
|
|
41
|
+
self.eq([
|
|
42
|
+
0,
|
|
43
|
+
'key-ret', 1,
|
|
44
|
+
'key-put',
|
|
45
|
+
'key-ret',
|
|
46
|
+
'key-put', 'key-put', 'key-ret',
|
|
47
|
+
None,
|
|
48
|
+
'return(`{$cache_key}-ret`)'
|
|
49
|
+
], rets)
|
|
50
|
+
|
|
51
|
+
# exhaust size
|
|
52
|
+
|
|
53
|
+
rets = await core.callStorm('''
|
|
54
|
+
$rets = ([])
|
|
55
|
+
$cache = $lib.cache.fixed("return(`{$cache_key}-ret`)", size=2)
|
|
56
|
+
|
|
57
|
+
$cache.put(one, one-put)
|
|
58
|
+
$cache.put(two, two-put)
|
|
59
|
+
$rets.append($cache.get(one))
|
|
60
|
+
$rets.append($cache.get(two))
|
|
61
|
+
|
|
62
|
+
$rets.append($cache.get(three))
|
|
63
|
+
$rets.append($cache.get(one))
|
|
64
|
+
|
|
65
|
+
return($rets)
|
|
66
|
+
''')
|
|
67
|
+
self.eq(['one-put', 'two-put', 'three-ret', 'one-ret'], rets)
|
|
68
|
+
|
|
69
|
+
# also accept a storm query object
|
|
70
|
+
|
|
71
|
+
ret = await core.callStorm('''
|
|
72
|
+
$cache = $lib.cache.fixed(${ $suf=ret return(`{$cache_key}-{$suf}`)})
|
|
73
|
+
return($cache.get(foo))
|
|
74
|
+
''')
|
|
75
|
+
self.eq('foo-ret', ret)
|
|
76
|
+
|
|
77
|
+
# callback runtime scoping
|
|
78
|
+
|
|
79
|
+
## a function still has the outer scope as its root
|
|
80
|
+
rets = await core.callStorm('''
|
|
81
|
+
$val = zero
|
|
82
|
+
$sent = $lib.null
|
|
83
|
+
|
|
84
|
+
function cb(key) {
|
|
85
|
+
$sent = $val
|
|
86
|
+
return(`{$key}-{$val}`)
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
$cache = $lib.cache.fixed("return($cb($cache_key))")
|
|
90
|
+
|
|
91
|
+
$rets = ([])
|
|
92
|
+
|
|
93
|
+
$rets.append($cache.get(foo))
|
|
94
|
+
$rets.append($sent)
|
|
95
|
+
|
|
96
|
+
$val = one
|
|
97
|
+
$rets.append($cache.get(bar))
|
|
98
|
+
$rets.append($sent)
|
|
99
|
+
|
|
100
|
+
return($rets)
|
|
101
|
+
''')
|
|
102
|
+
self.eq([
|
|
103
|
+
'foo-zero', 'zero',
|
|
104
|
+
'bar-one', 'one'
|
|
105
|
+
], rets)
|
|
106
|
+
|
|
107
|
+
## runtime also can modify refs to the outer scope
|
|
108
|
+
rets = await core.callStorm('''
|
|
109
|
+
$val = zero
|
|
110
|
+
$vals = ([])
|
|
111
|
+
$sent = $lib.null
|
|
112
|
+
|
|
113
|
+
$cache = $lib.cache.fixed(${
|
|
114
|
+
$sent = $val
|
|
115
|
+
$vals.append($cache_key)
|
|
116
|
+
return(`{$cache_key}-{$val}`)
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
$rets = ([])
|
|
120
|
+
|
|
121
|
+
$rets.append($cache.get(foo))
|
|
122
|
+
$rets.append($sent)
|
|
123
|
+
$rets.append($lib.str.join(",", $vals))
|
|
124
|
+
|
|
125
|
+
$val = one
|
|
126
|
+
$rets.append($cache.get(bar))
|
|
127
|
+
$rets.append($sent)
|
|
128
|
+
$rets.append($lib.str.join(",", $vals))
|
|
129
|
+
|
|
130
|
+
return($rets)
|
|
131
|
+
''')
|
|
132
|
+
self.eq([
|
|
133
|
+
'foo-zero', None, 'foo',
|
|
134
|
+
'bar-one', None, 'foo,bar',
|
|
135
|
+
], rets)
|
|
136
|
+
|
|
137
|
+
## default to null w/o a return
|
|
138
|
+
self.none(await core.callStorm('return($lib.cache.fixed("if (0) { return(yup) }").get(foo))'))
|
|
139
|
+
|
|
140
|
+
## control flow exceptions don't propagate up
|
|
141
|
+
rets = await core.callStorm('''
|
|
142
|
+
$cache = $lib.cache.fixed( ${ if ($cache_key < (2)) { return (`key={$cache_key}`) } else { break } } )
|
|
143
|
+
|
|
144
|
+
$rets = ([])
|
|
145
|
+
|
|
146
|
+
for $i in $lib.range(4) {
|
|
147
|
+
$rets.append($cache.get($i))
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
$rets.append(`i={$i}`)
|
|
151
|
+
return($rets)
|
|
152
|
+
''')
|
|
153
|
+
self.eq(['key=0', 'key=1', None, None, 'i=3'], rets)
|
|
154
|
+
|
|
155
|
+
## control flow scoped inside the callback
|
|
156
|
+
rets = await core.callStorm("""
|
|
157
|
+
$cache = $lib.cache.fixed('''
|
|
158
|
+
$vals = ([])
|
|
159
|
+
for $i in $lib.range(3) {
|
|
160
|
+
$vals.append(`key={$cache_key} i={$i}`)
|
|
161
|
+
break
|
|
162
|
+
}
|
|
163
|
+
return($vals)
|
|
164
|
+
''')
|
|
165
|
+
|
|
166
|
+
$rets = ([])
|
|
167
|
+
|
|
168
|
+
for $k in (foo, bar, baz) {
|
|
169
|
+
$rets.append($cache.get($k))
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return($rets)
|
|
173
|
+
""")
|
|
174
|
+
self.eq([('key=foo i=0',), ('key=bar i=0',), ('key=baz i=0',),], rets)
|
|
175
|
+
|
|
176
|
+
## coverage for the cb runtime emiting nodes
|
|
177
|
+
rets = await core.callStorm('''
|
|
178
|
+
$rets = ([])
|
|
179
|
+
$cache = $lib.cache.fixed(${ if (0) { return(yup) } [ inet:ipv4=$cache_key ] })
|
|
180
|
+
|
|
181
|
+
for $i in (0, 1) {
|
|
182
|
+
$rets.append($cache.get($i))
|
|
183
|
+
}
|
|
184
|
+
return($rets)
|
|
185
|
+
''')
|
|
186
|
+
self.eq([None, None], rets)
|
|
187
|
+
|
|
188
|
+
# stormrepr
|
|
189
|
+
|
|
190
|
+
msgs = await core.stormlist('''
|
|
191
|
+
$lib.print($lib.cache.fixed(${return(cool)}))
|
|
192
|
+
$lib.print($lib.cache.fixed($longq))
|
|
193
|
+
''', opts={'vars': {'longq': f'return({"a" * 150})'}})
|
|
194
|
+
self.stormIsInPrint('cache:fixed: size=10000 query="return(cool)"', msgs)
|
|
195
|
+
self.stormIsInPrint('aaaaa...', msgs)
|
|
196
|
+
|
|
197
|
+
# sad
|
|
198
|
+
|
|
199
|
+
## bad storm query
|
|
200
|
+
|
|
201
|
+
with self.raises(s_exc.BadArg) as ectx:
|
|
202
|
+
await core.nodes('$lib.cache.fixed("function x -> newp")')
|
|
203
|
+
self.isin('Invalid callback query', ectx.exception.errinfo.get('mesg'))
|
|
204
|
+
|
|
205
|
+
## no return
|
|
206
|
+
|
|
207
|
+
with self.raises(s_exc.BadArg) as ectx:
|
|
208
|
+
await core.nodes('$lib.cache.fixed("$x=1")')
|
|
209
|
+
self.eq('Callback query must return a value', ectx.exception.errinfo.get('mesg'))
|
|
210
|
+
|
|
211
|
+
## bad size
|
|
212
|
+
|
|
213
|
+
with self.raises(s_exc.BadArg) as ectx:
|
|
214
|
+
await core.nodes('$lib.cache.fixed("return()", size=(-1))')
|
|
215
|
+
self.eq('Cache size must be between 1-10000', ectx.exception.errinfo.get('mesg'))
|
|
216
|
+
|
|
217
|
+
with self.raises(s_exc.BadArg) as ectx:
|
|
218
|
+
await core.nodes('$lib.cache.fixed("return()", size=(1000000))')
|
|
219
|
+
self.eq('Cache size must be between 1-10000', ectx.exception.errinfo.get('mesg'))
|
|
220
|
+
|
|
221
|
+
## callback raises an exception
|
|
222
|
+
|
|
223
|
+
rets = await core.callStorm('''
|
|
224
|
+
function cb(key) {
|
|
225
|
+
if ($key = "sad") {
|
|
226
|
+
$lib.raise(Bad, Time)
|
|
227
|
+
}
|
|
228
|
+
return(`{$key}-happy`)
|
|
229
|
+
}
|
|
230
|
+
$cache = $lib.cache.fixed("return($cb($cache_key))")
|
|
231
|
+
|
|
232
|
+
$rets = ([])
|
|
233
|
+
|
|
234
|
+
try {
|
|
235
|
+
$rets.append($cache.get(sad))
|
|
236
|
+
} catch Bad as e {
|
|
237
|
+
$rets.append(badtime)
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
$rets.append($cache.get(foo))
|
|
241
|
+
|
|
242
|
+
return($rets)
|
|
243
|
+
''')
|
|
244
|
+
self.eq(['badtime', 'foo-happy'], rets)
|
|
245
|
+
|
|
246
|
+
with self.raises(s_exc.BadCast) as ectx:
|
|
247
|
+
await core.nodes('$lib.cache.fixed(${ return(($cache_key * 3)) }).get(foo)')
|
|
248
|
+
self.eq('Failed to make an integer from \'foo\'.', ectx.exception.errinfo.get('mesg'))
|
|
249
|
+
|
|
250
|
+
## mutable key
|
|
251
|
+
|
|
252
|
+
with self.raises(s_exc.BadArg) as ectx:
|
|
253
|
+
await core.nodes('$lib.cache.fixed("return()").get((foo,))')
|
|
254
|
+
self.eq('Mutable values are not allowed as cache keys', ectx.exception.errinfo.get('mesg'))
|
|
255
|
+
|
|
256
|
+
with self.raises(s_exc.BadArg) as ectx:
|
|
257
|
+
await core.nodes('$lib.cache.fixed("return()").put((foo,), bar)')
|
|
258
|
+
self.eq('Mutable values are not allowed as cache keys', ectx.exception.errinfo.get('mesg'))
|
|
259
|
+
|
|
260
|
+
with self.raises(s_exc.BadArg) as ectx:
|
|
261
|
+
await core.nodes('$lib.cache.fixed("return()").pop((foo,))')
|
|
262
|
+
self.eq('Mutable values are not allowed as cache keys', ectx.exception.errinfo.get('mesg'))
|
|
263
|
+
|
|
264
|
+
## non-primable key
|
|
265
|
+
|
|
266
|
+
with self.raises(s_exc.NoSuchType) as ectx:
|
|
267
|
+
await core.nodes('$cache = $lib.cache.fixed("return()") $cache.get($cache)')
|
|
268
|
+
self.eq('Unable to convert object to Storm primitive.', ectx.exception.errinfo.get('mesg'))
|
|
269
|
+
|
|
270
|
+
## missing use of $cache_key - no error
|
|
271
|
+
|
|
272
|
+
self.eq('newp', await core.callStorm('return($lib.cache.fixed("return(newp)").get(foo))'))
|
|
@@ -14,13 +14,13 @@ class StormlibCompressionTest(s_test.SynTest):
|
|
|
14
14
|
text = 'ohhai'
|
|
15
15
|
tenc = base64.urlsafe_b64encode((bz2.compress(text.encode()))).decode()
|
|
16
16
|
|
|
17
|
-
await core.nodes(f'[
|
|
18
|
-
await core.nodes(f'[
|
|
17
|
+
await core.nodes(f'[ tel:mob:telem=(node1,) :data="{tenc}" ]')
|
|
18
|
+
await core.nodes(f'[ tel:mob:telem=(node2,) :data="{text}" ]')
|
|
19
19
|
|
|
20
|
-
q = '
|
|
20
|
+
q = 'tel:mob:telem=(node1,) return($lib.compression.bzip2.un($lib.base64.decode(:data)).decode())'
|
|
21
21
|
self.eq(text, await core.callStorm(q))
|
|
22
22
|
|
|
23
|
-
q = '
|
|
23
|
+
q = 'tel:mob:telem=(node2,) return($lib.base64.encode($lib.compression.bzip2.en((:data).encode())))'
|
|
24
24
|
self.eq(tenc, await core.callStorm(q))
|
|
25
25
|
|
|
26
26
|
await self.asyncraises(s_exc.StormRuntimeError, core.nodes('$lib.compression.bzip2.en(foo)'))
|
|
@@ -31,13 +31,13 @@ class StormlibCompressionTest(s_test.SynTest):
|
|
|
31
31
|
text = 'ohhai'
|
|
32
32
|
tenc = base64.urlsafe_b64encode((gzip.compress(text.encode()))).decode()
|
|
33
33
|
|
|
34
|
-
await core.nodes(f'[
|
|
35
|
-
await core.nodes(f'[
|
|
34
|
+
await core.nodes(f'[ tel:mob:telem=(node1,) :data="{tenc}" ]')
|
|
35
|
+
await core.nodes(f'[ tel:mob:telem=(node2,) :data="{text}" ]')
|
|
36
36
|
|
|
37
|
-
q = '
|
|
37
|
+
q = 'tel:mob:telem=(node1,) return($lib.compression.gzip.un($lib.base64.decode(:data)).decode())'
|
|
38
38
|
self.eq(text, await core.callStorm(q))
|
|
39
39
|
|
|
40
|
-
q = '
|
|
40
|
+
q = 'tel:mob:telem=(node2,) return($lib.compression.gzip.en((:data).encode()))'
|
|
41
41
|
self.eq(text.encode(), gzip.decompress(await core.callStorm(q)))
|
|
42
42
|
|
|
43
43
|
await self.asyncraises(s_exc.StormRuntimeError, core.nodes('$lib.compression.gzip.en(foo)'))
|
|
@@ -48,13 +48,13 @@ class StormlibCompressionTest(s_test.SynTest):
|
|
|
48
48
|
text = 'ohhai'
|
|
49
49
|
tenc = base64.urlsafe_b64encode((zlib.compress(text.encode()))).decode()
|
|
50
50
|
|
|
51
|
-
await core.nodes(f'[
|
|
52
|
-
await core.nodes(f'[
|
|
51
|
+
await core.nodes(f'[ tel:mob:telem=(node1,) :data="{tenc}" ]')
|
|
52
|
+
await core.nodes(f'[ tel:mob:telem=(node2,) :data="{text}" ]')
|
|
53
53
|
|
|
54
|
-
q = '
|
|
54
|
+
q = 'tel:mob:telem=(node1,) return($lib.compression.zlib.un($lib.base64.decode(:data)).decode())'
|
|
55
55
|
self.eq(text, await core.callStorm(q))
|
|
56
56
|
|
|
57
|
-
q = '
|
|
57
|
+
q = 'tel:mob:telem=(node2,) return($lib.base64.encode($lib.compression.zlib.en((:data).encode())))'
|
|
58
58
|
self.eq(tenc, await core.callStorm(q))
|
|
59
59
|
|
|
60
60
|
await self.asyncraises(s_exc.StormRuntimeError, core.nodes('$lib.compression.zlib.en(foo)'))
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import json
|
|
2
2
|
|
|
3
|
+
import unittest.mock as mock
|
|
4
|
+
|
|
3
5
|
import aiohttp
|
|
4
6
|
|
|
5
7
|
import synapse.common as s_common
|
|
@@ -393,6 +395,9 @@ $request.reply(206, headers=$headers, body=({"no":"body"}))
|
|
|
393
395
|
return ( $api.iden )
|
|
394
396
|
'''
|
|
395
397
|
iden0 = await core.callStorm(q)
|
|
398
|
+
msgs = await core.stormlist('$lib.print($lib.cortex.httpapi.get($iden))', opts={'vars': {'iden': iden0}})
|
|
399
|
+
mesg = 'http:api: the hehe wildcard handler (********************************), path=hehe/([a-z0-9]*)'
|
|
400
|
+
self.stormIsInPrint(mesg, msgs, deguid=True)
|
|
396
401
|
|
|
397
402
|
q = '''
|
|
398
403
|
$api = $lib.cortex.httpapi.add('hehe/haha')
|
|
@@ -401,6 +406,7 @@ $request.reply(206, headers=$headers, body=({"no":"body"}))
|
|
|
401
406
|
$api.name = 'the hehe/haha handler'
|
|
402
407
|
$api.desc = 'beep boop zoop robot captain'
|
|
403
408
|
$api.runas = user
|
|
409
|
+
$api.pool = (true)
|
|
404
410
|
$api.perms = (
|
|
405
411
|
({"perm": ["hehe", "haha"]}),
|
|
406
412
|
({"perm": ["some", "thing"], "default": $lib.true}),
|
|
@@ -428,6 +434,9 @@ $request.reply(206, headers=$headers, body=({"no":"body"}))
|
|
|
428
434
|
return ( $api.iden )
|
|
429
435
|
'''
|
|
430
436
|
iden3 = await core.callStorm(q)
|
|
437
|
+
msgs = await core.stormlist('$lib.print($lib.cortex.httpapi.get($iden))', opts={'vars': {'iden': iden3}})
|
|
438
|
+
mesg = 'http:api: <no name> (********************************), path=wow'
|
|
439
|
+
self.stormIsInPrint(mesg, msgs, deguid=True)
|
|
431
440
|
|
|
432
441
|
# $lib.dict accessor methods
|
|
433
442
|
q = '$api=$lib.cortex.httpapi.get($iden) return ($lib.dict.keys($api.vars))'
|
|
@@ -473,6 +482,15 @@ $request.reply(206, headers=$headers, body=({"no":"body"}))
|
|
|
473
482
|
self.stormIsInPrint(f'2 {iden2}', msgs)
|
|
474
483
|
self.stormIsInPrint(f'3 {iden3}', msgs)
|
|
475
484
|
|
|
485
|
+
q = '''
|
|
486
|
+
$ret = $lib.null $api = $lib.cortex.httpapi.getByPath($path)
|
|
487
|
+
if $api { $ret = $api.iden}
|
|
488
|
+
return ( $ret )
|
|
489
|
+
'''
|
|
490
|
+
self.eq(iden0, await core.callStorm(q, opts={'vars': {'path': 'hehe/haha'}}))
|
|
491
|
+
self.eq(iden0, await core.callStorm(q, opts={'vars': {'path': 'hehe/ohmy'}}))
|
|
492
|
+
self.none(await core.callStorm(q, opts={'vars': {'path': 'newpnewpnewp'}}))
|
|
493
|
+
|
|
476
494
|
# Order matters. The hehe/haha path occurs after the wildcard.
|
|
477
495
|
async with self.getHttpSess(auth=('root', 'root'), port=hport) as sess:
|
|
478
496
|
resp = await sess.get(f'https://localhost:{hport}/api/ext/hehe/haha')
|
|
@@ -494,6 +512,9 @@ $request.reply(206, headers=$headers, body=({"no":"body"}))
|
|
|
494
512
|
msgs = await core.stormlist('cortex.httpapi.index $iden 1', opts={'vars': {'iden': iden0}})
|
|
495
513
|
self.stormIsInPrint(f'Set HTTP API {iden0} to index 1', msgs)
|
|
496
514
|
|
|
515
|
+
self.eq(iden1, await core.callStorm(q, opts={'vars': {'path': 'hehe/haha'}}))
|
|
516
|
+
self.eq(iden0, await core.callStorm(q, opts={'vars': {'path': 'hehe/ohmy'}}))
|
|
517
|
+
|
|
497
518
|
msgs = await core.stormlist('cortex.httpapi.list')
|
|
498
519
|
self.stormIsInPrint(f'0 {iden1}', msgs)
|
|
499
520
|
self.stormIsInPrint(f'1 {iden0}', msgs)
|
|
@@ -562,6 +583,7 @@ $request.reply(206, headers=$headers, body=({"no":"body"}))
|
|
|
562
583
|
self.stormIsInPrint('Owner: root', msgs)
|
|
563
584
|
self.stormIsInPrint('Runas: owner', msgs)
|
|
564
585
|
self.stormIsInPrint('Readonly: false', msgs)
|
|
586
|
+
self.stormIsInPrint('Pool enabled: false', msgs)
|
|
565
587
|
self.stormIsInPrint('Authenticated: true', msgs)
|
|
566
588
|
self.stormIsInPrint('Name: the hehe wildcard handler', msgs)
|
|
567
589
|
self.stormIsInPrint('Description: wildcard words', msgs)
|
|
@@ -596,6 +618,7 @@ $request.reply(206, headers=$headers, body=({"no":"body"}))
|
|
|
596
618
|
self.stormIsInPrint('Owner: root', msgs)
|
|
597
619
|
self.stormIsInPrint('Runas: user', msgs)
|
|
598
620
|
self.stormIsInPrint('Readonly: false', msgs)
|
|
621
|
+
self.stormIsInPrint('Pool enabled: true', msgs)
|
|
599
622
|
self.stormIsInPrint('Authenticated: true', msgs)
|
|
600
623
|
self.stormIsInPrint('Name: the hehe/haha handler', msgs)
|
|
601
624
|
self.stormIsInPrint('Description: beep boop zoop robot captain', msgs)
|
|
@@ -1373,3 +1396,51 @@ for $i in $values {
|
|
|
1373
1396
|
self.none(resp.headers.get('X-Content-Type-Options'))
|
|
1374
1397
|
# Server is still omitted though
|
|
1375
1398
|
self.none(resp.headers.get('Server'))
|
|
1399
|
+
|
|
1400
|
+
async def test_cortex_httpapi_pool(self):
|
|
1401
|
+
|
|
1402
|
+
# Test if we pass the mirror value in opts or not.
|
|
1403
|
+
async with self.getTestCore(conf={'https:headers': {'Key1': 'Valu1'}}) as core:
|
|
1404
|
+
await core.setUserPasswd(core.auth.rootuser.iden, 'root')
|
|
1405
|
+
addr, hport = await core.addHttpsPort(0)
|
|
1406
|
+
|
|
1407
|
+
q = '''$api = $lib.cortex.httpapi.add(stuff)
|
|
1408
|
+
$api.methods.get = ${
|
|
1409
|
+
$request.reply(200, headers=({"Weee": "valu"}) )
|
|
1410
|
+
}
|
|
1411
|
+
return ( ($api.iden) )'''
|
|
1412
|
+
iden00 = await core.callStorm(q)
|
|
1413
|
+
opts_iden00 = {'vars': {'iden': iden00}}
|
|
1414
|
+
|
|
1415
|
+
data = {}
|
|
1416
|
+
|
|
1417
|
+
oldstorm = core.storm
|
|
1418
|
+
async def storm(self, text, opts=None):
|
|
1419
|
+
data['opts'] = opts
|
|
1420
|
+
async for mesg in oldstorm(text, opts=opts):
|
|
1421
|
+
yield mesg
|
|
1422
|
+
|
|
1423
|
+
with mock.patch('synapse.cortex.Cortex.storm', new=storm) as patch:
|
|
1424
|
+
async with self.getHttpSess(auth=('root', 'root'), port=hport) as sess:
|
|
1425
|
+
resp = await sess.get(f'https://localhost:{hport}/api/ext/stuff') # type: aiohttp.ClientResponse
|
|
1426
|
+
self.eq(resp.status, 200)
|
|
1427
|
+
self.false(data['opts'].get('mirror'))
|
|
1428
|
+
data.clear()
|
|
1429
|
+
|
|
1430
|
+
q = '$api=$lib.cortex.httpapi.get($iden) $api.pool = (true) return ( $api.pack() ) '
|
|
1431
|
+
adef = await core.callStorm(q, opts=opts_iden00)
|
|
1432
|
+
self.true(adef.get('pool'))
|
|
1433
|
+
|
|
1434
|
+
resp = await sess.get(f'https://localhost:{hport}/api/ext/stuff') # type: aiohttp.ClientResponse
|
|
1435
|
+
self.eq(resp.status, 200)
|
|
1436
|
+
self.true(data['opts'].get('mirror'))
|
|
1437
|
+
data.clear()
|
|
1438
|
+
|
|
1439
|
+
q = '$api=$lib.cortex.httpapi.get($iden) $api.pool = (false) return ( $api.pack() ) '
|
|
1440
|
+
adef = await core.callStorm(q, opts=opts_iden00)
|
|
1441
|
+
self.false(adef.get('pool'))
|
|
1442
|
+
|
|
1443
|
+
resp = await sess.get(f'https://localhost:{hport}/api/ext/stuff') # type: aiohttp.ClientResponse
|
|
1444
|
+
self.eq(resp.status, 200)
|
|
1445
|
+
self.false(data['opts'].get('mirror'))
|
|
1446
|
+
data.clear()
|