synapse 2.177.0__py311-none-any.whl → 2.178.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/cortex.py +8 -4
- synapse/lib/aha.py +361 -88
- synapse/lib/base.py +27 -9
- synapse/lib/cell.py +167 -110
- synapse/lib/config.py +15 -11
- synapse/lib/coro.py +13 -0
- synapse/lib/layer.py +0 -5
- synapse/lib/link.py +1 -1
- synapse/lib/lmdbslab.py +3 -3
- synapse/lib/nexus.py +24 -12
- synapse/lib/stormlib/imap.py +6 -2
- synapse/lib/stormlib/smtp.py +12 -2
- synapse/lib/version.py +2 -2
- synapse/telepath.py +32 -17
- synapse/tests/files/aha/certs/cas/synapse.crt +28 -0
- synapse/tests/files/aha/certs/cas/synapse.key +51 -0
- synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.crt +30 -0
- synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.key +51 -0
- synapse/tests/files/aha/certs/users/root@synapse.crt +29 -0
- synapse/tests/files/aha/certs/users/root@synapse.key +51 -0
- synapse/tests/files/rstorm/testsvc.py +1 -1
- synapse/tests/test_axon.py +1 -1
- synapse/tests/test_cortex.py +22 -59
- synapse/tests/test_lib_agenda.py +3 -3
- synapse/tests/test_lib_aha.py +336 -490
- synapse/tests/test_lib_base.py +20 -0
- synapse/tests/test_lib_cell.py +49 -22
- synapse/tests/test_lib_config.py +4 -3
- synapse/tests/test_lib_nexus.py +8 -0
- synapse/tests/test_lib_stormlib_aha.py +35 -35
- synapse/tests/test_lib_stormlib_cell.py +4 -15
- synapse/tests/test_lib_stormlib_imap.py +14 -3
- synapse/tests/test_lib_stormlib_smtp.py +51 -0
- synapse/tests/test_tools_aha.py +78 -101
- synapse/tests/utils.py +86 -120
- synapse/tools/aha/clone.py +50 -0
- synapse/tools/aha/enroll.py +2 -1
- synapse/tools/backup.py +2 -2
- synapse/tools/changelog.py +3 -1
- {synapse-2.177.0.dist-info → synapse-2.178.0.dist-info}/METADATA +48 -48
- {synapse-2.177.0.dist-info → synapse-2.178.0.dist-info}/RECORD +44 -37
- {synapse-2.177.0.dist-info → synapse-2.178.0.dist-info}/LICENSE +0 -0
- {synapse-2.177.0.dist-info → synapse-2.178.0.dist-info}/WHEEL +0 -0
- {synapse-2.177.0.dist-info → synapse-2.178.0.dist-info}/top_level.txt +0 -0
synapse/tests/test_tools_aha.py
CHANGED
|
@@ -9,6 +9,7 @@ import synapse.lib.cell as s_cell
|
|
|
9
9
|
import synapse.tests.utils as s_t_utils
|
|
10
10
|
|
|
11
11
|
import synapse.tools.aha.list as s_a_list
|
|
12
|
+
import synapse.tools.aha.clone as s_a_clone
|
|
12
13
|
import synapse.tools.aha.enroll as s_a_enroll
|
|
13
14
|
import synapse.tools.aha.easycert as s_a_easycert
|
|
14
15
|
import synapse.tools.aha.provision.user as s_a_provision_user
|
|
@@ -17,40 +18,37 @@ class AhaToolsTest(s_t_utils.SynTest):
|
|
|
17
18
|
|
|
18
19
|
async def test_aha_list(self):
|
|
19
20
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
'dmon:listen': ephemeral_address}) as aha:
|
|
23
|
-
_, port = aha.sockaddr
|
|
24
|
-
ahaurl = f'tcp://root:root@127.0.0.1:{port}'
|
|
25
|
-
conf0 = {
|
|
26
|
-
'aha:registry': ahaurl,
|
|
27
|
-
'aha:name': 'cell0',
|
|
28
|
-
'aha:network': 'demo.net',
|
|
29
|
-
'dmon:listen': ephemeral_address,
|
|
30
|
-
}
|
|
31
|
-
conf1 = {
|
|
32
|
-
'aha:registry': ahaurl,
|
|
33
|
-
'aha:name': 'cell1',
|
|
34
|
-
'aha:network': 'example.net',
|
|
35
|
-
'dmon:listen': ephemeral_address,
|
|
36
|
-
}
|
|
21
|
+
async with self.getTestAha() as aha:
|
|
22
|
+
|
|
37
23
|
waiter = aha.waiter(2, 'aha:svcadd')
|
|
24
|
+
conf0 = {'aha:provision': await aha.addAhaSvcProv('cell0')}
|
|
25
|
+
|
|
26
|
+
provinfo = {'aha:network': 'example.net'}
|
|
27
|
+
conf1 = {'aha:provision': await aha.addAhaSvcProv('cell1', provinfo=provinfo)}
|
|
28
|
+
|
|
29
|
+
ahaurl = aha.getLocalUrl()
|
|
30
|
+
|
|
38
31
|
async with self.getTestCell(s_cell.Cell, conf=conf0) as cell0:
|
|
32
|
+
|
|
39
33
|
async with self.getTestCell(s_cell.Cell, conf=conf1) as cell1:
|
|
34
|
+
|
|
40
35
|
self.true(await waiter.wait(timeout=6))
|
|
36
|
+
|
|
41
37
|
argv = [ahaurl]
|
|
42
38
|
retn, outp = await self.execToolMain(s_a_list._main, argv)
|
|
43
39
|
self.eq(retn, 0)
|
|
44
|
-
|
|
45
|
-
outp.expect('
|
|
46
|
-
|
|
40
|
+
|
|
41
|
+
outp.expect('''
|
|
42
|
+
Service network leader
|
|
43
|
+
cell0 synapse None
|
|
44
|
+
cell1 example.net None
|
|
45
|
+
''', whitespace=False)
|
|
47
46
|
|
|
48
47
|
argv = [ahaurl, 'demo.net']
|
|
49
48
|
retn, outp = await self.execToolMain(s_a_list._main, argv)
|
|
50
49
|
self.eq(retn, 0)
|
|
51
|
-
outp.expect('Service network')
|
|
52
|
-
outp.expect('cell0 demo.net')
|
|
53
|
-
self.false(outp.expect('cell1 example.net', throw=False))
|
|
50
|
+
outp.expect('Service network', whitespace=False)
|
|
51
|
+
outp.expect('cell0 demo.net', whitespace=False)
|
|
54
52
|
|
|
55
53
|
async with self.getTestCore() as core:
|
|
56
54
|
curl = core.getLocalUrl()
|
|
@@ -94,102 +92,81 @@ class AhaToolsTest(s_t_utils.SynTest):
|
|
|
94
92
|
|
|
95
93
|
async def test_aha_enroll(self):
|
|
96
94
|
|
|
97
|
-
with self.
|
|
98
|
-
|
|
99
|
-
conf = {
|
|
100
|
-
'aha:name': 'aha',
|
|
101
|
-
'aha:network': 'loop.vertex.link',
|
|
102
|
-
'provision:listen': 'ssl://aha.loop.vertex.link:0',
|
|
103
|
-
'dmon:listen': 'tcp://0.0.0.0:0'
|
|
104
|
-
}
|
|
105
|
-
async with self.getTestAha(dirn=dirn, conf=conf) as aha:
|
|
106
|
-
|
|
107
|
-
addr, port = aha.provdmon.addr
|
|
108
|
-
aha.conf['provision:listen'] = f'ssl://aha.loop.vertex.link:{port}'
|
|
109
|
-
|
|
110
|
-
host_, ahaport = aha.sockaddr
|
|
111
|
-
self.eq(aha._getAhaUrls(), [f'ssl://aha.loop.vertex.link:{ahaport}'])
|
|
95
|
+
async with self.getTestAha() as aha:
|
|
112
96
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
provurl = str(outp).split(':', 1)[1].strip()
|
|
118
|
-
with self.getTestSynDir() as syndir:
|
|
97
|
+
argv = ['--url', aha.getLocalUrl(), '01.aha.loop.vertex.link']
|
|
98
|
+
retn, outp = await self.execToolMain(s_a_clone.main, argv)
|
|
99
|
+
self.eq(retn, 0)
|
|
100
|
+
self.isin('one-time use URL:', str(outp))
|
|
119
101
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
102
|
+
argv = ['--url', aha.getLocalUrl(), '01.aha.loop.vertex.link', '--only-url']
|
|
103
|
+
retn, outp = await self.execToolMain(s_a_clone.main, argv)
|
|
104
|
+
self.eq(retn, 0)
|
|
105
|
+
self.notin('one-time use URL:', str(outp))
|
|
123
106
|
|
|
124
|
-
|
|
125
|
-
|
|
107
|
+
argv = ['--url', 'newp://1.2.3.4', '01.aha.loop.vertex.link']
|
|
108
|
+
retn, outp = await self.execToolMain(s_a_clone.main, argv)
|
|
109
|
+
self.eq(retn, 1)
|
|
110
|
+
self.isin('ERROR: Invalid URL scheme: newp', str(outp))
|
|
126
111
|
|
|
127
|
-
|
|
128
|
-
|
|
112
|
+
argv = ['--url', aha.getLocalUrl(), 'visi']
|
|
113
|
+
retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
|
|
114
|
+
self.isin('one-time use URL:', str(outp))
|
|
129
115
|
|
|
130
|
-
|
|
131
|
-
|
|
116
|
+
provurl = str(outp).split(':', 1)[1].strip()
|
|
117
|
+
with self.getTestSynDir() as syndir:
|
|
132
118
|
|
|
133
|
-
|
|
134
|
-
|
|
119
|
+
capath = s_common.genpath(syndir, 'certs', 'cas', 'synapse.crt')
|
|
120
|
+
crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.crt')
|
|
121
|
+
keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.crt')
|
|
135
122
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
self.eq(teleyaml.get('aha:servers'), ('cell://aha', f'ssl://visi@aha.loop.vertex.link:{ahaport}'))
|
|
123
|
+
retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
|
|
124
|
+
self.eq(0, retn)
|
|
139
125
|
|
|
140
|
-
|
|
126
|
+
for path in (capath, crtpath, keypath):
|
|
127
|
+
self.true(os.path.isfile(path))
|
|
128
|
+
self.gt(os.path.getsize(path), 0)
|
|
141
129
|
|
|
142
|
-
|
|
143
|
-
|
|
130
|
+
teleyaml = s_common.yamlload(syndir, 'telepath.yaml')
|
|
131
|
+
self.eq(teleyaml.get('version'), 1)
|
|
132
|
+
self.len(1, teleyaml.get('aha:servers'))
|
|
144
133
|
|
|
145
|
-
|
|
134
|
+
shutil.rmtree(s_common.genpath(syndir, 'certs'))
|
|
146
135
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
provurl = str(outp).split(':', 1)[1].strip()
|
|
153
|
-
|
|
154
|
-
retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
|
|
155
|
-
|
|
156
|
-
servers = ['cell://aha',
|
|
157
|
-
'ssl://visi@aha.loop.vertex.link',
|
|
158
|
-
f'ssl://visi@aha.loop.vertex.link:{ahaport}']
|
|
136
|
+
argv = ['--again', '--url', aha.getLocalUrl(), 'visi']
|
|
137
|
+
retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
|
|
138
|
+
self.eq(retn, 0)
|
|
139
|
+
self.isin('one-time use URL:', str(outp))
|
|
159
140
|
|
|
160
|
-
|
|
161
|
-
self.sorteq(teleyaml.get('aha:servers'), servers)
|
|
141
|
+
provurl = str(outp).split(':', 1)[1].strip()
|
|
162
142
|
|
|
163
|
-
|
|
164
|
-
retn, outp = await self.execToolMain(s_a_provision_user.main, argv + ['--only-url'])
|
|
165
|
-
self.eq(retn, 0)
|
|
166
|
-
self.notin('one-time use URL:', str(outp))
|
|
167
|
-
self.isin('ssl://', str(outp))
|
|
143
|
+
retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
|
|
168
144
|
|
|
169
|
-
|
|
145
|
+
# Just return the URL
|
|
146
|
+
retn, outp = await self.execToolMain(s_a_provision_user.main, argv + ['--only-url'])
|
|
147
|
+
self.eq(retn, 0)
|
|
148
|
+
self.notin('one-time use URL:', str(outp))
|
|
149
|
+
self.isin('ssl://', str(outp))
|
|
170
150
|
|
|
171
|
-
|
|
172
|
-
retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
|
|
173
|
-
self.eq(retn, 0)
|
|
174
|
-
provurl = str(outp).split(':', 1)[1].strip()
|
|
151
|
+
with self.getTestSynDir() as syndir:
|
|
175
152
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
153
|
+
argv = ['--again', '--url', aha.getLocalUrl(), 'visi']
|
|
154
|
+
retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
|
|
155
|
+
self.eq(retn, 0)
|
|
156
|
+
provurl = str(outp).split(':', 1)[1].strip()
|
|
179
157
|
|
|
180
|
-
|
|
181
|
-
|
|
158
|
+
capath = s_common.genpath(syndir, 'certs', 'cas', 'synapse.crt')
|
|
159
|
+
crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.crt')
|
|
160
|
+
keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.key')
|
|
182
161
|
|
|
183
|
-
|
|
184
|
-
s_common.
|
|
162
|
+
for path in (capath, crtpath, keypath):
|
|
163
|
+
s_common.genfile(path)
|
|
185
164
|
|
|
186
|
-
|
|
187
|
-
|
|
165
|
+
retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
|
|
166
|
+
self.eq(0, retn)
|
|
188
167
|
|
|
189
|
-
|
|
190
|
-
|
|
168
|
+
for path in (capath, crtpath, keypath):
|
|
169
|
+
self.gt(os.path.getsize(path), 0)
|
|
191
170
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
self.eq(teleyaml.get('aha:servers'), ('cell://aha', 'cell://foo', ('cell://bar',),
|
|
195
|
-
f'ssl://visi@aha.loop.vertex.link:{ahaport}'))
|
|
171
|
+
teleyaml = s_common.yamlload(syndir, 'telepath.yaml')
|
|
172
|
+
self.eq(teleyaml.get('version'), 1)
|
synapse/tests/utils.py
CHANGED
|
@@ -677,7 +677,7 @@ class TstEnv:
|
|
|
677
677
|
|
|
678
678
|
class TstOutPut(s_output.OutPutStr):
|
|
679
679
|
|
|
680
|
-
def expect(self, substr, throw=True):
|
|
680
|
+
def expect(self, substr, throw=True, whitespace=True):
|
|
681
681
|
'''
|
|
682
682
|
Check if a string is present in the messages captured by the OutPutStr object.
|
|
683
683
|
|
|
@@ -689,6 +689,11 @@ class TstOutPut(s_output.OutPutStr):
|
|
|
689
689
|
bool: True if the string is present; False if the string is not present and throw is False.
|
|
690
690
|
'''
|
|
691
691
|
outs = str(self)
|
|
692
|
+
|
|
693
|
+
if not whitespace:
|
|
694
|
+
outs = ' '.join(outs.split())
|
|
695
|
+
substr = ' '.join(outs.split())
|
|
696
|
+
|
|
692
697
|
if outs.find(substr) == -1:
|
|
693
698
|
if throw:
|
|
694
699
|
mesg = 'TestOutPut.expect(%s) not in %s' % (substr, outs)
|
|
@@ -1297,34 +1302,20 @@ class SynTest(unittest.TestCase):
|
|
|
1297
1302
|
Returns:
|
|
1298
1303
|
s_cortex.Cortex: A Cortex object.
|
|
1299
1304
|
'''
|
|
1300
|
-
|
|
1301
|
-
conf = {
|
|
1302
|
-
'health:sysctl:checks': False,
|
|
1303
|
-
}
|
|
1304
|
-
|
|
1305
|
-
conf = copy.deepcopy(conf)
|
|
1305
|
+
conf = self.getCellConf(conf=conf)
|
|
1306
1306
|
|
|
1307
|
-
mods = conf.get('modules')
|
|
1308
|
-
|
|
1309
|
-
if mods is None:
|
|
1310
|
-
mods = []
|
|
1311
|
-
conf['modules'] = mods
|
|
1307
|
+
mods = list(conf.get('modules', ()))
|
|
1308
|
+
conf['modules'] = mods
|
|
1312
1309
|
|
|
1313
1310
|
mods.insert(0, ('synapse.tests.utils.TestModule', {'key': 'valu'}))
|
|
1314
1311
|
|
|
1315
1312
|
with self.withNexusReplay():
|
|
1316
1313
|
|
|
1317
|
-
|
|
1314
|
+
with self.mayTestDir(dirn) as dirn:
|
|
1318
1315
|
|
|
1319
1316
|
async with await s_cortex.Cortex.anit(dirn, conf=conf) as core:
|
|
1320
1317
|
yield core
|
|
1321
1318
|
|
|
1322
|
-
return
|
|
1323
|
-
|
|
1324
|
-
with self.getTestDir() as dirn:
|
|
1325
|
-
async with await s_cortex.Cortex.anit(dirn, conf=conf) as core:
|
|
1326
|
-
yield core
|
|
1327
|
-
|
|
1328
1319
|
@contextlib.asynccontextmanager
|
|
1329
1320
|
async def getTestCoreAndProxy(self, conf=None, dirn=None):
|
|
1330
1321
|
'''
|
|
@@ -1342,11 +1333,7 @@ class SynTest(unittest.TestCase):
|
|
|
1342
1333
|
@contextlib.asynccontextmanager
|
|
1343
1334
|
async def getTestJsonStor(self, dirn=None, conf=None):
|
|
1344
1335
|
|
|
1345
|
-
|
|
1346
|
-
conf = {
|
|
1347
|
-
'health:sysctl:checks': False,
|
|
1348
|
-
}
|
|
1349
|
-
conf = copy.deepcopy(conf)
|
|
1336
|
+
conf = self.getCellConf(conf=conf)
|
|
1350
1337
|
|
|
1351
1338
|
with self.withNexusReplay():
|
|
1352
1339
|
if dirn is not None:
|
|
@@ -1367,20 +1354,10 @@ class SynTest(unittest.TestCase):
|
|
|
1367
1354
|
Returns:
|
|
1368
1355
|
s_cryotank.CryoCell: Test cryocell.
|
|
1369
1356
|
'''
|
|
1370
|
-
|
|
1371
|
-
conf = {
|
|
1372
|
-
'health:sysctl:checks': False,
|
|
1373
|
-
}
|
|
1374
|
-
conf = copy.deepcopy(conf)
|
|
1357
|
+
conf = self.getCellConf(conf=conf)
|
|
1375
1358
|
|
|
1376
1359
|
with self.withNexusReplay():
|
|
1377
|
-
|
|
1378
|
-
async with await s_cryotank.CryoCell.anit(dirn, conf=conf) as cryo:
|
|
1379
|
-
yield cryo
|
|
1380
|
-
|
|
1381
|
-
return
|
|
1382
|
-
|
|
1383
|
-
with self.getTestDir() as dirn:
|
|
1360
|
+
with self.mayTestDir(dirn) as dirn:
|
|
1384
1361
|
async with await s_cryotank.CryoCell.anit(dirn, conf=conf) as cryo:
|
|
1385
1362
|
yield cryo
|
|
1386
1363
|
|
|
@@ -1409,26 +1386,13 @@ class SynTest(unittest.TestCase):
|
|
|
1409
1386
|
s_certdir.delCertPath(certpath)
|
|
1410
1387
|
|
|
1411
1388
|
@contextlib.asynccontextmanager
|
|
1412
|
-
async def getTestCell(self, ctor, conf=None, dirn=None):
|
|
1389
|
+
async def getTestCell(self, ctor=s_cell.Cell, conf=None, dirn=None):
|
|
1413
1390
|
'''
|
|
1414
1391
|
Get a test Cell.
|
|
1415
1392
|
'''
|
|
1416
|
-
|
|
1417
|
-
conf = {
|
|
1418
|
-
'health:sysctl:checks': False,
|
|
1419
|
-
}
|
|
1420
|
-
|
|
1421
|
-
conf = copy.deepcopy(conf)
|
|
1422
|
-
|
|
1393
|
+
conf = self.getCellConf(conf=conf)
|
|
1423
1394
|
with self.withNexusReplay():
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
async with await ctor.anit(dirn, conf=conf) as cell:
|
|
1427
|
-
yield cell
|
|
1428
|
-
|
|
1429
|
-
return
|
|
1430
|
-
|
|
1431
|
-
with self.getTestDir() as dirn:
|
|
1395
|
+
with self.mayTestDir(dirn) as dirn:
|
|
1432
1396
|
async with await ctor.anit(dirn, conf=conf) as cell:
|
|
1433
1397
|
yield cell
|
|
1434
1398
|
|
|
@@ -1451,62 +1415,77 @@ class SynTest(unittest.TestCase):
|
|
|
1451
1415
|
|
|
1452
1416
|
yield core, prox, testsvc
|
|
1453
1417
|
|
|
1418
|
+
@contextlib.contextmanager
|
|
1419
|
+
def mayTestDir(self, dirn: str | None) -> contextlib.AbstractContextManager[str, None, None]:
|
|
1420
|
+
'''
|
|
1421
|
+
Convenience method to make a temporary directory or use an existing one.
|
|
1422
|
+
|
|
1423
|
+
Args:
|
|
1424
|
+
dirn: Directory to use, or None.
|
|
1425
|
+
|
|
1426
|
+
Returns:
|
|
1427
|
+
The directory to use.
|
|
1428
|
+
'''
|
|
1429
|
+
if dirn is not None:
|
|
1430
|
+
yield dirn
|
|
1431
|
+
return
|
|
1432
|
+
|
|
1433
|
+
with self.getTestDir() as dirn:
|
|
1434
|
+
yield dirn
|
|
1435
|
+
|
|
1454
1436
|
@contextlib.asynccontextmanager
|
|
1455
|
-
async def getTestAha(self, conf=None, dirn=None):
|
|
1437
|
+
async def getTestAha(self, conf=None, dirn=None, ctor=None):
|
|
1456
1438
|
|
|
1457
1439
|
if conf is None:
|
|
1458
|
-
conf = {
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1440
|
+
conf = {}
|
|
1441
|
+
|
|
1442
|
+
if ctor is None:
|
|
1443
|
+
ctor = s_aha.AhaCell.anit
|
|
1444
|
+
|
|
1445
|
+
conf = s_msgpack.deepcopy(conf)
|
|
1446
|
+
|
|
1447
|
+
hasdmonlisn = 'dmon:listen' in conf
|
|
1448
|
+
hasprovlisn = 'provision:listen' in conf
|
|
1449
|
+
|
|
1450
|
+
network = conf.setdefault('aha:network', 'synapse')
|
|
1451
|
+
hostname = conf.setdefault('dns:name', '00.aha.loop.vertex.link')
|
|
1452
|
+
|
|
1453
|
+
if hostname:
|
|
1454
|
+
conf.setdefault('provision:listen', f'ssl://0.0.0.0:0?hostname={hostname}')
|
|
1455
|
+
conf.setdefault('dmon:listen', f'ssl://0.0.0.0:0?hostname={hostname}&ca={network}')
|
|
1456
|
+
|
|
1457
|
+
conf.setdefault('health:sysctl:checks', False)
|
|
1462
1458
|
|
|
1463
1459
|
with self.withNexusReplay():
|
|
1464
|
-
if dirn:
|
|
1465
|
-
async with await s_aha.AhaCell.anit(dirn, conf=conf) as aha:
|
|
1466
|
-
yield aha
|
|
1467
|
-
else:
|
|
1468
|
-
with self.getTestDir() as dirn:
|
|
1469
|
-
async with await s_aha.AhaCell.anit(dirn, conf=conf) as aha:
|
|
1470
|
-
yield aha
|
|
1471
1460
|
|
|
1472
|
-
|
|
1473
|
-
async def getTestAhaProv(self, conf=None, dirn=None):
|
|
1474
|
-
'''
|
|
1475
|
-
Get an Aha cell that is configured for provisioning on aha.loop.vertex.link.
|
|
1461
|
+
with self.mayTestDir(dirn) as dirn:
|
|
1476
1462
|
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1463
|
+
if conf.get('aha:network') == 'synapse':
|
|
1464
|
+
dstpath = os.path.join(dirn, 'certs')
|
|
1465
|
+
if not os.path.isdir(dstpath):
|
|
1466
|
+
srcpath = self.getTestFilePath('aha/certs')
|
|
1467
|
+
shutil.copytree(srcpath, os.path.join(dirn, 'certs'))
|
|
1480
1468
|
|
|
1481
|
-
|
|
1482
|
-
s_aha.AhaCell: The provisioned Aha cell.
|
|
1483
|
-
'''
|
|
1484
|
-
bconf = {
|
|
1485
|
-
'aha:name': 'aha',
|
|
1486
|
-
'aha:network': 'loop.vertex.link',
|
|
1487
|
-
'provision:listen': 'ssl://aha.loop.vertex.link:0'
|
|
1488
|
-
}
|
|
1469
|
+
async with await ctor(dirn, conf=conf) as aha:
|
|
1489
1470
|
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
for k, v in bconf.items():
|
|
1494
|
-
conf.setdefault(k, v)
|
|
1471
|
+
mods = {}
|
|
1472
|
+
if not hasdmonlisn and hostname:
|
|
1473
|
+
mods['dmon:listen'] = f'ssl://0.0.0.0:{aha.sockaddr[1]}?hostname={hostname}&ca={network}'
|
|
1495
1474
|
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
dnsname = f'{name}.{netw}'
|
|
1475
|
+
if not hasprovlisn and hostname:
|
|
1476
|
+
mods['provision:listen'] = f'ssl://0.0.0.0:{aha.provaddr[1]}?hostname={hostname}'
|
|
1499
1477
|
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
# update the config to reflect the dynamically bound port
|
|
1503
|
-
aha.conf['provision:listen'] = f'ssl://{dnsname}:{port}'
|
|
1478
|
+
if mods:
|
|
1479
|
+
aha.modCellConf(mods)
|
|
1504
1480
|
|
|
1505
|
-
|
|
1506
|
-
host, ahaport = await aha.dmon.listen(f'ssl://0.0.0.0:0?hostname={dnsname}&ca={netw}')
|
|
1507
|
-
aha.conf['aha:urls'] = (f'ssl://{dnsname}:{ahaport}',)
|
|
1481
|
+
yield aha
|
|
1508
1482
|
|
|
1509
|
-
|
|
1483
|
+
def getCellConf(self, conf=None):
|
|
1484
|
+
if conf is None:
|
|
1485
|
+
conf = {}
|
|
1486
|
+
conf = s_msgpack.deepcopy(conf)
|
|
1487
|
+
conf.setdefault('health:sysctl:checks', False)
|
|
1488
|
+
return conf
|
|
1510
1489
|
|
|
1511
1490
|
@contextlib.asynccontextmanager
|
|
1512
1491
|
async def addSvcToAha(self, aha, svcname, ctor,
|
|
@@ -1528,36 +1507,19 @@ class SynTest(unittest.TestCase):
|
|
|
1528
1507
|
The config data for the cell is pushed into dirn/cell.yaml.
|
|
1529
1508
|
The cells are created with the ``ctor.anit()`` function.
|
|
1530
1509
|
'''
|
|
1510
|
+
ahanetw = aha.conf.req('aha:network')
|
|
1511
|
+
svcfull = f'{svcname}.{ahanetw}'
|
|
1531
1512
|
onetime = await aha.addAhaSvcProv(svcname, provinfo=provinfo)
|
|
1532
1513
|
|
|
1533
|
-
|
|
1534
|
-
conf = {
|
|
1535
|
-
'health:sysctl:checks': False,
|
|
1536
|
-
}
|
|
1537
|
-
|
|
1514
|
+
conf = self.getCellConf(conf=conf)
|
|
1538
1515
|
conf['aha:provision'] = onetime
|
|
1539
1516
|
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
n = 1 # a simple svcname like "foo"
|
|
1543
|
-
if len(svcname.split('.')) > 1:
|
|
1544
|
-
n = 2 # A replica name like 00.foo
|
|
1545
|
-
if provinfo and 'mirror' in provinfo:
|
|
1546
|
-
n = 1 # A mirror like 01.foo with mirror: foo
|
|
1547
|
-
|
|
1548
|
-
waiter = aha.waiter(n, 'aha:svcadd')
|
|
1549
|
-
|
|
1550
|
-
if dirn:
|
|
1517
|
+
waiter = aha.waiter(1, f'aha:svcadd:{svcfull}')
|
|
1518
|
+
with self.mayTestDir(dirn) as dirn:
|
|
1551
1519
|
s_common.yamlsave(conf, dirn, 'cell.yaml')
|
|
1552
1520
|
async with await ctor.anit(dirn) as svc:
|
|
1553
|
-
self.
|
|
1521
|
+
self.true(await waiter.wait(timeout=12))
|
|
1554
1522
|
yield svc
|
|
1555
|
-
else:
|
|
1556
|
-
with self.getTestDir() as dirn:
|
|
1557
|
-
s_common.yamlsave(conf, dirn, 'cell.yaml')
|
|
1558
|
-
async with await ctor.anit(dirn) as svc:
|
|
1559
|
-
self.ge(len(await waiter.wait(timeout=12)), n)
|
|
1560
|
-
yield svc
|
|
1561
1523
|
|
|
1562
1524
|
async def addSvcToCore(self, svc, core, svcname='svc'):
|
|
1563
1525
|
'''
|
|
@@ -1592,7 +1554,7 @@ class SynTest(unittest.TestCase):
|
|
|
1592
1554
|
return s_telepath.openurl(f'tcp:///{name}', **kwargs)
|
|
1593
1555
|
|
|
1594
1556
|
@contextlib.contextmanager
|
|
1595
|
-
def getTestDir(self, mirror=None, copyfrom=None, chdir=False, startdir=None):
|
|
1557
|
+
def getTestDir(self, mirror=None, copyfrom=None, chdir=False, startdir=None) -> contextlib.AbstractContextManager[str, None, None]:
|
|
1596
1558
|
'''
|
|
1597
1559
|
Get a temporary directory for test purposes.
|
|
1598
1560
|
This destroys the directory afterwards.
|
|
@@ -2058,6 +2020,7 @@ class SynTest(unittest.TestCase):
|
|
|
2058
2020
|
Assert X is not None
|
|
2059
2021
|
'''
|
|
2060
2022
|
self.assertIsNotNone(x, msg=msg)
|
|
2023
|
+
return x
|
|
2061
2024
|
|
|
2062
2025
|
def none(self, x, msg=None):
|
|
2063
2026
|
'''
|
|
@@ -2154,7 +2117,7 @@ class SynTest(unittest.TestCase):
|
|
|
2154
2117
|
count += 1
|
|
2155
2118
|
self.eq(x, count, msg=msg)
|
|
2156
2119
|
|
|
2157
|
-
def stormIsInPrint(self, mesg, mesgs, deguid=False):
|
|
2120
|
+
def stormIsInPrint(self, mesg, mesgs, deguid=False, whitespace=True):
|
|
2158
2121
|
'''
|
|
2159
2122
|
Check if a string is present in all of the print messages from a stream of storm messages.
|
|
2160
2123
|
|
|
@@ -2166,6 +2129,9 @@ class SynTest(unittest.TestCase):
|
|
|
2166
2129
|
mesg = deguidify(mesg)
|
|
2167
2130
|
|
|
2168
2131
|
print_str = '\n'.join([m[1].get('mesg') for m in mesgs if m[0] == 'print'])
|
|
2132
|
+
if not whitespace:
|
|
2133
|
+
mesg = ' '.join(mesg.split())
|
|
2134
|
+
print_str = ' '.join(print_str.split())
|
|
2169
2135
|
|
|
2170
2136
|
if deguid:
|
|
2171
2137
|
print_str = deguidify(print_str)
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import asyncio
|
|
3
|
+
import argparse
|
|
4
|
+
|
|
5
|
+
import synapse.exc as s_exc
|
|
6
|
+
import synapse.telepath as s_telepath
|
|
7
|
+
|
|
8
|
+
import synapse.lib.output as s_output
|
|
9
|
+
|
|
10
|
+
descr = '''
|
|
11
|
+
Generate a new clone URL to deploy an AHA mirror.
|
|
12
|
+
|
|
13
|
+
Examples:
|
|
14
|
+
|
|
15
|
+
python -m synapse.tools.aha.clone 01.aha.loop.vertex.link
|
|
16
|
+
|
|
17
|
+
'''
|
|
18
|
+
|
|
19
|
+
async def main(argv, outp=s_output.stdout):
|
|
20
|
+
|
|
21
|
+
pars = argparse.ArgumentParser(prog='synapse.tools.aha.clone', description=descr)
|
|
22
|
+
pars.add_argument('dnsname', help='The DNS name of the new AHA server.')
|
|
23
|
+
pars.add_argument('--port', type=int, default=27492, help='The port that the new AHA server should listen on.')
|
|
24
|
+
pars.add_argument('--url', default='cell:///vertex/storage', help='The telepath URL to connect to the AHA service.')
|
|
25
|
+
pars.add_argument('--only-url', help='Only output the URL upon successful execution',
|
|
26
|
+
action='store_true', default=False)
|
|
27
|
+
opts = pars.parse_args(argv)
|
|
28
|
+
|
|
29
|
+
async with s_telepath.withTeleEnv():
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
async with await s_telepath.openurl(opts.url) as aha:
|
|
33
|
+
curl = await aha.addAhaClone(opts.dnsname, port=opts.port)
|
|
34
|
+
|
|
35
|
+
if opts.only_url:
|
|
36
|
+
outp.printf(curl)
|
|
37
|
+
else:
|
|
38
|
+
outp.printf(f'one-time use URL: {curl}')
|
|
39
|
+
return 0
|
|
40
|
+
|
|
41
|
+
except Exception as e:
|
|
42
|
+
mesg = repr(e)
|
|
43
|
+
if isinstance(e, s_exc.SynErr):
|
|
44
|
+
mesg = e.errinfo.get('mesg', repr(e))
|
|
45
|
+
|
|
46
|
+
outp.printf(f'ERROR: {mesg}')
|
|
47
|
+
return 1
|
|
48
|
+
|
|
49
|
+
if __name__ == '__main__': # pragma: no cover
|
|
50
|
+
sys.exit(asyncio.run(main(sys.argv[1:])))
|
synapse/tools/aha/enroll.py
CHANGED
|
@@ -76,7 +76,8 @@ async def main(argv, outp=s_output.stdout):
|
|
|
76
76
|
if isinstance(ahaurls, str):
|
|
77
77
|
ahaurls = (ahaurls,)
|
|
78
78
|
|
|
79
|
-
|
|
79
|
+
certname = f'{ahauser}@{ahanetw}'
|
|
80
|
+
ahaurls = set(s_telepath.modurl(ahaurls, certname=certname))
|
|
80
81
|
servers = teleyaml.get('aha:servers')
|
|
81
82
|
|
|
82
83
|
# repack the servers so lists are tuplized like values
|
synapse/tools/backup.py
CHANGED
|
@@ -148,7 +148,7 @@ def txnbackup(lmdbinfo, srcdir, dstdir, skipdirs=None):
|
|
|
148
148
|
logger.debug(f'Backup complete. Took [{tock-tick:.2f}] for [{srcdir}]')
|
|
149
149
|
return
|
|
150
150
|
|
|
151
|
-
def backup_lmdb(env, dstdir, txn=None):
|
|
151
|
+
def backup_lmdb(env: lmdb.Environment, dstdir: str, txn=None):
|
|
152
152
|
|
|
153
153
|
tick = time.time()
|
|
154
154
|
|
|
@@ -157,7 +157,7 @@ def backup_lmdb(env, dstdir, txn=None):
|
|
|
157
157
|
env.copy(dstdir, compact=True, txn=txn)
|
|
158
158
|
|
|
159
159
|
tock = time.time()
|
|
160
|
-
logger.info(f'backup took: {tock-tick:.2f} seconds')
|
|
160
|
+
logger.info(f'backup of: {env.path()} took: {tock-tick:.2f} seconds')
|
|
161
161
|
|
|
162
162
|
def main(argv):
|
|
163
163
|
args = parse_args(argv)
|
synapse/tools/changelog.py
CHANGED
|
@@ -466,6 +466,8 @@ async def gen(opts: argparse.Namespace,
|
|
|
466
466
|
def _gen_model_rst(version, model_ref, changes, current_model, outp: s_output.OutPut, width=80) -> s_autodoc.RstHelp:
|
|
467
467
|
rst = s_autodoc.RstHelp()
|
|
468
468
|
rst.addHead(f'{version} Model Updates', link=f'.. _{model_ref}:')
|
|
469
|
+
rst.addLines(f'The following model updates were made during the ``{version}`` Synapse release.')
|
|
470
|
+
|
|
469
471
|
if new_interfaces := changes.get('interfaces').get('new_interfaces'):
|
|
470
472
|
rst.addHead('New Interfaces', lvl=1)
|
|
471
473
|
for interface, info in new_interfaces.items():
|
|
@@ -849,7 +851,7 @@ async def format(opts: argparse.Namespace,
|
|
|
849
851
|
if opts.verbose:
|
|
850
852
|
outp.printf(model_text)
|
|
851
853
|
if opts.model_doc_dir:
|
|
852
|
-
fp = s_common.genpath(opts.model_doc_dir, f'
|
|
854
|
+
fp = s_common.genpath(opts.model_doc_dir, f'update_{clean_vers_ref}.rst')
|
|
853
855
|
with s_common.genfile(fp) as fd:
|
|
854
856
|
fd.truncate(0)
|
|
855
857
|
fd.write(model_text.encode())
|