synapse 2.177.0__py311-none-any.whl → 2.179.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (73) hide show
  1. synapse/cortex.py +170 -31
  2. synapse/datamodel.py +47 -1
  3. synapse/exc.py +1 -0
  4. synapse/lib/aha.py +362 -88
  5. synapse/lib/ast.py +26 -22
  6. synapse/lib/base.py +39 -12
  7. synapse/lib/cell.py +315 -119
  8. synapse/lib/config.py +15 -11
  9. synapse/lib/coro.py +27 -0
  10. synapse/lib/drive.py +551 -0
  11. synapse/lib/layer.py +0 -5
  12. synapse/lib/link.py +1 -1
  13. synapse/lib/lmdbslab.py +3 -3
  14. synapse/lib/nexus.py +24 -12
  15. synapse/lib/schemas.py +39 -0
  16. synapse/lib/snap.py +17 -7
  17. synapse/lib/storm.py +3 -1
  18. synapse/lib/stormhttp.py +1 -0
  19. synapse/lib/stormlib/imap.py +6 -2
  20. synapse/lib/stormlib/modelext.py +29 -3
  21. synapse/lib/stormlib/smtp.py +12 -2
  22. synapse/lib/stormlib/stix.py +40 -17
  23. synapse/lib/stormlib/vault.py +2 -2
  24. synapse/lib/stormtypes.py +1 -1
  25. synapse/lib/types.py +9 -0
  26. synapse/lib/version.py +2 -2
  27. synapse/lookup/pe.py +303 -38
  28. synapse/models/dns.py +24 -1
  29. synapse/models/geospace.py +4 -1
  30. synapse/models/infotech.py +26 -1
  31. synapse/telepath.py +32 -17
  32. synapse/tests/files/aha/certs/cas/synapse.crt +28 -0
  33. synapse/tests/files/aha/certs/cas/synapse.key +51 -0
  34. synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.crt +30 -0
  35. synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.key +51 -0
  36. synapse/tests/files/aha/certs/users/root@synapse.crt +29 -0
  37. synapse/tests/files/aha/certs/users/root@synapse.key +51 -0
  38. synapse/tests/files/rstorm/testsvc.py +1 -1
  39. synapse/tests/test_axon.py +1 -1
  40. synapse/tests/test_cortex.py +67 -60
  41. synapse/tests/test_lib_agenda.py +3 -3
  42. synapse/tests/test_lib_aha.py +353 -490
  43. synapse/tests/test_lib_base.py +20 -0
  44. synapse/tests/test_lib_cell.py +273 -22
  45. synapse/tests/test_lib_config.py +4 -3
  46. synapse/tests/test_lib_coro.py +12 -0
  47. synapse/tests/test_lib_nexus.py +8 -0
  48. synapse/tests/test_lib_stormhttp.py +40 -0
  49. synapse/tests/test_lib_stormlib_aha.py +35 -35
  50. synapse/tests/test_lib_stormlib_cell.py +4 -15
  51. synapse/tests/test_lib_stormlib_imap.py +14 -3
  52. synapse/tests/test_lib_stormlib_modelext.py +55 -3
  53. synapse/tests/test_lib_stormlib_smtp.py +51 -0
  54. synapse/tests/test_lib_stormlib_stix.py +15 -0
  55. synapse/tests/test_lib_stormlib_vault.py +11 -1
  56. synapse/tests/test_lib_stormtypes.py +5 -0
  57. synapse/tests/test_lib_types.py +9 -0
  58. synapse/tests/test_model_dns.py +8 -0
  59. synapse/tests/test_model_geospace.py +3 -1
  60. synapse/tests/test_model_infotech.py +47 -0
  61. synapse/tests/test_model_syn.py +11 -0
  62. synapse/tests/test_tools_aha.py +78 -101
  63. synapse/tests/test_utils_stormcov.py +1 -1
  64. synapse/tests/utils.py +86 -120
  65. synapse/tools/aha/clone.py +50 -0
  66. synapse/tools/aha/enroll.py +2 -1
  67. synapse/tools/backup.py +2 -2
  68. synapse/tools/changelog.py +31 -1
  69. {synapse-2.177.0.dist-info → synapse-2.179.0.dist-info}/METADATA +48 -48
  70. {synapse-2.177.0.dist-info → synapse-2.179.0.dist-info}/RECORD +73 -65
  71. {synapse-2.177.0.dist-info → synapse-2.179.0.dist-info}/WHEEL +1 -1
  72. {synapse-2.177.0.dist-info → synapse-2.179.0.dist-info}/LICENSE +0 -0
  73. {synapse-2.177.0.dist-info → synapse-2.179.0.dist-info}/top_level.txt +0 -0
@@ -9,6 +9,7 @@ import synapse.lib.cell as s_cell
9
9
  import synapse.tests.utils as s_t_utils
10
10
 
11
11
  import synapse.tools.aha.list as s_a_list
12
+ import synapse.tools.aha.clone as s_a_clone
12
13
  import synapse.tools.aha.enroll as s_a_enroll
13
14
  import synapse.tools.aha.easycert as s_a_easycert
14
15
  import synapse.tools.aha.provision.user as s_a_provision_user
@@ -17,40 +18,37 @@ class AhaToolsTest(s_t_utils.SynTest):
17
18
 
18
19
  async def test_aha_list(self):
19
20
 
20
- ephemeral_address = 'tcp://0.0.0.0:0/'
21
- async with self.getTestAha(conf={'auth:passwd': 'root',
22
- 'dmon:listen': ephemeral_address}) as aha:
23
- _, port = aha.sockaddr
24
- ahaurl = f'tcp://root:root@127.0.0.1:{port}'
25
- conf0 = {
26
- 'aha:registry': ahaurl,
27
- 'aha:name': 'cell0',
28
- 'aha:network': 'demo.net',
29
- 'dmon:listen': ephemeral_address,
30
- }
31
- conf1 = {
32
- 'aha:registry': ahaurl,
33
- 'aha:name': 'cell1',
34
- 'aha:network': 'example.net',
35
- 'dmon:listen': ephemeral_address,
36
- }
21
+ async with self.getTestAha() as aha:
22
+
37
23
  waiter = aha.waiter(2, 'aha:svcadd')
24
+ conf0 = {'aha:provision': await aha.addAhaSvcProv('cell0')}
25
+
26
+ provinfo = {'aha:network': 'example.net'}
27
+ conf1 = {'aha:provision': await aha.addAhaSvcProv('cell1', provinfo=provinfo)}
28
+
29
+ ahaurl = aha.getLocalUrl()
30
+
38
31
  async with self.getTestCell(s_cell.Cell, conf=conf0) as cell0:
32
+
39
33
  async with self.getTestCell(s_cell.Cell, conf=conf1) as cell1:
34
+
40
35
  self.true(await waiter.wait(timeout=6))
36
+
41
37
  argv = [ahaurl]
42
38
  retn, outp = await self.execToolMain(s_a_list._main, argv)
43
39
  self.eq(retn, 0)
44
- outp.expect('Service network leader')
45
- outp.expect('cell0 demo.net None')
46
- outp.expect('cell1 example.net None')
40
+
41
+ outp.expect('''
42
+ Service network leader
43
+ cell0 synapse None
44
+ cell1 example.net None
45
+ ''', whitespace=False)
47
46
 
48
47
  argv = [ahaurl, 'demo.net']
49
48
  retn, outp = await self.execToolMain(s_a_list._main, argv)
50
49
  self.eq(retn, 0)
51
- outp.expect('Service network')
52
- outp.expect('cell0 demo.net')
53
- self.false(outp.expect('cell1 example.net', throw=False))
50
+ outp.expect('Service network', whitespace=False)
51
+ outp.expect('cell0 demo.net', whitespace=False)
54
52
 
55
53
  async with self.getTestCore() as core:
56
54
  curl = core.getLocalUrl()
@@ -94,102 +92,81 @@ class AhaToolsTest(s_t_utils.SynTest):
94
92
 
95
93
  async def test_aha_enroll(self):
96
94
 
97
- with self.getTestDir() as dirn:
98
-
99
- conf = {
100
- 'aha:name': 'aha',
101
- 'aha:network': 'loop.vertex.link',
102
- 'provision:listen': 'ssl://aha.loop.vertex.link:0',
103
- 'dmon:listen': 'tcp://0.0.0.0:0'
104
- }
105
- async with self.getTestAha(dirn=dirn, conf=conf) as aha:
106
-
107
- addr, port = aha.provdmon.addr
108
- aha.conf['provision:listen'] = f'ssl://aha.loop.vertex.link:{port}'
109
-
110
- host_, ahaport = aha.sockaddr
111
- self.eq(aha._getAhaUrls(), [f'ssl://aha.loop.vertex.link:{ahaport}'])
95
+ async with self.getTestAha() as aha:
112
96
 
113
- argv = ['--url', aha.getLocalUrl(), 'visi']
114
- retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
115
- self.isin('one-time use URL:', str(outp))
116
-
117
- provurl = str(outp).split(':', 1)[1].strip()
118
- with self.getTestSynDir() as syndir:
97
+ argv = ['--url', aha.getLocalUrl(), '01.aha.loop.vertex.link']
98
+ retn, outp = await self.execToolMain(s_a_clone.main, argv)
99
+ self.eq(retn, 0)
100
+ self.isin('one-time use URL:', str(outp))
119
101
 
120
- capath = s_common.genpath(syndir, 'certs', 'cas', 'loop.vertex.link.crt')
121
- crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.crt')
122
- keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.key')
102
+ argv = ['--url', aha.getLocalUrl(), '01.aha.loop.vertex.link', '--only-url']
103
+ retn, outp = await self.execToolMain(s_a_clone.main, argv)
104
+ self.eq(retn, 0)
105
+ self.notin('one-time use URL:', str(outp))
123
106
 
124
- for path in (capath, crtpath, keypath):
125
- s_common.genfile(path)
107
+ argv = ['--url', 'newp://1.2.3.4', '01.aha.loop.vertex.link']
108
+ retn, outp = await self.execToolMain(s_a_clone.main, argv)
109
+ self.eq(retn, 1)
110
+ self.isin('ERROR: Invalid URL scheme: newp', str(outp))
126
111
 
127
- yamlpath = s_common.genpath(syndir, 'telepath.yaml')
128
- s_common.yamlsave({'aha:servers': 'cell://aha'}, yamlpath)
112
+ argv = ['--url', aha.getLocalUrl(), 'visi']
113
+ retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
114
+ self.isin('one-time use URL:', str(outp))
129
115
 
130
- retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
131
- self.eq(0, retn)
116
+ provurl = str(outp).split(':', 1)[1].strip()
117
+ with self.getTestSynDir() as syndir:
132
118
 
133
- for path in (capath, crtpath, keypath):
134
- self.gt(os.path.getsize(path), 0)
119
+ capath = s_common.genpath(syndir, 'certs', 'cas', 'synapse.crt')
120
+ crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.crt')
121
+ keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.crt')
135
122
 
136
- teleyaml = s_common.yamlload(syndir, 'telepath.yaml')
137
- self.eq(teleyaml.get('version'), 1)
138
- self.eq(teleyaml.get('aha:servers'), ('cell://aha', f'ssl://visi@aha.loop.vertex.link:{ahaport}'))
123
+ retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
124
+ self.eq(0, retn)
139
125
 
140
- shutil.rmtree(s_common.genpath(syndir, 'certs'))
126
+ for path in (capath, crtpath, keypath):
127
+ self.true(os.path.isfile(path))
128
+ self.gt(os.path.getsize(path), 0)
141
129
 
142
- def _strUrl(self):
143
- return 'ssl://aha.loop.vertex.link'
130
+ teleyaml = s_common.yamlload(syndir, 'telepath.yaml')
131
+ self.eq(teleyaml.get('version'), 1)
132
+ self.len(1, teleyaml.get('aha:servers'))
144
133
 
145
- with mock.patch('synapse.lib.aha.AhaCell._getAhaUrls', _strUrl):
134
+ shutil.rmtree(s_common.genpath(syndir, 'certs'))
146
135
 
147
- argv = ['--again', '--url', aha.getLocalUrl(), 'visi']
148
- retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
149
- self.eq(retn, 0)
150
- self.isin('one-time use URL:', str(outp))
151
-
152
- provurl = str(outp).split(':', 1)[1].strip()
153
-
154
- retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
155
-
156
- servers = ['cell://aha',
157
- 'ssl://visi@aha.loop.vertex.link',
158
- f'ssl://visi@aha.loop.vertex.link:{ahaport}']
136
+ argv = ['--again', '--url', aha.getLocalUrl(), 'visi']
137
+ retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
138
+ self.eq(retn, 0)
139
+ self.isin('one-time use URL:', str(outp))
159
140
 
160
- teleyaml = s_common.yamlload(syndir, 'telepath.yaml')
161
- self.sorteq(teleyaml.get('aha:servers'), servers)
141
+ provurl = str(outp).split(':', 1)[1].strip()
162
142
 
163
- # Just return the URL
164
- retn, outp = await self.execToolMain(s_a_provision_user.main, argv + ['--only-url'])
165
- self.eq(retn, 0)
166
- self.notin('one-time use URL:', str(outp))
167
- self.isin('ssl://', str(outp))
143
+ retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
168
144
 
169
- with self.getTestSynDir() as syndir:
145
+ # Just return the URL
146
+ retn, outp = await self.execToolMain(s_a_provision_user.main, argv + ['--only-url'])
147
+ self.eq(retn, 0)
148
+ self.notin('one-time use URL:', str(outp))
149
+ self.isin('ssl://', str(outp))
170
150
 
171
- argv = ['--again', '--url', aha.getLocalUrl(), 'visi']
172
- retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
173
- self.eq(retn, 0)
174
- provurl = str(outp).split(':', 1)[1].strip()
151
+ with self.getTestSynDir() as syndir:
175
152
 
176
- capath = s_common.genpath(syndir, 'certs', 'cas', 'loop.vertex.link.crt')
177
- crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.crt')
178
- keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.key')
153
+ argv = ['--again', '--url', aha.getLocalUrl(), 'visi']
154
+ retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
155
+ self.eq(retn, 0)
156
+ provurl = str(outp).split(':', 1)[1].strip()
179
157
 
180
- for path in (capath, crtpath, keypath):
181
- s_common.genfile(path)
158
+ capath = s_common.genpath(syndir, 'certs', 'cas', 'synapse.crt')
159
+ crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.crt')
160
+ keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.key')
182
161
 
183
- yamlpath = s_common.genpath(syndir, 'telepath.yaml')
184
- s_common.yamlsave({'aha:servers': ['cell://aha', 'cell://foo', ['cell://bar']]}, yamlpath)
162
+ for path in (capath, crtpath, keypath):
163
+ s_common.genfile(path)
185
164
 
186
- retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
187
- self.eq(0, retn)
165
+ retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
166
+ self.eq(0, retn)
188
167
 
189
- for path in (capath, crtpath, keypath):
190
- self.gt(os.path.getsize(path), 0)
168
+ for path in (capath, crtpath, keypath):
169
+ self.gt(os.path.getsize(path), 0)
191
170
 
192
- teleyaml = s_common.yamlload(syndir, 'telepath.yaml')
193
- self.eq(teleyaml.get('version'), 1)
194
- self.eq(teleyaml.get('aha:servers'), ('cell://aha', 'cell://foo', ('cell://bar',),
195
- f'ssl://visi@aha.loop.vertex.link:{ahaport}'))
171
+ teleyaml = s_common.yamlload(syndir, 'telepath.yaml')
172
+ self.eq(teleyaml.get('version'), 1)
@@ -78,7 +78,7 @@ class TestUtilsStormcov(s_utils.SynTest):
78
78
  await core.stormlist(s_files.getAssetStr('stormcov/lookup.storm'), opts={'mode': 'lookup'})
79
79
 
80
80
  orig = s_snap.Snap.nodesByPropValu
81
- async def nodesByPropValu(self, full, cmpr, valu):
81
+ async def nodesByPropValu(self, full, cmpr, valu, norm=True):
82
82
  frame = inspect.currentframe()
83
83
  if pivotracer.dynamic_source_filename(None, frame) is not None:
84
84
  assert (2, 2) == pivotracer.line_number_range(frame)
synapse/tests/utils.py CHANGED
@@ -677,7 +677,7 @@ class TstEnv:
677
677
 
678
678
  class TstOutPut(s_output.OutPutStr):
679
679
 
680
- def expect(self, substr, throw=True):
680
+ def expect(self, substr, throw=True, whitespace=True):
681
681
  '''
682
682
  Check if a string is present in the messages captured by the OutPutStr object.
683
683
 
@@ -689,6 +689,11 @@ class TstOutPut(s_output.OutPutStr):
689
689
  bool: True if the string is present; False if the string is not present and throw is False.
690
690
  '''
691
691
  outs = str(self)
692
+
693
+ if not whitespace:
694
+ outs = ' '.join(outs.split())
695
+ substr = ' '.join(outs.split())
696
+
692
697
  if outs.find(substr) == -1:
693
698
  if throw:
694
699
  mesg = 'TestOutPut.expect(%s) not in %s' % (substr, outs)
@@ -1297,34 +1302,20 @@ class SynTest(unittest.TestCase):
1297
1302
  Returns:
1298
1303
  s_cortex.Cortex: A Cortex object.
1299
1304
  '''
1300
- if conf is None:
1301
- conf = {
1302
- 'health:sysctl:checks': False,
1303
- }
1304
-
1305
- conf = copy.deepcopy(conf)
1305
+ conf = self.getCellConf(conf=conf)
1306
1306
 
1307
- mods = conf.get('modules')
1308
-
1309
- if mods is None:
1310
- mods = []
1311
- conf['modules'] = mods
1307
+ mods = list(conf.get('modules', ()))
1308
+ conf['modules'] = mods
1312
1309
 
1313
1310
  mods.insert(0, ('synapse.tests.utils.TestModule', {'key': 'valu'}))
1314
1311
 
1315
1312
  with self.withNexusReplay():
1316
1313
 
1317
- if dirn is not None:
1314
+ with self.mayTestDir(dirn) as dirn:
1318
1315
 
1319
1316
  async with await s_cortex.Cortex.anit(dirn, conf=conf) as core:
1320
1317
  yield core
1321
1318
 
1322
- return
1323
-
1324
- with self.getTestDir() as dirn:
1325
- async with await s_cortex.Cortex.anit(dirn, conf=conf) as core:
1326
- yield core
1327
-
1328
1319
  @contextlib.asynccontextmanager
1329
1320
  async def getTestCoreAndProxy(self, conf=None, dirn=None):
1330
1321
  '''
@@ -1342,11 +1333,7 @@ class SynTest(unittest.TestCase):
1342
1333
  @contextlib.asynccontextmanager
1343
1334
  async def getTestJsonStor(self, dirn=None, conf=None):
1344
1335
 
1345
- if conf is None:
1346
- conf = {
1347
- 'health:sysctl:checks': False,
1348
- }
1349
- conf = copy.deepcopy(conf)
1336
+ conf = self.getCellConf(conf=conf)
1350
1337
 
1351
1338
  with self.withNexusReplay():
1352
1339
  if dirn is not None:
@@ -1367,20 +1354,10 @@ class SynTest(unittest.TestCase):
1367
1354
  Returns:
1368
1355
  s_cryotank.CryoCell: Test cryocell.
1369
1356
  '''
1370
- if conf is None:
1371
- conf = {
1372
- 'health:sysctl:checks': False,
1373
- }
1374
- conf = copy.deepcopy(conf)
1357
+ conf = self.getCellConf(conf=conf)
1375
1358
 
1376
1359
  with self.withNexusReplay():
1377
- if dirn is not None:
1378
- async with await s_cryotank.CryoCell.anit(dirn, conf=conf) as cryo:
1379
- yield cryo
1380
-
1381
- return
1382
-
1383
- with self.getTestDir() as dirn:
1360
+ with self.mayTestDir(dirn) as dirn:
1384
1361
  async with await s_cryotank.CryoCell.anit(dirn, conf=conf) as cryo:
1385
1362
  yield cryo
1386
1363
 
@@ -1409,26 +1386,13 @@ class SynTest(unittest.TestCase):
1409
1386
  s_certdir.delCertPath(certpath)
1410
1387
 
1411
1388
  @contextlib.asynccontextmanager
1412
- async def getTestCell(self, ctor, conf=None, dirn=None):
1389
+ async def getTestCell(self, ctor=s_cell.Cell, conf=None, dirn=None):
1413
1390
  '''
1414
1391
  Get a test Cell.
1415
1392
  '''
1416
- if conf is None:
1417
- conf = {
1418
- 'health:sysctl:checks': False,
1419
- }
1420
-
1421
- conf = copy.deepcopy(conf)
1422
-
1393
+ conf = self.getCellConf(conf=conf)
1423
1394
  with self.withNexusReplay():
1424
- if dirn is not None:
1425
-
1426
- async with await ctor.anit(dirn, conf=conf) as cell:
1427
- yield cell
1428
-
1429
- return
1430
-
1431
- with self.getTestDir() as dirn:
1395
+ with self.mayTestDir(dirn) as dirn:
1432
1396
  async with await ctor.anit(dirn, conf=conf) as cell:
1433
1397
  yield cell
1434
1398
 
@@ -1451,62 +1415,77 @@ class SynTest(unittest.TestCase):
1451
1415
 
1452
1416
  yield core, prox, testsvc
1453
1417
 
1418
+ @contextlib.contextmanager
1419
+ def mayTestDir(self, dirn: str | None) -> contextlib.AbstractContextManager[str, None, None]:
1420
+ '''
1421
+ Convenience method to make a temporary directory or use an existing one.
1422
+
1423
+ Args:
1424
+ dirn: Directory to use, or None.
1425
+
1426
+ Returns:
1427
+ The directory to use.
1428
+ '''
1429
+ if dirn is not None:
1430
+ yield dirn
1431
+ return
1432
+
1433
+ with self.getTestDir() as dirn:
1434
+ yield dirn
1435
+
1454
1436
  @contextlib.asynccontextmanager
1455
- async def getTestAha(self, conf=None, dirn=None):
1437
+ async def getTestAha(self, conf=None, dirn=None, ctor=None):
1456
1438
 
1457
1439
  if conf is None:
1458
- conf = {
1459
- 'health:sysctl:checks': False,
1460
- }
1461
- conf = copy.deepcopy(conf)
1440
+ conf = {}
1441
+
1442
+ if ctor is None:
1443
+ ctor = s_aha.AhaCell.anit
1444
+
1445
+ conf = s_msgpack.deepcopy(conf)
1446
+
1447
+ hasdmonlisn = 'dmon:listen' in conf
1448
+ hasprovlisn = 'provision:listen' in conf
1449
+
1450
+ network = conf.setdefault('aha:network', 'synapse')
1451
+ hostname = conf.setdefault('dns:name', '00.aha.loop.vertex.link')
1452
+
1453
+ if hostname:
1454
+ conf.setdefault('provision:listen', f'ssl://0.0.0.0:0?hostname={hostname}')
1455
+ conf.setdefault('dmon:listen', f'ssl://0.0.0.0:0?hostname={hostname}&ca={network}')
1456
+
1457
+ conf.setdefault('health:sysctl:checks', False)
1462
1458
 
1463
1459
  with self.withNexusReplay():
1464
- if dirn:
1465
- async with await s_aha.AhaCell.anit(dirn, conf=conf) as aha:
1466
- yield aha
1467
- else:
1468
- with self.getTestDir() as dirn:
1469
- async with await s_aha.AhaCell.anit(dirn, conf=conf) as aha:
1470
- yield aha
1471
1460
 
1472
- @contextlib.asynccontextmanager
1473
- async def getTestAhaProv(self, conf=None, dirn=None):
1474
- '''
1475
- Get an Aha cell that is configured for provisioning on aha.loop.vertex.link.
1461
+ with self.mayTestDir(dirn) as dirn:
1476
1462
 
1477
- Args:
1478
- conf: Optional configuration information for the Aha cell.
1479
- dirn: Optional path to create the Aha cell in.
1463
+ if conf.get('aha:network') == 'synapse':
1464
+ dstpath = os.path.join(dirn, 'certs')
1465
+ if not os.path.isdir(dstpath):
1466
+ srcpath = self.getTestFilePath('aha/certs')
1467
+ shutil.copytree(srcpath, os.path.join(dirn, 'certs'))
1480
1468
 
1481
- Returns:
1482
- s_aha.AhaCell: The provisioned Aha cell.
1483
- '''
1484
- bconf = {
1485
- 'aha:name': 'aha',
1486
- 'aha:network': 'loop.vertex.link',
1487
- 'provision:listen': 'ssl://aha.loop.vertex.link:0'
1488
- }
1469
+ async with await ctor(dirn, conf=conf) as aha:
1489
1470
 
1490
- if conf is None:
1491
- conf = bconf
1492
- else:
1493
- for k, v in bconf.items():
1494
- conf.setdefault(k, v)
1471
+ mods = {}
1472
+ if not hasdmonlisn and hostname:
1473
+ mods['dmon:listen'] = f'ssl://0.0.0.0:{aha.sockaddr[1]}?hostname={hostname}&ca={network}'
1495
1474
 
1496
- name = conf.get('aha:name')
1497
- netw = conf.get('aha:network')
1498
- dnsname = f'{name}.{netw}'
1475
+ if not hasprovlisn and hostname:
1476
+ mods['provision:listen'] = f'ssl://0.0.0.0:{aha.provaddr[1]}?hostname={hostname}'
1499
1477
 
1500
- async with self.getTestAha(conf=conf, dirn=dirn) as aha:
1501
- addr, port = aha.provdmon.addr
1502
- # update the config to reflect the dynamically bound port
1503
- aha.conf['provision:listen'] = f'ssl://{dnsname}:{port}'
1478
+ if mods:
1479
+ aha.modCellConf(mods)
1504
1480
 
1505
- # do this config ex-post-facto due to port binding...
1506
- host, ahaport = await aha.dmon.listen(f'ssl://0.0.0.0:0?hostname={dnsname}&ca={netw}')
1507
- aha.conf['aha:urls'] = (f'ssl://{dnsname}:{ahaport}',)
1481
+ yield aha
1508
1482
 
1509
- yield aha
1483
+ def getCellConf(self, conf=None):
1484
+ if conf is None:
1485
+ conf = {}
1486
+ conf = s_msgpack.deepcopy(conf)
1487
+ conf.setdefault('health:sysctl:checks', False)
1488
+ return conf
1510
1489
 
1511
1490
  @contextlib.asynccontextmanager
1512
1491
  async def addSvcToAha(self, aha, svcname, ctor,
@@ -1528,36 +1507,19 @@ class SynTest(unittest.TestCase):
1528
1507
  The config data for the cell is pushed into dirn/cell.yaml.
1529
1508
  The cells are created with the ``ctor.anit()`` function.
1530
1509
  '''
1510
+ ahanetw = aha.conf.req('aha:network')
1511
+ svcfull = f'{svcname}.{ahanetw}'
1531
1512
  onetime = await aha.addAhaSvcProv(svcname, provinfo=provinfo)
1532
1513
 
1533
- if conf is None:
1534
- conf = {
1535
- 'health:sysctl:checks': False,
1536
- }
1537
-
1514
+ conf = self.getCellConf(conf=conf)
1538
1515
  conf['aha:provision'] = onetime
1539
1516
 
1540
- logger.info(f'Adding {svcname}')
1541
-
1542
- n = 1 # a simple svcname like "foo"
1543
- if len(svcname.split('.')) > 1:
1544
- n = 2 # A replica name like 00.foo
1545
- if provinfo and 'mirror' in provinfo:
1546
- n = 1 # A mirror like 01.foo with mirror: foo
1547
-
1548
- waiter = aha.waiter(n, 'aha:svcadd')
1549
-
1550
- if dirn:
1517
+ waiter = aha.waiter(1, f'aha:svcadd:{svcfull}')
1518
+ with self.mayTestDir(dirn) as dirn:
1551
1519
  s_common.yamlsave(conf, dirn, 'cell.yaml')
1552
1520
  async with await ctor.anit(dirn) as svc:
1553
- self.ge(len(await waiter.wait(timeout=12)), n)
1521
+ self.true(await waiter.wait(timeout=12))
1554
1522
  yield svc
1555
- else:
1556
- with self.getTestDir() as dirn:
1557
- s_common.yamlsave(conf, dirn, 'cell.yaml')
1558
- async with await ctor.anit(dirn) as svc:
1559
- self.ge(len(await waiter.wait(timeout=12)), n)
1560
- yield svc
1561
1523
 
1562
1524
  async def addSvcToCore(self, svc, core, svcname='svc'):
1563
1525
  '''
@@ -1592,7 +1554,7 @@ class SynTest(unittest.TestCase):
1592
1554
  return s_telepath.openurl(f'tcp:///{name}', **kwargs)
1593
1555
 
1594
1556
  @contextlib.contextmanager
1595
- def getTestDir(self, mirror=None, copyfrom=None, chdir=False, startdir=None):
1557
+ def getTestDir(self, mirror=None, copyfrom=None, chdir=False, startdir=None) -> contextlib.AbstractContextManager[str, None, None]:
1596
1558
  '''
1597
1559
  Get a temporary directory for test purposes.
1598
1560
  This destroys the directory afterwards.
@@ -2058,6 +2020,7 @@ class SynTest(unittest.TestCase):
2058
2020
  Assert X is not None
2059
2021
  '''
2060
2022
  self.assertIsNotNone(x, msg=msg)
2023
+ return x
2061
2024
 
2062
2025
  def none(self, x, msg=None):
2063
2026
  '''
@@ -2154,7 +2117,7 @@ class SynTest(unittest.TestCase):
2154
2117
  count += 1
2155
2118
  self.eq(x, count, msg=msg)
2156
2119
 
2157
- def stormIsInPrint(self, mesg, mesgs, deguid=False):
2120
+ def stormIsInPrint(self, mesg, mesgs, deguid=False, whitespace=True):
2158
2121
  '''
2159
2122
  Check if a string is present in all of the print messages from a stream of storm messages.
2160
2123
 
@@ -2166,6 +2129,9 @@ class SynTest(unittest.TestCase):
2166
2129
  mesg = deguidify(mesg)
2167
2130
 
2168
2131
  print_str = '\n'.join([m[1].get('mesg') for m in mesgs if m[0] == 'print'])
2132
+ if not whitespace:
2133
+ mesg = ' '.join(mesg.split())
2134
+ print_str = ' '.join(print_str.split())
2169
2135
 
2170
2136
  if deguid:
2171
2137
  print_str = deguidify(print_str)
@@ -0,0 +1,50 @@
1
+ import sys
2
+ import asyncio
3
+ import argparse
4
+
5
+ import synapse.exc as s_exc
6
+ import synapse.telepath as s_telepath
7
+
8
+ import synapse.lib.output as s_output
9
+
10
+ descr = '''
11
+ Generate a new clone URL to deploy an AHA mirror.
12
+
13
+ Examples:
14
+
15
+ python -m synapse.tools.aha.clone 01.aha.loop.vertex.link
16
+
17
+ '''
18
+
19
+ async def main(argv, outp=s_output.stdout):
20
+
21
+ pars = argparse.ArgumentParser(prog='synapse.tools.aha.clone', description=descr)
22
+ pars.add_argument('dnsname', help='The DNS name of the new AHA server.')
23
+ pars.add_argument('--port', type=int, default=27492, help='The port that the new AHA server should listen on.')
24
+ pars.add_argument('--url', default='cell:///vertex/storage', help='The telepath URL to connect to the AHA service.')
25
+ pars.add_argument('--only-url', help='Only output the URL upon successful execution',
26
+ action='store_true', default=False)
27
+ opts = pars.parse_args(argv)
28
+
29
+ async with s_telepath.withTeleEnv():
30
+
31
+ try:
32
+ async with await s_telepath.openurl(opts.url) as aha:
33
+ curl = await aha.addAhaClone(opts.dnsname, port=opts.port)
34
+
35
+ if opts.only_url:
36
+ outp.printf(curl)
37
+ else:
38
+ outp.printf(f'one-time use URL: {curl}')
39
+ return 0
40
+
41
+ except Exception as e:
42
+ mesg = repr(e)
43
+ if isinstance(e, s_exc.SynErr):
44
+ mesg = e.errinfo.get('mesg', repr(e))
45
+
46
+ outp.printf(f'ERROR: {mesg}')
47
+ return 1
48
+
49
+ if __name__ == '__main__': # pragma: no cover
50
+ sys.exit(asyncio.run(main(sys.argv[1:])))
@@ -76,7 +76,8 @@ async def main(argv, outp=s_output.stdout):
76
76
  if isinstance(ahaurls, str):
77
77
  ahaurls = (ahaurls,)
78
78
 
79
- ahaurls = set(s_telepath.modurl(ahaurls, user=ahauser))
79
+ certname = f'{ahauser}@{ahanetw}'
80
+ ahaurls = set(s_telepath.modurl(ahaurls, certname=certname))
80
81
  servers = teleyaml.get('aha:servers')
81
82
 
82
83
  # repack the servers so lists are tuplized like values
synapse/tools/backup.py CHANGED
@@ -148,7 +148,7 @@ def txnbackup(lmdbinfo, srcdir, dstdir, skipdirs=None):
148
148
  logger.debug(f'Backup complete. Took [{tock-tick:.2f}] for [{srcdir}]')
149
149
  return
150
150
 
151
- def backup_lmdb(env, dstdir, txn=None):
151
+ def backup_lmdb(env: lmdb.Environment, dstdir: str, txn=None):
152
152
 
153
153
  tick = time.time()
154
154
 
@@ -157,7 +157,7 @@ def backup_lmdb(env, dstdir, txn=None):
157
157
  env.copy(dstdir, compact=True, txn=txn)
158
158
 
159
159
  tock = time.time()
160
- logger.info(f'backup took: {tock-tick:.2f} seconds')
160
+ logger.info(f'backup of: {env.path()} took: {tock-tick:.2f} seconds')
161
161
 
162
162
  def main(argv):
163
163
  args = parse_args(argv)