synapse 2.223.0__py311-none-any.whl → 2.225.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/axon.py +10 -5
- synapse/common.py +2 -2
- synapse/cortex.py +52 -3
- synapse/datamodel.py +1 -1
- synapse/lib/cell.py +1 -1
- synapse/lib/const.py +4 -0
- synapse/lib/layer.py +6 -6
- synapse/lib/multislabseqn.py +36 -1
- synapse/lib/nexus.py +67 -8
- synapse/lib/queue.py +4 -1
- synapse/lib/rstorm.py +2 -2
- synapse/lib/schemas.py +13 -1
- synapse/lib/slabseqn.py +28 -0
- synapse/lib/storm.py +40 -2
- synapse/lib/stormhttp.py +7 -1
- synapse/lib/stormlib/imap.py +12 -4
- synapse/lib/stormlib/task.py +0 -1
- synapse/lib/stormtypes.py +19 -1
- synapse/lib/version.py +2 -2
- synapse/models/inet.py +29 -0
- synapse/models/media.py +4 -0
- synapse/models/proj.py +3 -0
- synapse/models/risk.py +9 -0
- synapse/models/syn.py +8 -0
- synapse/tests/test_common.py +4 -0
- synapse/tests/test_cortex.py +53 -2
- synapse/tests/test_lib_agenda.py +1 -1
- synapse/tests/test_lib_cell.py +1 -1
- synapse/tests/test_lib_certdir.py +1 -1
- synapse/tests/test_lib_httpapi.py +1 -1
- synapse/tests/test_lib_layer.py +1 -1
- synapse/tests/test_lib_multislabseqn.py +22 -0
- synapse/tests/test_lib_nexus.py +42 -1
- synapse/tests/test_lib_slabseqn.py +30 -1
- synapse/tests/test_lib_storm.py +156 -1
- synapse/tests/test_lib_stormhttp.py +16 -0
- synapse/tests/test_lib_stormlib_imap.py +14 -0
- synapse/tests/test_lib_stormlib_oauth.py +1 -1
- synapse/tests/test_lib_stormsvc.py +1 -1
- synapse/tests/test_lib_stormtypes.py +12 -0
- synapse/tests/test_lib_trigger.py +1 -1
- synapse/tests/test_model_inet.py +29 -0
- synapse/tests/test_model_media.py +4 -1
- synapse/tests/test_model_proj.py +3 -1
- synapse/tests/test_model_risk.py +12 -0
- synapse/tests/test_model_syn.py +54 -2
- synapse/tests/{test_tools_axon2axon.py → test_tools_axon_copy.py} +4 -4
- synapse/tests/{test_tools_pullfile.py → test_tools_axon_get.py} +4 -4
- synapse/tests/{test_tools_pushfile.py → test_tools_axon_put.py} +7 -7
- synapse/tests/{test_tools_csvtool.py → test_tools_cortex_csv.py} +12 -3
- synapse/tests/{test_tools_feed.py → test_tools_cortex_feed.py} +2 -2
- synapse/tests/{test_tools_apikey.py → test_tools_service_apikey.py} +1 -4
- synapse/tests/{test_tools_backup.py → test_tools_service_backup.py} +5 -5
- synapse/tests/{test_tools_demote.py → test_tools_service_demote.py} +1 -1
- synapse/tests/{test_tools_healthcheck.py → test_tools_service_healthcheck.py} +1 -1
- synapse/tests/{test_tools_livebackup.py → test_tools_service_livebackup.py} +1 -1
- synapse/tests/{test_tools_modrole.py → test_tools_service_modrole.py} +1 -1
- synapse/tests/{test_tools_moduser.py → test_tools_service_moduser.py} +1 -1
- synapse/tests/{test_tools_promote.py → test_tools_service_promote.py} +1 -1
- synapse/tests/{test_tools_reload.py → test_tools_service_reload.py} +1 -1
- synapse/tests/{test_tools_shutdown.py → test_tools_service_shutdown.py} +1 -1
- synapse/tests/{test_tools_snapshot.py → test_tools_service_snapshot.py} +1 -1
- synapse/tests/{test_tools_storm.py → test_tools_storm_cli.py} +1 -1
- synapse/tests/{test_tools_pkgs_gendocs.py → test_tools_storm_pkg_doc.py} +12 -3
- synapse/tests/{test_tools_genpkg.py → test_tools_storm_pkg_gen.py} +1 -1
- synapse/tests/{test_tools_autodoc.py → test_tools_utils_autodoc.py} +1 -1
- synapse/tests/test_tools_utils_changelog.py +454 -0
- synapse/tests/{test_tools_easycert.py → test_tools_utils_easycert.py} +48 -46
- synapse/tests/{test_tools_guid.py → test_tools_utils_guid.py} +3 -3
- synapse/tests/{test_tools_json2mpk.py → test_tools_utils_json2mpk.py} +3 -3
- synapse/tests/{test_tools_rstorm.py → test_tools_utils_rstorm.py} +6 -1
- synapse/tests/utils.py +3 -1
- synapse/tools/apikey.py +4 -83
- synapse/tools/autodoc.py +3 -1031
- synapse/tools/axon/copy.py +44 -0
- synapse/tools/axon/get.py +64 -0
- synapse/tools/axon/put.py +122 -0
- synapse/tools/axon2axon.py +3 -36
- synapse/tools/backup.py +6 -176
- synapse/tools/changelog.py +3 -1098
- synapse/tools/cortex/csv.py +236 -0
- synapse/tools/cortex/feed.py +151 -0
- synapse/tools/csvtool.py +3 -227
- synapse/tools/demote.py +4 -40
- synapse/tools/docker/validate.py +3 -3
- synapse/tools/easycert.py +4 -129
- synapse/tools/feed.py +3 -140
- synapse/tools/genpkg.py +3 -307
- synapse/tools/guid.py +7 -6
- synapse/tools/healthcheck.py +3 -101
- synapse/tools/json2mpk.py +6 -38
- synapse/tools/livebackup.py +4 -27
- synapse/tools/modrole.py +3 -108
- synapse/tools/moduser.py +3 -179
- synapse/tools/pkgs/gendocs.py +3 -164
- synapse/tools/promote.py +4 -41
- synapse/tools/pullfile.py +3 -56
- synapse/tools/pushfile.py +3 -114
- synapse/tools/reload.py +4 -61
- synapse/tools/rstorm.py +3 -26
- synapse/tools/service/__init__.py +0 -0
- synapse/tools/service/apikey.py +90 -0
- synapse/tools/service/backup.py +181 -0
- synapse/tools/service/demote.py +47 -0
- synapse/tools/service/healthcheck.py +109 -0
- synapse/tools/service/livebackup.py +34 -0
- synapse/tools/service/modrole.py +116 -0
- synapse/tools/service/moduser.py +184 -0
- synapse/tools/service/promote.py +48 -0
- synapse/tools/service/reload.py +68 -0
- synapse/tools/service/shutdown.py +51 -0
- synapse/tools/service/snapshot.py +64 -0
- synapse/tools/shutdown.py +5 -45
- synapse/tools/snapshot.py +4 -57
- synapse/tools/storm/__init__.py +0 -0
- synapse/tools/storm/__main__.py +5 -0
- synapse/tools/{storm.py → storm/_cli.py} +0 -3
- synapse/tools/storm/pkg/__init__.py +0 -0
- synapse/tools/{pkgs/pandoc_filter.py → storm/pkg/_pandoc_filter.py} +1 -1
- synapse/tools/storm/pkg/doc.py +176 -0
- synapse/tools/storm/pkg/gen.py +315 -0
- synapse/tools/utils/__init__.py +0 -0
- synapse/tools/utils/autodoc.py +1040 -0
- synapse/tools/utils/changelog.py +1124 -0
- synapse/tools/utils/easycert.py +136 -0
- synapse/tools/utils/guid.py +11 -0
- synapse/tools/utils/json2mpk.py +46 -0
- synapse/tools/utils/rstorm.py +35 -0
- {synapse-2.223.0.dist-info → synapse-2.225.0.dist-info}/METADATA +1 -1
- {synapse-2.223.0.dist-info → synapse-2.225.0.dist-info}/RECORD +134 -105
- synapse/tests/test_tools_changelog.py +0 -196
- /synapse/tests/{test_tools_axon.py → test_tools_axon_dump_load.py} +0 -0
- {synapse-2.223.0.dist-info → synapse-2.225.0.dist-info}/WHEEL +0 -0
- {synapse-2.223.0.dist-info → synapse-2.225.0.dist-info}/licenses/LICENSE +0 -0
- {synapse-2.223.0.dist-info → synapse-2.225.0.dist-info}/top_level.txt +0 -0
synapse/tools/healthcheck.py
CHANGED
|
@@ -1,109 +1,11 @@
|
|
|
1
|
-
import socket
|
|
2
|
-
import asyncio
|
|
3
|
-
|
|
4
|
-
import synapse.exc as s_exc
|
|
5
1
|
import synapse.common as s_common
|
|
6
|
-
import synapse.telepath as s_telepath
|
|
7
2
|
|
|
8
3
|
import synapse.lib.cmd as s_cmd
|
|
9
|
-
import synapse.lib.json as s_json
|
|
10
|
-
import synapse.lib.output as s_output
|
|
11
|
-
import synapse.lib.health as s_health
|
|
12
|
-
import synapse.lib.urlhelp as s_urlhelp
|
|
13
|
-
|
|
14
|
-
def serialize(ret):
|
|
15
|
-
return s_json.dumps(ret).decode()
|
|
16
|
-
|
|
17
|
-
def format_component(e, mesg: str) -> dict:
|
|
18
|
-
d = {
|
|
19
|
-
'name': 'error',
|
|
20
|
-
'status': s_health.FAILED,
|
|
21
|
-
'mesg': mesg,
|
|
22
|
-
'data': {
|
|
23
|
-
'errname': e.__class__.__name__,
|
|
24
|
-
'errmesg': str(e),
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
return d
|
|
29
|
-
|
|
30
|
-
async def main(argv, outp=s_output.stdout):
|
|
31
|
-
pars = getArgParser(outp)
|
|
32
|
-
try:
|
|
33
|
-
opts = pars.parse_args(argv)
|
|
34
|
-
except s_exc.ParserExit as e: # pragma: no cover
|
|
35
|
-
return e.get('status')
|
|
36
|
-
|
|
37
|
-
url = opts.cell
|
|
38
|
-
sanitized_url = s_urlhelp.sanitizeUrl(url)
|
|
39
|
-
|
|
40
|
-
try:
|
|
41
|
-
async with s_telepath.withTeleEnv():
|
|
42
|
-
|
|
43
|
-
prox = await s_common.wait_for(s_telepath.openurl(url),
|
|
44
|
-
timeout=opts.timeout)
|
|
45
|
-
except (s_exc.LinkErr, s_exc.NoSuchPath, socket.gaierror) as e:
|
|
46
|
-
mesg = f'Unable to connect to cell @ {sanitized_url}.'
|
|
47
|
-
ret = {'status': 'failed',
|
|
48
|
-
'iden': opts.cell,
|
|
49
|
-
'components': [format_component(e, mesg)],
|
|
50
|
-
}
|
|
51
|
-
outp.printf(serialize(ret))
|
|
52
|
-
return 1
|
|
53
|
-
except s_exc.SynErr as e:
|
|
54
|
-
mesg = 'Synapse error encountered.'
|
|
55
|
-
ret = {'status': s_health.FAILED,
|
|
56
|
-
'iden': opts.cell,
|
|
57
|
-
'components': [format_component(e, mesg)],
|
|
58
|
-
}
|
|
59
|
-
outp.printf(serialize(ret))
|
|
60
|
-
return 1
|
|
61
|
-
except asyncio.TimeoutError as e: # pragma: no cover
|
|
62
|
-
mesg = 'Timeout connecting to cell'
|
|
63
|
-
ret = {'status': s_health.FAILED,
|
|
64
|
-
'iden': opts.cell,
|
|
65
|
-
'components': [format_component(e, mesg)],
|
|
66
|
-
}
|
|
67
|
-
outp.printf(serialize(ret))
|
|
68
|
-
return 1
|
|
69
|
-
|
|
70
|
-
try:
|
|
71
|
-
ret = await s_common.wait_for(prox.getHealthCheck(),
|
|
72
|
-
timeout=opts.timeout)
|
|
73
|
-
except s_exc.SynErr as e:
|
|
74
|
-
mesg = 'Synapse error encountered.'
|
|
75
|
-
ret = {'status': s_health.FAILED,
|
|
76
|
-
'iden': opts.cell,
|
|
77
|
-
'components': [format_component(e, mesg)],
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
except asyncio.TimeoutError as e:
|
|
81
|
-
mesg = 'Timeout getting health information from cell.'
|
|
82
|
-
ret = {'status': s_health.FAILED,
|
|
83
|
-
'iden': opts.cell,
|
|
84
|
-
'components': [format_component(e, mesg)],
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
finally:
|
|
88
|
-
await prox.fini()
|
|
89
|
-
|
|
90
|
-
retval = 1
|
|
91
|
-
if ret.get('status') in (s_health.NOMINAL, s_health.DEGRADED):
|
|
92
|
-
retval = 0
|
|
93
4
|
|
|
94
|
-
|
|
95
|
-
return retval
|
|
5
|
+
from synapse.tools.service.healthcheck import main
|
|
96
6
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
synapse healthcheck tool
|
|
100
|
-
'''
|
|
101
|
-
pars = s_cmd.Parser(prog='synapse.tools.healthcheck', outp=outp, description=desc)
|
|
102
|
-
pars.add_argument('--cell', '-c', required=True, type=str,
|
|
103
|
-
help='Telepath path to the cell to check.')
|
|
104
|
-
pars.add_argument('--timeout', '-t', default=10, type=float,
|
|
105
|
-
help='Connection and call timeout')
|
|
106
|
-
return pars
|
|
7
|
+
s_common.deprecated('synapse.tools.healthcheck is deprecated. Please use synapse.tools.service.healthcheck instead.',
|
|
8
|
+
curv='v2.225.0')
|
|
107
9
|
|
|
108
10
|
if __name__ == '__main__': # pragma: no cover
|
|
109
11
|
s_cmd.exitmain(main)
|
synapse/tools/json2mpk.py
CHANGED
|
@@ -1,43 +1,11 @@
|
|
|
1
|
-
import
|
|
2
|
-
import sys
|
|
3
|
-
import argparse
|
|
1
|
+
import synapse.common as s_common
|
|
4
2
|
|
|
5
|
-
import synapse.lib.
|
|
6
|
-
import synapse.lib.output as s_output
|
|
7
|
-
import synapse.lib.msgpack as s_msgpack
|
|
3
|
+
import synapse.lib.cmd as s_cmd
|
|
8
4
|
|
|
9
|
-
|
|
10
|
-
pars = argparse.ArgumentParser(description='Convert files from json lines to msgpack')
|
|
11
|
-
pars.add_argument('--rm', action='store_true', help='Remove json files once the conversion is complete')
|
|
12
|
-
pars.add_argument('paths', nargs='+', help='json files or directories full of json files')
|
|
13
|
-
return pars
|
|
5
|
+
from synapse.tools.utils.json2mpk import main
|
|
14
6
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
opts = pars.parse_args(argv)
|
|
18
|
-
|
|
19
|
-
for path in opts.paths:
|
|
20
|
-
|
|
21
|
-
if not path.endswith('.json'):
|
|
22
|
-
outp.printf('skip: %s (not .json extension)' % (path,))
|
|
23
|
-
continue
|
|
24
|
-
|
|
25
|
-
if not os.path.isfile(path):
|
|
26
|
-
outp.printf('skip: %s (not a file)' % (path,))
|
|
27
|
-
continue
|
|
28
|
-
|
|
29
|
-
base = path[:-5]
|
|
30
|
-
newp = base + '.mpk'
|
|
31
|
-
|
|
32
|
-
outp.printf('converting: %s -> .mpk' % (path,))
|
|
33
|
-
with open(path, 'r', encoding='utf8') as fd:
|
|
34
|
-
with open(newp, 'wb') as pk:
|
|
35
|
-
for line in fd:
|
|
36
|
-
item = s_json.loads(line)
|
|
37
|
-
pk.write(s_msgpack.en(item))
|
|
38
|
-
|
|
39
|
-
if opts.rm:
|
|
40
|
-
os.unlink(path)
|
|
7
|
+
s_common.deprecated('synapse.tools.json2mpk is deprecated. Please use synapse.tools.utils.json2mpk instead.',
|
|
8
|
+
curv='v2.225.0')
|
|
41
9
|
|
|
42
10
|
if __name__ == '__main__': # pragma: no cover
|
|
43
|
-
|
|
11
|
+
s_cmd.exitmain(main)
|
synapse/tools/livebackup.py
CHANGED
|
@@ -1,34 +1,11 @@
|
|
|
1
|
-
import synapse.
|
|
1
|
+
import synapse.common as s_common
|
|
2
2
|
|
|
3
3
|
import synapse.lib.cmd as s_cmd
|
|
4
|
-
import synapse.lib.output as s_output
|
|
5
4
|
|
|
6
|
-
|
|
7
|
-
Generate a new backup of a running Synapse service.
|
|
5
|
+
from synapse.tools.service.livebackup import main
|
|
8
6
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
# Generate a backup from inside a Synapse service container
|
|
12
|
-
python -m synapse.tools.livebackup
|
|
13
|
-
|
|
14
|
-
'''
|
|
15
|
-
|
|
16
|
-
async def main(argv, outp=s_output.stdout):
|
|
17
|
-
|
|
18
|
-
pars = s_cmd.Parser(prog='synapse.tools.livebackup', outp=outp, description=descr)
|
|
19
|
-
|
|
20
|
-
pars.add_argument('--url', default='cell:///vertex/storage', help='The telepath URL of the Synapse service.')
|
|
21
|
-
pars.add_argument('--name', default=None, help='Specify a name for the backup. Defaults to an automatically generated timestamp.')
|
|
22
|
-
|
|
23
|
-
opts = pars.parse_args(argv)
|
|
24
|
-
|
|
25
|
-
async with s_telepath.withTeleEnv():
|
|
26
|
-
|
|
27
|
-
async with await s_telepath.openurl(opts.url) as cell:
|
|
28
|
-
outp.printf(f'Running backup of: {opts.url}')
|
|
29
|
-
name = await cell.runBackup(name=opts.name)
|
|
30
|
-
outp.printf(f'...backup created: {name}')
|
|
31
|
-
return 0
|
|
7
|
+
s_common.deprecated('synapse.tools.livebackup is deprecated. Please use synapse.tools.service.livebackup instead.',
|
|
8
|
+
curv='v2.225.0')
|
|
32
9
|
|
|
33
10
|
if __name__ == '__main__': # pragma: no cover
|
|
34
11
|
s_cmd.exitmain(main)
|
synapse/tools/modrole.py
CHANGED
|
@@ -1,116 +1,11 @@
|
|
|
1
1
|
import synapse.common as s_common
|
|
2
|
-
import synapse.telepath as s_telepath
|
|
3
2
|
|
|
4
3
|
import synapse.lib.cmd as s_cmd
|
|
5
|
-
import synapse.lib.output as s_output
|
|
6
4
|
|
|
7
|
-
|
|
8
|
-
Add or modify a role in a Synapse service.
|
|
9
|
-
'''
|
|
5
|
+
from synapse.tools.service.modrole import main
|
|
10
6
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
outp.printf(f'Role: {role.get("name")} ({role.get("iden")})')
|
|
14
|
-
outp.printf('')
|
|
15
|
-
outp.printf(' Rules:')
|
|
16
|
-
for indx, rule in enumerate(role.get('rules')):
|
|
17
|
-
outp.printf(f' [{str(indx).ljust(3)}] - {s_common.reprauthrule(rule)}')
|
|
18
|
-
|
|
19
|
-
outp.printf('')
|
|
20
|
-
outp.printf(' Gates:')
|
|
21
|
-
for gateiden, gateinfo in role.get('authgates', {}).items():
|
|
22
|
-
outp.printf(f' {gateiden}')
|
|
23
|
-
outp.printf(f' Admin: {gateinfo.get("admin") == True}')
|
|
24
|
-
for indx, rule in enumerate(gateinfo.get('rules', ())):
|
|
25
|
-
outp.printf(f' [{str(indx).ljust(3)}] - {s_common.reprauthrule(rule)}')
|
|
26
|
-
|
|
27
|
-
async def main(argv, outp=s_output.stdout):
|
|
28
|
-
|
|
29
|
-
pars = s_cmd.Parser(prog='synapse.tools.modrole', outp=outp, description=descr)
|
|
30
|
-
pars.add_argument('--svcurl', default='cell:///vertex/storage', help='The telepath URL of the Synapse service.')
|
|
31
|
-
pars.add_argument('--add', default=False, action='store_true', help='Add the role if they do not already exist.')
|
|
32
|
-
pars.add_argument('--del', dest='delete', default=False, action='store_true', help='Delete the role if it exists.')
|
|
33
|
-
pars.add_argument('--list', default=False, action='store_true',
|
|
34
|
-
help='List existing roles, or rules of a specific role.')
|
|
35
|
-
pars.add_argument('--allow', default=[], action='append', help='A permission string to allow for the role.')
|
|
36
|
-
pars.add_argument('--deny', default=[], action='append', help='A permission string to deny for the role.')
|
|
37
|
-
pars.add_argument('--gate', default=None, help='The iden of an auth gate to add/del rules on.')
|
|
38
|
-
pars.add_argument('rolename', nargs='?', help='The rolename to add/edit.')
|
|
39
|
-
|
|
40
|
-
opts = pars.parse_args(argv)
|
|
41
|
-
|
|
42
|
-
if opts.add and opts.delete:
|
|
43
|
-
outp.printf('ERROR: Cannot specify --add and --del together.')
|
|
44
|
-
return 1
|
|
45
|
-
|
|
46
|
-
async with s_telepath.withTeleEnv():
|
|
47
|
-
|
|
48
|
-
async with await s_telepath.openurl(opts.svcurl) as cell:
|
|
49
|
-
|
|
50
|
-
if opts.list:
|
|
51
|
-
if opts.rolename:
|
|
52
|
-
role = await cell.getRoleDefByName(opts.rolename)
|
|
53
|
-
if role is None:
|
|
54
|
-
outp.printf(f'ERROR: Role not found: {opts.rolename}')
|
|
55
|
-
return 1
|
|
56
|
-
|
|
57
|
-
printrole(role, outp)
|
|
58
|
-
|
|
59
|
-
else:
|
|
60
|
-
outp.printf('Roles:')
|
|
61
|
-
for role in await cell.getRoleDefs():
|
|
62
|
-
outp.printf(f' {role.get("iden")} - {role.get("name")}')
|
|
63
|
-
|
|
64
|
-
return 0
|
|
65
|
-
elif opts.rolename is None:
|
|
66
|
-
outp.printf(f'ERROR: A rolename argument is required when --list is not specified.')
|
|
67
|
-
return 1
|
|
68
|
-
|
|
69
|
-
if opts.gate:
|
|
70
|
-
gate = await cell.getAuthGate(opts.gate)
|
|
71
|
-
if gate is None:
|
|
72
|
-
outp.printf(f'ERROR: No auth gate found with iden: {opts.gate}')
|
|
73
|
-
return 1
|
|
74
|
-
|
|
75
|
-
role = await cell.getRoleDefByName(opts.rolename)
|
|
76
|
-
if role is not None:
|
|
77
|
-
outp.printf(f'Modifying role: {opts.rolename}')
|
|
78
|
-
|
|
79
|
-
if role is None:
|
|
80
|
-
if not opts.add:
|
|
81
|
-
outp.printf(f'ERROR: Role not found (need --add?): {opts.rolename}')
|
|
82
|
-
return 1
|
|
83
|
-
|
|
84
|
-
outp.printf(f'Adding role: {opts.rolename}')
|
|
85
|
-
role = await cell.addRole(opts.rolename)
|
|
86
|
-
|
|
87
|
-
roleiden = role.get('iden')
|
|
88
|
-
|
|
89
|
-
if opts.delete:
|
|
90
|
-
outp.printf(f'...deleting role: {opts.rolename}')
|
|
91
|
-
await cell.delRole(roleiden)
|
|
92
|
-
return 0
|
|
93
|
-
|
|
94
|
-
for allow in opts.allow:
|
|
95
|
-
perm = allow.lower().split('.')
|
|
96
|
-
mesg = f'...adding allow rule: {allow}'
|
|
97
|
-
if opts.gate:
|
|
98
|
-
mesg += f' on gate {opts.gate}'
|
|
99
|
-
|
|
100
|
-
outp.printf(mesg)
|
|
101
|
-
if not await cell.isRoleAllowed(roleiden, perm, gateiden=opts.gate):
|
|
102
|
-
await cell.addRoleRule(roleiden, (True, perm), indx=0, gateiden=opts.gate)
|
|
103
|
-
|
|
104
|
-
for deny in opts.deny:
|
|
105
|
-
perm = deny.lower().split('.')
|
|
106
|
-
mesg = f'...adding deny rule: {deny}'
|
|
107
|
-
if opts.gate:
|
|
108
|
-
mesg += f' on gate {opts.gate}'
|
|
109
|
-
|
|
110
|
-
outp.printf(mesg)
|
|
111
|
-
if await cell.isRoleAllowed(roleiden, perm, gateiden=opts.gate):
|
|
112
|
-
await cell.addRoleRule(roleiden, (False, perm), indx=0, gateiden=opts.gate)
|
|
113
|
-
return 0
|
|
7
|
+
s_common.deprecated('synapse.tools.modrole is deprecated. Please use synapse.tools.service.modrole instead.',
|
|
8
|
+
curv='v2.225.0')
|
|
114
9
|
|
|
115
10
|
if __name__ == '__main__': # pragma: no cover
|
|
116
11
|
s_cmd.exitmain(main)
|
synapse/tools/moduser.py
CHANGED
|
@@ -1,187 +1,11 @@
|
|
|
1
1
|
import synapse.common as s_common
|
|
2
|
-
import synapse.telepath as s_telepath
|
|
3
2
|
|
|
4
3
|
import synapse.lib.cmd as s_cmd
|
|
5
|
-
import synapse.lib.output as s_output
|
|
6
4
|
|
|
7
|
-
|
|
8
|
-
Add, modify, or list users of a Synapse service.
|
|
9
|
-
'''
|
|
5
|
+
from synapse.tools.service.moduser import main
|
|
10
6
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
admin = user.get('admin')
|
|
14
|
-
authtype = user.get('type')
|
|
15
|
-
|
|
16
|
-
outp.printf(f'User: {user.get("name")} ({user.get("iden")})')
|
|
17
|
-
outp.printf('')
|
|
18
|
-
outp.printf(f' Locked: {user.get("locked")}')
|
|
19
|
-
outp.printf(f' Admin: {user.get("admin")}')
|
|
20
|
-
outp.printf(f' Email: {user.get("email")}')
|
|
21
|
-
outp.printf(' Rules:')
|
|
22
|
-
for indx, rule in enumerate(user.get('rules')):
|
|
23
|
-
outp.printf(f' [{str(indx).ljust(3)}] - {s_common.reprauthrule(rule)}')
|
|
24
|
-
|
|
25
|
-
outp.printf('')
|
|
26
|
-
outp.printf(' Roles:')
|
|
27
|
-
for role in user.get('roles'):
|
|
28
|
-
outp.printf(f' {role.get("iden")} - {role.get("name")}')
|
|
29
|
-
|
|
30
|
-
outp.printf('')
|
|
31
|
-
outp.printf(' Gates:')
|
|
32
|
-
for gateiden, gateinfo in user.get('authgates', {}).items():
|
|
33
|
-
outp.printf(f' {gateiden}')
|
|
34
|
-
outp.printf(f' Admin: {gateinfo.get("admin") == True}')
|
|
35
|
-
for indx, rule in enumerate(gateinfo.get('rules', ())):
|
|
36
|
-
outp.printf(f' [{str(indx).ljust(3)}] - {s_common.reprauthrule(rule)}')
|
|
37
|
-
|
|
38
|
-
async def main(argv, outp=s_output.stdout):
|
|
39
|
-
|
|
40
|
-
pars = s_cmd.Parser(prog='synapse.tools.moduser', outp=outp, description=descr)
|
|
41
|
-
pars.add_argument('--svcurl', default='cell:///vertex/storage', help='The telepath URL of the Synapse service.')
|
|
42
|
-
pars.add_argument('--add', default=False, action='store_true', help='Add the user if they do not already exist.')
|
|
43
|
-
pars.add_argument('--del', dest='delete', default=False, action='store_true', help='Delete the user if they exist.')
|
|
44
|
-
pars.add_argument('--list', default=False, action='store_true',
|
|
45
|
-
help='List existing users of the service, or details of a specific user.')
|
|
46
|
-
pars.add_argument('--admin', choices=('true', 'false'), default=None, help='Set the user admin status.')
|
|
47
|
-
pars.add_argument('--passwd', action='store', type=str, help='A password to set for the user.')
|
|
48
|
-
pars.add_argument('--email', action='store', type=str, help='An email to set for the user.')
|
|
49
|
-
pars.add_argument('--locked', choices=('true', 'false'), default=None, help='Set the user locked status.')
|
|
50
|
-
pars.add_argument('--grant', default=[], action='append', help='A role to grant to the user.')
|
|
51
|
-
pars.add_argument('--revoke', default=[], action='append', help='A role to revoke from the user.')
|
|
52
|
-
pars.add_argument('--allow', default=[], action='append', help='A permission string to allow for the user.')
|
|
53
|
-
pars.add_argument('--deny', default=[], action='append', help='A permission string to deny for the user.')
|
|
54
|
-
pars.add_argument('--gate', default=None, help='The iden of an auth gate to add/del rules or set admin status on.')
|
|
55
|
-
pars.add_argument('username', nargs='?', help='The username to add/edit or show details.')
|
|
56
|
-
|
|
57
|
-
opts = pars.parse_args(argv)
|
|
58
|
-
|
|
59
|
-
if opts.add and opts.delete:
|
|
60
|
-
outp.printf('ERROR: Cannot specify --add and --del together.')
|
|
61
|
-
return 1
|
|
62
|
-
|
|
63
|
-
async with s_telepath.withTeleEnv():
|
|
64
|
-
|
|
65
|
-
async with await s_telepath.openurl(opts.svcurl) as cell:
|
|
66
|
-
|
|
67
|
-
if opts.list:
|
|
68
|
-
if opts.username:
|
|
69
|
-
user = await cell.getUserDefByName(opts.username)
|
|
70
|
-
if user is None:
|
|
71
|
-
outp.printf(f'ERROR: User not found: {opts.username}')
|
|
72
|
-
return 1
|
|
73
|
-
|
|
74
|
-
printuser(user, outp)
|
|
75
|
-
|
|
76
|
-
else:
|
|
77
|
-
outp.printf('Users:')
|
|
78
|
-
for user in await cell.getUserDefs():
|
|
79
|
-
outp.printf(f' {user.get("name")}')
|
|
80
|
-
|
|
81
|
-
return 0
|
|
82
|
-
|
|
83
|
-
elif opts.username is None:
|
|
84
|
-
outp.printf(f'ERROR: A username argument is required when --list is not specified.')
|
|
85
|
-
return 1
|
|
86
|
-
|
|
87
|
-
if opts.gate:
|
|
88
|
-
gate = await cell.getAuthGate(opts.gate)
|
|
89
|
-
if gate is None:
|
|
90
|
-
outp.printf(f'ERROR: No auth gate found with iden: {opts.gate}')
|
|
91
|
-
return 1
|
|
92
|
-
|
|
93
|
-
grants = []
|
|
94
|
-
revokes = []
|
|
95
|
-
|
|
96
|
-
for rolename in opts.grant:
|
|
97
|
-
role = await cell.getRoleDefByName(rolename)
|
|
98
|
-
if role is None:
|
|
99
|
-
outp.printf(f'ERROR: Role not found: {rolename}')
|
|
100
|
-
return 1
|
|
101
|
-
grants.append(role)
|
|
102
|
-
|
|
103
|
-
for rolename in opts.revoke:
|
|
104
|
-
role = await cell.getRoleDefByName(rolename)
|
|
105
|
-
if role is None:
|
|
106
|
-
outp.printf(f'ERROR: Role not found: {rolename}')
|
|
107
|
-
return 1
|
|
108
|
-
revokes.append(role)
|
|
109
|
-
|
|
110
|
-
user = await cell.getUserDefByName(opts.username)
|
|
111
|
-
|
|
112
|
-
if user is None:
|
|
113
|
-
if not opts.add:
|
|
114
|
-
outp.printf(f'ERROR: User not found (need --add?): {opts.username}')
|
|
115
|
-
return 1
|
|
116
|
-
|
|
117
|
-
outp.printf(f'Adding user: {opts.username}')
|
|
118
|
-
user = await cell.addUser(opts.username)
|
|
119
|
-
|
|
120
|
-
else:
|
|
121
|
-
outp.printf(f'Modifying user: {opts.username}')
|
|
122
|
-
|
|
123
|
-
useriden = user.get('iden')
|
|
124
|
-
if not s_common.isguid(useriden): # pragma: no cover
|
|
125
|
-
outp.printf(f'ERROR: Invalid useriden: {useriden}')
|
|
126
|
-
return 1
|
|
127
|
-
|
|
128
|
-
if opts.delete:
|
|
129
|
-
outp.printf(f'...deleting user: {opts.username}')
|
|
130
|
-
await cell.delUser(useriden)
|
|
131
|
-
return 0
|
|
132
|
-
|
|
133
|
-
if opts.admin is not None:
|
|
134
|
-
admin = s_common.yamlloads(opts.admin)
|
|
135
|
-
mesg = f'...setting admin: {opts.admin}'
|
|
136
|
-
if opts.gate:
|
|
137
|
-
mesg += f' on gate {opts.gate}'
|
|
138
|
-
|
|
139
|
-
outp.printf(mesg)
|
|
140
|
-
await cell.setUserAdmin(useriden, admin, gateiden=opts.gate)
|
|
141
|
-
|
|
142
|
-
if opts.locked is not None:
|
|
143
|
-
locked = s_common.yamlloads(opts.locked)
|
|
144
|
-
outp.printf(f'...setting locked: {opts.locked}')
|
|
145
|
-
await cell.setUserLocked(useriden, locked)
|
|
146
|
-
|
|
147
|
-
if opts.passwd is not None:
|
|
148
|
-
outp.printf(f'...setting passwd: {opts.passwd}')
|
|
149
|
-
await cell.setUserPasswd(useriden, opts.passwd)
|
|
150
|
-
|
|
151
|
-
if opts.email is not None:
|
|
152
|
-
outp.printf(f'...setting email: {opts.email}')
|
|
153
|
-
await cell.setUserEmail(useriden, opts.email)
|
|
154
|
-
|
|
155
|
-
for role in grants:
|
|
156
|
-
rolename = role.get('name')
|
|
157
|
-
outp.printf(f'...granting role: {rolename}')
|
|
158
|
-
await cell.addUserRole(useriden, role.get('iden'))
|
|
159
|
-
|
|
160
|
-
for role in revokes:
|
|
161
|
-
rolename = role.get('name')
|
|
162
|
-
outp.printf(f'...revoking role: {rolename}')
|
|
163
|
-
await cell.delUserRole(useriden, role.get('iden'))
|
|
164
|
-
|
|
165
|
-
for allow in opts.allow:
|
|
166
|
-
perm = allow.lower().split('.')
|
|
167
|
-
mesg = f'...adding allow rule: {allow}'
|
|
168
|
-
if opts.gate:
|
|
169
|
-
mesg += f' on gate {opts.gate}'
|
|
170
|
-
|
|
171
|
-
outp.printf(mesg)
|
|
172
|
-
if not await cell.isUserAllowed(useriden, perm, gateiden=opts.gate):
|
|
173
|
-
await cell.addUserRule(useriden, (True, perm), indx=0, gateiden=opts.gate)
|
|
174
|
-
|
|
175
|
-
for deny in opts.deny:
|
|
176
|
-
perm = deny.lower().split('.')
|
|
177
|
-
mesg = f'...adding deny rule: {deny}'
|
|
178
|
-
if opts.gate:
|
|
179
|
-
mesg += f' on gate {opts.gate}'
|
|
180
|
-
|
|
181
|
-
outp.printf(mesg)
|
|
182
|
-
if await cell.isUserAllowed(useriden, perm, gateiden=opts.gate):
|
|
183
|
-
await cell.addUserRule(useriden, (False, perm), indx=0, gateiden=opts.gate)
|
|
184
|
-
return 0
|
|
7
|
+
s_common.deprecated('synapse.tools.moduser is deprecated. Please use synapse.tools.service.moduser instead.',
|
|
8
|
+
curv='v2.225.0')
|
|
185
9
|
|
|
186
10
|
if __name__ == '__main__': # pragma: no cover
|
|
187
11
|
s_cmd.exitmain(main)
|
synapse/tools/pkgs/gendocs.py
CHANGED
|
@@ -1,174 +1,13 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import shutil
|
|
3
1
|
import logging
|
|
4
|
-
import subprocess
|
|
5
2
|
|
|
6
|
-
import regex as re
|
|
7
|
-
|
|
8
|
-
import synapse.exc as s_exc
|
|
9
3
|
import synapse.common as s_common
|
|
10
4
|
|
|
11
5
|
import synapse.lib.cmd as s_cmd
|
|
12
|
-
import synapse.lib.output as s_output
|
|
13
|
-
|
|
14
|
-
import synapse.tools.rstorm as s_rstorm
|
|
15
|
-
import synapse.tools.autodoc as s_autodoc
|
|
16
|
-
|
|
17
|
-
logger = logging.getLogger(__name__)
|
|
18
|
-
|
|
19
|
-
_TOOLDIR = os.path.split(__file__)[0]
|
|
20
|
-
PANDOC_FILTER = os.path.join(_TOOLDIR, 'pandoc_filter.py')
|
|
21
|
-
|
|
22
|
-
# see https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html#file-wide-metadata
|
|
23
|
-
re_sphinx_metadata_fields = re.compile(r'^:(tocdepth|nocomments|orphan|nosearch):( \w+)?\n\n',
|
|
24
|
-
flags=re.MULTILINE)
|
|
25
|
-
|
|
26
|
-
def hasPandoc():
|
|
27
|
-
if os.system('pandoc --version') == 0:
|
|
28
|
-
return True
|
|
29
|
-
return False
|
|
30
|
-
|
|
31
|
-
async def buildPkgDocs(outp, pkgpath: str, rst_only: bool =False):
|
|
32
|
-
|
|
33
|
-
logger.info(f'Building pkg for {pkgpath}')
|
|
34
|
-
pkgdef = s_common.yamlload(pkgpath)
|
|
35
|
-
if pkgdef is None:
|
|
36
|
-
raise s_exc.BadArg(mesg=f'Package does not exist or does not contain yaml: {pkgpath}')
|
|
37
|
-
|
|
38
|
-
dirn = os.path.dirname(s_common.genpath(pkgpath))
|
|
39
|
-
|
|
40
|
-
docsdir = os.path.join(dirn, 'docs')
|
|
41
|
-
builddir = os.path.join(dirn, 'docs', '_build')
|
|
42
|
-
|
|
43
|
-
shutil.rmtree(builddir, ignore_errors=True)
|
|
44
|
-
|
|
45
|
-
s_common.gendir(builddir)
|
|
46
|
-
|
|
47
|
-
# touch any files we need in order to load a package, due to
|
|
48
|
-
# rstorm needing to load the package using genpkg tool. This
|
|
49
|
-
# does mean that standalone builds of a storm package from this
|
|
50
|
-
# must be done after using this buildpkg tool.
|
|
51
|
-
stormpkg_md_present = False
|
|
52
|
-
for dnfo in pkgdef.get('docs', ()):
|
|
53
|
-
fpath = dnfo.get('path')
|
|
54
|
-
with s_common.genfile(dirn, fpath) as fd:
|
|
55
|
-
pass
|
|
56
|
-
if fpath.endswith('stormpackage.md'):
|
|
57
|
-
stormpkg_md_present = True
|
|
58
|
-
|
|
59
|
-
# Generate the build .RST for stormpackage.md
|
|
60
|
-
if stormpkg_md_present:
|
|
61
|
-
logger.info(f'Generating stormpackage.rst for {pkgpath}')
|
|
62
|
-
pkgdocs, pkgname = await s_autodoc.docStormpkg(pkgpath)
|
|
63
|
-
with s_common.genfile(docsdir, 'stormpackage.rst') as fd:
|
|
64
|
-
text = pkgdocs.getRstText()
|
|
65
|
-
if rst_only is False:
|
|
66
|
-
# Leave this in place if we're only generating RST
|
|
67
|
-
text = text.replace('.. highlight:: none\n', '')
|
|
68
|
-
fd.write(text.encode())
|
|
69
|
-
logger.info('Generated the stormpackage.rst file!')
|
|
70
|
-
|
|
71
|
-
for name in os.listdir(docsdir):
|
|
72
|
-
|
|
73
|
-
if not name.endswith('.rst'):
|
|
74
|
-
continue
|
|
75
|
-
|
|
76
|
-
docpath = os.path.join(docsdir, name)
|
|
77
|
-
|
|
78
|
-
basename = name.rsplit('.', 1)[0]
|
|
79
|
-
|
|
80
|
-
builtmd = os.path.join(builddir, f'{basename}.md')
|
|
81
|
-
builtrst = os.path.join(builddir, name)
|
|
82
|
-
|
|
83
|
-
argv = (docpath, '--save', builtrst)
|
|
84
|
-
logger.info(f'Executing rstorm for {argv}')
|
|
85
|
-
await s_rstorm.main(argv)
|
|
86
|
-
|
|
87
|
-
if rst_only:
|
|
88
|
-
logger.info(f'rst_only enabled, done processing {name}')
|
|
89
|
-
continue
|
|
90
|
-
|
|
91
|
-
logger.info('Preprocessing rstorm output')
|
|
92
|
-
with s_common.genfile(builtrst) as fd:
|
|
93
|
-
buf = fd.read().decode()
|
|
94
|
-
|
|
95
|
-
# Remove highglight:: none directives
|
|
96
|
-
buf = buf.replace('.. highlight:: none\n', '')
|
|
97
|
-
|
|
98
|
-
# Remove sphinx metadata fields
|
|
99
|
-
buf = re_sphinx_metadata_fields.sub('', buf)
|
|
100
|
-
|
|
101
|
-
lines = buf.splitlines(keepends=True)
|
|
102
|
-
|
|
103
|
-
# Remove lines which start with explicit sphinx rst targets
|
|
104
|
-
nlines1 = []
|
|
105
|
-
for line in lines:
|
|
106
|
-
if line.startswith('.. _') and line.strip().endswith(':'):
|
|
107
|
-
logger.info(f'Dropping: [{line.strip()}]')
|
|
108
|
-
continue
|
|
109
|
-
nlines1.append(line)
|
|
110
|
-
|
|
111
|
-
buf = ''.join(nlines1)
|
|
112
|
-
|
|
113
|
-
with s_common.genfile(builtrst) as fd:
|
|
114
|
-
fd.truncate()
|
|
115
|
-
_ = fd.write(buf.encode())
|
|
116
|
-
|
|
117
|
-
logger.info(f'Converting {builtrst} to markdown')
|
|
118
|
-
if name == 'stormpackage.rst':
|
|
119
|
-
args = ['pandoc', '--filter', PANDOC_FILTER, '-f', 'rst', '-t', 'markdown', '-o', builtmd, builtrst]
|
|
120
|
-
else:
|
|
121
|
-
args = ['pandoc', '-f', 'rst', '-t', 'markdown', '-o', builtmd, builtrst]
|
|
122
|
-
|
|
123
|
-
r = subprocess.run(args, capture_output=True)
|
|
124
|
-
|
|
125
|
-
# Re-write stderr (logging) to our outp
|
|
126
|
-
for line in r.stderr.decode().splitlines():
|
|
127
|
-
outp.printf(f'ERR: {line}')
|
|
128
|
-
|
|
129
|
-
if r.returncode != 0:
|
|
130
|
-
raise s_exc.SynErr(mesg=f'Error converting {builtrst} to {builtmd}')
|
|
131
|
-
|
|
132
|
-
logger.info(f'Done converting {builtrst} to {builtmd}')
|
|
133
|
-
|
|
134
|
-
# Strip out / manipulate the md content
|
|
135
|
-
with s_common.genfile(builtmd) as fd:
|
|
136
|
-
buf = fd.read().decode()
|
|
137
|
-
|
|
138
|
-
lines = buf.splitlines(keepends=True)
|
|
139
|
-
|
|
140
|
-
# Remove lines which only have a single `:` left in them
|
|
141
|
-
nlines1 = [line for line in lines if line.strip() != ':']
|
|
142
|
-
|
|
143
|
-
buf = ''.join(nlines1)
|
|
144
|
-
|
|
145
|
-
with s_common.genfile(builtmd) as fd:
|
|
146
|
-
fd.truncate()
|
|
147
|
-
_ = fd.write(buf.encode())
|
|
148
|
-
|
|
149
|
-
logger.info('Done manipulating markdown')
|
|
150
|
-
|
|
151
|
-
logger.info(f'buildPkgDocs complete for {pkgpath}.')
|
|
152
|
-
|
|
153
|
-
prog = 'synapse.tools.pkgs.gendocs'
|
|
154
|
-
desc = 'A tool for building storm package docs from RStorm into markdown. This tool requires pandoc to be available.'
|
|
155
|
-
|
|
156
|
-
async def main(argv, outp=s_output.stdout):
|
|
157
|
-
|
|
158
|
-
pars = s_cmd.Parser(prog=prog, outp=outp, description=desc)
|
|
159
|
-
pars.add_argument('pkgfile', metavar='<pkgfile>', help='Path to a storm package prototype yml file.')
|
|
160
|
-
pars.add_argument('--rst-only', default=False, action='store_true',
|
|
161
|
-
help='Stops building after the .rst files have been generated.')
|
|
162
|
-
|
|
163
|
-
opts = pars.parse_args(argv)
|
|
164
|
-
|
|
165
|
-
if opts.rst_only is False and not hasPandoc():
|
|
166
|
-
logger.error('Pandoc is not available, can only run rst/rstorm output.')
|
|
167
|
-
return 1
|
|
168
6
|
|
|
169
|
-
|
|
7
|
+
from synapse.tools.storm.pkg.doc import logger, main
|
|
170
8
|
|
|
171
|
-
|
|
9
|
+
s_common.deprecated('synapse.tools.pkgs.gendocs is deprecated. Please use synapse.tools.storm.pkg.doc instead.',
|
|
10
|
+
curv='v2.225.0')
|
|
172
11
|
|
|
173
12
|
if __name__ == '__main__': # pragma: no cover
|
|
174
13
|
s_common.setlogging(logger, 'DEBUG')
|