synapse 2.224.0__py311-none-any.whl → 2.225.0__py311-none-any.whl

synapse/tools/promote.py

@@ -1,48 +1,11 @@
-import synapse.exc as s_exc
-
-import synapse.telepath as s_telepath
+import synapse.common as s_common
 
 import synapse.lib.cmd as s_cmd
-import synapse.lib.output as s_output
-import synapse.lib.urlhelp as s_urlhelp
-
-descr = '''
-Promote a mirror to the leader.
-
-Example (being run from a Cortex mirror docker container):
-    python -m synapse.tools.promote
-'''
-
-async def main(argv, outp=s_output.stdout):
-
-    pars = s_cmd.Parser(prog='synapse.tools.promote', outp=outp, description=descr)
-
-    pars.add_argument('--svcurl', default='cell:///vertex/storage',
-                      help='The telepath URL of the Synapse service.')
-
-    pars.add_argument('--failure', default=False, action='store_true',
-                      help='Promotion is due to leader being offline. Graceful handoff is not possible.')
-
-    opts = pars.parse_args(argv)
-
-    async with s_telepath.withTeleEnv():
-
-        async with await s_telepath.openurl(opts.svcurl) as cell:
-
-            graceful = not opts.failure
 
-            outp.printf(f'Promoting to leader: {opts.svcurl}')
-            try:
-                await cell.promote(graceful=graceful)
-            except s_exc.BadState as e:
-                mesg = f'Failed to promote service to being a leader; {e.get("mesg")}'
-                outp.printf(mesg)
-                return 1
-            except s_exc.SynErr as e:
-                outp.printf(f'Failed to promote service {s_urlhelp.sanitizeUrl(opts.svcurl)}: {e}')
-                return 1
+from synapse.tools.service.promote import main
 
-    return 0
+s_common.deprecated('synapse.tools.promote is deprecated. Please use synapse.tools.service.promote instead.',
+                    curv='v2.225.0')
 
 if __name__ == '__main__':  # pragma: no cover
     s_cmd.exitmain(main)

synapse/tools/pullfile.py

@@ -1,64 +1,11 @@
-import pathlib
-
 import synapse.common as s_common
-import synapse.telepath as s_telepath
 
 import synapse.lib.cmd as s_cmd
-import synapse.lib.output as s_output
-
-
-async def main(argv, outp=s_output.stdout):
-    pars = getArgParser(outp)
-    opts = pars.parse_args(argv)
-
-    if opts.output is None:
-        opts.output = '.'
-
-    outdir = pathlib.Path(opts.output)
-
-    s_common.gendir(opts.output)
-
-    async with s_telepath.withTeleEnv():
-
-        async with await s_telepath.openurl(opts.axon) as axon:
-
-            # reminder: these are the hashes *not* available
-
-            awants = await axon.wants([s_common.uhex(h) for h in opts.hashes])
-            for a in awants:
-                outp.printf(f'{s_common.ehex(a)} not in axon store')
-
-            exists = [h for h in opts.hashes if s_common.uhex(h) not in awants]
-
-            for h in exists:
-
-                try:
-                    outp.printf(f'Fetching {h} to file')
-
-                    with open(outdir.joinpath(h), 'wb') as fd:
-                        async for b in axon.get(s_common.uhex(h)):
-                            fd.write(b)
-
-                    outp.printf(f'Fetched {h} to file')
-
-                except Exception as e:
-                    outp.printf('Error: Hit Exception: %s' % (str(e),))
-                    continue
-
-    return 0
-
 
-def getArgParser(outp):
-    desc = 'Fetches file from the given axon'
-    pars = s_cmd.Parser(prog='synapse.tools.pullfile', outp=outp, description=desc)
-    pars.add_argument('-a', '--axon', type=str, dest='axon', required=True,
-                      help='URL to the axon blob store')
-    pars.add_argument('-o', '--output', type=str, dest='output',
-                      help='Directory to output files to')
-    pars.add_argument('-l', '--list-hashes', dest='hashes', action='append', default=[],
-                      help='List of hashes to pull from axon')
+from synapse.tools.axon.get import main
 
-    return pars
+s_common.deprecated('synapse.tools.pullfile is deprecated. Please use synapse.tools.axon.get instead.',
+                    curv='v2.225.0')
 
 if __name__ == '__main__':  # pragma: no cover
     s_cmd.exitmain(main)

synapse/tools/pushfile.py

@@ -1,122 +1,11 @@
-import os
-import glob
-import logging
-
-import synapse.exc as s_exc
 import synapse.common as s_common
-import synapse.telepath as s_telepath
 
 import synapse.lib.cmd as s_cmd
-import synapse.lib.output as s_output
-import synapse.lib.hashset as s_hashset
-
-logger = logging.getLogger(__name__)
-
-
-async def main(argv, outp=s_output.stdout):
-    pars = getArgParser(outp)
-    opts = pars.parse_args(argv)
-
-    async with s_telepath.withTeleEnv():
-
-        axon = await s_telepath.openurl(opts.axon)
-
-        core = None
-        if opts.cortex:
-            core = await s_telepath.openurl(opts.cortex)
-
-            tags = set()
-            if opts.tags:
-                for tag in opts.tags.split(','):
-                    tags.add(tag)
-
-            tags = tuple(tags)
-            if tags:
-                outp.printf(f'adding tags: {tags}')
-
-        filepaths = set()
-        for item in opts.filenames:
-            paths = glob.glob(item, recursive=opts.recursive)
-
-            if not paths:
-                outp.printf(f'filepath does not contain any files: {item}')
-                continue
-
-            filepaths.update([path for path in paths if os.path.isfile(path)])
-
-        for path in filepaths:
-
-            bname = os.path.basename(path)
-
-            hset = s_hashset.HashSet()
-            with s_common.reqfile(path) as fd:
-                hset.eatfd(fd)
-
-            fhashes = {htyp: hasher.hexdigest() for htyp, hasher in hset.hashes}
-
-            sha256 = fhashes.get('sha256')
-            bsha256 = s_common.uhex(sha256)
-
-            if not await axon.has(bsha256):
-
-                async with await axon.upload() as upfd:
-
-                    with s_common.genfile(path) as fd:
-                        for byts in s_common.iterfd(fd):
-                            await upfd.write(byts)
-
-                    size, hashval = await upfd.save()
-
-                if hashval != bsha256:  # pragma: no cover
-                    raise s_exc.SynErr(mesg='hashes do not match',
-                                       ehash=s_common.ehex(hashval),
-                                       ahash=hashval)
-
-                outp.printf(f'Uploaded [{bname}] to axon')
-            else:
-                outp.printf(f'Axon already had [{bname}]')
-
-            if core:
-                opts = {'vars': {
-                    'md5': fhashes.get('md5'),
-                    'sha1': fhashes.get('sha1'),
-                    'sha256': fhashes.get('sha256'),
-                    'size': hset.size,
-                    'name': bname,
-                    'tags': tags,
-                }}
-
-                q = '[file:bytes=$sha256 :md5=$md5 :sha1=$sha1 :size=$size :name=$name] ' \
-                    '{ for $tag in $tags { [+#$tag] } }'
-
-                msgs = await core.storm(q, opts=opts).list()
-                node = [m[1] for m in msgs if m[0] == 'node'][0]
-
-                iden = node[0][1]
-                size = node[1]['props']['size']
-                name = node[1]['props']['name']
-                mesg = f'file: {bname} ({size}) added to core ({iden}) as {name}'
-                outp.printf(mesg)
-
-        await axon.fini()
-        if core:
-            await core.fini()
 
-    return 0
+from synapse.tools.axon.put import main
 
-def getArgParser(outp):
-    desc = 'Command line tool for uploading files to an Axon and making ' \
-           'file:bytes in a Cortex.'
-    pars = s_cmd.Parser(prog='synapse.tools.pushfile', outp=outp, description=desc)
-    pars.add_argument('-a', '--axon', required=True, type=str, dest='axon',
-                   help='URL for a target Axon to store files at.')
-    pars.add_argument('-c', '--cortex', default=None, type=str, dest='cortex',
-                   help='URL for a target Cortex to make file:bytes nodes.')
-    pars.add_argument('filenames', nargs='+', help='File names (or glob patterns) to upload')
-    pars.add_argument('-r', '--recursive', action='store_true',
-                      help='Recursively search paths to upload files.')
-    pars.add_argument('-t', '--tags', help='comma separated list of tags to add to the nodes')
-    return pars
+s_common.deprecated('synapse.tools.pushfile is deprecated. Please use synapse.tools.axon.put instead.',
+                    curv='v2.225.0')
 
 if __name__ == '__main__':  # pragma: no cover
     s_cmd.exitmain(main)

synapse/tools/reload.py

@@ -1,68 +1,11 @@
-import synapse.telepath as s_telepath
+import synapse.common as s_common
 
 import synapse.lib.cmd as s_cmd
-import synapse.lib.output as s_output
-import synapse.lib.urlhelp as s_urlhelp
 
-descr = '''
-List or execute reload subsystems on a Synapse service.
-'''
+from synapse.tools.service.reload import main
 
-async def main(argv, outp=s_output.stdout):
-
-    pars = getArgParser(outp)
-    opts = pars.parse_args(argv)
-
-    async with s_telepath.withTeleEnv():
-
-        async with await s_telepath.openurl(opts.svcurl) as cell:
-
-            if opts.cmd == 'list':
-                names = await cell.getReloadableSystems()
-                if names:
-                    outp.printf(f'Cell at {s_urlhelp.sanitizeUrl(opts.svcurl)} has the following reload subsystems:')
-                    for name in names:
-                        outp.printf(name)
-                else:
-                    outp.printf(f'Cell at {s_urlhelp.sanitizeUrl(opts.svcurl)} has no registered reload subsystems.')
-
-            if opts.cmd == 'reload':
-                outp.printf(f'Reloading cell at {s_urlhelp.sanitizeUrl(opts.svcurl)}')
-                try:
-                    ret = await cell.reload(subsystem=opts.name)
-                except Exception as e:
-                    outp.printf(f'Error reloading cell: {e}')
-                    return 1
-
-                if not ret:
-                    outp.printf('No subsystems reloaded.')
-                else:
-                    outp.printf(f'{"Name:".ljust(40)}{"Result:".ljust(10)}Value:')
-                    for name, (isok, valu) in ret.items():
-                        if isok:
-                            mesg = str(valu)
-                            result = 'Success'
-                        else:
-                            mesg = valu[1].get('mesg')
-                            if mesg is None:
-                                mesg = valu[0]
-                            result = 'Failed'
-
-                        outp.printf(f'{name.ljust(40)}{result.ljust(10)}{mesg}')
-    return 0
-
-def getArgParser(outp):
-    pars = s_cmd.Parser(prog='synapse.tools.reload', outp=outp, description=descr)
-    pars.add_argument('--svcurl', default='cell:///vertex/storage', help='The telepath URL of the Synapse service.')
-
-    subpars = pars.add_subparsers(required=True,
-                                  title='subcommands',
-                                  dest='cmd',)
-    pars_list = subpars.add_parser('list', help='List subsystems which can be reloaded.')
-    reld_list = subpars.add_parser('reload', help='Reload registered subsystems.')
-    reld_list.add_argument('-n', '--name', type=str, help='Name of a subsystem to reload.')
-
-    return pars
+s_common.deprecated('synapse.tools.reload is deprecated. Please use synapse.tools.service.reload instead.',
+                    curv='v2.225.0')
 
 if __name__ == '__main__':  # pragma: no cover
     s_cmd.exitmain(main)

synapse/tools/rstorm.py

@@ -1,34 +1,11 @@
-import logging
-
 import synapse.common as s_common
 
 import synapse.lib.cmd as s_cmd
-import synapse.lib.output as s_output
-import synapse.lib.rstorm as s_rstorm
-
-logger = logging.getLogger(__name__)
-
-prog = 'synapse.tools.rstorm'
-descr = 'An RST pre-processor that allows you to embed storm directives.'
-
-async def main(argv, outp=s_output.stdout):
-
-    pars = s_cmd.Parser(prog=prog, outp=outp, description=descr)
-    pars.add_argument('rstfile', help='Input RST file with storm directives.')
-    pars.add_argument('--save', help='Output file to save (default: stdout)')
-
-    opts = pars.parse_args(argv)
 
-    async with await s_rstorm.StormRst.anit(opts.rstfile) as rstorm:
-        lines = await rstorm.run()
+from synapse.tools.utils.rstorm import logger, main
 
-    if opts.save:
-        with open(s_common.genpath(opts.save), 'w') as fd:
-            fd.truncate(0)
-            [fd.write(line) for line in lines]
-    else:
-        for line in lines:
-            outp.printf(line, addnl=False)
+s_common.deprecated('synapse.tools.rstorm is deprecated. Please use synapse.tools.utils.rstorm instead.',
+                    curv='v2.225.0')
 
 if __name__ == '__main__':  # pragma: no cover
     s_common.setlogging(logger)

synapse/tools/service/apikey.py

@@ -0,0 +1,90 @@
+import synapse.exc as s_exc
+import synapse.telepath as s_telepath
+
+import synapse.lib.cmd as s_cmd
+import synapse.lib.time as s_time
+import synapse.lib.output as s_output
+
+descr = '''
+Add, list, or delete user API keys from a Synapse service.
+'''
+
+def printkey(outp, info, apikey=None):
+    iden = info.get('iden')
+    name = info.get('name')
+    created = info.get('created')
+    updated = info.get('updated')
+    expires = info.get('expires')
+
+    outp.printf(f'Iden: {iden}')
+    if apikey:
+        outp.printf(f'  API Key: {apikey}')
+    outp.printf(f'  Name: {name}')
+    outp.printf(f'  Created: {s_time.repr(created)}')
+    outp.printf(f'  Updated: {s_time.repr(updated)}')
+    if expires:
+        outp.printf(f'  Expires: {s_time.repr(expires)}')
+
+    outp.printf('')
+
+async def main(argv, outp=s_output.stdout):
+
+    pars = s_cmd.Parser(prog='synapse.tools.service.apikey', outp=outp, description=descr)
+    pars.add_argument('--svcurl', default='cell:///vertex/storage', help='The telepath URL of the Synapse service.')
+
+    subpars = pars.add_subparsers(dest='action', required=True)
+
+    addp = subpars.add_parser('add', help='Add a user API key.')
+    addp.add_argument('-d', '--duration', type=int, help='The duration of the API key in seconds.')
+    addp.add_argument('-u', '--username', type=str, help='The username to add an API key to (restricted to admins).')
+    addp.add_argument('name', help='The name of the API key to add.')
+
+    listp = subpars.add_parser('list', help='List user API keys.')
+    listp.add_argument('-u', '--username', type=str, help='The username to list API keys for (restricted to admins).')
+
+    delp = subpars.add_parser('del', help='Delete a user API key.')
+    delp.add_argument('iden', help='The iden of the API key to delete.')
+
+    opts = pars.parse_args(argv)
+
+    async with s_telepath.withTeleEnv():
+
+        async with await s_telepath.openurl(opts.svcurl) as cell:
+
+            try:
+                useriden = None
+                if opts.action in ('add', 'list') and opts.username:
+                    user = await cell.getUserInfo(opts.username)
+                    useriden = user.get('iden')
+
+                if opts.action == 'add':
+                    if (duration := opts.duration) is not None:
+                        # Convert from seconds to milliseconds
+                        duration *= 1000
+
+                    apikey, info = await cell.addUserApiKey(opts.name, duration=duration, useriden=useriden)
+                    outp.printf(f'Successfully added API key with name={opts.name}.')
+                    printkey(outp, info, apikey)
+
+                elif opts.action == 'del':
+                    await cell.delUserApiKey(opts.iden)
+                    outp.printf(f'Successfully deleted API key with iden={opts.iden}.')
+
+                elif opts.action == 'list':
+                    apikeys = await cell.listUserApiKeys(useriden=useriden)
+                    if not apikeys:
+                        outp.printf('No API keys found.')
+                        return 0
+
+                    for info in apikeys:
+                        printkey(outp, info)
+
+            except s_exc.SynErr as exc:
+                mesg = exc.get('mesg')
+                outp.printf(f'ERROR: {exc.__class__.__name__}: {mesg}')
+                return 1
+
+    return 0
+
+if __name__ == '__main__':  # pragma: no cover
+    s_cmd.exitmain(main)

synapse/tools/service/backup.py

@@ -0,0 +1,181 @@
+import os
+import sys
+import glob
+import time
+import shutil
+import fnmatch
+import logging
+import argparse
+import contextlib
+
+import lmdb
+
+import synapse.common as s_common
+
+import synapse.lib.cmd as s_cmd
+
+logger = logging.getLogger(__name__)
+
+def backup(srcdir, dstdir, skipdirs=None):
+    '''
+    Create a backup of a Synapse application.
+
+    Args:
+        srcdir (str): Path to the directory to backup.
+        dstdir (str): Path to backup target directory.
+        skipdirs (list or None): Optional list of relative directory name glob patterns to exclude from the backup.
+
+    Note:
+        Running this method from the same process as a running user of the directory may lead to a segmentation fault
+    '''
+    with capturelmdbs(srcdir, skipdirs=skipdirs) as lmdbinfo:
+        txnbackup(lmdbinfo, srcdir, dstdir, skipdirs=skipdirs)
+
+@contextlib.contextmanager
+def capturelmdbs(srcdir, skipdirs=None, onlydirs=None):
+    '''
+    A context manager that opens all the lmdb files under a srcdir and makes a read transaction.  All transactions are
+    aborted and environments closed when the context is exited.
+
+    Yields:
+        Dict[str, Tuple[lmdb.Environment, lmdb.Transaction]]: Maps path to environment, transaction
+    '''
+    if onlydirs:
+        lmdbpaths = onlydirs
+
+    else:
+        if skipdirs is None:
+            skipdirs = []
+
+        srcdir = glob.escape(os.path.abspath(srcdir))
+        skipdirs.append(os.path.join(srcdir, 'tmp/*'))
+        skipdirs.append(os.path.join(srcdir, '*/tmp/*'))
+
+        srcdirglob = s_common.genpath(srcdir, '**/data.mdb')
+        fniter = glob.iglob(srcdirglob, recursive=True)
+        lmdbpaths = [os.path.dirname(fn) for fn in fniter if not
+                     any([fnmatch.fnmatch(fn, pattern) for pattern in skipdirs])]
+
+    lmdbinfo = {}
+
+    with contextlib.ExitStack() as stack:
+        for path in lmdbpaths:
+            logger.debug(f'Capturing txn for {path}')
+            datafile = os.path.join(path, 'data.mdb')
+            stat = os.stat(datafile)
+            map_size = stat.st_size
+            env = stack.enter_context(
+                lmdb.open(path, map_size=map_size, max_dbs=16384, create=False, readonly=True))
+            txn = stack.enter_context(env.begin())
+            assert path not in lmdbinfo
+            lmdbinfo[path] = (env, txn)
+
+        yield lmdbinfo
+
+def txnbackup(lmdbinfo, srcdir, dstdir, skipdirs=None):
+    '''
+    Create a backup of a Synapse application under a (hopefully consistent) set of transactions.
+
+    Args:
+        lmdbinfo(Dict[str, Tuple[lmdb.Environment, lmdb.Transaction]]): Maps of path to environment, transaction
+        srcdir (str): Path to the directory to backup.
+        dstdir (str): Path to backup target directory.
+        skipdirs (list or None): Optional list of relative directory name glob patterns to exclude from the backup.
+
+    Note:
+        Running this method from the same process as a running user of the directory may lead to a segmentation fault
+    '''
+    tick = s_common.now()
+
+    srcdir = s_common.reqdir(srcdir)
+    dstdir = s_common.gendir(dstdir)
+
+    if skipdirs is None:
+        skipdirs = []
+
+    # Always avoid backing up temporary and backup directories
+    skipdirs.append('**/tmp')
+    skipdirs.append('**/backups')
+
+    logger.debug(f'Starting backup of [{srcdir}]')
+    logger.debug(f'Destination dir: [{dstdir}]')
+
+    for root, dnames, fnames in os.walk(srcdir, topdown=True):
+
+        relpath = os.path.relpath(root, start=srcdir)
+
+        for name in list(dnames):
+
+            srcpath = s_common.genpath(root, name)
+
+            relname = os.path.join(relpath, name)
+
+            if any([fnmatch.fnmatch(relname, pattern) for pattern in skipdirs]):
+                logger.debug(f'skipping dir:{srcpath}')
+                dnames.remove(name)
+                continue
+
+            dstpath = s_common.genpath(dstdir, relname)
+
+            info = lmdbinfo.get(os.path.abspath(srcpath))
+
+            if info is not None:
+                logger.debug('backing up lmdb file: %s', srcpath)
+                dnames.remove(name)
+                env, txn = info
+                backup_lmdb(env, dstpath, txn=txn)
+                continue
+
+            if name.endswith('.lmdb'):
+                logger.warning('lmdb file %s not copied', srcpath)
+                dnames.remove(name)
+                continue
+
+            logger.debug(f'making dir:{dstpath}')
+            s_common.gendir(dstpath)
+
+        for name in fnames:
+
+            srcpath = s_common.genpath(root, name)
+            # skip unix sockets etc...
+            if not os.path.isfile(srcpath):
+                continue
+
+            dstpath = s_common.genpath(dstdir, relpath, name)
+            logger.debug(f'copying: {srcpath} -> {dstpath}')
+            shutil.copy(srcpath, dstpath)
+
+    tock = s_common.now()
+
+    logger.debug(f'Backup complete. Took [{tock-tick:.2f}] for [{srcdir}]')
+    return
+
+def backup_lmdb(env: lmdb.Environment, dstdir: str, txn=None):
+
+    tick = time.time()
+
+    s_common.gendir(dstdir)
+
+    env.copy(dstdir, compact=True, txn=txn)
+
+    tock = time.time()
+    logger.info(f'backup of: {env.path()} took: {tock-tick:.2f} seconds')
+
+async def main(argv):
+    args = parse_args(argv)
+    backup(args.srcdir, args.dstdir, args.skipdirs)
+    return 0
+
+def parse_args(argv):
+    desc = 'Create an optimized backup of a Synapse directory.'
+    parser = argparse.ArgumentParser('synapse.tools.service.backup', description=desc)
+    parser.add_argument('srcdir', help='Path to the Synapse directory to backup.')
+    parser.add_argument('dstdir', help='Path to the backup target directory.')
+    parser.add_argument('--skipdirs', nargs='+',
+                        help='Glob patterns of relative directory names to exclude from the backup.')
+    args = parser.parse_args(argv)
+    return args
+
+if __name__ == '__main__':  # pragma: no cover
+    s_common.setlogging(logger, defval='DEBUG')
+    s_cmd.exitmain(main)

synapse/tools/service/demote.py

@@ -0,0 +1,47 @@
+import synapse.exc as s_exc
+
+import synapse.telepath as s_telepath
+
+import synapse.lib.cmd as s_cmd
+import synapse.lib.output as s_output
+import synapse.lib.urlhelp as s_urlhelp
+
+descr = '''
+Automatically select a new leader and demote this service.
+
+Example:
+    python -m synapse.tools.service.demote
+'''
+
+async def main(argv, outp=s_output.stdout):
+
+    pars = s_cmd.Parser(prog='synapse.tools.service.demote', outp=outp, description=descr)
+
+    pars.add_argument('--url', default='cell:///vertex/storage',
+                      help='The telepath URL of the Synapse service.')
+
+    pars.add_argument('--timeout', type=int, default=60,
+                      help='The timeout to use awaiting network connections.')
+
+    opts = pars.parse_args(argv)
+
+    async with s_telepath.withTeleEnv():
+
+        try:
+
+            async with await s_telepath.openurl(opts.url) as cell:
+
+                outp.printf(f'Demoting leader: {opts.url}')
+
+                if await cell.demote(timeout=opts.timeout):
+                    return 0
+
+        except s_exc.SynErr as e:
+            outp.printf(f'Error while demoting service {s_urlhelp.sanitizeUrl(opts.url)}: {e}')
+            return 1
+
+        outp.printf(f'Failed to demote service {s_urlhelp.sanitizeUrl(opts.url)}')
+        return 1
+
+if __name__ == '__main__':  # pragma: no cover
+    s_cmd.exitmain(main)