synapse 2.224.0__py311-none-any.whl → 2.225.0__py311-none-any.whl

synapse/tools/axon/copy.py

@@ -0,0 +1,44 @@
+import logging
+
+import synapse.common as s_common
+import synapse.telepath as s_telepath
+
+import synapse.exc as s_exc
+import synapse.lib.cmd as s_cmd
+import synapse.lib.base as s_base
+import synapse.lib.output as s_output
+
+logger = logging.getLogger(__name__)
+
+async def main(argv, outp=s_output.stdout):
+
+    pars = s_cmd.Parser(prog='synapse.tools.axon.copy', outp=outp)
+    pars.add_argument('--offset', type=int, default=0, help='An offset within the source axon to start from.')
+    pars.add_argument('src_axon', help='The telepath URL of the source axon.')
+    pars.add_argument('dst_axon', help='The telepath URL of the destination axon.')
+
+    try:
+        opts = pars.parse_args(argv)
+    except s_exc.ParserExit:
+        return 1
+
+    async with s_telepath.withTeleEnv():
+        async with await s_base.Base.anit() as base:
+
+            srcaxon = await base.enter_context(await s_telepath.openurl(opts.src_axon))
+            dstaxon = await base.enter_context(await s_telepath.openurl(opts.dst_axon))
+
+            outp.printf(f'Starting transfer at offset: {opts.offset}')
+
+            async for (offs, (sha256, size)) in srcaxon.hashes(opts.offset):
+                offstext = str(offs).rjust(10)
+                sha2text = s_common.ehex(sha256)
+                outp.printf(f'[{offstext}] - {sha2text} ({size})')
+                async with await dstaxon.upload() as fd:
+                    async for byts in srcaxon.get(sha256):
+                        await fd.write(byts)
+                    await fd.save()
+    return 0
+
+if __name__ == '__main__':  # pragma: no cover
+    s_cmd.exitmain(main)

synapse/tools/axon/get.py

@@ -0,0 +1,64 @@
+import pathlib
+
+import synapse.common as s_common
+import synapse.telepath as s_telepath
+
+import synapse.lib.cmd as s_cmd
+import synapse.lib.output as s_output
+
+
+async def main(argv, outp=s_output.stdout):
+    pars = getArgParser(outp)
+    opts = pars.parse_args(argv)
+
+    if opts.output is None:
+        opts.output = '.'
+
+    outdir = pathlib.Path(opts.output)
+
+    s_common.gendir(opts.output)
+
+    async with s_telepath.withTeleEnv():
+
+        async with await s_telepath.openurl(opts.axon) as axon:
+
+            # reminder: these are the hashes *not* available
+
+            awants = await axon.wants([s_common.uhex(h) for h in opts.hashes])
+            for a in awants:
+                outp.printf(f'{s_common.ehex(a)} not in axon store')
+
+            exists = [h for h in opts.hashes if s_common.uhex(h) not in awants]
+
+            for h in exists:
+
+                try:
+                    outp.printf(f'Fetching {h} to file')
+
+                    with open(outdir.joinpath(h), 'wb') as fd:
+                        async for b in axon.get(s_common.uhex(h)):
+                            fd.write(b)
+
+                    outp.printf(f'Fetched {h} to file')
+
+                except Exception as e:
+                    outp.printf('Error: Hit Exception: %s' % (str(e),))
+                    continue
+
+    return 0
+
+
+def getArgParser(outp):
+    desc = 'Fetches file from the given axon'
+    pars = s_cmd.Parser(prog='synapse.tools.axon.get', outp=outp, description=desc)
+    pars.add_argument('-a', '--axon', type=str, dest='axon', required=True,
+                      help='URL to the axon blob store')
+    pars.add_argument('-o', '--output', type=str, dest='output',
+                      help='Directory to output files to')
+    pars.add_argument('-l', '--list-hashes', dest='hashes', action='append', default=[],
+                      help='List of hashes to pull from axon')
+
+    return pars
+
+if __name__ == '__main__':  # pragma: no cover
+    s_cmd.exitmain(main)

synapse/tools/axon/put.py

@@ -0,0 +1,122 @@
+import os
+import glob
+import logging
+
+import synapse.exc as s_exc
+import synapse.common as s_common
+import synapse.telepath as s_telepath
+
+import synapse.lib.cmd as s_cmd
+import synapse.lib.output as s_output
+import synapse.lib.hashset as s_hashset
+
+logger = logging.getLogger(__name__)
+
+
+async def main(argv, outp=s_output.stdout):
+    pars = getArgParser(outp)
+    opts = pars.parse_args(argv)
+
+    async with s_telepath.withTeleEnv():
+
+        axon = await s_telepath.openurl(opts.axon)
+
+        core = None
+        if opts.cortex:
+            core = await s_telepath.openurl(opts.cortex)
+
+            tags = set()
+            if opts.tags:
+                for tag in opts.tags.split(','):
+                    tags.add(tag)
+
+            tags = tuple(tags)
+            if tags:
+                outp.printf(f'adding tags: {tags}')
+
+        filepaths = set()
+        for item in opts.filenames:
+            paths = glob.glob(item, recursive=opts.recursive)
+
+            if not paths:
+                outp.printf(f'filepath does not contain any files: {item}')
+                continue
+
+            filepaths.update([path for path in paths if os.path.isfile(path)])
+
+        for path in filepaths:
+
+            bname = os.path.basename(path)
+
+            hset = s_hashset.HashSet()
+            with s_common.reqfile(path) as fd:
+                hset.eatfd(fd)
+
+            fhashes = {htyp: hasher.hexdigest() for htyp, hasher in hset.hashes}
+
+            sha256 = fhashes.get('sha256')
+            bsha256 = s_common.uhex(sha256)
+
+            if not await axon.has(bsha256):
+
+                async with await axon.upload() as upfd:
+
+                    with s_common.genfile(path) as fd:
+                        for byts in s_common.iterfd(fd):
+                            await upfd.write(byts)
+
+                    size, hashval = await upfd.save()
+
+                if hashval != bsha256:  # pragma: no cover
+                    raise s_exc.SynErr(mesg='hashes do not match',
+                                       ehash=s_common.ehex(hashval),
+                                       ahash=hashval)
+
+                outp.printf(f'Uploaded [{bname}] to axon')
+            else:
+                outp.printf(f'Axon already had [{bname}]')
+
+            if core:
+                opts = {'vars': {
+                    'md5': fhashes.get('md5'),
+                    'sha1': fhashes.get('sha1'),
+                    'sha256': fhashes.get('sha256'),
+                    'size': hset.size,
+                    'name': bname,
+                    'tags': tags,
+                }}
+
+                q = '[file:bytes=$sha256 :md5=$md5 :sha1=$sha1 :size=$size :name=$name] ' \
+                    '{ for $tag in $tags { [+#$tag] } }'
+
+                msgs = await core.storm(q, opts=opts).list()
+                node = [m[1] for m in msgs if m[0] == 'node'][0]
+
+                iden = node[0][1]
+                size = node[1]['props']['size']
+                name = node[1]['props']['name']
+                mesg = f'file: {bname} ({size}) added to core ({iden}) as {name}'
+                outp.printf(mesg)
+
+        await axon.fini()
+        if core:
+            await core.fini()
+
+    return 0
+
+def getArgParser(outp):
+    desc = 'Command line tool for uploading files to an Axon and making ' \
+           'file:bytes in a Cortex.'
+    pars = s_cmd.Parser(prog='synapse.tools.axon.put', outp=outp, description=desc)
+    pars.add_argument('-a', '--axon', required=True, type=str, dest='axon',
+                   help='URL for a target Axon to store files at.')
+    pars.add_argument('-c', '--cortex', default=None, type=str, dest='cortex',
+                   help='URL for a target Cortex to make file:bytes nodes.')
+    pars.add_argument('filenames', nargs='+', help='File names (or glob patterns) to upload')
+    pars.add_argument('-r', '--recursive', action='store_true',
+                      help='Recursively search paths to upload files.')
+    pars.add_argument('-t', '--tags', help='comma separated list of tags to add to the nodes')
+    return pars
+
+if __name__ == '__main__':  # pragma: no cover
+    s_cmd.exitmain(main)

synapse/tools/axon2axon.py

@@ -1,44 +1,11 @@
-import logging
-
 import synapse.common as s_common
-import synapse.telepath as s_telepath
 
-import synapse.exc as s_exc
 import synapse.lib.cmd as s_cmd
-import synapse.lib.base as s_base
-import synapse.lib.output as s_output
-
-logger = logging.getLogger(__name__)
-
-async def main(argv, outp=s_output.stdout):
-
-    pars = s_cmd.Parser(prog='synapse.tools.axon2axon', outp=outp)
-    pars.add_argument('--offset', type=int, default=0, help='An offset within the source axon to start from.')
-    pars.add_argument('src_axon', help='The telepath URL of the source axon.')
-    pars.add_argument('dst_axon', help='The telepath URL of the destination axon.')
-
-    try:
-        opts = pars.parse_args(argv)
-    except s_exc.ParserExit:
-        return 1
-
-    async with s_telepath.withTeleEnv():
-        async with await s_base.Base.anit() as base:
-
-            srcaxon = await base.enter_context(await s_telepath.openurl(opts.src_axon))
-            dstaxon = await base.enter_context(await s_telepath.openurl(opts.dst_axon))
 
-            outp.printf(f'Starting transfer at offset: {opts.offset}')
+from synapse.tools.axon.copy import main
 
-            async for (offs, (sha256, size)) in srcaxon.hashes(opts.offset):
-                offstext = str(offs).rjust(10)
-                sha2text = s_common.ehex(sha256)
-                outp.printf(f'[{offstext}] - {sha2text} ({size})')
-                async with await dstaxon.upload() as fd:
-                    async for byts in srcaxon.get(sha256):
-                        await fd.write(byts)
-                    await fd.save()
-    return 0
+s_common.deprecated('synapse.tools.axon2axon is deprecated. Please use synapse.tools.axon.copy instead.',
+                    curv='v2.225.0')
 
 if __name__ == '__main__':  # pragma: no cover
     s_cmd.exitmain(main)

synapse/tools/backup.py

@@ -1,182 +1,12 @@
-import os
-import sys
-import glob
-import time
-import shutil
-import fnmatch
-import logging
-import argparse
-import contextlib
-
-import lmdb
-
 import synapse.common as s_common
 
-logger = logging.getLogger(__name__)
-
-def backup(srcdir, dstdir, skipdirs=None):
-    '''
-    Create a backup of a Synapse application.
-
-    Args:
-        srcdir (str): Path to the directory to backup.
-        dstdir (str): Path to backup target directory.
-        skipdirs (list or None): Optional list of relative directory name glob patterns to exclude from the backup.
-
-    Note:
-        Running this method from the same process as a running user of the directory may lead to a segmentation fault
-    '''
-    with capturelmdbs(srcdir, skipdirs=skipdirs) as lmdbinfo:
-        txnbackup(lmdbinfo, srcdir, dstdir, skipdirs=skipdirs)
-
-@contextlib.contextmanager
-def capturelmdbs(srcdir, skipdirs=None, onlydirs=None):
-    '''
-    A context manager that opens all the lmdb files under a srcdir and makes a read transaction.  All transactions are
-    aborted and environments closed when the context is exited.
-
-    Yields:
-        Dict[str, Tuple[lmdb.Environment, lmdb.Transaction]]: Maps path to environment, transaction
-    '''
-    if onlydirs:
-        lmdbpaths = onlydirs
-
-    else:
-        if skipdirs is None:
-            skipdirs = []
-
-        srcdir = glob.escape(os.path.abspath(srcdir))
-        skipdirs.append(os.path.join(srcdir, 'tmp/*'))
-        skipdirs.append(os.path.join(srcdir, '*/tmp/*'))
-
-        srcdirglob = s_common.genpath(srcdir, '**/data.mdb')
-        fniter = glob.iglob(srcdirglob, recursive=True)
-        lmdbpaths = [os.path.dirname(fn) for fn in fniter if not
-                     any([fnmatch.fnmatch(fn, pattern) for pattern in skipdirs])]
-
-    lmdbinfo = {}
-
-    with contextlib.ExitStack() as stack:
-        for path in lmdbpaths:
-            logger.debug(f'Capturing txn for {path}')
-            datafile = os.path.join(path, 'data.mdb')
-            stat = os.stat(datafile)
-            map_size = stat.st_size
-            env = stack.enter_context(
-                lmdb.open(path, map_size=map_size, max_dbs=16384, create=False, readonly=True))
-            txn = stack.enter_context(env.begin())
-            assert path not in lmdbinfo
-            lmdbinfo[path] = (env, txn)
-
-        yield lmdbinfo
-
-def txnbackup(lmdbinfo, srcdir, dstdir, skipdirs=None):
-    '''
-    Create a backup of a Synapse application under a (hopefully consistent) set of transactions.
-
-    Args:
-        lmdbinfo(Dict[str, Tuple[lmdb.Environment, lmdb.Transaction]]): Maps of path to environment, transaction
-        srcdir (str): Path to the directory to backup.
-        dstdir (str): Path to backup target directory.
-        skipdirs (list or None): Optional list of relative directory name glob patterns to exclude from the backup.
-
-    Note:
-        Running this method from the same process as a running user of the directory may lead to a segmentation fault
-    '''
-    tick = s_common.now()
-
-    srcdir = s_common.reqdir(srcdir)
-    dstdir = s_common.gendir(dstdir)
-
-    if skipdirs is None:
-        skipdirs = []
-
-    # Always avoid backing up temporary and backup directories
-    skipdirs.append('**/tmp')
-    skipdirs.append('**/backups')
-
-    logger.debug(f'Starting backup of [{srcdir}]')
-    logger.debug(f'Destination dir: [{dstdir}]')
-
-    for root, dnames, fnames in os.walk(srcdir, topdown=True):
-
-        relpath = os.path.relpath(root, start=srcdir)
+import synapse.lib.cmd as s_cmd
 
-        for name in list(dnames):
+from synapse.tools.service.backup import logger, main, backup
 
-            srcpath = s_common.genpath(root, name)
-
-            relname = os.path.join(relpath, name)
-
-            if any([fnmatch.fnmatch(relname, pattern) for pattern in skipdirs]):
-                logger.debug(f'skipping dir:{srcpath}')
-                dnames.remove(name)
-                continue
-
-            dstpath = s_common.genpath(dstdir, relname)
-
-            info = lmdbinfo.get(os.path.abspath(srcpath))
-
-            if info is not None:
-                logger.debug('backing up lmdb file: %s', srcpath)
-                dnames.remove(name)
-                env, txn = info
-                backup_lmdb(env, dstpath, txn=txn)
-                continue
-
-            if name.endswith('.lmdb'):
-                logger.warning('lmdb file %s not copied', srcpath)
-                dnames.remove(name)
-                continue
-
-            logger.debug(f'making dir:{dstpath}')
-            s_common.gendir(dstpath)
-
-        for name in fnames:
-
-            srcpath = s_common.genpath(root, name)
-            # skip unix sockets etc...
-            if not os.path.isfile(srcpath):
-                continue
-
-            dstpath = s_common.genpath(dstdir, relpath, name)
-            logger.debug(f'copying: {srcpath} -> {dstpath}')
-            shutil.copy(srcpath, dstpath)
-
-    tock = s_common.now()
-
-    logger.debug(f'Backup complete. Took [{tock-tick:.2f}] for [{srcdir}]')
-    return
-
-def backup_lmdb(env: lmdb.Environment, dstdir: str, txn=None):
-
-    tick = time.time()
-
-    s_common.gendir(dstdir)
-
-    env.copy(dstdir, compact=True, txn=txn)
-
-    tock = time.time()
-    logger.info(f'backup of: {env.path()} took: {tock-tick:.2f} seconds')
-
-def main(argv):
-    args = parse_args(argv)
-    backup(args.srcdir, args.dstdir, args.skipdirs)
-    return 0
-
-def parse_args(argv):
-    desc = 'Create an optimized backup of a Synapse directory.'
-    parser = argparse.ArgumentParser('synapse.tools.backup', description=desc)
-    parser.add_argument('srcdir', help='Path to the Synapse directory to backup.')
-    parser.add_argument('dstdir', help='Path to the backup target directory.')
-    parser.add_argument('--skipdirs', nargs='+',
-                        help='Glob patterns of relative directory names to exclude from the backup.')
-    args = parser.parse_args(argv)
-    return args
-
-def _main(argv):  # pragma: no cover
-    s_common.setlogging(logger, defval='DEBUG')
-    return main(argv)
+s_common.deprecated('synapse.tools.backup is deprecated. Please use synapse.tools.service.backup instead.',
+                    curv='v2.225.0')
 
 if __name__ == '__main__':  # pragma: no cover
-    sys.exit(_main(sys.argv[1:]))
+    s_common.setlogging(logger, defval='DEBUG')
+    s_cmd.exitmain(main)