synapse 2.213.0__py311-none-any.whl → 2.214.0__py311-none-any.whl

synapse/tools/cortex/layer/dump.py

@@ -0,0 +1,184 @@
+import os
+import tempfile
+import contextlib
+
+import synapse.exc as s_exc
+import synapse.common as s_common
+import synapse.telepath as s_telepath
+
+import synapse.lib.cmd as s_cmd
+import synapse.lib.output as s_output
+import synapse.lib.msgpack as s_msgpack
+
+descr = '''
+Export node edits from a Synapse layer.
+'''
+
+
+@contextlib.contextmanager
+def _tmpfile(dirn: str | None = None, prefix: str | None = None):
+    '''
+    Context manager to create a temporary file and close it when finished. If an
+    error occurs within the scope of the context manager, the tempfile will be
+    automatically deleted.
+
+    Args:
+        dirn: The optional directory name to create the tempfile in.
+        prefix: The optional tempfile name prefix.
+    '''
+    (_fd, path) = tempfile.mkstemp(dir=dirn, prefix=prefix)
+
+    try:
+        with contextlib.closing(os.fdopen(_fd, 'wb+')) as fd:
+            yield (fd, path)
+
+    except Exception: # pragma: no cover
+        os.unlink(path)
+        raise
+
+async def exportLayer(opts, outp):
+
+    async with await s_telepath.openurl(opts.url) as cell:
+
+        info = await cell.getCellInfo()
+
+        if (celltype := info['cell']['type']) != 'cortex':
+            mesg = f'Layer dump tool only works on cortexes, not {celltype}'
+            raise s_exc.TypeMismatch(mesg=mesg)
+
+        celliden = info['cell']['iden']
+        cellvers = info['cell']['version']
+
+    # Find and read state file
+    state = {}
+    statefile = opts.statefile
+    if statefile is None:
+        statefile = s_common.genpath(opts.outdir, f'{celliden}.{opts.iden}.yaml')
+
+    if (data := s_common.yamlload(statefile)) is not None:
+        state = data
+
+    if (soffs := opts.offset) is None:
+        soffs = state.get('offset:next', 0)
+
+    eoffs = None
+
+    async with await s_telepath.openurl(opts.url, name=f'*/layer/{opts.iden}') as layer:
+
+        # Handle no edits to export
+        if soffs >= await layer.getEditIndx():
+            mesg = f'No edits to export starting from offset ({soffs})'
+            raise s_exc.BadArg(mesg=mesg)
+
+        finished = False
+
+        genr = layer.syncNodeEdits2(soffs, wait=False)
+
+        nodeiter = aiter(genr)
+
+        while not finished:
+
+            try:
+                # Pull the first edit so we can get the starting offset
+                first = await anext(nodeiter)
+            except StopAsyncIteration: # pragma: no cover
+                break
+
+            soffs = first[0]
+
+            with _tmpfile(dirn=opts.outdir, prefix='layer.dump') as (fd, tmppath):
+
+                # Write header to file
+                fd.write(s_msgpack.en((
+                    'init',
+                    {
+                        'hdrvers': 1,
+                        'celliden': celliden,
+                        'cellvers': cellvers,
+                        'layriden': opts.iden,
+                        'offset': soffs,
+                        'chunksize': opts.chunksize,
+                        'tick': s_common.now(),
+                    }
+                )))
+
+                # Now write the first edit that we already pulled
+                fd.write(s_msgpack.en(('edit', first)))
+
+                count = 1
+
+                async for nodeedit in nodeiter:
+
+                    # Write individual edits to file
+                    fd.write(s_msgpack.en(('edit', nodeedit)))
+
+                    eoffs = nodeedit[0]
+
+                    count += 1
+
+                    if opts.chunksize and count % opts.chunksize == 0:
+                        break
+
+                else:
+                    finished = True
+
+                # Write footer to file
+                fd.write(s_msgpack.en(('fini', {
+                    'offset': eoffs,
+                    'tock': s_common.now(),
+                })))
+
+                path = s_common.genpath(opts.outdir, f'{celliden}.{opts.iden}.{soffs}-{eoffs}.nodeedits')
+                os.rename(tmppath, path)
+                outp.printf(f'Wrote layer node edits {soffs}-{eoffs} to {path}.')
+
+            # Save state file after each export file
+            state['offset:next'] = eoffs + 1
+            s_common.yamlsave(state, statefile)
+
+    return 0
+
+async def main(argv, outp=s_output.stdout):
+
+    pars = s_cmd.Parser(prog='layer.dump', outp=outp, description=descr)
+    pars.add_argument('--url', default='cell:///vertex/storage',
+                      help='The telepath URL of the Synapse service.')
+    pars.add_argument('--offset', default=None, type=int,
+                      help='The starting offset of the node edits to export.')
+    pars.add_argument('--chunksize', default=0, type=int,
+                      help='The number of node edits to store in a single file. Zero to disable chunking.')
+    pars.add_argument('--statefile', type=str, default=None,
+                      help='Path to the state tracking file for this layer dump.')
+
+    pars.add_argument('iden', help='The iden of the layer to export.')
+    pars.add_argument('outdir', help='The directory to save the exported node edits to.')
+
+    opts = pars.parse_args(argv)
+
+    if os.path.exists(opts.outdir) and not os.path.isdir(opts.outdir):
+        mesg = f'Specified output directory {opts.outdir} exists but is not a directory.'
+        outp.printf(f'ERROR: {mesg}')
+        return 1
+
+    os.makedirs(opts.outdir, exist_ok=True)
+
+    async with s_telepath.withTeleEnv():
+        try:
+            await exportLayer(opts, outp)
+
+        except s_exc.SynErr as exc:
+            mesg = exc.get('mesg')
+            outp.printf(f'ERROR: {mesg}.')
+            return 1
+
+        except Exception as exc: # pragma: no cover
+            mesg = str(exc)
+            outp.printf(f'ERROR: {mesg}.')
+            return 1
+
+    outp.printf(f'Successfully exported layer {opts.iden}.')
+
+    return 0
+
+if __name__ == '__main__':  # pragma: no cover
+    s_cmd.exitmain(main)

synapse/tools/cortex/layer/load.py

@@ -0,0 +1,129 @@
+import os
+
+import synapse.exc as s_exc
+import synapse.telepath as s_telepath
+
+import synapse.lib.cmd as s_cmd
+import synapse.lib.time as s_time
+import synapse.lib.output as s_output
+import synapse.lib.msgpack as s_msgpack
+import synapse.lib.version as s_version
+
+descr = '''
+Import node edits to a Synapse layer.
+'''
+
+async def importLayer(infiles, opts, outp):
+
+    async with await s_telepath.openurl(opts.url) as cell:
+
+        info = await cell.getCellInfo()
+
+        if (celltype := info['cell']['type']) != 'cortex':
+            mesg = f'Layer load tool only works on cortexes, not {celltype}.'
+            raise s_exc.TypeMismatch(mesg=mesg)
+
+        # Get the highest cellvers from all the input files
+        reqver = max([infile[0].get('cellvers') for infile in infiles])
+
+        if (synver := info.get('cell').get('version')) < reqver:
+            synstr = s_version.fmtVersion(*synver)
+            reqstr = s_version.fmtVersion(*reqver)
+            mesg = f'Synapse version mismatch ({synstr} < {reqstr}).'
+            raise s_exc.BadVersion(mesg=mesg)
+
+    async with await s_telepath.openurl(opts.url, name=f'*/layer/{opts.iden}') as layer:
+        for header, filename, genr in infiles:
+            soffs = header.get('offset')
+            tick = header.get('tick')
+
+            outp.printf(f'Loading {filename}, offset={soffs}, tick={s_time.repr(tick)}.')
+
+            eoffs = soffs
+            fini = None
+
+            for item in genr:
+                match item:
+                    case ('edit', (eoffs, edit, meta)):
+                        if opts.dryrun:
+                            continue
+
+                        await layer.saveNodeEdits(edit, meta=meta)
+
+                    case ('fini', info):
+                        fini = info
+                        break
+
+                    case _:
+                        mtype = item[0]
+                        mesg = f'Unexpected message type: {mtype}.'
+                        raise s_exc.BadMesgFormat(mesg=mesg)
+
+            if fini is None:
+                mesg = f'Incomplete/corrupt export: {filename}.'
+                raise s_exc.BadDataValu(mesg=mesg)
+
+            elif (offset := fini.get('offset')) != eoffs:
+                mesg = f'Incomplete/corrupt export: {filename}. Expected offset {offset}, got {eoffs}.'
+                raise s_exc.BadDataValu(mesg=mesg)
+
+            else:
+                if opts.dryrun:
+                    outp.printf(f'Successfully read {filename} as a dryrun test.')
+                else:
+                    outp.printf(f'Successfully loaded {filename} with {eoffs + 1 - soffs} edits ({soffs} - {eoffs}).')
+
+async def main(argv, outp=s_output.stdout):
+
+    pars = s_cmd.Parser(prog='layer.load', outp=outp, description=descr)
+    pars.add_argument('--dryrun', action='store_true', help="Process files but don't apply changes.")
+    pars.add_argument('--url', default='cell:///vertex/storage', help='The telepath URL of the Synapse service.')
+
+    pars.add_argument('iden', help='The iden of the layer to import to.')
+    pars.add_argument('files', nargs='+', help='The .nodeedits files to import from.')
+
+    opts = pars.parse_args(argv)
+
+    infiles = []
+
+    try:
+        # Load the files
+        for filename in opts.files:
+            if not os.path.exists(filename) or not os.path.isfile(filename):
+                mesg = f'Invalid input file specified: {filename}.'
+                raise s_exc.NoSuchFile(mesg=mesg)
+
+            genr = s_msgpack.iterfile(filename)
+            header = next(genr)
+            if header[0] != 'init':
+                mesg = f'Invalid header in {filename}.'
+                raise s_exc.BadMesgFormat(mesg=mesg)
+
+            infiles.append((header[1], filename, genr))
+
+        # Sort the files based on their offset
+        infiles = sorted(infiles, key=lambda x: x[0].get('offset'))
+
+        outp.printf('Processing the following nodeedits:')
+        outp.printf('Offset           | Filename')
+        outp.printf('-----------------|----------')
+        for header, filename, genr in infiles:
+            offset = header.get('offset')
+            outp.printf(f'{offset:<16d} | {filename}')
+
+        async with s_telepath.withTeleEnv():
+            await importLayer(infiles, opts, outp)
+
+        return 0
+
+    except s_exc.SynErr as exc:
+        mesg = exc.get('mesg')
+        outp.printf(f'ERROR: {mesg}.')
+        return 1
+
+    except Exception as exc: # pragma: no cover
+        outp.printf(f'ERROR: {str(exc)}.')
+        return 1
+
+if __name__ == '__main__':  # pragma: no cover
+    s_cmd.exitmain(main)

synapse/tools/demote.py

@@ -0,0 +1,52 @@
+import sys
+import asyncio
+import argparse
+
+import synapse.exc as s_exc
+
+import synapse.telepath as s_telepath
+
+import synapse.lib.cmd as s_cmd
+import synapse.lib.output as s_output
+import synapse.lib.urlhelp as s_urlhelp
+
+descr = '''
+Automatically select a new leader and demote this service.
+
+Example:
+    python -m synapse.tools.demote
+'''
+
+async def main(argv, outp=s_output.stdout):
+
+    pars = argparse.ArgumentParser(prog='synapse.tools.demote', description=descr,
+                        formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    pars.add_argument('--url', default='cell:///vertex/storage',
+                      help='The telepath URL of the Synapse service.')
+
+    pars.add_argument('--timeout', type=int, default=60,
+                      help='The timeout to use awaiting network connections.')
+
+    opts = pars.parse_args(argv)
+
+    async with s_telepath.withTeleEnv():
+
+        try:
+
+            async with await s_telepath.openurl(opts.url) as cell:
+
+                outp.printf(f'Demoting leader: {opts.url}')
+
+                if await cell.demote(timeout=opts.timeout):
+                    return 0
+
+        except s_exc.SynErr as e:
+            outp.printf(f'Error while demoting service {s_urlhelp.sanitizeUrl(opts.url)}: {e}')
+            return 1
+
+        outp.printf(f'Failed to demote service {s_urlhelp.sanitizeUrl(opts.url)}')
+        return 1
+
+if __name__ == '__main__':  # pragma: no cover
+    s_cmd.exitmain(main)

synapse/tools/healthcheck.py

@@ -44,7 +44,7 @@ async def main(argv, outp=s_output.stdout):
 
             prox = await s_common.wait_for(s_telepath.openurl(url),
                                            timeout=opts.timeout)
-    except (FileNotFoundError, ConnectionError, socket.gaierror) as e:
+    except (s_exc.LinkErr, s_exc.NoSuchPath, socket.gaierror) as e:
         mesg = f'Unable to connect to cell @ {sanitized_url}.'
         ret = {'status': 'failed',
                'iden': opts.cell,

synapse/tools/pkgs/gendocs.py

@@ -0,0 +1,176 @@
+import os
+import shutil
+import logging
+import subprocess
+
+import regex as re
+
+import synapse.exc as s_exc
+import synapse.common as s_common
+
+import synapse.lib.cmd as s_cmd
+import synapse.lib.output as s_output
+
+import synapse.tools.rstorm as s_rstorm
+import synapse.tools.autodoc as s_autodoc
+
+logger = logging.getLogger(__name__)
+
+_TOOLDIR = os.path.split(__file__)[0]
+PANDOC_FILTER = os.path.join(_TOOLDIR, 'pandoc_filter.py')
+
+# see https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html#file-wide-metadata
+re_sphinx_metadata_fields = re.compile(r'^:(tocdepth|nocomments|orphan|nosearch):( \w+)?\n\n',
+                                       flags=re.MULTILINE)
+
+def hasPandoc():
+    if os.system('pandoc --version') == 0:
+        return True
+    return False
+
+async def buildPkgDocs(outp: s_output.OutPut, pkgpath: str, rst_only: bool =False):
+
+    logger.info(f'Building pkg for {pkgpath}')
+    pkgdef = s_common.yamlload(pkgpath)
+    if pkgdef is None:
+        raise s_exc.BadArg(mesg=f'Package does not exist or does not contain yaml: {pkgpath}')
+
+    dirn = os.path.dirname(s_common.genpath(pkgpath))
+
+    docsdir = os.path.join(dirn, 'docs')
+    builddir = os.path.join(dirn, 'docs', '_build')
+
+    shutil.rmtree(builddir, ignore_errors=True)
+
+    s_common.gendir(builddir)
+
+    # touch any files we need in order to load a package, due to
+    # rstorm needing to load the package using genpkg tool. This
+    # does mean that standalone builds of a storm package from this
+    # must be done after using this buildpkg tool.
+    stormpkg_md_present = False
+    for dnfo in pkgdef.get('docs', ()):
+        fpath = dnfo.get('path')
+        with s_common.genfile(dirn, fpath) as fd:
+            pass
+        if fpath.endswith('stormpackage.md'):
+            stormpkg_md_present = True
+
+    # Generate the build .RST for stormpackage.md
+    if stormpkg_md_present:
+        logger.info(f'Generating stormpackage.rst for {pkgpath}')
+        pkgdocs, pkgname = await s_autodoc.docStormpkg(pkgpath)
+        with s_common.genfile(docsdir, 'stormpackage.rst') as fd:
+            text = pkgdocs.getRstText()
+            if rst_only is False:
+                # Leave this in place if we're only generating RST
+                text = text.replace('.. highlight:: none\n', '')
+            fd.write(text.encode())
+        logger.info('Generated the stormpackage.rst file!')
+
+    for name in os.listdir(docsdir):
+
+        if not name.endswith('.rst'):
+            continue
+
+        docpath = os.path.join(docsdir, name)
+
+        basename = name.rsplit('.', 1)[0]
+
+        builtmd = os.path.join(builddir, f'{basename}.md')
+        builtrst = os.path.join(builddir, name)
+
+        argv = (docpath, '--save', builtrst)
+        logger.info(f'Executing rstorm for {argv}')
+        await s_rstorm.main(argv)
+
+        if rst_only:
+            logger.info(f'rst_only enabled, done processing {name}')
+            continue
+
+        logger.info('Preprocessing rstorm output')
+        with s_common.genfile(builtrst) as fd:
+            buf = fd.read().decode()
+
+        # Remove highglight:: none directives
+        buf = buf.replace('.. highlight:: none\n', '')
+
+        # Remove sphinx metadata fields
+        buf = re_sphinx_metadata_fields.sub('', buf)
+
+        lines = buf.splitlines(keepends=True)
+
+        # Remove lines which start with explicit sphinx rst targets
+        nlines1 = []
+        for line in lines:
+            if line.startswith('.. _') and line.strip().endswith(':'):
+                logger.info(f'Dropping: [{line.strip()}]')
+                continue
+            nlines1.append(line)
+
+        buf = ''.join(nlines1)
+
+        with s_common.genfile(builtrst) as fd:
+            fd.truncate()
+            _ = fd.write(buf.encode())
+
+        logger.info(f'Converting {builtrst} to markdown')
+        if name == 'stormpackage.rst':
+            args = ['pandoc', '--filter', PANDOC_FILTER, '-f', 'rst', '-t', 'markdown', '-o', builtmd, builtrst]
+        else:
+            args = ['pandoc', '-f', 'rst', '-t', 'markdown', '-o', builtmd, builtrst]
+
+        r = subprocess.run(args, capture_output=True)
+
+        # Re-write stderr (logging) to our outp
+        for line in r.stderr.decode().splitlines():
+            outp.printf(f'ERR: {line}')
+
+        if r.returncode != 0:
+            raise s_exc.SynErr(mesg=f'Error converting {builtrst} to {builtmd}')
+
+        logger.info(f'Done converting {builtrst} to {builtmd}')
+
+        # Strip out / manipulate the md content
+        with s_common.genfile(builtmd) as fd:
+            buf = fd.read().decode()
+
+        lines = buf.splitlines(keepends=True)
+
+        # Remove lines which only have a single `:` left in them
+        nlines1 = [line for line in lines if line.strip() != ':']
+
+        buf = ''.join(nlines1)
+
+        with s_common.genfile(builtmd) as fd:
+            fd.truncate()
+            _ = fd.write(buf.encode())
+
+        logger.info('Done manipulating markdown')
+
+    logger.info(f'buildPkgDocs complete for {pkgpath}.')
+
+prog = 'synapse.tools.pkgs.gendocs'
+desc = 'A tool for building storm package docs from RStorm into markdown. This tool requires pandoc to be available.'
+
+async def main(argv, outp=s_output.stdout):
+
+    pars = s_cmd.Parser(prog=prog, outp=outp, description=desc)
+    pars.add_argument('pkgfile', metavar='<pkgfile>', help='Path to a storm package prototype yml file.')
+    pars.add_argument('--rst-only', default=False, action='store_true',
+                      help='Stops building after the .rst files have been generated.')
+
+    opts = pars.parse_args(argv)
+
+    if opts.rst_only is False and not hasPandoc():
+        logger.error('Pandoc is not available, can only run rst/rstorm output.')
+        return 1
+
+    await buildPkgDocs(outp, opts.pkgfile, rst_only=opts.rst_only)
+
+    return 0
+
+if __name__ == '__main__':  # pragma: no cover
+    s_common.setlogging(logger, 'DEBUG')
+    logging.getLogger('vcr').setLevel(logging.WARNING)
+    s_cmd.exitmain(main)

synapse/tools/pkgs/pandoc_filter.py

@@ -0,0 +1,79 @@
+import sys
+import json
+
+import packaging.version as p_version
+import packaging.specifiers as p_specifiers
+
+PANDOC_API_REQVERS = '>=1.23.0,<1.24.0'
+
+def walk(elem):
+    '''
+    Walk the pandoc AST, yielding (type, content) tuples.
+    Ref: https://pandoc.org/using-the-pandoc-api.html#walking-the-ast
+    '''
+
+    if isinstance(elem, list):
+        for subelem in elem:
+            yield from walk(subelem)
+        return
+
+    if isinstance(elem, dict):
+        if 't' in elem:
+            yield elem['t'], elem.get('c')
+        for v in elem.values():
+            yield from walk(v)
+        return
+
+def main():
+    '''
+    A pandoc filter reads the intermediate JSON-formatted AST generated from the source, makes any modifications,
+    and then writes the JSON-formatted AST to be used to generate the target.
+
+    Ref: https://pandoc.org/filters.html
+
+    Usage:
+
+       pandoc -f rst -t markdown --filter ./synapse/tools/pkg/pandoc_filter.py -o foo.md foo.rst
+    '''
+
+    ast = json.load(sys.stdin)
+
+    spec = p_specifiers.SpecifierSet(PANDOC_API_REQVERS)
+    vers = p_version.Version('.'.join(str(part) for part in ast['pandoc-api-version']))
+    if vers not in spec:
+        raise Exception(f'Pandoc API version {vers} does not match required version {PANDOC_API_REQVERS}')
+
+    for type_, content in walk(ast['blocks']):
+
+        if type_ != 'DefinitionList':
+            continue
+
+        # An RST term with multiple definitions gets combined into one -> split
+        # ( Only Para types should get split )
+        for term, defs in content:
+
+            newdefs = []
+            newdef = []
+
+            for def_ in defs[0]:
+
+                if def_['t'] == 'Para' and newdef:
+                    # we are on a new paragraph so save the
+                    # previous term/def group
+                    newdefs.append(newdef.copy())
+                    newdef.clear()
+
+                newdef.append(def_)
+
+            if newdef:
+                newdefs.append(newdef.copy())
+
+            defs.clear()
+            defs.extend(newdefs)
+
+    sys.stdout.write(json.dumps(ast))
+
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

synapse/tools/shutdown.py

@@ -0,0 +1,52 @@
+import synapse.exc as s_exc
+import synapse.common as s_common
+import synapse.telepath as s_telepath
+
+import synapse.lib.cmd as s_cmd
+import synapse.lib.output as s_output
+
+desc = '''
+Initiate a graceful shutdown of a service.
+
+This tool is designed to put the service into a state where
+any non-background tasks will be allowed to complete while ensuring
+no new tasks are created. Without a timeout, it can block forever if
+tasks do not exit.
+
+The command exits with code 0 if the graceful shutdown was successful and
+exit code 1 if a timeout was specified and was hit. Upon hitting the timeout
+the system resumes normal operation.
+
+NOTE: This will also demote the service if run on a leader with mirrors.
+'''
+
+async def main(argv, outp=s_output.stdout):
+
+    pars = s_cmd.Parser('synapse.tools.shutdown', outp=outp, description=desc)
+
+    pars.add_argument('--url', default='cell:///vertex/storage',
+                        help='The telepath URL to connect to the service.')
+
+    pars.add_argument('--timeout', default=None, type=int,
+                        help='An optional timeout in seconds. If timeout is reached, the shutdown is aborted.')
+
+    opts = pars.parse_args(argv)
+
+    async with s_telepath.withTeleEnv():
+
+        try:
+
+            async with await s_telepath.openurl(opts.url) as proxy:
+
+                if await proxy.shutdown(timeout=opts.timeout):
+                    return 0
+
+                return 1
+
+        except Exception as e: # pragma: no cover
+            text = s_exc.reprexc(e)
+            outp.printf(f'Error while attempting graceful shutdown: {text}')
+            return 1
+
+if __name__ == '__main__': # pragma: no cover
+    s_cmd.exitmain(main)

{synapse-2.213.0.dist-info → synapse-2.214.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 2.213.0
+Version: 2.214.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0