synapse 2.224.0__py311-none-any.whl → 2.225.0__py311-none-any.whl

synapse/tools/shutdown.py

@@ -1,51 +1,11 @@
-import synapse.exc as s_exc
-import synapse.telepath as s_telepath
+import synapse.common as s_common
 
 import synapse.lib.cmd as s_cmd
-import synapse.lib.output as s_output
 
-desc = '''
-Initiate a graceful shutdown of a service.
+from synapse.tools.service.shutdown import main
 
-This tool is designed to put the service into a state where
-any non-background tasks will be allowed to complete while ensuring
-no new tasks are created. Without a timeout, it can block forever if
-tasks do not exit.
+s_common.deprecated('synapse.tools.shutdown is deprecated. Please use synapse.tools.service.shutdown instead.',
+                    curv='v2.225.0')
 
-The command exits with code 0 if the graceful shutdown was successful and
-exit code 1 if a timeout was specified and was hit. Upon hitting the timeout
-the system resumes normal operation.
-
-NOTE: This will also demote the service if run on a leader with mirrors.
-'''
-
-async def main(argv, outp=s_output.stdout):
-
-    pars = s_cmd.Parser(prog='synapse.tools.shutdown', outp=outp, description=desc)
-
-    pars.add_argument('--url', default='cell:///vertex/storage',
-                        help='The telepath URL to connect to the service.')
-
-    pars.add_argument('--timeout', default=None, type=int,
-                        help='An optional timeout in seconds. If timeout is reached, the shutdown is aborted.')
-
-    opts = pars.parse_args(argv)
-
-    async with s_telepath.withTeleEnv():
-
-        try:
-
-            async with await s_telepath.openurl(opts.url) as proxy:
-
-                if await proxy.shutdown(timeout=opts.timeout):
-                    return 0
-
-                return 1
-
-        except Exception as e: # pragma: no cover
-            text = s_exc.reprexc(e)
-            outp.printf(f'Error while attempting graceful shutdown: {text}')
-            return 1
-
-if __name__ == '__main__': # pragma: no cover
+if __name__ == '__main__':  # pragma: no cover
     s_cmd.exitmain(main)

synapse/tools/snapshot.py

@@ -1,64 +1,11 @@
-import logging
-import synapse.exc as s_exc
-import synapse.telepath as s_telepath
+import synapse.common as s_common
 
 import synapse.lib.cmd as s_cmd
-import synapse.lib.output as s_output
 
-logger = logging.getLogger(__name__)
+from synapse.tools.service.snapshot import main
 
-desc = '''
-Command line tool to freeze/resume service operations to allow
-system admins to generate a transactionally consistent volume
-snapshot using 3rd party tools.
-
-The use pattern should be::
-
-    python -m synapse.tools.snapshot freeze
-
-    <generate volume snapshot using 3rd party tools>
-
-    python -m synapse.tools.snapshot resume
-
-The tool will set the process exit code to 0 on success.
-'''
-
-async def main(argv, outp=s_output.stdout):
-
-    pars = s_cmd.Parser(prog='synapse.tools.snapshot', outp=outp, description=desc)
-
-    subs = pars.add_subparsers(required=True, title='commands', dest='cmd')
-
-    freeze = subs.add_parser('freeze', help='Suspend edits and sync changes to disk.')
-    freeze.add_argument('--timeout', type=int, default=120,
-                        help='Maximum time to wait for the nexus lock.')
-
-    freeze.add_argument('--svcurl', default='cell:///vertex/storage',
-                        help='The telepath URL of the Synapse service.')
-
-    resume = subs.add_parser('resume', help='Resume edits and continue normal operation.')
-    resume.add_argument('--svcurl', default='cell:///vertex/storage',
-                        help='The telepath URL of the Synapse service.')
-
-    opts = pars.parse_args(argv)
-
-    try:
-        async with s_telepath.withTeleEnv():
-
-            async with await s_telepath.openurl(opts.svcurl) as proxy:
-
-                if opts.cmd == 'freeze':
-                    await proxy.freeze(timeout=opts.timeout)
-                    return 0
-
-                if opts.cmd == 'resume':
-                    await proxy.resume()
-                    return 0
-
-    except s_exc.SynErr as e:
-        mesg = e.errinfo.get('mesg')
-        outp.printf(f'ERROR {e.__class__.__name__}: {mesg}')
-        return 1
+s_common.deprecated('synapse.tools.snapshot is deprecated. Please use synapse.tools.service.snapshot instead.',
+                    curv='v2.225.0')
 
 if __name__ == '__main__':  # pragma: no cover
     s_cmd.exitmain(main)

synapse/tools/storm/__main__.py

@@ -0,0 +1,5 @@
+import synapse.lib.cmd as s_cmd
+import synapse.tools.storm._cli as s_t_storm
+
+if __name__ == '__main__':  # pragma: no cover
+    s_cmd.exitmain(s_t_storm.main)

synapse/tools/{storm.py → storm/_cli.py}

@@ -645,6 +645,3 @@ async def main(argv, outp=s_output.stdout):
 
                     await cli.addSignalHandlers()
                     await cli.runCmdLoop()
-
-if __name__ == '__main__':  # pragma: no cover
-    s_cmd.exitmain(main)

synapse/tools/{pkgs/pandoc_filter.py → storm/pkg/_pandoc_filter.py}

@@ -33,7 +33,7 @@ def main():
 
     Usage:
 
-       pandoc -f rst -t markdown --filter ./synapse/tools/pkg/pandoc_filter.py -o foo.md foo.rst
+       pandoc -f rst -t markdown --filter ./synapse/tools/storm/pkg/_pandoc_filter.py -o foo.md foo.rst
     '''
 
     ast = json.load(sys.stdin)

synapse/tools/storm/pkg/doc.py

@@ -0,0 +1,176 @@
+import os
+import shutil
+import logging
+import subprocess
+
+import regex as re
+
+import synapse.exc as s_exc
+import synapse.common as s_common
+
+import synapse.lib.cmd as s_cmd
+import synapse.lib.output as s_output
+
+import synapse.tools.utils.rstorm as s_rstorm
+import synapse.tools.utils.autodoc as s_autodoc
+
+logger = logging.getLogger(__name__)
+
+_TOOLDIR = os.path.split(__file__)[0]
+PANDOC_FILTER = os.path.join(_TOOLDIR, '_pandoc_filter.py')
+
+# see https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html#file-wide-metadata
+re_sphinx_metadata_fields = re.compile(r'^:(tocdepth|nocomments|orphan|nosearch):( \w+)?\n\n',
+                                       flags=re.MULTILINE)
+
+def hasPandoc():
+    if os.system('pandoc --version') == 0:
+        return True
+    return False
+
+async def buildPkgDocs(outp, pkgpath: str, rst_only: bool =False):
+
+    logger.info(f'Building pkg for {pkgpath}')
+    pkgdef = s_common.yamlload(pkgpath)
+    if pkgdef is None:
+        raise s_exc.BadArg(mesg=f'Package does not exist or does not contain yaml: {pkgpath}')
+
+    dirn = os.path.dirname(s_common.genpath(pkgpath))
+
+    docsdir = os.path.join(dirn, 'docs')
+    builddir = os.path.join(dirn, 'docs', '_build')
+
+    shutil.rmtree(builddir, ignore_errors=True)
+
+    s_common.gendir(builddir)
+
+    # touch any files we need in order to load a package, due to
+    # rstorm needing to load the package using genpkg tool. This
+    # does mean that standalone builds of a storm package from this
+    # must be done after using this buildpkg tool.
+    stormpkg_md_present = False
+    for dnfo in pkgdef.get('docs', ()):
+        fpath = dnfo.get('path')
+        with s_common.genfile(dirn, fpath) as fd:
+            pass
+        if fpath.endswith('stormpackage.md'):
+            stormpkg_md_present = True
+
+    # Generate the build .RST for stormpackage.md
+    if stormpkg_md_present:
+        logger.info(f'Generating stormpackage.rst for {pkgpath}')
+        pkgdocs, pkgname = await s_autodoc.docStormpkg(pkgpath)
+        with s_common.genfile(docsdir, 'stormpackage.rst') as fd:
+            text = pkgdocs.getRstText()
+            if rst_only is False:
+                # Leave this in place if we're only generating RST
+                text = text.replace('.. highlight:: none\n', '')
+            fd.write(text.encode())
+        logger.info('Generated the stormpackage.rst file!')
+
+    for name in os.listdir(docsdir):
+
+        if not name.endswith('.rst'):
+            continue
+
+        docpath = os.path.join(docsdir, name)
+
+        basename = name.rsplit('.', 1)[0]
+
+        builtmd = os.path.join(builddir, f'{basename}.md')
+        builtrst = os.path.join(builddir, name)
+
+        argv = (docpath, '--save', builtrst)
+        logger.info(f'Executing rstorm for {argv}')
+        await s_rstorm.main(argv)
+
+        if rst_only:
+            logger.info(f'rst_only enabled, done processing {name}')
+            continue
+
+        logger.info('Preprocessing rstorm output')
+        with s_common.genfile(builtrst) as fd:
+            buf = fd.read().decode()
+
+        # Remove highglight:: none directives
+        buf = buf.replace('.. highlight:: none\n', '')
+
+        # Remove sphinx metadata fields
+        buf = re_sphinx_metadata_fields.sub('', buf)
+
+        lines = buf.splitlines(keepends=True)
+
+        # Remove lines which start with explicit sphinx rst targets
+        nlines1 = []
+        for line in lines:
+            if line.startswith('.. _') and line.strip().endswith(':'):
+                logger.info(f'Dropping: [{line.strip()}]')
+                continue
+            nlines1.append(line)
+
+        buf = ''.join(nlines1)
+
+        with s_common.genfile(builtrst) as fd:
+            fd.truncate()
+            _ = fd.write(buf.encode())
+
+        logger.info(f'Converting {builtrst} to markdown')
+        if name == 'stormpackage.rst':
+            args = ['pandoc', '--filter', PANDOC_FILTER, '-f', 'rst', '-t', 'markdown', '-o', builtmd, builtrst]
+        else:
+            args = ['pandoc', '-f', 'rst', '-t', 'markdown', '-o', builtmd, builtrst]
+
+        r = subprocess.run(args, capture_output=True)
+
+        # Re-write stderr (logging) to our outp
+        for line in r.stderr.decode().splitlines():
+            outp.printf(f'ERR: {line}')
+
+        if r.returncode != 0:
+            raise s_exc.SynErr(mesg=f'Error converting {builtrst} to {builtmd}')
+
+        logger.info(f'Done converting {builtrst} to {builtmd}')
+
+        # Strip out / manipulate the md content
+        with s_common.genfile(builtmd) as fd:
+            buf = fd.read().decode()
+
+        lines = buf.splitlines(keepends=True)
+
+        # Remove lines which only have a single `:` left in them
+        nlines1 = [line for line in lines if line.strip() != ':']
+
+        buf = ''.join(nlines1)
+
+        with s_common.genfile(builtmd) as fd:
+            fd.truncate()
+            _ = fd.write(buf.encode())
+
+        logger.info('Done manipulating markdown')
+
+    logger.info(f'buildPkgDocs complete for {pkgpath}.')
+
+prog = 'synapse.tools.storm.pkg.doc'
+desc = 'A tool for building storm package docs from RStorm into markdown. This tool requires pandoc to be available.'
+
+async def main(argv, outp=s_output.stdout):
+
+    pars = s_cmd.Parser(prog=prog, outp=outp, description=desc)
+    pars.add_argument('pkgfile', metavar='<pkgfile>', help='Path to a storm package prototype yml file.')
+    pars.add_argument('--rst-only', default=False, action='store_true',
+                      help='Stops building after the .rst files have been generated.')
+
+    opts = pars.parse_args(argv)
+
+    if opts.rst_only is False and not hasPandoc():
+        logger.error('Pandoc is not available, can only run rst/rstorm output.')
+        return 1
+
+    await buildPkgDocs(outp, opts.pkgfile, rst_only=opts.rst_only)
+
+    return 0
+
+if __name__ == '__main__':  # pragma: no cover
+    s_common.setlogging(logger, 'DEBUG')
+    logging.getLogger('vcr').setLevel(logging.WARNING)
+    s_cmd.exitmain(main)

synapse/tools/storm/pkg/gen.py

@@ -0,0 +1,315 @@
+import io
+import os
+import base64
+import logging
+
+import regex
+
+import synapse.exc as s_exc
+import synapse.common as s_common
+import synapse.telepath as s_telepath
+
+import synapse.lib.cmd as s_cmd
+import synapse.lib.json as s_json
+import synapse.lib.output as s_output
+import synapse.lib.certdir as s_certdir
+import synapse.lib.dyndeps as s_dyndeps
+import synapse.lib.schemas as s_schemas
+import synapse.lib.version as s_version
+
+logger = logging.getLogger(__name__)
+
+wflownamere = regex.compile(r'^([\w-]+)\.yaml$')
+
+def getStormStr(fn):
+    if not os.path.isfile(fn):
+        raise s_exc.NoSuchFile(mesg='Storm file {} not found'.format(fn), path=fn)
+
+    with open(fn, 'rb') as f:
+        return f.read().decode()
+
+def loadOpticFiles(pkgdef, path):
+
+    pkgfiles = pkgdef['optic']['files']
+
+    abspath = s_common.genpath(path)
+    for root, dirs, files, in os.walk(path):
+
+        for name in files:
+
+            if name.startswith('.'):  # pragma: no cover
+                continue
+
+            fullname = s_common.genpath(root, name)
+            if not os.path.isfile(fullname):  # pragma: no cover
+                continue
+
+            pkgfname = fullname[len(abspath) + 1:]
+
+            with open(fullname, 'rb') as fd:
+                pkgfiles[pkgfname] = {
+                    'file': base64.b64encode(fd.read()).decode(),
+                }
+
+def loadOpticWorkflows(pkgdef, path):
+
+    wdefs = pkgdef['optic']['workflows']
+
+    for root, dirs, files in os.walk(path):
+
+        for name in files:
+
+            match = wflownamere.match(name)
+
+            if match is None:
+                logger.warning('Skipping workflow "%s" that does not match pattern "%s"' % (name, wflownamere.pattern))
+                continue
+
+            wname = match.groups()[0]
+
+            fullname = s_common.genpath(root, name)
+            if not os.path.isfile(fullname):  # pragma: no cover
+                continue
+
+            wdefs[wname] = s_common.yamlload(fullname)
+
+def tryLoadPkgProto(fp, opticdir=None, readonly=False):
+    '''
+    Try to get a Storm Package prototype from disk with or without inline documentation.
+
+    Args:
+        fp (str): Path to the package .yaml file on disk.
+        opticdir (str): Path to optional Optic module code to add to the Storm Package.
+        readonly (bool): If set, open files in read-only mode. If files are missing, that will raise a NoSuchFile
+                         exception.
+
+    Returns:
+        dict: A Storm package definition.
+    '''
+    try:
+        return loadPkgProto(fp, opticdir=opticdir, readonly=readonly)
+    except s_exc.NoSuchFile:
+        return loadPkgProto(fp, opticdir=opticdir, no_docs=True, readonly=readonly)
+
+def loadPkgProto(path, opticdir=None, no_docs=False, readonly=False):
+    '''
+    Get a Storm Package definition from disk.
+
+    Args:
+        path (str): Path to the package .yaml file on disk.
+        opticdir (str): Path to optional Optic module code to add to the Storm Package.
+        no_docs (bool): If true, omit inline documentation content if it is not present on disk.
+        readonly (bool): If set, open files in read-only mode. If files are missing, that will raise a NoSuchFile
+                         exception.
+
+    Returns:
+        dict: A Storm package definition.
+    '''
+
+    full = s_common.genpath(path)
+    pkgdef = s_common.yamlload(full)
+    if pkgdef is None:
+        raise s_exc.NoSuchFile(mesg=f'File {full} does not exist or is empty.', path=full)
+
+    version = pkgdef.get('version')
+    if isinstance(version, (tuple, list)):
+        pkgdef['version'] = '%d.%d.%d' % tuple(version)
+
+    protodir = os.path.dirname(full)
+    pkgname = pkgdef.get('name')
+
+    genopts = pkgdef.pop('genopts', {})
+
+    # Stamp build info into the pkgdef if it doesn't already exist
+    pkgdef.setdefault('build', {})
+    pkgdef['build'].setdefault('time', s_common.now())
+    pkgdef['build'].setdefault('synapse:version', s_version.verstring)
+    pkgdef['build'].setdefault('synapse:commit', s_version.commit)
+
+    logodef = pkgdef.get('logo')
+    if logodef is not None:
+
+        path = logodef.pop('path', None)
+
+        if path is not None:
+            with s_common.reqfile(protodir, path) as fd:
+                logodef['file'] = base64.b64encode(fd.read()).decode()
+
+        if logodef.get('mime') is None:
+            mesg = 'Mime type must be specified for logo file.'
+            raise s_exc.BadPkgDef(mesg=mesg)
+
+        if logodef.get('file') is None:
+            mesg = 'Logo def must contain path or file.'
+            raise s_exc.BadPkgDef(mesg=mesg)
+
+    for docdef in pkgdef.get('docs', ()):
+
+        if docdef.get('title') is None:
+            mesg = 'Each entry in docs must have a title.'
+            raise s_exc.BadPkgDef(mesg=mesg)
+
+        if no_docs:
+            docdef['content'] = ''
+            continue
+
+        path = docdef.pop('path', None)
+        if path is not None:
+            with s_common.reqfile(protodir, path) as fd:
+                docdef['content'] = fd.read().decode()
+
+        if docdef.get('content') is None:
+            mesg = 'Docs entry has no path or content.'
+            raise s_exc.BadPkgDef(mesg=mesg)
+
+    for mod in pkgdef.get('modules', ()):
+
+        name = mod.get('name')
+
+        basename = name
+        if genopts.get('dotstorm', False):
+            basename = f'{basename}.storm'
+
+        mod_path = s_common.genpath(protodir, 'storm', 'modules', basename)
+        if readonly:
+            mod['storm'] = getStormStr(mod_path)
+        else:
+            with s_common.genfile(mod_path) as fd:
+                mod['storm'] = fd.read().decode()
+
+    for extmod in pkgdef.get('external_modules', ()):
+        fpth = extmod.get('file_path')
+        if fpth is not None:
+            extmod['storm'] = getStormStr(fpth)
+        else:
+            path = extmod.get('package_path')
+            extpkg = s_dyndeps.tryDynMod(extmod.get('package'))
+            extmod['storm'] = extpkg.getAssetStr(path)
+
+        extname = extmod.get('name')
+        extmod['name'] = f'{pkgname}.{extname}'
+
+        pkgdef.setdefault('modules', [])
+        pkgdef['modules'].append(extmod)
+
+    pkgdef.pop('external_modules', None)
+
+    for cmd in pkgdef.get('commands', ()):
+        name = cmd.get('name')
+
+        basename = name
+        if genopts.get('dotstorm'):
+            basename = f'{basename}.storm'
+
+        cmd_path = s_common.genpath(protodir, 'storm', 'commands', basename)
+        if readonly:
+            cmd['storm'] = getStormStr(cmd_path)
+        else:
+            with s_common.genfile(cmd_path) as fd:
+                cmd['storm'] = fd.read().decode()
+
+    for gdef in pkgdef.get('graphs', ()):
+        gdef['iden'] = s_common.guid((pkgname, gdef.get('name')))
+        gdef['scope'] = 'power-up'
+        gdef['power-up'] = pkgname
+
+    inits = pkgdef.get('inits')
+    if inits is not None:
+        lastver = None
+        for initdef in inits.get('versions'):
+            curver = initdef.get('version')
+            if lastver is not None and not curver > lastver:
+                raise s_exc.BadPkgDef(mesg='Init versions must be monotonically increasing.', version=curver)
+            lastver = curver
+
+    wflowdir = s_common.genpath(protodir, 'workflows')
+    if os.path.isdir(wflowdir):
+        pkgdef.setdefault('optic', {})
+        pkgdef['optic'].setdefault('workflows', {})
+        loadOpticWorkflows(pkgdef, wflowdir)
+
+    if opticdir is None:
+        opticdir = s_common.genpath(protodir, 'optic')
+
+    if os.path.isdir(opticdir):
+        pkgdef.setdefault('optic', {})
+        pkgdef['optic'].setdefault('files', {})
+        loadOpticFiles(pkgdef, opticdir)
+
+    s_schemas.reqValidPkgdef(pkgdef)
+
+    # Ensure the package is json safe and tuplify it.
+    s_json.reqjsonsafe(pkgdef, strict=True)
+    pkgdef = s_common.tuplify(pkgdef)
+    return pkgdef
+
+
+desc = 'A tool for generating/pushing storm packages from YAML prototypes.'
+
+async def main(argv, outp=s_output.stdout):
+
+    pars = s_cmd.Parser(prog='synapse.tools.storm.pkg.gen', outp=outp, description=desc)
+    pars.add_argument('--push', metavar='<url>', help='A telepath URL of a Cortex or PkgRepo.')
+    pars.add_argument('--push-verify', default=False, action='store_true',
+                      help='Tell the Cortex to verify the package signature.')
+    pars.add_argument('--save', metavar='<path>', help='Save the completed package JSON to a file.')
+    pars.add_argument('--optic', metavar='<path>', help='Load Optic module files from a directory.')
+    pars.add_argument('--signas', metavar='<name>', help='Specify a code signing identity to use from ~/.syn/certs/code.')
+    pars.add_argument('--certdir', metavar='<dir>', default='~/.syn/certs',
+                      help='Specify an alternate certdir to ~/.syn/certs.')
+    pars.add_argument('--no-build', action='store_true',
+                      help='Treat pkgfile argument as an already-built package')
+    pars.add_argument('--no-docs', default=False, action='store_true',
+                      help='Do not require docs to be present and replace any doc content with empty strings.')
+    pars.add_argument('pkgfile', metavar='<pkgfile>',
+                      help='Path to a storm package prototype .yaml file, or a completed package .json/.yaml file.')
+
+    opts = pars.parse_args(argv)
+
+    if opts.no_build:
+        pkgdef = s_common.yamlload(opts.pkgfile)
+        if not pkgdef:
+            outp.printf(f'Unable to load pkgdef from [{opts.pkgfile}]')
+            return 1
+        if opts.save:
+            outp.printf(f'File {opts.pkgfile} is treated as already built (--no-build); incompatible with --save.')
+            return 1
+    else:
+        pkgdef = loadPkgProto(opts.pkgfile, opticdir=opts.optic, no_docs=opts.no_docs)
+
+    if opts.signas is not None:
+
+        s_certdir.addCertPath(opts.certdir)
+        certdir = s_certdir.getCertDir()
+
+        pkey = certdir.getCodeKey(opts.signas)
+        with io.open(certdir.getCodeCertPath(opts.signas)) as fd:
+            cert = fd.read()
+
+        sign = s_common.ehex(pkey.signitem(pkgdef))
+
+        pkgdef['codesign'] = {
+            'cert': cert,
+            'sign': sign,
+        }
+
+    s_schemas.reqValidPkgdef(pkgdef)
+
+    if not opts.save and not opts.push:
+        outp.printf('Neither --push nor --save provided.  Nothing to do.')
+        return 1
+
+    if opts.save:
+        s_json.jssave(pkgdef, opts.save)
+
+    if opts.push:
+
+        async with s_telepath.withTeleEnv():
+
+            async with await s_telepath.openurl(opts.push) as core:
+                await core.addStormPkg(pkgdef, verify=opts.push_verify)
+
+    return 0
+
+if __name__ == '__main__':  # pragma: no cover
+    s_cmd.exitmain(main)