synapse 2.224.0__py311-none-any.whl → 2.226.0__py311-none-any.whl

synapse/tools/docker/validate.py

@@ -1,5 +1,4 @@
 import re
-import sys
 import base64
 import pprint
 import argparse
@@ -9,6 +8,7 @@ import synapse.exc as s_exc
 import synapse.data as s_data
 import synapse.common as s_common
 
+import synapse.lib.cmd as s_cmd
 import synapse.lib.json as s_json
 import synapse.lib.output as s_outp
 import synapse.lib.certdir as s_certdir
@@ -108,7 +108,7 @@ def checkCosignSignature(outp, pubk_byts, image_to_verify):
         outp.printf(pprint.pformat(blob))
         return True
 
-def main(argv, outp=s_outp.stdout):  # pragma: no cover
+async def main(argv, outp=s_outp.stdout):  # pragma: no cover
     pars = getArgParser()
     opts = pars.parse_args(argv)
 
@@ -151,4 +151,4 @@ def getArgParser():  # pragma: no cover
     return pars
 
 if __name__ == '__main__':  # pragma: no cover
-    sys.exit(main(sys.argv[1:]))
+    s_cmd.exitmain(main)

synapse/tools/easycert.py

@@ -1,136 +1,11 @@
-import argparse
+import synapse.common as s_common
 
-import synapse.exc as s_exc
 import synapse.lib.cmd as s_cmd
-import synapse.lib.output as s_output
-import synapse.lib.certdir as s_certdir
 
-descr = '''
-Command line tool to generate simple x509 certs
-'''
+from synapse.tools.utils.easycert import main
 
-def main(argv, outp=s_output.stdout):
-
-    pars = argparse.ArgumentParser(prog='easycert', description=descr)
-    pars.add_argument('--certdir', default='~/.syn/certs', help='Directory for certs/keys')
-    pars.add_argument('--importfile', choices=('cas', 'hosts', 'users'), help='import certs and/or keys into local certdir')
-
-    pars.add_argument('--ca', default=False, action='store_true', help='mark the certificate as a CA/CRL signer')
-    pars.add_argument('--crl', default=False, action='store_true', help='Generate a new CRL for the given CA name.')
-    pars.add_argument('--p12', default=False, action='store_true', help='mark the certificate as a p12 archive')
-    pars.add_argument('--code', default=False, action='store_true', help='mark the certificate for use in code signing.')
-    pars.add_argument('--server', default=False, action='store_true', help='mark the certificate as a server')
-    pars.add_argument('--server-sans', help='server cert subject alternate names')
-
-    pars.add_argument('--csr', default=False, action='store_true', help='generate a cert signing request')
-    pars.add_argument('--sign-csr', default=False, action='store_true', help='sign a cert signing request')
-    pars.add_argument('--signas', help='sign the new cert with the given cert name')
-    pars.add_argument('--revokeas', help='Revoke a cert as the given CA and add it to the CSR.')
-
-    pars.add_argument('name', help='common name for the certificate (or filename for CSR signing)')
-
-    opts = pars.parse_args(argv)
-
-    cdir = s_certdir.CertDir(path=opts.certdir)
-
-    try:
-
-        if opts.crl:
-
-            crl = cdir.genCaCrl(opts.name)
-            crl._save()
-
-            outp.printf(f'CRL saved: {crl.path}')
-
-            return 0
-
-        if opts.revokeas:
-
-            if opts.code:
-                cert = cdir.getCodeCert(opts.name)
-
-            elif opts.server:
-                cert = cdir.getHostCert(opts.name)
-
-            elif opts.ca:
-                cert = cdir.getCaCert(opts.name)
-
-            else:
-                cert = cdir.getUserCert(opts.name)
-
-            if cert is None:
-                outp.printf(f'Certificate not found: {opts.name}')
-                return 1
-
-            crl = cdir.genCaCrl(opts.revokeas)
-            try:
-                crl.revoke(cert)
-            except s_exc.SynErr as e:
-                outp.printf(f'Failed to revoke certificate: {e.get("mesg")}')
-                return 1
-
-            outp.printf(f'Certificate revoked: {opts.name}')
-            outp.printf(f'CRL updated: {opts.revokeas}')
-
-            return 0
-
-        if opts.importfile:
-            cdir.importFile(opts.name, opts.importfile, outp=outp)
-            return 0
-
-        if opts.p12:
-
-            cdir.genClientCert(opts.name, outp=outp)
-            return 0
-
-        if opts.sign_csr:
-
-            if opts.signas is None:
-                outp.printf('--sign-csr requires --signas')
-                return 1
-
-            xcsr = cdir._loadCsrPath(opts.name)
-            if xcsr is None:
-                outp.printf('csr not found: %s' % (opts.name,))
-                return 1
-
-            if opts.server:
-                cdir.signHostCsr(xcsr, opts.signas, outp=outp)
-                return 0
-
-            cdir.signUserCsr(xcsr, opts.signas, outp=outp)
-            return 0
-
-        if opts.csr:
-
-            if opts.ca:
-                raise NotImplementedError('CSR for CA cert not supported (yet)')
-
-            if opts.server:
-                cdir.genHostCsr(opts.name, outp=outp)
-                return 0
-
-            cdir.genUserCsr(opts.name, outp=outp)
-            return 0
-
-        if opts.ca:
-            cdir.genCaCert(opts.name, signas=opts.signas, outp=outp)
-            return 0
-
-        if opts.server:
-            cdir.genHostCert(opts.name, signas=opts.signas, outp=outp, sans=opts.server_sans)
-            return 0
-
-        if opts.code:
-            cdir.genCodeCert(opts.name, signas=opts.signas, outp=outp)
-            return 0
-
-        cdir.genUserCert(opts.name, signas=opts.signas, outp=outp)
-        return 0
-
-    except s_exc.DupFileName as e:
-        outp.printf('file exists: %s' % (e.errinfo.get('path'),))
-        return 1
+s_common.deprecated('synapse.tools.easycert is deprecated. Please use synapse.tools.utils.easycert instead.',
+                    curv='v2.225.0')
 
 if __name__ == '__main__':  # pragma: no cover
     s_cmd.exitmain(main)

synapse/tools/feed.py

@@ -1,148 +1,11 @@
-import os
-import time
-import logging
-
-import synapse.exc as s_exc
 import synapse.common as s_common
-import synapse.cortex as s_cortex
-import synapse.telepath as s_telepath
 
 import synapse.lib.cmd as s_cmd
-import synapse.lib.cmdr as s_cmdr
-import synapse.lib.json as s_json
-import synapse.lib.output as s_output
-import synapse.lib.msgpack as s_msgpack
-import synapse.lib.version as s_version
-import synapse.lib.encoding as s_encoding
-
-logger = logging.getLogger(__name__)
-
-reqver = '>=0.2.0,<3.0.0'
-
-def getItems(*paths):
-    items = []
-    for path in paths:
-        if path.endswith('.json'):
-            item = s_json.jsload(path)
-            if not isinstance(item, list):
-                item = [item]
-            items.append((path, item))
-        elif path.endswith('.jsonl'):
-            with s_common.genfile(path) as fd:
-                item = list(s_encoding.iterdata(fd, False, format='jsonl'))
-                items.append((path, item))
-        elif path.endswith(('.yaml', '.yml')):
-            item = s_common.yamlload(path)
-            if not isinstance(item, list):
-                item = [item]
-            items.append((path, item))
-        elif path.endswith('.mpk') or path.endswith('.nodes'):
-            genr = s_msgpack.iterfile(path)
-            items.append((path, genr))
-        else:  # pragma: no cover
-            logger.warning('Unsupported file path: [%s]', path)
-    return items
-
-async def addFeedData(core, outp, feedformat, debug=False, *paths, chunksize=1000, offset=0, viewiden=None):
-
-    items = getItems(*paths)
-    for path, item in items:
-
-        bname = os.path.basename(path)
-
-        tick = time.time()
-        outp.printf(f'Adding items from [{path}]')
-
-        foff = 0
-        for chunk in s_common.chunks(item, chunksize):
-
-            clen = len(chunk)
-            if offset and foff + clen < offset:
-                # We have not yet encountered a chunk which
-                # will include the offset size.
-                foff += clen
-                continue
-
-            await core.addFeedData(feedformat, chunk, viewiden=viewiden)
-
-            foff += clen
-            outp.printf(f'Added [{clen}] items from [{bname}] - offset [{foff}]')
-
-        tock = time.time()
-
-        outp.printf(f'Done consuming from [{bname}]')
-        outp.printf(f'Took [{tock - tick}] seconds.')
-
-    if debug:
-        await s_cmdr.runItemCmdr(core, outp, True)
-
-async def main(argv, outp=s_output.stdout):
-
-    pars = getArgParser(outp)
-    opts = pars.parse_args(argv)
-
-    if opts.offset:
-        if len(opts.files) > 1:
-            outp.printf('Cannot start from a arbitrary offset for more than 1 file.')
-            return 1
-
-        outp.printf(f'Starting from offset [{opts.offset}] - it may take a while'
-                    f' to get to that location in the input file.')
-
-    if opts.test:
-        async with s_cortex.getTempCortex(mods=opts.modules) as prox:
-            await addFeedData(prox, outp, opts.format, opts.debug,
-                        chunksize=opts.chunksize,
-                        offset=opts.offset,
-                        *opts.files)
-
-    elif opts.cortex:
-        async with s_telepath.withTeleEnv():
-            async with await s_telepath.openurl(opts.cortex) as core:
-                try:
-                    s_version.reqVersion(core._getSynVers(), reqver)
-                except s_exc.BadVersion as e:
-                    valu = s_version.fmtVersion(*e.get('valu'))
-                    outp.printf(f'Cortex version {valu} is outside of the feed tool supported range ({reqver}).')
-                    outp.printf(f'Please use a version of Synapse which supports {valu}; '
-                          f'current version is {s_version.verstring}.')
-                    return 1
-                await addFeedData(core, outp, opts.format, opts.debug,
-                                  chunksize=opts.chunksize,
-                                  offset=opts.offset, viewiden=opts.view,
-                                  *opts.files)
-
-    else:  # pragma: no cover
-        outp.printf('No valid options provided [%s]', opts)
-        return 1
-
-    return 0
-
-def getArgParser(outp):
-    desc = 'Command line tool for ingesting data into a cortex'
-    pars = s_cmd.Parser(prog='synapse.tools.feed', outp=outp, description=desc)
-
-    muxp = pars.add_mutually_exclusive_group(required=True)
-    muxp.add_argument('--cortex', '-c', type=str,
-                      help='Cortex to connect and add nodes too.')
-    muxp.add_argument('--test', '-t', default=False, action='store_true',
-                      help='Perform a local ingest against a temporary cortex.')
 
-    pars.add_argument('--debug', '-d', default=False, action='store_true',
-                      help='Drop to interactive prompt to inspect cortex after loading data.')
-    pars.add_argument('--format', '-f', type=str, action='store', default='syn.nodes',
-                      help='Feed format to use for the ingested data.')
-    pars.add_argument('--modules', '-m', type=str, action='append', default=[],
-                      help='Additional modules to load locally with a test Cortex.')
-    pars.add_argument('--chunksize', type=int, action='store', default=1000,
-                      help='Default chunksize for iterating over items.')
-    pars.add_argument('--offset', type=int, action='store', default=0,
-                      help='Item offset to start consuming data from.')
-    pars.add_argument('--view', type=str, action='store', default=None,
-                      help='The View to ingest the data into.')
-    pars.add_argument('files', nargs='*', help='json/yaml/msgpack feed files')
+from synapse.tools.cortex.feed import logger, main
 
-    return pars
+s_common.deprecated('synapse.tools.feed is deprecated. Please use synapse.tools.cortex.feed instead.',
+                    curv='v2.225.0')
 
 if __name__ == '__main__':  # pragma: no cover
     s_common.setlogging(logger, 'DEBUG')

synapse/tools/genpkg.py

@@ -1,315 +1,11 @@
-import io
-import os
-import base64
-import logging
-
-import regex
-
-import synapse.exc as s_exc
 import synapse.common as s_common
-import synapse.telepath as s_telepath
 
 import synapse.lib.cmd as s_cmd
-import synapse.lib.json as s_json
-import synapse.lib.output as s_output
-import synapse.lib.certdir as s_certdir
-import synapse.lib.dyndeps as s_dyndeps
-import synapse.lib.schemas as s_schemas
-import synapse.lib.version as s_version
-
-logger = logging.getLogger(__name__)
-
-wflownamere = regex.compile(r'^([\w-]+)\.yaml$')
-
-def getStormStr(fn):
-    if not os.path.isfile(fn):
-        raise s_exc.NoSuchFile(mesg='Storm file {} not found'.format(fn), path=fn)
-
-    with open(fn, 'rb') as f:
-        return f.read().decode()
-
-def loadOpticFiles(pkgdef, path):
-
-    pkgfiles = pkgdef['optic']['files']
-
-    abspath = s_common.genpath(path)
-    for root, dirs, files, in os.walk(path):
-
-        for name in files:
-
-            if name.startswith('.'):  # pragma: no cover
-                continue
-
-            fullname = s_common.genpath(root, name)
-            if not os.path.isfile(fullname):  # pragma: no cover
-                continue
-
-            pkgfname = fullname[len(abspath) + 1:]
-
-            with open(fullname, 'rb') as fd:
-                pkgfiles[pkgfname] = {
-                    'file': base64.b64encode(fd.read()).decode(),
-                }
-
-def loadOpticWorkflows(pkgdef, path):
-
-    wdefs = pkgdef['optic']['workflows']
-
-    for root, dirs, files in os.walk(path):
-
-        for name in files:
-
-            match = wflownamere.match(name)
-
-            if match is None:
-                logger.warning('Skipping workflow "%s" that does not match pattern "%s"' % (name, wflownamere.pattern))
-                continue
-
-            wname = match.groups()[0]
-
-            fullname = s_common.genpath(root, name)
-            if not os.path.isfile(fullname):  # pragma: no cover
-                continue
-
-            wdefs[wname] = s_common.yamlload(fullname)
-
-def tryLoadPkgProto(fp, opticdir=None, readonly=False):
-    '''
-    Try to get a Storm Package prototype from disk with or without inline documentation.
-
-    Args:
-        fp (str): Path to the package .yaml file on disk.
-        opticdir (str): Path to optional Optic module code to add to the Storm Package.
-        readonly (bool): If set, open files in read-only mode. If files are missing, that will raise a NoSuchFile
-                         exception.
-
-    Returns:
-        dict: A Storm package definition.
-    '''
-    try:
-        return loadPkgProto(fp, opticdir=opticdir, readonly=readonly)
-    except s_exc.NoSuchFile:
-        return loadPkgProto(fp, opticdir=opticdir, no_docs=True, readonly=readonly)
-
-def loadPkgProto(path, opticdir=None, no_docs=False, readonly=False):
-    '''
-    Get a Storm Package definition from disk.
-
-    Args:
-        path (str): Path to the package .yaml file on disk.
-        opticdir (str): Path to optional Optic module code to add to the Storm Package.
-        no_docs (bool): If true, omit inline documentation content if it is not present on disk.
-        readonly (bool): If set, open files in read-only mode. If files are missing, that will raise a NoSuchFile
-                         exception.
-
-    Returns:
-        dict: A Storm package definition.
-    '''
-
-    full = s_common.genpath(path)
-    pkgdef = s_common.yamlload(full)
-    if pkgdef is None:
-        raise s_exc.NoSuchFile(mesg=f'File {full} does not exist or is empty.', path=full)
-
-    version = pkgdef.get('version')
-    if isinstance(version, (tuple, list)):
-        pkgdef['version'] = '%d.%d.%d' % tuple(version)
-
-    protodir = os.path.dirname(full)
-    pkgname = pkgdef.get('name')
-
-    genopts = pkgdef.pop('genopts', {})
-
-    # Stamp build info into the pkgdef if it doesn't already exist
-    pkgdef.setdefault('build', {})
-    pkgdef['build'].setdefault('time', s_common.now())
-    pkgdef['build'].setdefault('synapse:version', s_version.verstring)
-    pkgdef['build'].setdefault('synapse:commit', s_version.commit)
-
-    logodef = pkgdef.get('logo')
-    if logodef is not None:
-
-        path = logodef.pop('path', None)
-
-        if path is not None:
-            with s_common.reqfile(protodir, path) as fd:
-                logodef['file'] = base64.b64encode(fd.read()).decode()
-
-        if logodef.get('mime') is None:
-            mesg = 'Mime type must be specified for logo file.'
-            raise s_exc.BadPkgDef(mesg=mesg)
-
-        if logodef.get('file') is None:
-            mesg = 'Logo def must contain path or file.'
-            raise s_exc.BadPkgDef(mesg=mesg)
-
-    for docdef in pkgdef.get('docs', ()):
-
-        if docdef.get('title') is None:
-            mesg = 'Each entry in docs must have a title.'
-            raise s_exc.BadPkgDef(mesg=mesg)
-
-        if no_docs:
-            docdef['content'] = ''
-            continue
-
-        path = docdef.pop('path', None)
-        if path is not None:
-            with s_common.reqfile(protodir, path) as fd:
-                docdef['content'] = fd.read().decode()
-
-        if docdef.get('content') is None:
-            mesg = 'Docs entry has no path or content.'
-            raise s_exc.BadPkgDef(mesg=mesg)
-
-    for mod in pkgdef.get('modules', ()):
-
-        name = mod.get('name')
-
-        basename = name
-        if genopts.get('dotstorm', False):
-            basename = f'{basename}.storm'
-
-        mod_path = s_common.genpath(protodir, 'storm', 'modules', basename)
-        if readonly:
-            mod['storm'] = getStormStr(mod_path)
-        else:
-            with s_common.genfile(mod_path) as fd:
-                mod['storm'] = fd.read().decode()
-
-    for extmod in pkgdef.get('external_modules', ()):
-        fpth = extmod.get('file_path')
-        if fpth is not None:
-            extmod['storm'] = getStormStr(fpth)
-        else:
-            path = extmod.get('package_path')
-            extpkg = s_dyndeps.tryDynMod(extmod.get('package'))
-            extmod['storm'] = extpkg.getAssetStr(path)
-
-        extname = extmod.get('name')
-        extmod['name'] = f'{pkgname}.{extname}'
-
-        pkgdef.setdefault('modules', [])
-        pkgdef['modules'].append(extmod)
-
-    pkgdef.pop('external_modules', None)
-
-    for cmd in pkgdef.get('commands', ()):
-        name = cmd.get('name')
-
-        basename = name
-        if genopts.get('dotstorm'):
-            basename = f'{basename}.storm'
-
-        cmd_path = s_common.genpath(protodir, 'storm', 'commands', basename)
-        if readonly:
-            cmd['storm'] = getStormStr(cmd_path)
-        else:
-            with s_common.genfile(cmd_path) as fd:
-                cmd['storm'] = fd.read().decode()
-
-    for gdef in pkgdef.get('graphs', ()):
-        gdef['iden'] = s_common.guid((pkgname, gdef.get('name')))
-        gdef['scope'] = 'power-up'
-        gdef['power-up'] = pkgname
-
-    inits = pkgdef.get('inits')
-    if inits is not None:
-        lastver = None
-        for initdef in inits.get('versions'):
-            curver = initdef.get('version')
-            if lastver is not None and not curver > lastver:
-                raise s_exc.BadPkgDef(mesg='Init versions must be monotonically increasing.', version=curver)
-            lastver = curver
-
-    wflowdir = s_common.genpath(protodir, 'workflows')
-    if os.path.isdir(wflowdir):
-        pkgdef.setdefault('optic', {})
-        pkgdef['optic'].setdefault('workflows', {})
-        loadOpticWorkflows(pkgdef, wflowdir)
-
-    if opticdir is None:
-        opticdir = s_common.genpath(protodir, 'optic')
-
-    if os.path.isdir(opticdir):
-        pkgdef.setdefault('optic', {})
-        pkgdef['optic'].setdefault('files', {})
-        loadOpticFiles(pkgdef, opticdir)
-
-    s_schemas.reqValidPkgdef(pkgdef)
-
-    # Ensure the package is json safe and tuplify it.
-    s_json.reqjsonsafe(pkgdef, strict=True)
-    pkgdef = s_common.tuplify(pkgdef)
-    return pkgdef
-
-
-desc = 'A tool for generating/pushing storm packages from YAML prototypes.'
-
-async def main(argv, outp=s_output.stdout):
-
-    pars = s_cmd.Parser(prog='synapse.tools.genpkg', outp=outp, description=desc)
-    pars.add_argument('--push', metavar='<url>', help='A telepath URL of a Cortex or PkgRepo.')
-    pars.add_argument('--push-verify', default=False, action='store_true',
-                      help='Tell the Cortex to verify the package signature.')
-    pars.add_argument('--save', metavar='<path>', help='Save the completed package JSON to a file.')
-    pars.add_argument('--optic', metavar='<path>', help='Load Optic module files from a directory.')
-    pars.add_argument('--signas', metavar='<name>', help='Specify a code signing identity to use from ~/.syn/certs/code.')
-    pars.add_argument('--certdir', metavar='<dir>', default='~/.syn/certs',
-                      help='Specify an alternate certdir to ~/.syn/certs.')
-    pars.add_argument('--no-build', action='store_true',
-                      help='Treat pkgfile argument as an already-built package')
-    pars.add_argument('--no-docs', default=False, action='store_true',
-                      help='Do not require docs to be present and replace any doc content with empty strings.')
-    pars.add_argument('pkgfile', metavar='<pkgfile>',
-                      help='Path to a storm package prototype .yaml file, or a completed package .json/.yaml file.')
-
-    opts = pars.parse_args(argv)
-
-    if opts.no_build:
-        pkgdef = s_common.yamlload(opts.pkgfile)
-        if not pkgdef:
-            outp.printf(f'Unable to load pkgdef from [{opts.pkgfile}]')
-            return 1
-        if opts.save:
-            outp.printf(f'File {opts.pkgfile} is treated as already built (--no-build); incompatible with --save.')
-            return 1
-    else:
-        pkgdef = loadPkgProto(opts.pkgfile, opticdir=opts.optic, no_docs=opts.no_docs)
-
-    if opts.signas is not None:
-
-        s_certdir.addCertPath(opts.certdir)
-        certdir = s_certdir.getCertDir()
-
-        pkey = certdir.getCodeKey(opts.signas)
-        with io.open(certdir.getCodeCertPath(opts.signas)) as fd:
-            cert = fd.read()
-
-        sign = s_common.ehex(pkey.signitem(pkgdef))
-
-        pkgdef['codesign'] = {
-            'cert': cert,
-            'sign': sign,
-        }
-
-    s_schemas.reqValidPkgdef(pkgdef)
-
-    if not opts.save and not opts.push:
-        outp.printf('Neither --push nor --save provided.  Nothing to do.')
-        return 1
-
-    if opts.save:
-        s_json.jssave(pkgdef, opts.save)
-
-    if opts.push:
-
-        async with s_telepath.withTeleEnv():
 
-            async with await s_telepath.openurl(opts.push) as core:
-                await core.addStormPkg(pkgdef, verify=opts.push_verify)
+from synapse.tools.storm.pkg.gen import main, tryLoadPkgProto, loadPkgProto
 
-    return 0
+s_common.deprecated('synapse.tools.genpkg is deprecated. Please use synapse.tools.storm.pkg.gen instead.',
+                    curv='v2.225.0')
 
 if __name__ == '__main__':  # pragma: no cover
     s_cmd.exitmain(main)

synapse/tools/guid.py

@@ -1,10 +1,11 @@
-import sys
-
 import synapse.common as s_common
-import synapse.lib.output as s_output
 
-def main(argv, outp=s_output.stdout):
-    outp.printf(s_common.guid())
+import synapse.lib.cmd as s_cmd
+
+from synapse.tools.utils.guid import main
+
+s_common.deprecated('synapse.tools.guid is deprecated. Please use synapse.tools.utils.guid instead.',
+                    curv='v2.225.0')
 
 if __name__ == '__main__':  # pragma: no cover
-    sys.exit(main(sys.argv[1:]))
+    s_cmd.exitmain(main)