synapse 2.224.0__py311-none-any.whl → 2.226.0__py311-none-any.whl

synapse/tools/cortex/csv.py

@@ -0,0 +1,236 @@
+import csv
+
+import synapse.exc as s_exc
+import synapse.cortex as s_cortex
+import synapse.common as s_common
+import synapse.telepath as s_telepath
+
+import synapse.lib.cmd as s_cmd
+import synapse.lib.cmdr as s_cmdr
+import synapse.lib.coro as s_coro
+import synapse.lib.json as s_json
+import synapse.lib.output as s_output
+import synapse.lib.version as s_version
+
+reqver = '>=0.2.0,<3.0.0'
+prog = 'synapse.tools.cortex.csv'
+desc = '''Command line tool for ingesting csv files into a cortex
+
+The storm file is run with the CSV rows specified in the variable "rows" so most
+storm files will use a variable based for loop to create edit nodes.  For example:
+
+for ($fqdn, $ipv4, $tag) in $rows {
+
+    [ inet:dns:a=($fqdn, $ipv4) +#$tag ]
+
+}
+
+More advanced uses may include switch cases to provide different logic based on
+a column value.
+
+for ($type, $valu, $info) in $rows {
+
+    switch $type {
+
+        fqdn: {
+            [ inet:fqdn=$valu ]
+        }
+
+        "person name": {
+            [ ps:name=$valu ]
+        }
+
+        *: {
+            // default case...
+        }
+
+    }
+
+    switch $info {
+        "known malware": { [+#cno.mal] }
+    }
+
+}
+'''
+
+async def runCsvExport(opts, outp, text, stormopts):
+    if not opts.cortex:
+        outp.printf('--export requires --cortex')
+        return 1
+
+    if len(opts.csvfiles) != 1:
+        outp.printf('--export requires exactly 1 csvfile')
+        return 1
+
+    path = s_common.genpath(opts.csvfiles[0])
+    outp.printf(f'Exporting CSV rows to: {path}')
+
+    async with await s_telepath.openurl(opts.cortex) as core:
+
+        try:
+            s_version.reqVersion(core._getSynVers(), reqver)
+        except s_exc.BadVersion as e:
+            valu = s_version.fmtVersion(*e.get('valu'))
+            outp.printf(f'Cortex version {valu} is outside of the {prog} supported range ({reqver}).')
+            outp.printf(f'Please use a version of Synapse which supports {valu}; '
+                        f'current version is {s_version.verstring}.')
+            return 1
+
+        with open(path, 'w') as fd:
+
+            wcsv = csv.writer(fd)
+            # prevent streaming nodes by limiting shown events
+            stormopts['show'] = ('csv:row', 'print', 'warn', 'err')
+            count = 0
+            async for name, info in core.storm(text, opts=stormopts):
+
+                if name == 'csv:row':
+                    count += 1
+                    wcsv.writerow(info['row'])
+                    continue
+
+                if name in ('init', 'fini'):
+                    continue
+
+                outp.printf('%s: %r' % (name, info))
+
+            outp.printf(f'exported {count} csv rows.')
+
+    return 0
+
+async def runCsvImport(opts, outp, text, stormopts):
+
+    def iterrows():
+        for path in opts.csvfiles:
+
+            with open(path, 'r', encoding='utf8') as fd:
+
+                if opts.csv_header:
+                    fd.readline()
+
+                def genr():
+
+                    for row in csv.reader(fd):
+                        yield row
+
+                for rows in s_common.chunks(genr(), 1000):
+                    yield rows
+
+    rowgenr = iterrows()
+
+    logfd = None
+    if opts.logfile is not None:
+        logfd = s_common.genfile(opts.logfile)
+        logfd.seek(0, 2)
+
+    async def addCsvData(core):
+
+        nodecount = 0
+
+        stormopts['editformat'] = 'nodeedits'
+
+        vars = stormopts.setdefault('vars', {})
+
+        for rows in rowgenr:
+
+            vars['rows'] = rows
+
+            async for mesg in core.storm(text, opts=stormopts):
+
+                if mesg[0] == 'node':
+                    nodecount += 1
+
+                elif mesg[0] == 'err' and not opts.debug:
+                    outp.printf(repr(mesg))
+
+                elif mesg[0] == 'print':
+                    outp.printf(mesg[1].get('mesg'))
+
+                if opts.debug:
+                    outp.printf(repr(mesg))
+
+                if logfd is not None:
+                    logfd.write(s_json.dumps(mesg, newline=True))
+
+        if opts.cli:
+            await s_cmdr.runItemCmdr(core, outp, True)
+
+        return nodecount
+
+    if opts.test:
+        async with s_cortex.getTempCortex() as core:
+            nodecount = await addCsvData(core)
+
+    else:
+        async with await s_telepath.openurl(opts.cortex) as core:
+
+            try:
+                s_version.reqVersion(core._getSynVers(), reqver)
+            except s_exc.BadVersion as e:
+                valu = s_version.fmtVersion(*e.get('valu'))
+                outp.printf(f'Cortex version {valu} is outside of the {prog} supported range ({reqver}).')
+                outp.printf(f'Please use a version of Synapse which supports {valu}; '
+                            f'current version is {s_version.verstring}.')
+                return 1
+
+            nodecount = await addCsvData(core)
+
+    if logfd is not None:
+        logfd.close()
+
+    outp.printf('%d nodes.' % (nodecount, ))
+    return 0
+
+async def main(argv, outp=s_output.stdout):
+    pars = makeargparser(outp)
+
+    try:
+        opts = pars.parse_args(argv)
+    except s_exc.ParserExit as e:
+        return e.get('status')
+
+    with open(opts.stormfile, 'r', encoding='utf8') as fd:
+        text = fd.read()
+
+    stormopts = {}
+    if opts.optsfile:
+        stormopts = s_common.yamlload(opts.optsfile)
+
+    if opts.view:
+        if not s_common.isguid(opts.view):
+            outp.printf(f'View is not a guid {opts.view}')
+            return 1
+        stormopts['view'] = opts.view
+
+    async with s_telepath.withTeleEnv():
+
+        if opts.export:
+            return await runCsvExport(opts, outp, text, stormopts)
+        else:
+            return await runCsvImport(opts, outp, text, stormopts)
+
+def makeargparser(outp):
+    pars = s_cmd.Parser(prog=prog, description=desc, outp=outp)
+    pars.add_argument('--logfile', help='Set a log file to get JSON lines from the server events.')
+    pars.add_argument('--csv-header', default=False, action='store_true',
+                      help='Skip the first line from each CSV file.')
+    pars.add_argument('--cli', default=False, action='store_true',
+                      help='Drop into a cli session after loading data.')
+    pars.add_argument('--debug', default=False, action='store_true', help='Enable verbose debug output.')
+    muxp = pars.add_mutually_exclusive_group(required=True)
+    muxp.add_argument('--cortex', '-c', type=str,
+                      help='The telepath URL for the cortex ( or alias from ~/.syn/aliases ).')
+    muxp.add_argument('--test', '-t', default=False, action='store_true',
+                      help='Perform a local CSV ingest against a temporary cortex.')
+    pars.add_argument('--export', default=False, action='store_true',
+                      help='Export CSV data to file from storm using $lib.csv.emit(...) events.')
+    pars.add_argument('--view', default=None, action='store',
+                      help='Optional view to work in.')
+    pars.add_argument('--optsfile', default=None, action='store',
+                      help='Path to an opts file (.yaml) on disk.')
+    pars.add_argument('stormfile', help='A Storm script describing how to create nodes from rows.')
+    pars.add_argument('csvfiles', nargs='+', help='CSV files to load.')
+    return pars
+
+if __name__ == '__main__':  # pragma: no cover
+    s_cmd.exitmain(main)

synapse/tools/cortex/feed.py

@@ -0,0 +1,151 @@
+import os
+import time
+import logging
+
+import synapse.exc as s_exc
+import synapse.common as s_common
+import synapse.cortex as s_cortex
+import synapse.telepath as s_telepath
+
+import synapse.lib.cmd as s_cmd
+import synapse.lib.cmdr as s_cmdr
+import synapse.lib.json as s_json
+import synapse.lib.output as s_output
+import synapse.lib.msgpack as s_msgpack
+import synapse.lib.version as s_version
+import synapse.lib.encoding as s_encoding
+
+logger = logging.getLogger(__name__)
+
+reqver = '>=0.2.0,<3.0.0'
+
+prog = 'synapse.tools.cortex.feed'
+
+def getItems(*paths):
+    items = []
+    for path in paths:
+        if path.endswith('.json'):
+            item = s_json.jsload(path)
+            if not isinstance(item, list):
+                item = [item]
+            items.append((path, item))
+        elif path.endswith('.jsonl'):
+            with s_common.genfile(path) as fd:
+                item = list(s_encoding.iterdata(fd, False, format='jsonl'))
+                items.append((path, item))
+        elif path.endswith(('.yaml', '.yml')):
+            item = s_common.yamlload(path)
+            if not isinstance(item, list):
+                item = [item]
+            items.append((path, item))
+        elif path.endswith('.mpk') or path.endswith('.nodes'):
+            genr = s_msgpack.iterfile(path)
+            items.append((path, genr))
+        else:  # pragma: no cover
+            logger.warning('Unsupported file path: [%s]', path)
+    return items
+
+async def addFeedData(core, outp, feedformat, debug=False, *paths, chunksize=1000, offset=0, viewiden=None):
+
+    items = getItems(*paths)
+    for path, item in items:
+
+        bname = os.path.basename(path)
+
+        tick = time.time()
+        outp.printf(f'Adding items from [{path}]')
+
+        foff = 0
+        for chunk in s_common.chunks(item, chunksize):
+
+            clen = len(chunk)
+            if offset and foff + clen < offset:
+                # We have not yet encountered a chunk which
+                # will include the offset size.
+                foff += clen
+                continue
+
+            await core.addFeedData(feedformat, chunk, viewiden=viewiden)
+
+            foff += clen
+            outp.printf(f'Added [{clen}] items from [{bname}] - offset [{foff}]')
+
+        tock = time.time()
+
+        outp.printf(f'Done consuming from [{bname}]')
+        outp.printf(f'Took [{tock - tick}] seconds.')
+
+    if debug:
+        await s_cmdr.runItemCmdr(core, outp, True)
+
+async def main(argv, outp=s_output.stdout):
+
+    pars = getArgParser(outp)
+    opts = pars.parse_args(argv)
+
+    if opts.offset:
+        if len(opts.files) > 1:
+            outp.printf('Cannot start from a arbitrary offset for more than 1 file.')
+            return 1
+
+        outp.printf(f'Starting from offset [{opts.offset}] - it may take a while'
+                    f' to get to that location in the input file.')
+
+    if opts.test:
+        async with s_cortex.getTempCortex(mods=opts.modules) as prox:
+            await addFeedData(prox, outp, opts.format, opts.debug,
+                        chunksize=opts.chunksize,
+                        offset=opts.offset,
+                        *opts.files)
+
+    elif opts.cortex:
+        async with s_telepath.withTeleEnv():
+            async with await s_telepath.openurl(opts.cortex) as core:
+                try:
+                    s_version.reqVersion(core._getSynVers(), reqver)
+                except s_exc.BadVersion as e:
+                    valu = s_version.fmtVersion(*e.get('valu'))
+                    outp.printf(f'Cortex version {valu} is outside of the {prog} supported range ({reqver}).')
+                    outp.printf(f'Please use a version of Synapse which supports {valu}; '
+                          f'current version is {s_version.verstring}.')
+                    return 1
+                await addFeedData(core, outp, opts.format, opts.debug,
+                                  chunksize=opts.chunksize,
+                                  offset=opts.offset, viewiden=opts.view,
+                                  *opts.files)
+
+    else:  # pragma: no cover
+        outp.printf('No valid options provided [%s]', opts)
+        return 1
+
+    return 0
+
+def getArgParser(outp):
+    desc = 'Command line tool for ingesting data into a cortex'
+    pars = s_cmd.Parser(prog=prog, outp=outp, description=desc)
+
+    muxp = pars.add_mutually_exclusive_group(required=True)
+    muxp.add_argument('--cortex', '-c', type=str,
+                      help='Cortex to connect and add nodes too.')
+    muxp.add_argument('--test', '-t', default=False, action='store_true',
+                      help='Perform a local ingest against a temporary cortex.')
+
+    pars.add_argument('--debug', '-d', default=False, action='store_true',
+                      help='Drop to interactive prompt to inspect cortex after loading data.')
+    pars.add_argument('--format', '-f', type=str, action='store', default='syn.nodes',
+                      help='Feed format to use for the ingested data.')
+    pars.add_argument('--modules', '-m', type=str, action='append', default=[],
+                      help='Additional modules to load locally with a test Cortex.')
+    pars.add_argument('--chunksize', type=int, action='store', default=1000,
+                      help='Default chunksize for iterating over items.')
+    pars.add_argument('--offset', type=int, action='store', default=0,
+                      help='Item offset to start consuming data from.')
+    pars.add_argument('--view', type=str, action='store', default=None,
+                      help='The View to ingest the data into.')
+    pars.add_argument('files', nargs='*', help='json/yaml/msgpack feed files')
+
+    return pars
+
+if __name__ == '__main__':  # pragma: no cover
+    s_common.setlogging(logger, 'DEBUG')
+    s_cmd.exitmain(main)

synapse/tools/csvtool.py

@@ -1,235 +1,11 @@
-import csv
-
-import synapse.exc as s_exc
-import synapse.cortex as s_cortex
 import synapse.common as s_common
-import synapse.telepath as s_telepath
 
 import synapse.lib.cmd as s_cmd
-import synapse.lib.cmdr as s_cmdr
-import synapse.lib.coro as s_coro
-import synapse.lib.json as s_json
-import synapse.lib.output as s_output
-import synapse.lib.version as s_version
-
-reqver = '>=0.2.0,<3.0.0'
-desc = '''Command line tool for ingesting csv files into a cortex
-
-The storm file is run with the CSV rows specified in the variable "rows" so most
-storm files will use a variable based for loop to create edit nodes.  For example:
-
-for ($fqdn, $ipv4, $tag) in $rows {
-
-    [ inet:dns:a=($fqdn, $ipv4) +#$tag ]
-
-}
-
-More advanced uses may include switch cases to provide different logic based on
-a column value.
-
-for ($type, $valu, $info) in $rows {
-
-    switch $type {
-
-        fqdn: {
-            [ inet:fqdn=$valu ]
-        }
-
-        "person name": {
-            [ ps:name=$valu ]
-        }
-
-        *: {
-            // default case...
-        }
-
-    }
-
-    switch $info {
-        "known malware": { [+#cno.mal] }
-    }
-
-}
-'''
-
-async def runCsvExport(opts, outp, text, stormopts):
-    if not opts.cortex:
-        outp.printf('--export requires --cortex')
-        return 1
-
-    if len(opts.csvfiles) != 1:
-        outp.printf('--export requires exactly 1 csvfile')
-        return 1
-
-    path = s_common.genpath(opts.csvfiles[0])
-    outp.printf(f'Exporting CSV rows to: {path}')
-
-    async with await s_telepath.openurl(opts.cortex) as core:
-
-        try:
-            s_version.reqVersion(core._getSynVers(), reqver)
-        except s_exc.BadVersion as e:
-            valu = s_version.fmtVersion(*e.get('valu'))
-            outp.printf(f'Cortex version {valu} is outside of the csvtool supported range ({reqver}).')
-            outp.printf(f'Please use a version of Synapse which supports {valu}; '
-                        f'current version is {s_version.verstring}.')
-            return 1
-
-        with open(path, 'w') as fd:
-
-            wcsv = csv.writer(fd)
-            # prevent streaming nodes by limiting shown events
-            stormopts['show'] = ('csv:row', 'print', 'warn', 'err')
-            count = 0
-            async for name, info in core.storm(text, opts=stormopts):
-
-                if name == 'csv:row':
-                    count += 1
-                    wcsv.writerow(info['row'])
-                    continue
-
-                if name in ('init', 'fini'):
-                    continue
-
-                outp.printf('%s: %r' % (name, info))
-
-            outp.printf(f'exported {count} csv rows.')
-
-    return 0
-
-async def runCsvImport(opts, outp, text, stormopts):
-
-    def iterrows():
-        for path in opts.csvfiles:
-
-            with open(path, 'r', encoding='utf8') as fd:
-
-                if opts.csv_header:
-                    fd.readline()
-
-                def genr():
-
-                    for row in csv.reader(fd):
-                        yield row
-
-                for rows in s_common.chunks(genr(), 1000):
-                    yield rows
-
-    rowgenr = iterrows()
-
-    logfd = None
-    if opts.logfile is not None:
-        logfd = s_common.genfile(opts.logfile)
-        logfd.seek(0, 2)
-
-    async def addCsvData(core):
-
-        nodecount = 0
-
-        stormopts['editformat'] = 'nodeedits'
-
-        vars = stormopts.setdefault('vars', {})
-
-        for rows in rowgenr:
-
-            vars['rows'] = rows
-
-            async for mesg in core.storm(text, opts=stormopts):
-
-                if mesg[0] == 'node':
-                    nodecount += 1
-
-                elif mesg[0] == 'err' and not opts.debug:
-                    outp.printf(repr(mesg))
-
-                elif mesg[0] == 'print':
-                    outp.printf(mesg[1].get('mesg'))
-
-                if opts.debug:
-                    outp.printf(repr(mesg))
-
-                if logfd is not None:
-                    logfd.write(s_json.dumps(mesg, newline=True))
-
-        if opts.cli:
-            await s_cmdr.runItemCmdr(core, outp, True)
-
-        return nodecount
-
-    if opts.test:
-        async with s_cortex.getTempCortex() as core:
-            nodecount = await addCsvData(core)
-
-    else:
-        async with await s_telepath.openurl(opts.cortex) as core:
-
-            try:
-                s_version.reqVersion(core._getSynVers(), reqver)
-            except s_exc.BadVersion as e:
-                valu = s_version.fmtVersion(*e.get('valu'))
-                outp.printf(f'Cortex version {valu} is outside of the csvtool supported range ({reqver}).')
-                outp.printf(f'Please use a version of Synapse which supports {valu}; '
-                            f'current version is {s_version.verstring}.')
-                return 1
-
-            nodecount = await addCsvData(core)
-
-    if logfd is not None:
-        logfd.close()
-
-    outp.printf('%d nodes.' % (nodecount, ))
-    return 0
-
-async def main(argv, outp=s_output.stdout):
-    pars = makeargparser(outp)
-
-    try:
-        opts = pars.parse_args(argv)
-    except s_exc.ParserExit as e:
-        return e.get('status')
-
-    with open(opts.stormfile, 'r', encoding='utf8') as fd:
-        text = fd.read()
-
-    stormopts = {}
-    if opts.optsfile:
-        stormopts = s_common.yamlload(opts.optsfile)
-
-    if opts.view:
-        if not s_common.isguid(opts.view):
-            outp.printf(f'View is not a guid {opts.view}')
-            return 1
-        stormopts['view'] = opts.view
-
-    async with s_telepath.withTeleEnv():
 
-        if opts.export:
-            return await runCsvExport(opts, outp, text, stormopts)
-        else:
-            return await runCsvImport(opts, outp, text, stormopts)
+from synapse.tools.cortex.csv import main
 
-def makeargparser(outp):
-    pars = s_cmd.Parser(prog='synapse.tools.csvtool', description=desc, outp=outp)
-    pars.add_argument('--logfile', help='Set a log file to get JSON lines from the server events.')
-    pars.add_argument('--csv-header', default=False, action='store_true',
-                      help='Skip the first line from each CSV file.')
-    pars.add_argument('--cli', default=False, action='store_true',
-                      help='Drop into a cli session after loading data.')
-    pars.add_argument('--debug', default=False, action='store_true', help='Enable verbose debug output.')
-    muxp = pars.add_mutually_exclusive_group(required=True)
-    muxp.add_argument('--cortex', '-c', type=str,
-                      help='The telepath URL for the cortex ( or alias from ~/.syn/aliases ).')
-    muxp.add_argument('--test', '-t', default=False, action='store_true',
-                      help='Perform a local CSV ingest against a temporary cortex.')
-    pars.add_argument('--export', default=False, action='store_true',
-                      help='Export CSV data to file from storm using $lib.csv.emit(...) events.')
-    pars.add_argument('--view', default=None, action='store',
-                      help='Optional view to work in.')
-    pars.add_argument('--optsfile', default=None, action='store',
-                      help='Path to an opts file (.yaml) on disk.')
-    pars.add_argument('stormfile', help='A Storm script describing how to create nodes from rows.')
-    pars.add_argument('csvfiles', nargs='+', help='CSV files to load.')
-    return pars
+s_common.deprecated('synapse.tools.csvtool is deprecated. Please use synapse.tools.cortex.csv instead.',
+                    curv='v2.225.0')
 
 if __name__ == '__main__':  # pragma: no cover
     s_cmd.exitmain(main)

synapse/tools/demote.py

@@ -1,47 +1,11 @@
-import synapse.exc as s_exc
-
-import synapse.telepath as s_telepath
+import synapse.common as s_common
 
 import synapse.lib.cmd as s_cmd
-import synapse.lib.output as s_output
-import synapse.lib.urlhelp as s_urlhelp
-
-descr = '''
-Automatically select a new leader and demote this service.
-
-Example:
-    python -m synapse.tools.demote
-'''
-
-async def main(argv, outp=s_output.stdout):
-
-    pars = s_cmd.Parser(prog='synapse.tools.demote', outp=outp, description=descr)
-
-    pars.add_argument('--url', default='cell:///vertex/storage',
-                      help='The telepath URL of the Synapse service.')
-
-    pars.add_argument('--timeout', type=int, default=60,
-                      help='The timeout to use awaiting network connections.')
-
-    opts = pars.parse_args(argv)
-
-    async with s_telepath.withTeleEnv():
-
-        try:
-
-            async with await s_telepath.openurl(opts.url) as cell:
-
-                outp.printf(f'Demoting leader: {opts.url}')
-
-                if await cell.demote(timeout=opts.timeout):
-                    return 0
 
-        except s_exc.SynErr as e:
-            outp.printf(f'Error while demoting service {s_urlhelp.sanitizeUrl(opts.url)}: {e}')
-            return 1
+from synapse.tools.service.demote import main
 
-        outp.printf(f'Failed to demote service {s_urlhelp.sanitizeUrl(opts.url)}')
-        return 1
+s_common.deprecated('synapse.tools.demote is deprecated. Please use synapse.tools.service.demote instead.',
+                    curv='v2.225.0')
 
 if __name__ == '__main__':  # pragma: no cover
     s_cmd.exitmain(main)