partis-bcr 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

partis_bcr-1.0.1.data/scripts/parse-output.py

@@ -0,0 +1,277 @@
+#!python
+from __future__ import absolute_import, division, unicode_literals
+from __future__ import print_function
+import sys
+import csv
+from io import open
+csv.field_size_limit(sys.maxsize)  # make sure we can write very large csv fields
+import os
+import argparse
+import colored_traceback.always
+from pathlib import Path
+
+# if you move this script, you'll need to change this method of getting the imports
+partis_dir = str(Path(__file__).parent.parent)
+sys.path.insert(1, partis_dir) # + '/python')
+
+import python.utils as utils
+import python.glutils as glutils
+from python.clusterpath import ClusterPath
+import python.paircluster as paircluster
+
+# ----------------------------------------------------------------------------------------
+def count_plot(tglfo, tlist, plotdir, paired_loci=None):
+    if len(tlist) == 0:
+        return
+    if args.plot_tree_mut_stats:
+        import python.plotting as plotting
+        plotting.plot_tree_mut_stats(plotdir, tlist, args.is_simu, only_leaves=args.only_plot_leaves, treefname=args.treefname)
+        plotting.make_html(plotdir)
+        return
+    if args.only_count_correlations:
+        from python.corrcounter import CorrCounter
+        ccounter = CorrCounter(paired_loci=paired_loci)
+        for line in tlist:
+            l_info = None
+            if paired_loci is not None:
+                line, l_info = line
+            ccounter.increment(line, l_info=l_info)
+        ccounter.plot(plotdir + '/correlations', only_csv=args.only_csv_plots, debug=args.debug)
+        return
+    if args.simfname is not None:
+        simglfo, true_antn_list, _ = utils.read_output(args.simfname)
+        true_antn_dict = {}
+        for true_line in true_antn_list:
+            for iseq, uid in enumerate(true_line['unique_ids']):
+                true_antn_dict[uid] = utils.synthesize_single_seq_line(true_line, iseq)
+        # true_antn_dict = utils.get_annotation_dict(true_antn_list)
+        from python.performanceplotter import PerformancePlotter
+        perfplotter = PerformancePlotter('hmm')
+        n_failed = 0
+        for line in tlist:
+            if line['invalid']:
+                n_failed += 1
+                continue
+            for iseq, uid in enumerate(line['unique_ids']):  # NOTE this counts rearrangement-level parameters once for every mature sequence, which is inconsistent with the pcounters... but I think might make more sense here?
+                _ = perfplotter.evaluate(true_antn_dict[uid], utils.synthesize_single_seq_line(line, iseq), simglfo=simglfo)
+        perfplotter.plot(args.plotdir, only_csv=args.only_csv_plots)
+        if n_failed > 0:
+            print('  %s %d / %d failed queries' % (utils.color('yellow', 'warning'), n_failed, len([u for l in tlist for u in l['unique_ids']])))
+        if args.only_plot_performance:
+            return
+    assert not args.paired  # only handled for correlation counting atm
+    from python.parametercounter import ParameterCounter
+    setattr(args, 'region_end_exclusions', {r : [0 for e in ['5p', '3p']] for r in utils.regions})  # hackity hackity hackity
+    pcounter = ParameterCounter(tglfo, args)  # NOTE doesn't count correlations by default
+    for line in tlist:
+        pcounter.increment(line)
+    pcounter.plot(plotdir, only_csv=args.only_csv_plots, only_overall=args.only_overall_plots) #, make_per_base_plots=True) , make_per_base_plots=True
+
+# ----------------------------------------------------------------------------------------
+helpstr = """
+Extract sequences from a partis output file and write them to a fasta, csv, or tsv file, optionally with a limited amount of extra information for each sequence.
+For details of partis output files, see the manual.
+To view the partitions and annotations in a partis output file, use the partis \'view-output\' action.
+Example usage:
+    bin/parse-output.py test/reference-results/partition-new-simu.yaml out.fa
+    bin/parse-output.py test/paired/ref-results/partition-new-simu outdir --paired
+"""
+class MultiplyInheritedFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter):
+    pass
+formatter_class = MultiplyInheritedFormatter
+parser = argparse.ArgumentParser(formatter_class=MultiplyInheritedFormatter, description=helpstr)
+parser.add_argument('infile', help='partis output file from which to read input')
+parser.add_argument('outfile', help='File to which to write output extracted from <infile> (fasta or csv/tsv). If --paired is set, this must be a directory, to which will be written a fasta with all sequences, a yaml with pairing info, and a csv with h/l sequence pairs.')
+parser.add_argument('--paired', action='store_true', help='if set, <infile> should be a paired output dir, rather than a single file')
+parser.add_argument('--extra-columns', help='colon-separated list of additional partis output columns (beyond sequences), to write to the output file. If writing to a fasta file, the column values are appended after the sequence name, separated by --fasta-info-separator. If writing to csv/tsv, they\'re written as proper, labeled columns.')
+parser.add_argument('--partition-index', type=int, help='if set, use the partition at this index in the cluster path, rather than the default of using the best partition')
+parser.add_argument('--seed-unique-id', help='if set, take sequences only from the cluster containing this seed sequence, rather than the default of taking all sequences from all clusters')
+parser.add_argument('--cluster-index', type=int, help='if set, take sequences only from the cluster at this index in the partition, rather than the default of taking all sequences from all clusters. This index is with respect to the cluster order found in the file (which, in contrast to plots made by --plotdir, is *not* sorted by size)')
+parser.add_argument('--sort-by-size', action='store_true', help='if set, sort clusters in partition by decreasing size before applying --cluster-index')
+parser.add_argument('--indel-reversed-seqs', action='store_true', help='if set, take sequences that have had any shm indels "reversed" (i.e. insertions are reversed, and deletions are replaced with the germline bases) rather than the default of using sequences from the original input file. Indel-reversed sequences can be convenient because they are by definition the same length as and aligned to the naive sequence.')
+parser.add_argument('--glfo-dir', help='Directory with germline info. Only necessary for old-style csv output files. Equivalent to a parameter dir with \'/hmm/germline-sets\' appended.')
+parser.add_argument('--template-glfo-dir', help='use this glfo dir as a template when reading --glfo-dir (only used for airr input atm)')
+parser.add_argument('--locus', default='igh', help='only used for old-style csv output files')
+parser.add_argument('--plotdir', help='if set, plot annotation parameters from infile to --plotdir and exit (you still have to set outfile, sorry, since it\'s nice having it be a positional arg, but it doesn\'t get used for this). To add e.g. per-gene-per-position plots comment/uncomment args in the call below.')
+parser.add_argument('--only-count-correlations', action='store_true', help='')
+parser.add_argument('--only-plot-performance', action='store_true', help='')
+parser.add_argument('--fasta-info-separator', default=' ', help='character to use ')
+parser.add_argument('--debug', type=int, default=0)
+parser.add_argument('--airr-input', action='store_true', help='read input in AIRR tsv format, and if output file suffix is .yaml write partis output.')
+parser.add_argument('--airr-output', action='store_true', help='write output in AIRR tsv format')
+parser.add_argument('--skip-other-locus', action='store_true', help='if --airr-output is set, this tells us to skip lines from the other locus')
+parser.add_argument('--skip-columns', help='don\'t write these columns to output (atm only implemented for airr output, since we need to remove the clone_id column so scoper doesn\'t crash)')
+parser.add_argument('--simfname', help='simulation file corresponding to input file (i.e. presumably <infile> is inference that was performed on --simfname')
+parser.add_argument('--only-csv-plots', action='store_true', help='only write csv versions of plots (not svg), which is a lot faster')
+parser.add_argument('--only-make-plots', action='store_true', help='if --plotdir is set, set this to only do plotting, i.e. don\'t do the usual/default file reading/conversion')
+parser.add_argument('--plot-tree-mut-stats', action='store_true', help='plot tree mutation stats and exit')
+parser.add_argument('--only-plot-leaves', action='store_true', help='only affects --plot-tree-mut-stats')
+parser.add_argument('--is-simu', action='store_true', help='only affects --plot-tree-mut-stats')
+parser.add_argument('--only-overall-plots', action='store_true', help='TODO')
+parser.add_argument('--treefname', help='only affects --plot-tree-mut-stats')
+parser.add_argument('--meta-info-key-to-color', help='see partis help')
+parser.add_argument('--meta-emph-formats', help='see partis help')
+parser.add_argument('--meta-info-to-emphasize', help='see partis help')
+
+if 'extract-fasta.py' in sys.argv[0]:  # if they're trying to run this old script, which is now just a link to this one, print a warning and rejigger the arguments so it still works
+    print('  note: running deprecated script %s, which currently is just a link pointing to %s' % (os.path.basename(sys.argv[0]), os.path.basename(os.path.realpath( __file__))))
+    print('  note: transferring deprecated arguments --input-file and --fasta-output-file to the first two positional arguments (this will continue to work, you only need to change things if you want this warning to go away)')
+    utils.insert_in_arglist(sys.argv, [utils.get_val_from_arglist(sys.argv, '--input-file'), utils.get_val_from_arglist(sys.argv, '--fasta-output-file')], sys.argv[0])
+    utils.remove_from_arglist(sys.argv, '--input-file', has_arg=True)
+    utils.remove_from_arglist(sys.argv, '--fasta-output-file', has_arg=True)
+
+args = parser.parse_args()
+args.extra_columns = utils.get_arg_list(args.extra_columns)
+args.meta_emph_formats = utils.get_arg_list(args.meta_emph_formats, key_val_pairs=True)
+utils.meta_emph_arg_process(args)
+if args.paired:
+    if utils.getsuffix(args.outfile) != '':
+        raise Exception('--outfile \'%s\' must be a directory, but it has a non-empty suffix \'%s\'' % (args.outfile, utils.getsuffix(args.outfile)))
+else:
+    assert utils.getsuffix(args.outfile) in ['.csv', '.tsv', '.fa', '.fasta', '.yaml'] # or args.airr_input and utils.getsuffix(args.outfile) == '.yaml'
+
+default_glfo_dir = partis_dir + '/data/germlines/human'
+if utils.getsuffix(args.infile) in ['.csv', '.tsv'] and args.glfo_dir is None:
+    print('  note: reading csv/tsv format without germline info, so need to get germline info from a separate directory; --glfo-dir was not set, so using default %s. If it doesn\'t crash, it\'s probably ok.' % default_glfo_dir)
+    args.glfo_dir = default_glfo_dir
+
+# ----------------------------------------------------------------------------------------
+# read input
+if args.paired:
+    if not os.path.isdir(args.infile):
+        raise Exception('--infile \'%s\' either doesn\'t exist or it isn\'t a directory' % args.infile)
+    lp_infos = paircluster.read_paired_dir(args.infile)
+else:
+    if args.airr_input:
+        glfo, glfd = None, args.glfo_dir
+        if args.template_glfo_dir is not None:  # NOTE only handled for airr input at the moment, cause that's what i need it for right now
+            glfo = glutils.read_glfo(args.glfo_dir, args.locus, template_glfo=glutils.read_glfo(args.template_glfo_dir, args.locus))
+            # glutils.write_glfo(args.glfo_dir + '-parsed', glfo, debug=True)
+            glfd = None
+        glfo, annotation_list, cpath = utils.read_airr_output(args.infile, locus=args.locus, glfo=glfo, glfo_dir=glfd, skip_other_locus=args.skip_other_locus)
+    else:
+        glfo, annotation_list, cpath = utils.read_output(args.infile, glfo_dir=args.glfo_dir, locus=args.locus)
+
+# plot
+if args.plotdir is not None:
+    if args.paired:
+        for lpair in utils.locus_pairs['ig']:
+            if lp_infos[tuple(lpair)]['glfos'] is None:
+                continue
+            for ltmp in lpair:
+                count_plot(lp_infos[tuple(lpair)]['glfos'][ltmp], lp_infos[tuple(lpair)]['antn_lists'][ltmp], '%s/%s/%s'%(args.plotdir, '+'.join(lpair), ltmp))
+            antn_pairs = paircluster.find_cluster_pairs(lp_infos, lpair) #, debug=True)
+            count_plot(None, antn_pairs, '%s/%s'%(args.plotdir, '+'.join(lpair)), paired_loci=[l['loci'][0] for l in antn_pairs[0]])
+    else:
+        count_plot(glfo, annotation_list, args.plotdir)
+    if args.only_make_plots:
+        sys.exit(0)
+
+if args.paired:
+    glfos, antn_lists, cpaths = paircluster.concat_heavy_chain(utils.locus_pairs['ig'], lp_infos, dont_deep_copy=True)  # NOTE this is a pretty arbitrary way to combine the partitions for the seqs with uncertain pairing info, but whatever
+    outfos, metafos = paircluster.get_combined_outmetafos(antn_lists)
+    paircluster.write_combined_fasta_and_meta('%s/all-seqs.fa'%args.outfile, '%s/meta.yaml'%args.outfile, outfos, metafos)
+    outfos = paircluster.find_seq_pairs(antn_lists)
+    print('    writing sequence id pairs to %s' % '%s/seq-pairs.csv'%args.outfile)
+    with open('%s/seq-pairs.csv'%args.outfile, utils.csv_wmode()) as cfile:
+        okeys = ['%s_%s'%(c, s) for s in ('id', 'locus', 'seq') for c in 'hl']
+        writer = csv.DictWriter(cfile, okeys)  # sorted(outfos[0].keys()))
+        writer.writeheader()
+        for ofo in outfos:
+            writer.writerow({k : ofo[k] for k in okeys})
+    if args.airr_output:
+        for ltmp in sorted(glfos):
+            utils.write_airr_output('%s/%s.tsv'%(args.outfile, ltmp), antn_lists[ltmp], cpath=cpaths[ltmp], glfo=glfos[ltmp])
+    sys.exit(0)
+
+# restrict to certain partitions/clusters
+if cpath is None or cpath.i_best is None:
+    clusters_to_use = [l['unique_ids'] for l in annotation_list]
+    print('  no cluster path in input file, so just using all %d sequences (in %d clusters) in annotations' % (sum(len(c) for c in clusters_to_use), len(clusters_to_use)))
+else:
+    ipartition = cpath.i_best if args.partition_index is None else args.partition_index
+    print('  found %d clusters with %d seqs in %s' % (len(cpath.partitions[ipartition]), sum(len(c) for c in cpath.partitions[ipartition]), 'best partition' if args.partition_index is None else 'partition at index %d (of %d)' % (ipartition, len(cpath.partitions))))
+    modified = False
+    if args.cluster_index is None:
+        clusters_to_use = cpath.partitions[ipartition]
+        print('    taking all %d clusters' % len(clusters_to_use))
+    else:
+        ptn = cpath.partitions[ipartition]
+        if args.sort_by_size:
+            ptn = sorted(cpath.partitions[ipartition], key=len, reverse=True)
+        clusters_to_use = [ptn[args.cluster_index]]
+        modified = True
+        print('    taking cluster at index %d with size %d%s' % (args.cluster_index, len(clusters_to_use[0]), ' after sorting by size' if args.sort_by_size else ''))
+    if args.seed_unique_id is not None:
+        clusters_to_use = [c for c in clusters_to_use if args.seed_unique_id in c]  # NOTE can result in more than one cluster with the seed sequence (e.g. if this file contains intermediate annotations from seed partitioning))
+        modified = True
+        print('    removing clusters not containing sequence \'%s\' (leaving %d)' % (args.seed_unique_id, len(clusters_to_use)))
+    if modified:
+        cpath = ClusterPath(partition=clusters_to_use, seed_unique_id=args.seed_unique_id)
+        antn_dict = utils.get_annotation_dict(annotation_list)
+        annotation_list = [antn_dict[':'.join(c)] for c in clusters_to_use if ':'.join(c) in antn_dict]
+
+if not os.path.exists(os.path.dirname(os.path.abspath(args.outfile))):
+    os.makedirs(os.path.dirname(os.path.abspath(args.outfile)))
+
+if args.airr_output:
+    print('  writing %d annotations%s to %s' % (len(annotation_list), '' if cpath is None else ' (with partition: %d seqs in %d clusters)'%(sum(len(c) for c in cpath.best()), len(cpath.best())), args.outfile))
+    utils.write_airr_output(args.outfile, annotation_list, cpath=cpath, extra_columns=args.extra_columns, skip_columns=args.skip_columns)
+    sys.exit(0)
+
+# condense partis info into <seqfos> for fasta/csv output
+n_skipped, n_failed_to_add = 0, 0
+seqfos = []
+antn_dict = utils.get_annotation_dict(annotation_list)
+for cluster in clusters_to_use:
+    if ':'.join(cluster) not in antn_dict:
+        n_skipped += 1
+        # print '  %s cluster with size %d not in annotations, so skipping it' % (utils.color('yellow', 'warning'), len(cluster))
+        continue
+    cluster_annotation = antn_dict[':'.join(cluster)]
+    newfos = [{'name' : u, 'seq' : s} for u, s in zip(cluster_annotation['unique_ids'], cluster_annotation['seqs' if args.indel_reversed_seqs else 'input_seqs'])]
+    if args.extra_columns is not None:
+        for ecol in args.extra_columns:
+            if ecol not in cluster_annotation:
+                utils.add_extra_column(ecol, cluster_annotation, cluster_annotation, glfo=glfo)
+            if ecol not in cluster_annotation:
+                n_failed_to_add += 1
+                cluster_annotation[ecol] = None
+            for iseq in range(len(newfos)):
+                ival = cluster_annotation[ecol]
+                if ival is not None and ecol in utils.linekeys['per_seq']:
+                    ival = ival[iseq]
+                newfos[iseq][ecol] = ival
+    seqfos += newfos
+if n_skipped > 0:
+    print('  missing annotations for %d sequences' % n_skipped)
+if n_failed_to_add > 0:
+    print('  %s couldn\'t add \'%s\' to %d / %d annotations' % (utils.wrnstr(), ecol, n_failed_to_add, len(clusters_to_use) - n_skipped))
+
+# write output
+with open(args.outfile, utils.csv_wmode()) as ofile:
+    if utils.getsuffix(args.outfile) in ['.csv', '.tsv']:
+        print('  writing %d sequences to %s' % (len(seqfos), args.outfile))
+        writer = csv.DictWriter(ofile, list(seqfos[0].keys()), delimiter=str(',') if utils.getsuffix(args.outfile)=='.csv' else '\t')
+        writer.writeheader()
+        for sfo in seqfos:
+            writer.writerow(sfo)
+    elif utils.getsuffix(args.outfile) in ['.fa', '.fasta']:
+        print('  writing %d sequences to %s' % (len(seqfos), args.outfile))
+        for sfo in seqfos:
+            estr = ''
+            if args.extra_columns is not None:
+                estr = args.fasta_info_separator
+                estr += args.fasta_info_separator.join(str(sfo[c]) for c in args.extra_columns)
+            ofile.write('>%s%s\n%s\n' % (sfo['name'], estr, sfo['seq']))
+    elif utils.getsuffix(args.outfile) == '.yaml':
+        true_partition = None
+        if args.simfname is not None:
+            print('    reading true partition from %s' % args.simfname)
+            _, _, true_cpath = utils.read_output(args.simfname, skip_annotations=True)
+            true_partition = true_cpath.best()
+        plines = cpath.get_partition_lines(true_partition=true_partition, calc_missing_values='none' if true_partition is None else 'best')
+        print('  writing %d annotations with %d partition%s to %s' % (len(annotation_list), len(plines), utils.plural(len(plines)), args.outfile))
+        utils.write_annotations(args.outfile, glfo, annotation_list, utils.add_lists(utils.annotation_headers, args.extra_columns), partition_lines=plines)
+    else:
+        assert False

partis_bcr-1.0.1.data/scripts/split-loci.py

@@ -0,0 +1,317 @@
+#!python
+from __future__ import absolute_import, division, unicode_literals
+from __future__ import print_function
+import json
+import csv
+import os
+import sys
+import argparse
+import operator
+import colored_traceback.always
+import collections
+import copy
+from collections import defaultdict
+import random
+import numpy
+from io import open
+import time
+from pathlib import Path
+
+# if you move this script, you'll need to change this method of getting the imports
+partis_dir = str(Path(__file__).parent.parent)
+sys.path.insert(1, partis_dir) # + '/python')
+
+import python.utils as utils
+import python.paircluster as paircluster
+import python.glutils as glutils
+from python.clusterpath import ClusterPath
+import python.seqfileopener as seqfileopener
+
+# ----------------------------------------------------------------------------------------
+dstr = """
+Uses vsearch (or the \'locus\' key in --input-metfname) to split the sequences in <fname> according to their loci, writing each locus to its own fasta file <locus>.fa.
+If \'paired-uids\' are available in --input-metafname, also splits the heavy sequences according to the light chain locus with which they\'re paired, resulting in subdirectories e.g. igh+igk/ and igh+igl/.
+Use --reverse-negative-strands to check both senses for each input sequence.
+"""
+parser = argparse.ArgumentParser(description=dstr,
+                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)  # why tf isn't this printing the defaults?
+parser.add_argument('fname', help='fasta input file')
+parser.add_argument('--outdir', help='directory to which to write output files (if not set, output is written to directory of <fname>)')
+parser.add_argument('--reverse-negative-strands', action='store_true', help='align every sequence both forwards and revcomp\'d, then for each sequence keep the sense with better alignment.')
+parser.add_argument('--species', default='human', choices=('human', 'macaque', 'mouse'), help='Which species?')
+parser.add_argument('--germline-dir', default=utils.get_partis_dir() + '/data/germlines', help='doesn\'t need to be the germlines corresponding to this sample since it\'s just so it can figure out which is igh vs igk vs igl, so the default is probably fine')
+parser.add_argument('--workdir', default=utils.choose_random_subdir('/tmp/%s/partis' % os.getenv('USER', default='partis-work')), help='working directory for vsearch')
+parser.add_argument('--vsearch-binary', help='Path to vsearch binary (vsearch binaries for linux and darwin are included in partis/bin/, so leaving this unset should work, but for other systems you need to get your own)')
+parser.add_argument('--vsearch-threshold', type=float, default=0.4, help='default identity threshold for vsearch')
+parser.add_argument('--debug', type=int, default=1)
+parser.add_argument('--overwrite', action='store_true')
+parser.add_argument('--random-seed', type=int, default=1)
+parser.add_argument('--guess-pairing-info', action='store_true', help=utils.did_help['guess'])
+parser.add_argument('--droplet-id-separators', help=utils.did_help['seps'])
+parser.add_argument('--droplet-id-indices', help=utils.did_help['indices'])
+parser.add_argument('--fasta-info-index', type=int, help='zero-based index in fasta info/meta string of sequence name/uid (e.g. if name line is \'>stuff more-stuff NAME extra-stuff\' the index should be 2)')
+parser.add_argument('--allowed-contigs-per-droplet', help='if set, discard sequences from droplets that contain any number of contigs not in this colon-separated list')
+parser.add_argument('--allowed-meta-keys-values', help='if set, require that the kept contigs from --allowed-contigs-per-droplet have these key:value pairs (colon-separated list of comma-separated key:value pairs)')
+parser.add_argument('--input-metafname', help='yaml file with meta information keyed by sequence id. See same argument in main partis help, and https://github.com/psathyrella/partis/blob/master/docs/subcommands.md#input-meta-info for an example.')
+parser.add_argument('--for-testing-n-max-queries', type=int, default=-1, help='only for testing, applied when reading initial fasta file, just in case it\'s huge and you want to run quickly without having to read the whole file')
+parser.add_argument('--n-max-queries', type=int, default=-1, help='see partis help (although here it applies to droplets, not individual seqs)')
+parser.add_argument('--n-random-queries', type=int, help='see partis help (although here it applies to droplets, not individual seqs)')
+parser.add_argument('--ig-or-tr', default='ig', choices=list(utils.locus_pairs.keys()), help='antibodies or TCRs?')
+
+# ----------------------------------------------------------------------------------------
+def use_rev_comp(pline, rline):  # decide whether positive sense <pline> or negative sense <rline> has better alignment
+    assert pline['unique_ids'][0] == rline['unique_ids'][0]
+    if rline.get('invalid', False):
+        return False
+    elif pline.get('invalid', False):
+        return True
+    elif rline['score'] > pline['score']:
+        return True
+    else:
+        return False
+
+# ----------------------------------------------------------------------------------------
+def run_vsearch(seqfos):  # run vsearch to see if you can get a match for each locus for every sequence
+    print('  running vsearch on %d sequences:' % len(seqfos))
+    n_rev_compd, n_total = 0, 0
+    for locus in utils.sub_loci(args.ig_or_tr):
+        lglfo = glutils.read_glfo(args.germline_dir, locus)
+        annotations = utils.run_vsearch_with_duplicate_uids('search', seqfos, args.workdir + '/vsearch', args.vsearch_threshold, glfo=lglfo, print_time=True, vsearch_binary=args.vsearch_binary, get_annotations=True, expect_failure=True, extra_str='   %s  fwd:'%utils.color('blue', locus) if args.reverse_negative_strands else '    %s: '%locus) #, debug=args.debug>1)
+        assert len(annotations) == len(seqfos)
+        if args.reverse_negative_strands:  # it might be nicer to user vsearch options to run on both senses at once, but otoh this might be nicer
+            revnotations = utils.run_vsearch_with_duplicate_uids('search', revfos, args.workdir + '/vsearch', args.vsearch_threshold, glfo=lglfo, print_time=True, vsearch_binary=args.vsearch_binary, get_annotations=True, expect_failure=True, extra_str='        rev:') #, debug=args.debug>1)
+            assert len(revnotations) == len(seqfos)
+        for il, (sfo, line) in enumerate(zip(seqfos, annotations)):
+            assert sfo['name'] == line['unique_ids'][0]  # note that they're not full annotations, they just have a couple keys
+            if args.reverse_negative_strands and use_rev_comp(line, revnotations[il]):
+                sfo['seq'] = revfos[il]['seq']
+                line = revnotations[il]
+                n_rev_compd += 1
+            sfo[locus] = line  # add info for each locus to the input seqfos
+            n_total += 1
+    if args.reverse_negative_strands:
+        print('  used rev comp for %d/%d locus results (for %d seqs)' % (n_rev_compd, n_total, len(seqfos)))
+
+# ----------------------------------------------------------------------------------------
+def write_locus_file(locus, ofos, lpair=None, extra_str='  ', totstr=''):
+    ofn = paircluster.paired_fn(args.outdir, locus=locus, lpair=lpair)
+    if utils.output_exists(args, ofn, leave_zero_len=len(ofos)==0, offset=4):  # NOTE not really sure this does anything (or if i want it) now that I'm cleaning/looking for the whole dir at the start of this script
+        return
+    if not os.path.exists(os.path.dirname(ofn)):
+        os.makedirs(os.path.dirname(ofn))
+    if len(ofos) == 0:
+        # print '%s%s: nothing to write' % (extra_str, locus)
+        open(ofn, 'w').close()
+        return
+    print('%s%s: %d%s to %s/%s' % (extra_str, locus, len(ofos), totstr, os.path.basename(os.path.dirname(ofn)), os.path.basename(ofn)))
+    with open(ofn, 'w') as lfile:
+        for sfo in ofos:
+            lfile.write('>%s\n%s\n' % (sfo['name'], sfo['seq']))
+
+# ----------------------------------------------------------------------------------------
+def read_meta_info(seqfos):  # read all input meta info, and add pairing info (if present) to <paired_uids>
+    dummy_annotation_list = [{'unique_ids' : [sfo['name']]} for sfo in seqfos]
+    seqfileopener.read_input_metafo([args.input_metafname], dummy_annotation_list)  # , required_keys=['paired-uids'])
+    for line in dummy_annotation_list:
+        uid = utils.get_single_entry(line['unique_ids'])
+        if 'loci' in line:
+            meta_loci[uid] = line['loci'][0]
+        if 'paired-uids' in line:
+            paired_uids[uid] = line['paired-uids'][0]
+    if len(paired_uids) > 0:
+        print('    read pairing info for %d seqs from input meta file' % len(paired_uids))
+        if len(paired_uids) < len(seqfos):
+            print('      %s only read pairing info for %d/%d seqfos' % (utils.color('yellow', 'warning'), len(paired_uids), len(seqfos)))
+    if len(meta_loci) > 0:
+        print('    read loci for %d sequences from input meta file (so not running vsearch)' % len(meta_loci))
+        if len(meta_loci) < len(seqfos):
+            print('      %s only read locus info for %d/%d seqfos' % (utils.color('yellow', 'warning'), len(meta_loci), len(seqfos)))
+    input_metafos = utils.read_json_yaml(args.input_metafname)
+    for uid in input_metafos:  # we want to copy over any additional meta info (not paired uids or loci) to the output meta info file (since if we're guessing pair info, the uid names will change, so the original one is no good)
+        additional_mfo = {k : v for k, v in input_metafos[uid].items() if k not in ['loci', 'paired-uids']}
+        if len(additional_mfo) > 0:
+            input_metafos[uid] = additional_mfo
+    return input_metafos
+
+# ----------------------------------------------------------------------------------------
+def print_pairing_info(outfos, paired_uids):
+    loci_by_uid = {sfo['name'] : l for l in outfos for sfo in outfos[l]}  # locus of each sequence, just for counting below
+    print_cutoff = 0.01
+    n_missing = 0
+    print('            count  frac  paired with')
+    for locus in utils.sub_loci(args.ig_or_tr):
+        plocicounts = {}
+        for sfo in outfos[locus]:
+            if sfo['name'] not in paired_uids:
+                n_missing += 1
+                continue
+            plstr = ' '.join(utils.locstr(l) for l in sorted([loci_by_uid.get(pid, '?') for pid in paired_uids[sfo['name']]]))
+            if plstr not in plocicounts:
+                plocicounts[plstr] = 0
+            plocicounts[plstr] += 1
+        total = sum(plocicounts.values())
+        n_skipped = 0
+        for ipl, (plstr, counts) in enumerate(sorted(list(plocicounts.items()), key=operator.itemgetter(1), reverse=True)):
+            if counts / float(total) < print_cutoff:
+                n_skipped += counts
+                continue
+            print('       %s  %6d  %5.2f   %s' % (utils.locstr(locus) if ipl==0 else ' ', counts, counts / float(total), plstr))
+        if n_skipped > 0:
+            print('                +%d counts skipped with <%.3f each' % (n_skipped , print_cutoff)) # utils.color('yellow', 'note
+    if n_missing > 0:
+        print('          %s %d uids missing from paired uids' % (utils.wrnstr(), n_missing))
+
+# ----------------------------------------------------------------------------------------
+args = parser.parse_args()
+random.seed(args.random_seed)
+numpy.random.seed(args.random_seed)
+if os.path.dirname(args.fname) == '':
+    args.fname = '%s/%s' % (os.getcwd(), args.fname)
+if args.outdir is None:
+    args.outdir = utils.getprefix(args.fname)
+args.droplet_id_indices = utils.get_arg_list(args.droplet_id_indices, intify=True)
+args.allowed_contigs_per_droplet = utils.get_arg_list(args.allowed_contigs_per_droplet, intify=True)
+args.allowed_meta_keys_values = utils.get_arg_list(args.allowed_meta_keys_values, key_val_pairs=True)
+args.input_metafname = utils.fpath(args.input_metafname)
+
+if any(os.path.exists(ofn) for ofn in paircluster.paired_dir_fnames(args.outdir)):
+    if args.overwrite:
+        paircluster.clean_paired_dir(args.outdir)
+    else:
+        print('  split-loci.py output exists and --overwrite was not set, so not doing anything: %s' % args.outdir)
+        sys.exit(0)
+
+seqfos = utils.read_fastx(args.fname, n_max_queries=args.for_testing_n_max_queries)
+if args.n_max_queries != -1 or args.n_random_queries is not None:
+    seqfos = utils.subset_paired_queries(seqfos, args.droplet_id_separators, args.droplet_id_indices, n_max_queries=args.n_max_queries, n_random_queries=args.n_random_queries)
+if args.fasta_info_index is not None:
+    for sfo in seqfos:
+        sfo['name'] = sfo['infostrs'][args.fasta_info_index]
+if args.reverse_negative_strands:
+    revfos = [{'name' : s['name'], 'seq' : utils.revcomp(s['seq'])} for s in seqfos]  # NOTE this is not on an equal footing with <seqfos>, since we add all the vsearch info to <seqfos>, then use it do decide on locus, and then to write output
+
+if os.path.exists(args.germline_dir + '/' + args.species):  # ick that is hackey
+    args.germline_dir += '/' + args.species
+
+# read input meta file and/or run vsearch
+paired_uids, meta_loci, input_metafos = {}, {}, {}
+if args.input_metafname is not None:
+    input_metafos = read_meta_info(seqfos)
+if len(meta_loci) == 0:  # default: no input locus info
+    run_vsearch(seqfos)
+
+# then, for each sequence, choose the locus with the best-scoring match (in practice i doubt you ever really get multiple loci with matches)
+outfos = collections.OrderedDict(((l, []) for l in utils.sub_loci(args.ig_or_tr)))
+failed_seqs = []
+if args.debug > 1:
+    print('    printing scores for locus determination:')
+    n_skipped = 0
+for sfo in seqfos:
+    if len(meta_loci) == 0:  # default: use vsearch match scores
+        lscores = {l : sfo[l]['score'] if 'invalid' not in sfo[l] else 0 for l in utils.sub_loci(args.ig_or_tr)}
+        locus, max_score = sorted(list(lscores.items()), key=operator.itemgetter(1), reverse=True)[0]
+        if max_score == 0:
+            failed_seqs.append(sfo)
+            continue
+    else:  # if we were passed input locus info
+        locus = meta_loci[sfo['name']]
+    outfos[locus].append(sfo)
+    if args.debug > 1:
+        def lpstr(spair):
+            l, s = spair
+            return '%s %s' % (utils.locstr(l) if l==locus else l.replace('ig', ''), utils.color('red' if s!=0 else None, '%3d'%s))
+        if list(lscores.values()).count(0) == 2:
+            n_skipped += 1
+        else:
+            print('       %s   %s' % ('  '.join(lpstr(s) for s in sorted(list(lscores.items()), key=operator.itemgetter(1), reverse=True)), sfo['name']))
+if args.debug > 1 and n_skipped > 0:
+    print('      skipped %d seqs with non-zero scores from only one locus' % n_skipped)
+
+print('totals: %s%s' % (' '.join(('%s %d'%(l, len(sfos))) for l, sfos in outfos.items()), '' if len(failed_seqs) == 0 else ' (%s: %d)'%(utils.color('yellow', 'failed'), len(failed_seqs))))
+assert sum(len(ofo) for ofo in outfos.values()) + len(failed_seqs) == len(seqfos)
+
+if args.guess_pairing_info:
+    if len(paired_uids) > 0:
+        raise Exception('can\'t/shouldn\'t guess pairing info if we already have it from elsewhere')
+    for locus in outfos:
+        for ofo in outfos[locus]:
+            new_name = ofo['name']
+            if '-' not in new_name or ofo['name'].split('-')[-1] != locus:  # add locus (e.g. '-igh') to name, unless it's already there
+                new_name = ofo['name'] + '-' + locus
+            if ofo['name'] in input_metafos:
+                input_metafos[new_name] = input_metafos[ofo['name']]
+                del input_metafos[ofo['name']]
+            ofo['name'] = new_name
+    guessed_metafos = utils.extract_pairing_info(seqfos, droplet_id_separators=args.droplet_id_separators, droplet_id_indices=args.droplet_id_indices, debug=max(1, args.debug))
+    for uid in set(guessed_metafos) & set(input_metafos):
+        guessed_metafos[uid].update(input_metafos[uid])
+    for uid, mfo in guessed_metafos.items():
+        paired_uids[uid] = mfo['paired-uids']
+
+removed_uids = set()
+if args.allowed_contigs_per_droplet is not None:
+    new_outfos = collections.OrderedDict(((l, []) for l in utils.sub_loci(args.ig_or_tr)))
+    for locus in outfos:
+        n_ctg_removed, n_meta_removed, n_meta_added = defaultdict(int), 0, 0
+        for ofo in outfos[locus]:
+            skip = False
+            n_contigs = len(paired_uids[ofo['name']]) + 1  # total n contigs in the droplet
+            if args.allowed_meta_keys_values is not None:
+                mv_uids = [ofo['name']] + copy.copy(paired_uids[ofo['name']])  # uids in this droplet that have the required meta info values
+                for mkey, mval in args.allowed_meta_keys_values.items():
+                    mv_uids = [u for u in mv_uids if input_metafos[u][mkey] == mval]  # reduce mv_uids to the uids that have the required meta value
+                if len(mv_uids) != n_contigs:
+                    # print('    reducing n_contigs with %s=%s: %d %d (%s  -->  %s)' % (mkey, mval, n_contigs, len(mv_uids), [guessed_metafos[u][mkey] for u in [ofo['name']] + paired_uids[ofo['name']]], [guessed_metafos[u][mkey] for u in mv_uids]))
+                    if n_contigs in args.allowed_contigs_per_droplet and len(mv_uids) not in args.allowed_contigs_per_droplet:
+                        n_meta_removed += 1  # keep track of how many were removed only because of the meta info requirements
+                    if n_contigs not in args.allowed_contigs_per_droplet and len(mv_uids) in args.allowed_contigs_per_droplet:
+                        n_meta_added += 1  # and how many were added only because of the meta info requirements
+                    n_contigs = len(mv_uids)  # n contigs that have the required meta values
+            if n_contigs in args.allowed_contigs_per_droplet:
+                new_outfos[locus].append(ofo)
+            else:
+                n_ctg_removed[n_contigs] += 1
+                removed_uids.add(ofo['name'])
+        if sum(n_ctg_removed.values()) > 0:
+            print('    %s --allowed-contigs-per-droplet: removed %d / %d contigs that were in droplets that didn\'t have an allowed number of contigs (%s): %s' % (utils.locstr(locus), sum(n_ctg_removed.values()), len(outfos[locus]), ' '.join(str(n) for n in args.allowed_contigs_per_droplet), '  '.join('%s: %d'%(k, v) for k, v in n_ctg_removed.items())))
+            if args.allowed_meta_keys_values is not None and n_meta_removed > 0:
+                print('          --allowed-meta-keys-values: %d were removed (and %d were kept) because of the meta info requirements: %s' % (n_meta_removed, n_meta_added, args.allowed_meta_keys_values))
+    outfos = new_outfos
+
+removed_uids |= set(s['name'] for s in failed_seqs)
+if len(removed_uids) > 0:
+    start = time.time()
+    n_removed = 0
+    for fid in removed_uids:
+        if fid in paired_uids:
+            del paired_uids[fid]
+            n_removed += 1
+    paired_uids = {uid : sorted(set(paired_uids[uid]) - removed_uids) for uid in paired_uids}
+    print('  removed %d uids from paired_uids (%d failed, %d removed b/c of disallowed N contigs) in %.1fs' % (n_removed, len(failed_seqs), len(removed_uids) - len(failed_seqs), time.time() - start))
+
+if args.debug and len(paired_uids) > 0:
+    print_pairing_info(outfos, paired_uids)
+
+print('writing to %s/' % args.outdir)
+if len(failed_seqs) > 0:
+    write_locus_file('failed', failed_seqs)
+
+for locus in outfos:  # first write the single files with all seqs for each locus
+    write_locus_file(locus, outfos[locus])
+
+omfname = '%s/meta.yaml' % args.outdir
+if args.guess_pairing_info:
+    utils.jsdump(omfname, guessed_metafos)  # NOTE file name duplicates code in bin/partis
+elif args.input_metafname is not None and not os.path.exists(omfname):
+    utils.makelink(os.path.dirname(omfname), args.input_metafname, omfname)
+
+if len(paired_uids) == 0:
+    print('no pairing info')
+else:
+    print('writing to paired subdirs')
+    for lpair in utils.locus_pairs[args.ig_or_tr]:
+        print('    %s:' % '+'.join(lpair))
+        for l_other, ltmp in [lpair, reversed(lpair)]:
+            all_paired_uids = set(pid for s in outfos[ltmp] if s['name'] in paired_uids for pid in paired_uids[s['name']])  # all uids that are paired with any <ltmp> uid (not necessarily *correctly* paired, at this stage it likely just means they're in the same droplet)
+            other_outfo = [sfo for sfo in outfos[l_other] if sfo['name'] in all_paired_uids]  # any <l_other> locus seq that was in a droplet with an <ltmp> uid
+            write_locus_file(l_other, other_outfo, lpair=lpair, extra_str='      ', totstr=' / %s'%len(outfos[l_other]))