partis-bcr 1.0.5__py3-none-any.whl → 1.0.6__py3-none-any.whl

test/new-results/test.log

@@ -1,265 +1,31 @@
-cache-parameters-simu            partis cache-parameters --dont-write-git-info --infname test/ref-results/test/simu.yaml --parameter-dir test/new-results/test/parameters/simu --sw-cachefname test/new-results/test/parameters/simu/sw-cache.yaml --is-simu --random-seed 1 --n-procs 10
+cache-parameters-quick-new-simu   partis cache-parameters --dont-write-git-info --infname test/ref-results/test/simu.yaml --parameter-dir test/new-results/test/parameters/simu --sw-cachefname test/new-results/test/parameters/simu/sw-cache.yaml --n-max-queries 10 --random-seed 1 --n-procs 10
+  --n-max-queries: stopped after reading 10 queries from input file
 caching parameters
-  vsearch: 36 / 36 v annotations (0 failed) with 7 v genes in 0.0 sec
-    keeping 7 / 198 v genes
-    missing 8 simulation genes (counts): hv1-202 0  hv2-502 0  hv2-70D04 0  hv3-1104 0  hv3-69-102 0  hv3-902 0  hv4-3401 0  hv4-3909 0
+  vsearch: 10 / 10 v annotations (0 failed) with 2 v genes in 0.0 sec
+    keeping 2 / 198 v genes
 smith-waterman  (new-allele fitting)
-  vsearch: 36 / 36 v annotations (0 failed) with 7 v genes in 0.0 sec
+  vsearch: 10 / 10 v annotations (0 failed) with 2 v genes in 0.0 sec
   removing old sw cache test/new-results/test/parameters/simu/sw-cache.yaml
-    running 10 procs for 36 seqs
-    running 11 procs for 1 seq
-      info for 36 / 36 = 1.000   (removed: 0 failed)
-      kept 18 (0.500) unproductive
-    water time: 0.1  (ig-sw 0.0  processing 0.0)
+    running 10 procs for 10 seqs
+      info for 10 / 10 = 1.000   (removed: 0 failed)
+      kept 10 (1.000) unproductive
+    water time: 0.0  (ig-sw 0.0  processing 0.0)
 smith-waterman  (writing parameters)
-  vsearch: 36 / 36 v annotations (0 failed) with 7 v genes in 0.0 sec
-    running 10 procs for 36 seqs
-    running 11 procs for 1 seq
-      info for 36 / 36 = 1.000   (removed: 0 failed)
-      kept 18 (0.500) unproductive
+  vsearch: 10 / 10 v annotations (0 failed) with 2 v genes in 0.0 sec
+    running 10 procs for 10 seqs
+      info for 10 / 10 = 1.000   (removed: 0 failed)
+      kept 10 (1.000) unproductive
         writing sw results to test/new-results/test/parameters/simu/sw-cache.yaml
-    writing parameters to test/new-results/test/parameters/simu/sw (0.9 sec)
-    water time: 1.0  (ig-sw 0.9  processing 0.0)
-  writing hmms     removed 30 genes from glfo (leaving v 7  d 5  j 3)
-(0.2 sec)
+    writing parameters to test/new-results/test/parameters/simu/sw (0.4 sec)
+    water time: 0.5  (ig-sw 0.5  processing 0.0)
+  writing hmms     removed 35 genes from glfo (leaving v 2  d 1  j 2)
+(0.1 sec)
 hmm
     reading output
-        read 36 hmm output lines with 36 sequences in 36 events  (0 failures)
-    writing parameters to test/new-results/test/parameters/simu/hmm (0.7 sec)
-    writing parameters to test/new-results/test/parameters/simu/true (0.6 sec)
-         infra time: 1.4
-      hmm step time: 1.8
-  writing hmms     removed 0  genes from glfo (leaving v 7  d 5  j 3)
-(0.2 sec)
-      total time: 3.4
-annotate-new-simu                partis annotate --dont-write-git-info --infname test/ref-results/test/simu.yaml --parameter-dir test/new-results/test/parameters/simu --sw-cachefname test/new-results/test/parameters/simu/sw-cache.yaml --plot-annotation-performance --is-simu --plotdir test/new-results/annotate-new-simu-annotation-performance --only-csv-plots --random-seed 1 --n-procs 10 --outfname test/new-results/annotate-new-simu.yaml
-annotating    (with test/new-results/test/parameters/simu/hmm)
-smith-waterman
-  vsearch: 36 / 36 v annotations (0 failed) with 7 v genes in 0.0 sec
-        reading sw results from test/new-results/test/parameters/simu/sw-cache.yaml
-      info for 36 / 36 = 1.000   (removed: 0 failed, 0 duplicates [removed when cache file was written])
-      kept 18 (0.500) unproductive
-  plotting performance 
-    warning skipped annotation performance evaluation on 1 queries with different true and inferred net shm indel lengths: d8761c689a
-(0.0 sec)
-        water time: 0.3
-hmm
-    reading output
-        read 36 hmm output lines with 36 sequences in 36 events  (0 failures)
-  plotting performance 
-    warning skipped annotation performance evaluation on 1 queries with different true and inferred net shm indel lengths: d8761c689a
-(0.0 sec)
-      hmm step time: 0.4
-      total time: 0.7
-multi-annotate-new-simu          partis annotate --dont-write-git-info --infname test/ref-results/test/simu.yaml --parameter-dir test/new-results/test/parameters/simu --sw-cachefname test/new-results/test/parameters/simu/sw-cache.yaml --plot-annotation-performance --simultaneous-true-clonal-seqs --is-simu --plotdir test/new-results/multi-annotate-new-simu-annotation-performance --only-csv-plots --random-seed 1 --n-procs 10 --outfname test/new-results/multi-annotate-new-simu.yaml
-annotating    (with test/new-results/test/parameters/simu/hmm)
-smith-waterman
-  vsearch: 36 / 36 v annotations (0 failed) with 7 v genes in 0.0 sec
-  note: not running msa indel stuff for --simultaneous-true-clonal-seqs, so any families with shm indels within cdr3 will be split up before running the hmm. To fix this you'll either need to run set_msa_info() (which is fine and easy, but slow, and requires deciding whether to make sure to run parameter caching with the arg, or else rerun smith waterman with the msa indels
-        reading sw results from test/new-results/test/parameters/simu/sw-cache.yaml
-      info for 36 / 36 = 1.000   (removed: 0 failed, 0 duplicates [removed when cache file was written])
-      kept 18 (0.500) unproductive
-  plotting performance 
-    warning skipped annotation performance evaluation on 1 queries with different true and inferred net shm indel lengths: d8761c689a
-(0.0 sec)
-        water time: 0.3
-hmm
-  --simultaneous-true-clonal-seqs: grouping seqs according to true partition
-  warning split apart 1 cluster that contained multiple cdr3 lengths (total clusters: 10 --> 11)
-      cluster splits:   8 --> 7 1
-  subcluster annotating 11 clusters with steps:  (0.2s) (0.1s)   plotting performance 
-    warning skipped annotation performance evaluation on 1 queries with different true and inferred net shm indel lengths: d8761c689a
-(0.0 sec)
-
-    subcluster annotation time 0.3
-      total time: 0.7
-partition-new-simu               partis partition --dont-write-git-info --infname test/ref-results/test/simu.yaml --parameter-dir test/new-results/test/parameters/simu --sw-cachefname test/new-results/test/parameters/simu/sw-cache.yaml --plot-annotation-performance --max-ccf-fail-frac 0.10 --is-simu --persistent-cachefname test/new-results/cache-new-partition.csv --plotdir test/new-results/partition-new-simu-annotation-performance --only-csv-plots --no-partition-plots --random-seed 1 --n-procs 10 --outfname test/new-results/partition-new-simu.yaml
-partitioning     (with test/new-results/test/parameters/simu/hmm)
-smith-waterman
-  vsearch: 36 / 36 v annotations (0 failed) with 7 v genes in 0.0 sec
-        reading sw results from test/new-results/test/parameters/simu/sw-cache.yaml
-      info for 36 / 36 = 1.000   (removed: 0 failed, 0 duplicates [removed when cache file was written])
-      kept 18 (0.500) unproductive
-  plotting performance 
-    warning skipped annotation performance evaluation on 1 queries with different true and inferred net shm indel lengths: d8761c689a
-(0.0 sec)
-        water time: 0.3
-hmm
-  --persistent-cachefname: using existing hmm cache file test/new-results/cache-new-partition.csv
-  caching all 36 naive sequences (0.4s)
-    collapsed 36 queries into 12 clusters with identical naive seqs (0.0 sec)
-  12 clusters with 10 procs (0.1s)
-   12 clusters with 7 procs (0.0s)
-   12 clusters with 5 procs (0.0s)
-   12 clusters with 3 procs (0.0s)
-   12 clusters with 2 procs (0.1s)
-   11 clusters with 1 proc (0.4s)
-       partition loop time: 0.7
-getting annotations for final partition
-  subcluster annotating 10 clusters with steps:  (0.2s) (0.1s) (0.1s)   plotting performance 
-    warning skipped annotation performance evaluation on 1 queries with different true and inferred net shm indel lengths: d8761c689a
-(0.0 sec)
-
-    subcluster annotation time 0.4
-      total time: 1.7
-seed-partition-new-simu          partis partition --dont-write-git-info --infname test/ref-results/test/simu.yaml --parameter-dir test/new-results/test/parameters/simu --sw-cachefname test/new-results/test/parameters/simu/sw-cache.yaml --max-ccf-fail-frac 0.10 --is-simu --persistent-cachefname test/new-results/cache-new-partition.csv --seed-unique-id 8024ed1a22 --random-seed 1 --n-procs 10 --outfname test/new-results/seed-partition-new-simu.yaml
-partitioning     (with test/new-results/test/parameters/simu/hmm)
-smith-waterman
-  vsearch: 36 / 36 v annotations (0 failed) with 7 v genes in 0.0 sec
-        reading sw results from test/new-results/test/parameters/simu/sw-cache.yaml
-      info for 36 / 36 = 1.000   (removed: 0 failed, 0 duplicates [removed when cache file was written])
-      kept 18 (0.500) unproductive
-      removed 25 / 36 = 0.69 sequences with cdr3 length different from seed sequence (leaving 11)
-        water time: 0.0
-hmm
-  --persistent-cachefname: using existing hmm cache file test/new-results/cache-new-partition.csv
-   collapsed 11 queries into 2 clusters with identical naive seqs (0.0 sec)
-  reducing n procs to number of clusters: 10 --> 2
-  2 clusters with 2 procs (0.0s)
-       removed 0 sequences in unseeded clusters, split 2 seeded clusters into 11 singletons, and merged these into 2 clusters with identical naive seqs (0.0 sec)
-        new n_procs 1 (initial seqs/proc: 5.50   new seqs/proc: 2.00
-  2 clusters with 1 proc (0.1s)
-       partition loop time: 0.1
-getting annotations for final partition
-  subcluster annotating 1 cluster with steps:  (0.1s) (0.1s) (0.1s) 
-    subcluster annotation time 0.2
-  seed cluster size in best partition: 11, excluding seed seq: 10
-      total time: 0.4
-vsearch-partition-new-simu       partis partition --dont-write-git-info --infname test/ref-results/test/simu.yaml --parameter-dir test/new-results/test/parameters/simu --sw-cachefname test/new-results/test/parameters/simu/sw-cache.yaml --naive-vsearch --is-simu --persistent-cachefname test/new-results/cache-new-partition.csv --random-seed 1 --n-procs 10 --outfname test/new-results/vsearch-partition-new-simu.yaml
-partitioning     (with test/new-results/test/parameters/simu/hmm)
-smith-waterman
-  vsearch: 36 / 36 v annotations (0 failed) with 7 v genes in 0.0 sec
-        reading sw results from test/new-results/test/parameters/simu/sw-cache.yaml
-      info for 36 / 36 = 1.000   (removed: 0 failed, 0 duplicates [removed when cache file was written])
-      kept 18 (0.500) unproductive
-        water time: 0.0
-hmm
-  --persistent-cachefname: using existing hmm cache file test/new-results/cache-new-partition.csv
-        collapsed 36 sequences (36 initial naive seqs/clusters) into 12 unique naive sequences over 9 cdr3 lengths (0.0 sec)
-    using hfrac bound for vsearch 0.013
-    running vsearch 9 times (once for each cdr3 length class): . . . . . . . . . 
-      vsearch time: 0.1
-getting annotations for final partition
-      calculating hmm annotations even though --fast/--naive-vsearch was set (you can set --use-sw-annotations to get faster (but substantially less accurate) annotations, or set --dont-calculate-annotations if you only need the partition)
-  subcluster annotating 10 clusters with steps:  (0.2s) (0.1s) (0.1s) 
-    subcluster annotation time 0.4
-      total time: 0.5
-get-selection-metrics-new-simu   partis get-selection-metrics --dont-write-git-info --existing-output-run-cfg paired --min-selection-metric-cluster-size 3 --min-paired-cluster-size-to-read 3 --random-seed 1 --n-procs 10 --outfname test/new-results/partition-new-simu.yaml --selection-metric-fname test/new-results/get-selection-metrics-new-simu.yaml
-  note: --parameter-dir not set, so using default: x-dummy-x
-  note: By default we print/operate on *all* annotations in the output file, which in general can include annotations from non-best partititons and non-seed clusters (e.g. if --n-final-clusters was set).
-        If you want to restrict to particular annotations, use one of --only-print-best-partition, --only-print-seed-clusters, or --only-print-queries-to-include-clusters (or, if set during partitioning, --n-final-clusters or --min-largest-cluster-size).
-    getting selection metrics (lbi lbr aa-lbi aa-lbr cons-dist-aa) with inferred lines: 4 clusters with sizes 11 7 7 3
-      skipped 6 smaller than 3 (controlled by --min-selection-metric-cluster-size)
-    getting trees for 3 clusters with sizes: 11 7 7
-      all fasttree outputs exist, not rerunning (e.g. /home/dralph/work/partis/test/new-results/partition-new-simu/fasttree/iclust-0/fasttree.out)
-    tree origins: 3 ran fasttree
-    skipped 1/4 clusters smaller than 4
-      setting default tau to 1 / 352 = 0.0028
-    calculating normalized lb index and lb ratio and lb fraction with tau 0.0028
-      warning get_aa_tree() (AA inf tree, iclust 0): skipped 13/18 edges for which we didn't have sequences for both nodes (i.e. left the original branch length unmodified). Missing nodes: amz bev bnw bop dkn ikm
-      setting default tau to 1 / 352 = 0.0028
-    calculating normalized lb index and lb ratio and lb fraction on aa tree with tau 0.0028
-                iclust 1: warning tree depth and mfreq differ by more than 25% for 2/7 nodes (note that this is expected if these are single chain sequences being compared to a paired h+l tree)
-                    mean values:  tree depth 0.001  mfreq 0.001  diff 0.000  abs(frac diff) 25%  ratio 1.3
-          10 with highest abs(diff):
-            tree depth   mfreq      ratio    diff
-              0.0034    0.0027       1.3     0.0007     20a0129b57
-              0.0034    0.0027       1.3     0.0007     97366cf0d8
-          10 with highest+lowest ratios:
-              0.0034    0.0027       1.3     0.0007     20a0129b57
-              0.0034    0.0027       1.3     0.0007     97366cf0d8
-              0.0034    0.0027       1.3     0.0007     20a0129b57
-              0.0034    0.0027       1.3     0.0007     97366cf0d8
-      warning get_aa_tree() (AA inf tree, iclust 1): skipped 3/9 edges for which we didn't have sequences for both nodes (i.e. left the original branch length unmodified). Missing nodes: eov
-      warning get_aa_tree() (AA inf tree, iclust 2): skipped 11/13 edges for which we didn't have sequences for both nodes (i.e. left the original branch length unmodified). Missing nodes: apr dhq fnv nox uvx
-    skipped 1/4 clusters that had no uids in common with tree
-  writing selection metrics to test/new-results/get-selection-metrics-new-simu.yaml
-      total time: 0.1
-cache-parameters-data            partis cache-parameters --dont-write-git-info --infname test/mishmash.fa --parameter-dir test/new-results/test/parameters/data --sw-cachefname test/new-results/test/parameters/data/sw-cache.yaml --n-max-queries 50 --input-metafnames test/input-meta.yaml --random-seed 1 --n-procs 10
-    stopped after reading 50 sequences from test/mishmash.fa
-  --n-max-queries: stopped after reading 50 queries from input file
-  --input-metafnames: added meta info for 4 sequences from test/input-meta.yaml: multiplicities
-caching parameters
-  vsearch: 44 / 50 v annotations (6 failed) with 54 v genes in 0.0 sec
-    keeping 24 / 198 v genes
-smith-waterman  (new-allele fitting)
-  vsearch: 45 / 50 v annotations (5 failed) with 23 v genes in 0.0 sec
-  removing old sw cache test/new-results/test/parameters/data/sw-cache.yaml
-    running 10 procs for 50 seqs
-    running 12 procs for 5 seqs
-      info for 46 / 50 = 0.920   (removed: 4 failed)
-      kept 16 (0.320) unproductive
-    water time: 0.1  (ig-sw 0.0  processing 0.0)
-smith-waterman  (writing parameters)
-  vsearch: 45 / 50 v annotations (5 failed) with 23 v genes in 0.0 sec
-    running 10 procs for 46 seqs
-    running 11 procs for 1 seq
-      info for 46 / 50 = 0.920   (removed: 4 failed)
-      kept 16 (0.320) unproductive
-        writing sw results to test/new-results/test/parameters/data/sw-cache.yaml
-    writing parameters to test/new-results/test/parameters/data/sw (1.0 sec)
-    water time: 1.2  (ig-sw 1.1  processing 0.0)
-  writing hmms     removed 15 genes from glfo (leaving v 18  d 22  j 7)
-(0.4 sec)
-hmm
-    reading output
-        read 46 hmm output lines with 46 sequences in 46 events  (0 failures)
-    writing parameters to test/new-results/test/parameters/data/hmm (0.8 sec)
-         infra time: 0.9
-      hmm step time: 1.8
-  writing hmms     removed 6  genes from glfo (leaving v 15  d 19  j 7)
-(0.4 sec)
-      total time: 4.1
-simulate                         partis simulate --dont-write-git-info --parameter-dir test/new-results/test/parameters/data --n-sim-events 10 --n-trees 10 --n-leaf-distribution geometric --n-leaves 5 --random-seed 1 --n-procs 10 --outfname test/new-results/test/simu.yaml --indel-frequency 0.2
-simulating
-Traceback (most recent call last):
-  File "/home/dralph/.local/pipx/venvs/partis-bcr/lib/python3.10/site-packages/bin/partis", line 1874, in <module>
-    args.func(args)
-  File "/home/dralph/.local/pipx/venvs/partis-bcr/lib/python3.10/site-packages/bin/partis", line 195, in run_simulation
-    cmdfos = run_sub_cmds(working_gldir)
-  File "/home/dralph/.local/pipx/venvs/partis-bcr/lib/python3.10/site-packages/bin/partis", line 120, in run_sub_cmds
-    utils.run_cmds(cmdfos, batch_system=args.batch_system, batch_options=args.batch_options, batch_config_fname=args.batch_config_fname, debug='print')
-  File "/home/dralph/.local/pipx/venvs/partis-bcr/lib/python3.10/site-packages/python/utils.py", line 5510, in run_cmds
-    status, dbgstrs[iproc] = finish_process(iproc, procs, n_tries_list[iproc], cmdfos[iproc], n_max_tries, dbgfo=cmdfos[iproc].get('dbgfo'), batch_system=batch_system, debug=debug, ignore_stderr=ignore_stderr, clean_on_success=clean_on_success, allow_failure=allow_failure)
-  File "/home/dralph/.local/pipx/venvs/partis-bcr/lib/python3.10/site-packages/python/utils.py", line 5630, in finish_process
-    raise Exception(failstr)
-Exception: exceeded max number of tries (1 >= 1) for subprocess with command:
-        partis simulate --dont-write-git-info --parameter-dir test/new-results/test/parameters/data --n-sim-events 1 --n-trees 10 --n-leaf-distribution geometric --n-leaves 5 --random-seed 1 --n-procs 1 --outfname /tmp/dralph/hmms/852975/sub-0/simu.yaml --indel-frequency 0.2 --im-a-subproc --workdir /tmp/dralph/hmms/852975/sub-0
-        stdout:           /tmp/dralph/hmms/852975/log-0/out
-                reco params: test/new-results/test/parameters/data/hmm     shm params: test/new-results/test/parameters/data/hmm
-            
-        stderr:           /tmp/dralph/hmms/852975/log-0/err
-            Traceback (most recent call last):
-              File "/home/dralph/.local/pipx/venvs/partis-bcr/lib/python3.10/site-packages/python/utils.py", line 5228, in check_cmd
-                subprocess.check_call([cmd] + options, stdout=open('/dev/null'))
-              File "/usr/lib/python3.10/subprocess.py", line 364, in check_call
-                retcode = call(*popenargs, **kwargs)
-              File "/usr/lib/python3.10/subprocess.py", line 345, in call
-                with Popen(*popenargs, **kwargs) as p:
-              File "/usr/lib/python3.10/subprocess.py", line 971, in __init__
-                self._execute_child(args, executable, preexec_fn, close_fds,
-              File "/usr/lib/python3.10/subprocess.py", line 1863, in _execute_child
-                raise child_exception_type(errno_num, err_msg, err_filename)
-            FileNotFoundError: [Errno 2] No such file or directory: 'R'
-            
-            During handling of the above exception, another exception occurred:
-            
-            Traceback (most recent call last):
-              File "/home/dralph/.local/pipx/venvs/partis-bcr/lib/python3.10/site-packages/bin/partis", line 1874, in <module>
-                args.func(args)
-              File "/home/dralph/.local/pipx/venvs/partis-bcr/lib/python3.10/site-packages/bin/partis", line 193, in run_simulation
-                make_events(args.n_sim_events, args.workdir, args.outfname, [random.randint(0, numpy.iinfo(numpy.int32).max) for _ in range(args.n_sim_events)], heavy_chain_events=heavy_chain_events)
-              File "/home/dralph/.local/pipx/venvs/partis-bcr/lib/python3.10/site-packages/bin/partis", line 177, in make_events
-                reco = Recombinator(args, glfo, seed=args.random_seed, workdir=workdir, heavy_chain_events=heavy_chain_events)
-              File "/home/dralph/.local/pipx/venvs/partis-bcr/lib/python3.10/site-packages/python/recombinator.py", line 78, in __init__
-                treegen.generate_trees(seed, self.treefname, self.workdir)
-              File "/home/dralph/.local/pipx/venvs/partis-bcr/lib/python3.10/site-packages/python/treegenerator.py", line 257, in generate_trees
-                ages, treestrs = self.run_treesim(seed, outfname, workdir)
-              File "/home/dralph/.local/pipx/venvs/partis-bcr/lib/python3.10/site-packages/python/treegenerator.py", line 205, in run_treesim
-                utils.run_r(cmd_lines, workdir, print_time='tree generation' if self.args.debug else None, remove_cmdfile=True)
-              File "/home/dralph/.local/pipx/venvs/partis-bcr/lib/python3.10/site-packages/python/utils.py", line 5248, in run_r
-                check_cmd('R', options=['--slave', '--version'])
-              File "/home/dralph/.local/pipx/venvs/partis-bcr/lib/python3.10/site-packages/python/utils.py", line 5235, in check_cmd
-                raise Exception('command \'%s\' not found in path (maybe not installed?)' % cmd)
-            Exception: command 'R' not found in path (maybe not installed?)
-            
+        read 10 hmm output lines with 10 sequences in 10 events  (0 failures)
+    writing parameters to test/new-results/test/parameters/simu/hmm (0.2 sec)
+         infra time: 0.2
+      hmm step time: 0.3
+  writing hmms     removed 0  genes from glfo (leaving v 2  d 1  j 2)
+(0.1 sec)
+      total time: 1.2

test/test.py

@@ -33,7 +33,10 @@ class Tester(object):
     # ----------------------------------------------------------------------------------------
     def dirs(self, tstr, force_paired=False):
         assert tstr in ['ref', 'new']
-        return 'test%s/%s-results%s' % ('/paired' if (args.paired or force_paired) else '', tstr, '-slow' if args.slow else '')
+        bdir = 'test%s' % ('/paired' if (args.paired or force_paired) else '')
+        if not os.path.exists(bdir):
+            bdir = '%s/%s' % (utils.get_partis_dir(), bdir)
+        return '%s/%s-results%s' % (bdir, tstr, '-slow' if args.slow else '')
     # ----------------------------------------------------------------------------------------
     def nqr(self, act):
         if act == 'quick':  # bears no relation to the others, so makes sense to handle it differently