seqtree 1.0.0__tar.gz → 1.1.0__tar.gz

{seqtree-1.0.0 → seqtree-1.1.0}/.gitignore

@@ -16,3 +16,15 @@ bench/cache/
 .coverage
 .coverage.*
 htmlcov/
+
+# scratch worktrees and probe output
+tmp/
+
+# Local agent instructions and planning notes
+/AGENTS.md
+/CLAUDE.md
+/ROADMAP.md
+/SOURCES.md
+/skills/
+/add_bindings.py
+/add_native.py

{seqtree-1.0.0 → seqtree-1.1.0}/CHANGELOG.md

@@ -3,6 +3,129 @@
 All notable changes to `seqtree`. Dates are release dates. From 1.0.0 the project follows semantic
 versioning: breaking changes need a **major** bump.
 
+## [1.1.0] — 2026-10-11
+
+### Added
+
+- Native batched edit-count histograms, with an exact bit-vector fast path for bounded unit-cost searches.
+- Gap-block length profiles and symmetric per-input positional weights, optional group penalties,
+  exhaustive balls, linked sum-distance counts, integer linear mass, and streaming group audits.
+- TextIndex E-values and per-query mismatch cutoffs using distinct-record counts, with native
+  reduction of repeated offsets and rejection of truncated results.
+
+### Fixed
+
+- Stable small Poisson tails and tiny expectation probabilities.
+- Shared native batches respect detected CPU allocations and reject oversubscribed
+  worker counts; serial budgets execute inline and all pools close before returning.
+- Setup combines requested test and benchmark extras and propagates installation errors.
+- Local agent instructions, planning notes and scratch scripts are excluded from distributions.
+
+### API details
+
+- **E-values for substring hits.** `text_evalues` and `text_threshold_for_evalue`
+  calibrate a `TextIndex` hit against a background text, the way `evalues` already
+  does for an `Index`:
+
+  ```python
+  proteome = seqtree.TextIndex.build(records, "aa", k=5)
+  decoy    = seqtree.TextIndex.build(shuffled, "aa", k=5)
+  for q, r in zip(peptides, seqtree.text_evalues(proteome, decoy, peptides, max_subs=2)):
+      if r["p_enrichment"] < 0.01:
+          ...
+  ```
+
+  The arithmetic is `evalue.evalue_result` unchanged; what is specific to text search
+  is two counting choices. **Counts are per distinct record, not per
+  offset**: one query typically hits a record at several overlapping offsets, those
+  events are strongly dependent, and counting them separately gives the
+  over-dispersed compound-Poisson count `appendix/evalue.tex` collapses away by
+  counting unique clonotypes rather than reads. `N` and `M` are `num_refs`. **The ball
+  is `max_subs`, never `TextResult.score`**: that score is a similarity, `0` at an
+  exact match and more negative for a worse one, the opposite sign to the
+  non-negative penalty `thetas_from_scores` inverts; `n_subs` is non-negative,
+  integer and monotone in the scope, so it nests the way the proof requires.
+
+  Two limits, both refusals rather than partial answers. A search whose hits were
+  capped by `max_hits` raises, because a truncated count is a floor and an E-value
+  computed from it is simply wrong. And there is no `load_control` for texts: the
+  background must be composition- and length-matched to the target, that assumption
+  is what the whole calibration rests on, and nothing in the library can check it, so
+  it stays the caller's explicit choice.
+
+- **Bounded exhaustive gapblock reductions.** `topk_batch` keeps
+  stable `(score, ref_id)` top-k rows without a dense score matrix; `count_batch`
+  counts all ties at arbitrary per-query thresholds. Linked two-view variants use
+  the maximum lane penalty and preserve both lane scores. Exact identity exclusion
+  precedes selection and removes only both-view identity for linked inputs.
+  `Index.ref_seqs()` retrieves stored reference strings in ID order in one native
+  call. All scoring reuses the existing single-gap-block cell; no calibration,
+  domain payload or runtime dependency is introduced.
+
+- **`TextIndex` searches with insertions and deletions.** `search_batch` takes `max_indels`
+  alongside `max_subs`, capped independently — `max_subs=2, max_indels=1` accepts two
+  substitutions *and* one gap, not three edits of any kind.
+
+  ```python
+  res = ix.search_batch(peptides, max_subs=1, max_indels=1, threads=0)
+  h = res[0][0]
+  ix.ref_seq(h.ref_id)[h.offset : h.offset + h.length]   # the matched text
+  h.n_subs, h.n_ins, h.n_dels
+  ```
+
+  **The index, the build and the probe enumeration are unchanged.** With
+  `b = max_subs + max_indels + 1` disjoint blocks the total edit count cannot reach the block
+  count, so the same pigeonhole argument still leaves a **zero-error** block — and a zero-error
+  block matches the text exactly whether or not the occurrence carries gaps, so the existing
+  exact seed lookup finds it. Verified before any of this was written: 2,800 planted
+  occurrences with random insertions and deletions, **recall 1.000** from the unmodified exact
+  lookup.
+
+  What changes is verification. A seed no longer pins the start — the prefix before the block
+  may have gained or lost up to `max_indels` residues — so every start in
+  `[pos - base - max_indels, pos - base + max_indels]` is aligned by a banded DP over the states
+  `(query position, net offset, indels spent)`. An alignment must begin and end on an aligned
+  pair: a gap at either edge would stretch the reported interval over a residue that matches
+  nothing, and a leading gap is the same occurrence starting one residue over, so allowing it
+  would report one match several times.
+
+  `TextHit` gains `n_ins`, `n_dels` and **`length`** — a gapped match is no longer `len(query)`
+  residues wide, and `length` is `len(query) + n_dels - n_ins`. `TextResult.arrays()` gains the
+  three matching columns.
+
+  Two limits, both refusals rather than partial answers. Every seed block must be exact, so a
+  query must be at least `(max_subs + max_indels + 1) * k` long — at `k = 4` with
+  `max_subs=1, max_indels=1` that is 12 residues — and a shorter one raises with the bound in
+  the message. And `mismatches` is empty on this path, with `matrix=` ignored: recovering which
+  columns were substituted needs an alignment traceback the verifier does not keep.
+
+  Measured on a synthetic 5,000,000-residue corpus at `k = 4`, one thread, against the same
+  queries with `max_indels = 0`:
+
+  | query length | `max_subs` | ms/query, no indels | ms/query, `max_indels=1` |
+  |--:|--:|--:|--:|
+  | 16 | 1 | 0.001 | 0.028 |
+  | 16 | 2 | 0.001 | 0.043 |
+  | 20 | 3 | 0.002 | 0.065 |
+  | 24 | 3 | 0.002 | 0.067 |
+
+  25–37× an ungapped search at the same length and `max_subs`, and still under 0.1 ms/query.
+  An early exit when every banded state has already breached a cap is worth 2.5–3× of that; the
+  DP was otherwise walking all L rows on candidates the Hamming path rejects after two residues.
+
+  Completeness is held to the same standard as the substitution path: **set equality against an
+  independent brute force**, over `k` ∈ {3, 4, 5} × `max_subs` 0–3 × `max_indels` 1–2,
+  **6,732 occurrences, zero missing and zero extra**. `max_indels=0` is byte-identical to the
+  previous substitution path, hit for hit, and pays none of the cost.
+
+## [1.0.1] — 2026-10-09
+
+### Fixed
+
+- A one-thread batch executes on the calling thread instead of launching a worker.
+- Worker scratch-allocation exceptions propagate to the caller rather than terminating the process.
+- A partially started worker pool joins its running workers if thread creation fails.
+
 ## [1.0.0] — 2026-09-06
 
 ### Added
@@ -572,7 +695,7 @@ Numbers that constrain the API, all reproducible from `bench/` and the downstrea
 
 ### Docs
 
-- `skills/seqtree/SKILL.md` — public API surface, invariants, and the gotchas that have bitten.
+- Public API surface, invariants, and gotchas are documented in `docs/api.rst` and the API guides.
 - `docs/gapblock.rst` — a worked guide: why one gap block, how to choose its position, why a
   placement rule is a column frame, and why a fixed score cutoff is not a calibrated one.
 - README corrected: `seqtrie` is a full-width DP that ignores per-type caps, not a banded one, and

{seqtree-1.0.0 → seqtree-1.1.0}/CMakeLists.txt

@@ -65,6 +65,7 @@ if(SEQTREE_TESTS)
     tests/cpp/test_positional.cpp
     tests/cpp/test_kmer_index.cpp
     tests/cpp/test_text_index.cpp
+    tests/cpp/test_parallel.cpp
   )
   target_include_directories(seqtree_tests PRIVATE tests/cpp src)
   target_link_libraries(seqtree_tests PRIVATE seqtree_core)

{seqtree-1.0.0 → seqtree-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: seqtree
-Version: 1.0.0
+Version: 1.1.0
 Summary: Fast fuzzy search over biological sequences (C++ core, Python bindings)
 Keywords: sequence-search,fuzzy-matching,CDR3,immunology,bioinformatics,trie
 Author-Email: ISALGO laboratory <mikhail.shugay@gmail.com>
@@ -135,7 +135,10 @@ Beyond search, seqtree ships:
   completeness pinned by brute-force **set equality** over L 6–30 × `max_subs` 0–3 ×
   k ∈ {3,4,5}, and by the answer being identical across `k`. On the human proteome
   (69,578,135 residues) one thread answers a 9-mer within 2 substitutions in **1.3 ms** and a
-  15-mer within 3 in **1.5 ms**. Results come back as flat CSR arrays with zero-copy
+  15-mer within 3 in **1.5 ms**. `max_indels` adds insertions and deletions on the same index
+  and the same scheme — with `max_subs + max_indels + 1` blocks the edit count is still below
+  the block count, so pigeonhole leaves a zero-error block and the *unchanged* exact seed lookup
+  finds gapped occurrences too; only verification becomes a banded DP. Results come back as flat CSR arrays with zero-copy
   `numpy` views, mismatches as `(pos, query_aa, text_aa)` pairs, an optional fold onto
   caller-supplied group ids that makes a tie explicit, and a cap that is always reported.
 - **E-values / significance** — calibrate hit counts against a background control repertoire

{seqtree-1.0.0 → seqtree-1.1.0}/README.md

@@ -103,7 +103,10 @@ Beyond search, seqtree ships:
   completeness pinned by brute-force **set equality** over L 6–30 × `max_subs` 0–3 ×
   k ∈ {3,4,5}, and by the answer being identical across `k`. On the human proteome
   (69,578,135 residues) one thread answers a 9-mer within 2 substitutions in **1.3 ms** and a
-  15-mer within 3 in **1.5 ms**. Results come back as flat CSR arrays with zero-copy
+  15-mer within 3 in **1.5 ms**. `max_indels` adds insertions and deletions on the same index
+  and the same scheme — with `max_subs + max_indels + 1` blocks the edit count is still below
+  the block count, so pigeonhole leaves a zero-error block and the *unchanged* exact seed lookup
+  finds gapped occurrences too; only verification becomes a banded DP. Results come back as flat CSR arrays with zero-copy
   `numpy` views, mismatches as `(pos, query_aa, text_aa)` pairs, an optional fold onto
   caller-supplied group ids that makes a tie explicit, and a cap that is always reported.
 - **E-values / significance** — calibrate hit counts against a background control repertoire

{seqtree-1.0.0 → seqtree-1.1.0}/include/seqtree/kmer_index.hpp

@@ -39,7 +39,7 @@ public:
     // For each query (its k-mer list), return candidates with >= min_shared shared k-mers,
     // ranked by (shared_kmers desc, best_score asc). `params` drives the per-k-mer fuzzy
     // match (max_subs / pos_matrix / engine). allele_filter >= 0 keeps only peptides with
-    // that allele tag; < 0 means no filter. threads <= 0 => hardware_concurrency.
+    // that allele tag; < 0 means no filter. threads <= 0 => available CPU budget.
     std::vector<std::vector<Candidate>> seed_and_gather(
         const std::vector<std::vector<std::string>>& query_kmers, const SearchParams& params,
         uint32_t min_shared = 1, int64_t allele_filter = -1, int threads = 0) const;

seqtree-1.1.0/include/seqtree/parallel.hpp

@@ -0,0 +1,134 @@
+#pragma once
+// One parallel-for scaffold for the whole tree. Every batch entry point in seqtree used to
+// carry its own copy of this: an atomic cursor, an adaptive chunk, an exception_ptr, a vector
+// of threads, and a rethrow after the join. Seven copies drifted -- one of them had no
+// exception plumbing at all.
+#include <algorithm>
+#include <atomic>
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <exception>
+#include <fstream>
+#include <mutex>
+#include <stdexcept>
+#include <thread>
+#include <vector>
+
+#ifdef __linux__
+#include <sched.h>
+#elif defined(__APPLE__)
+#include <sys/sysctl.h>
+#elif defined(_WIN32)
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#include <windows.h>
+#endif
+
+namespace seqtree {
+
+// Resolve once per batch; affinity/quota limits can change between calls.
+inline unsigned available_cpus() {
+    unsigned count = std::max(1u, std::thread::hardware_concurrency());
+#ifdef __linux__
+    cpu_set_t affinity;
+    if (sched_getaffinity(0, sizeof(affinity), &affinity) == 0)
+        count = std::min(count, unsigned(std::max(1, CPU_COUNT(&affinity))));
+    std::ifstream limits("/sys/fs/cgroup/cpu.max");
+    uint64_t quota, period;
+    if (limits >> quota >> period && period)
+        count = unsigned(std::min<uint64_t>(count, std::max<uint64_t>(1, quota / period)));
+    for (const char* name : {"SLURM_CPUS_PER_TASK", "SLURM_CPUS_ON_NODE"}) {
+        if (const char* value = std::getenv(name)) {
+            char* end = nullptr;
+            const auto allocated = std::strtoul(value, &end, 10);
+            if (end != value && !*end && allocated)
+                count = unsigned(std::min<unsigned long>(count, allocated));
+        }
+    }
+#elif defined(__APPLE__)
+    unsigned physical = count;
+    size_t size = sizeof(physical);
+    if (sysctlbyname("hw.physicalcpu", &physical, &size, nullptr, 0) == 0 && physical)
+        count = std::min(count, physical);
+#elif defined(_WIN32)
+    DWORD_PTR process_mask, system_mask;
+    if (GetProcessAffinityMask(GetCurrentProcess(), &process_mask, &system_mask)) {
+        unsigned allowed = 0;
+        for (; process_mask; process_mask &= process_mask - 1) ++allowed;
+        if (allowed) count = std::min(count, allowed);
+    }
+#endif
+    return count;
+}
+
+// Runs body(i, local) for every i in [0, n), across nt workers pulling contiguous chunks off
+// one atomic cursor. `threads <= 0` means the detected available CPU budget.
+//
+// `make_local()` is called once per worker and its result handed to every body() that worker
+// runs -- that is how a Searcher or a scratch buffer gets allocated once per thread rather
+// than once per item. Return a dummy for kernels that need no state.
+//
+// The chunk is adaptive: large batches keep ~1024 items per grab so the atomic is not the
+// bottleneck, small ones split ~8 chunks per thread so no worker goes idle. Raise `chunk_min`
+// when the per-item output struct is small enough that neighbouring workers would share a
+// cache line (TextIndex's QueryOut is ~80 bytes, so it passes 8).
+//
+// Workers catch, the first exception is kept, and it is rethrown after the join. This is not
+// optional: an exception escaping a std::thread entry function calls std::terminate, which
+// from Python is an uncatchable SIGABRT rather than a raise.
+template <class MakeLocal, class Body>
+void parallel_for(size_t n, int threads, MakeLocal make_local, Body body,
+                  size_t chunk_min = 1, size_t chunk_max = 1024) {
+    const unsigned available = available_cpus();
+    unsigned nt = threads > 0 ? unsigned(threads) : available;
+    if (nt > available || nt >= 1000)
+        throw std::invalid_argument("worker count exceeds the available CPU budget; request fewer threads");
+    if (n == 0) return;
+    nt = unsigned(std::min<size_t>(nt, n));  // Do not start idle workers for absent work.
+
+    // A serial budget runs on the calling thread; spawning one worker only adds overhead.
+    if (nt == 1) {
+        auto local = make_local();
+        for (size_t i = 0; i < n; ++i) body(i, local);
+        return;
+    }
+
+    std::atomic<size_t> next{0};
+    const size_t chunk = std::clamp<size_t>(n / (size_t(nt) * 8), chunk_min, chunk_max);
+    std::exception_ptr err;
+    std::mutex emu;
+
+    auto worker = [&] {
+        // Scratch construction can allocate and throw too. Nothing may escape the entry point.
+        try {
+            auto local = make_local();
+            for (;;) {
+                const size_t start = next.fetch_add(chunk);
+                if (start >= n) break;
+                const size_t end = std::min(n, start + chunk);
+                for (size_t i = start; i < end; ++i) {
+                    body(i, local);
+                }
+            }
+        } catch (...) {
+            std::lock_guard<std::mutex> lk(emu);
+            if (!err) err = std::current_exception();
+        }
+    };
+
+    // A failed thread launch must also join workers already started before unwinding.
+    std::vector<std::jthread> pool;
+    pool.reserve(nt);
+    // The caller is one compute thread: a budget B creates only B-1 OS workers.
+    for (unsigned t = 1; t < nt; ++t) pool.emplace_back(worker);
+    worker();
+    for (auto& th : pool) th.join();
+    if (err) std::rethrow_exception(err);
+}
+
+}  // namespace seqtree

{seqtree-1.0.0 → seqtree-1.1.0}/include/seqtree/seqtree.hpp

@@ -5,6 +5,7 @@
 #include <memory>
 #include <string>
 #include <string_view>
+#include <tuple>
 #include <vector>
 
 namespace seqtree {
@@ -120,10 +121,15 @@ public:
     Alignment align(std::string_view query, uint32_t ref_id, const SearchParams&) const;
 
     // Parallel batch: pure C++ (no Python), one Searcher per worker thread.
-    // threads <= 0 => hardware_concurrency. Results aligned with input order.
+    // threads <= 0 => available CPU budget. Results aligned with input order.
     std::vector<std::vector<Hit>> search_batch(const std::vector<std::string>& queries,
                                                const SearchParams&, int threads = 0) const;
 
+    // Exact edit-count bins 0..max_total_edits; one reusable hit buffer per worker.
+    std::vector<std::vector<uint64_t>> edit_histogram_batch(
+        const std::vector<std::string>& queries, const SearchParams&,
+        int threads = 0, bool exclude_exact = false) const;
+
     // Per-query count of seqtm collisions: how often a reference was re-reached via a
     // different edit path during branch-and-bound (0 for the seqtrie engine / Hamming).
     std::vector<uint64_t> collisions_batch(const std::vector<std::string>& queries,
@@ -194,7 +200,7 @@ Alignment align_pair(std::string_view query, std::string_view ref, const Substit
                      Alphabet, AlignMode, int32_t gap_open, int32_t gap_extend);
 
 // Dense N*K similarity matrix, row-major (row i is queries[i] vs every ref). Parallel;
-// threads <= 0 => hardware_concurrency.
+// threads <= 0 => available CPU budget.
 std::vector<int32_t> align_score_matrix(const std::vector<std::string>& queries,
                                         const std::vector<std::string>& refs,
                                         const SubstitutionMatrix&, Alphabet, AlignMode,
@@ -214,12 +220,106 @@ std::vector<int32_t> align_dist_matrix(const std::vector<std::string>& queries,
 // The block of length d = |m-n| is placed at whichever of the min(m,n)+1 positions minimises
 // substitution cost plus `prior`. `prior` is the gap prior flattened to [m][d][i] with stride
 // (prior_width+1); pass it empty for no prior. It is read only when d > 0, which is what keeps
-// the diagonal zero. `matrix` may be null for unit cost. threads <= 0 => hardware_concurrency.
+// the diagonal zero. `matrix` may be null for unit cost. threads <= 0 => available CPU budget.
+// Optional integer weights are indexed [longer length][longer position], symmetric in q/r.
+// All input frames must be covered; empty tables retain the existing unweighted scorer.
+// Alternatively, per-input rows index each sequence's full coordinates. Aligned
+// substitution costs use max(query weight, reference weight); gaps are unchanged.
+// Both axes are required; these rows cannot combine with length profiles.
+// Optional group IDs index a nonnegative rectangular penalty table added per pair.
+// Exact exclusion remains encoded junction identity, independent of these groups.
 std::vector<int32_t> gapblock_matrix(const std::vector<std::string>& queries,
                                      const std::vector<std::string>& refs, Alphabet,
                                      const SubstitutionMatrix* matrix, int32_t gap_open,
                                      int32_t gap_extend, const std::vector<int32_t>& prior,
-                                     uint32_t prior_width, int threads = 0);
+                                     uint32_t prior_width, int threads = 0,
+                                     const std::vector<std::vector<int32_t>>& weights = {},
+                                     const std::vector<int32_t>& query_groups = {},
+                                     const std::vector<int32_t>& reference_groups = {},
+                                     const std::vector<std::vector<int32_t>>& group_distances = {},
+        const std::vector<std::vector<int32_t>>& query_weights = {},
+        const std::vector<std::vector<int32_t>>& reference_weights = {});
+
+// Exhaustive ball rows in reference order; hit edit fields are unset (zero).
+std::vector<std::vector<Hit>> gapblock_ball_batch(const std::vector<std::string>& queries,
+        const std::vector<std::string>& refs, const std::vector<int32_t>& thresholds,
+        Alphabet alphabet, const SubstitutionMatrix* matrix, int32_t go, int32_t ge,
+        const std::vector<int32_t>& prior, uint32_t width, int threads, bool exclude_exact,
+        const std::vector<std::vector<int32_t>>& weights = {},
+        const std::vector<int32_t>& query_groups = {},
+        const std::vector<int32_t>& reference_groups = {},
+        const std::vector<std::vector<int32_t>>& group_distances = {},
+        const std::vector<std::vector<int32_t>>& query_weights = {},
+        const std::vector<std::vector<int32_t>>& reference_weights = {});
+
+// Exhaustive bounded reductions using the same gapblock cell. Top-k ties use
+// (penalty, ref_id); paired penalty is max(lane scores). Exact exclusion uses
+// encoded sequence identity (both lanes for pairs), never zero penalty alone.
+// Count the component ball before/after group offsets; minimum full score within it.
+// One threshold per query; nearest=-1 denotes an empty component ball. O(Q) output.
+std::tuple<std::vector<uint64_t>, std::vector<uint64_t>, std::vector<int32_t>>
+gapblock_group_audit_batch(const std::vector<std::string>& queries,
+        const std::vector<std::string>& refs, const std::vector<int32_t>& thresholds,
+        Alphabet alphabet, const SubstitutionMatrix* matrix, int32_t go, int32_t ge,
+        const std::vector<int32_t>& prior, uint32_t width, int threads, bool exclude_exact,
+        const std::vector<std::vector<int32_t>>& weights = {},
+        const std::vector<int32_t>& query_groups = {},
+        const std::vector<int32_t>& reference_groups = {},
+        const std::vector<std::vector<int32_t>>& group_distances = {},
+        const std::vector<std::vector<int32_t>>& query_weights = {},
+        const std::vector<std::vector<int32_t>>& reference_weights = {});
+
+std::vector<std::vector<Hit>> gapblock_topk_batch(const std::vector<std::string>& queries,
+        const std::vector<std::string>& refs, uint32_t k, Alphabet alphabet,
+        const SubstitutionMatrix* matrix, int32_t go, int32_t ge, const std::vector<int32_t>& prior,
+        uint32_t width, int threads, bool exclude_exact);
+std::vector<std::vector<std::tuple<uint32_t, int32_t, int32_t>>> gapblock_paired_topk_batch(
+        const std::vector<std::string>& qa, const std::vector<std::string>& qb,
+        const std::vector<std::string>& ra, const std::vector<std::string>& rb,
+        uint32_t k, Alphabet alphabet, const SubstitutionMatrix* matrix, int32_t go, int32_t ge,
+        const std::vector<int32_t>& prior, uint32_t width, int threads, bool exclude_exact);
+std::vector<std::vector<uint64_t>> gapblock_count_batch(const std::vector<std::string>& queries,
+        const std::vector<std::string>& refs, const std::vector<std::vector<int32_t>>& thresholds,
+        Alphabet alphabet, const SubstitutionMatrix* matrix, int32_t go, int32_t ge,
+        const std::vector<int32_t>& prior, uint32_t width, int threads, bool exclude_exact,
+        const std::vector<std::vector<int32_t>>& weights = {},
+        const std::vector<int32_t>& query_groups = {},
+        const std::vector<int32_t>& reference_groups = {},
+        const std::vector<std::vector<int32_t>>& group_distances = {},
+        const std::vector<std::vector<int32_t>>& query_weights = {},
+        const std::vector<std::vector<int32_t>>& reference_weights = {});
+std::pair<std::vector<std::vector<uint64_t>>, std::vector<std::vector<uint64_t>>> gapblock_count_mass_batch(const std::vector<std::string>& queries,
+        const std::vector<std::string>& refs, const std::vector<std::vector<int32_t>>& thresholds,
+        Alphabet alphabet, const SubstitutionMatrix* matrix, int32_t go, int32_t ge,
+        const std::vector<int32_t>& prior, uint32_t width, int threads, bool exclude_exact,
+        const std::vector<std::vector<int32_t>>& weights = {},
+        const std::vector<int32_t>& query_groups = {},
+        const std::vector<int32_t>& reference_groups = {},
+        const std::vector<std::vector<int32_t>>& group_distances = {},
+        const std::vector<std::vector<int32_t>>& query_weights = {},
+        const std::vector<std::vector<int32_t>>& reference_weights = {});
+std::vector<std::vector<uint64_t>> gapblock_paired_count_batch(
+        const std::vector<std::string>& qa, const std::vector<std::string>& qb,
+        const std::vector<std::string>& ra, const std::vector<std::string>& rb,
+        const std::vector<std::vector<int32_t>>& thresholds,
+        Alphabet alphabet, const SubstitutionMatrix* matrix, int32_t go, int32_t ge,
+        const std::vector<int32_t>& prior, uint32_t width, int threads, bool exclude_exact);
+std::pair<std::vector<std::vector<uint64_t>>, std::vector<std::vector<uint64_t>>> gapblock_paired_sum_count_batch(
+        const std::vector<std::string>& qa, const std::vector<std::string>& qb,
+        const std::vector<std::string>& ra, const std::vector<std::string>& rb,
+        const std::vector<std::vector<int32_t>>& thresholds,
+        Alphabet alphabet, const SubstitutionMatrix* matrix, int32_t go, int32_t ge,
+        const std::vector<int32_t>& prior, uint32_t width, int threads, bool exclude_exact,
+        const std::vector<std::vector<int32_t>>& weights_alpha,
+        const std::vector<std::vector<int32_t>>& weights_beta,
+        const std::vector<int32_t>& query_groups_alpha, const std::vector<int32_t>& query_groups_beta,
+        const std::vector<int32_t>& reference_groups_alpha, const std::vector<int32_t>& reference_groups_beta,
+        const std::vector<std::vector<int32_t>>& group_distances_alpha,
+        const std::vector<std::vector<int32_t>>& group_distances_beta,
+        const std::vector<std::vector<int32_t>>& query_weights_alpha = {},
+        const std::vector<std::vector<int32_t>>& query_weights_beta = {},
+        const std::vector<std::vector<int32_t>>& reference_weights_alpha = {},
+        const std::vector<std::vector<int32_t>>& reference_weights_beta = {});
 
 // ---------------------------------------------------------------------------------------
 // Plain string edit distances on raw characters, unit costs. Deliberately alphabet-agnostic:
@@ -235,7 +335,7 @@ int hamming(std::string_view a, std::string_view b);
 int levenshtein(std::string_view a, std::string_view b);
 
 // Dense N*K distance matrices, row-major (row i is a[i] vs every b[k]). Parallel;
-// threads <= 0 => hardware_concurrency. hamming_matrix throws on any length-mismatched pair.
+// threads <= 0 => available CPU budget. hamming_matrix throws on any length-mismatched pair.
 std::vector<int32_t> hamming_matrix(const std::vector<std::string>& a,
                                     const std::vector<std::string>& b, int threads = 0);
 std::vector<int32_t> levenshtein_matrix(const std::vector<std::string>& a,

{seqtree-1.0.0 → seqtree-1.1.0}/include/seqtree/text_index.hpp

@@ -13,12 +13,24 @@ struct TextStore;  // internal, src/text_index.cpp
 // One row of the optional group fold: hits collapsed onto the build-time group ids.
 struct TextGroup {
     uint32_t group_id = 0;
-    uint16_t min_subs = 0;  // best (smallest) Hamming distance reaching this group
+    uint16_t min_subs = 0;  // best (smallest) substitution count reaching this group
     uint32_t n_hits   = 0;  // how many hits fell in it, at any distance
 };
 
 struct TextQueryOpts {
     uint16_t max_subs      = 0;
+    // Insertions + deletions allowed, capped independently of max_subs. 0 keeps the pure
+    // Hamming predicate and its verification path untouched.
+    //
+    // The seed table does not change: with b = max_subs + max_indels + 1 disjoint blocks the
+    // total edit count is below the block count, so by pigeonhole SOME block still matches
+    // EXACTLY -- the existing exact lookup finds every occurrence, indels included. What that
+    // costs is verification, which becomes a banded alignment over the 2*max_indels+1 possible
+    // starts rather than one Hamming scan at a fixed start.
+    //
+    // Because every block must be exact, this needs L >= (max_subs + max_indels + 1) * k;
+    // search_batch refuses a shorter query rather than answering incompletely.
+    uint16_t max_indels    = 0;
     bool     exclude_exact = false;  // drop 0-mismatch hits (a candidate that IS the text)
     bool     best_only     = false;  // keep only the minimum-n_subs shell (still ALL of it)
     bool     group_by      = false;  // also fold hits onto the build-time group ids
@@ -40,6 +52,11 @@ struct TextResult {
     std::vector<uint32_t> query_begin;  // n_queries + 1, CSR into the hit arrays
     std::vector<uint32_t> ref_id, offset;
     std::vector<uint16_t> n_subs;
+    // Indels in the alignment, and the matched substring's length in the text. With
+    // max_indels = 0 these are 0, 0 and the query length for every hit. With indels the match
+    // is no longer len(query) residues wide, so `length` is what bounds it:
+    // ref_seq(ref_id)[offset : offset + length].
+    std::vector<uint16_t> n_ins, n_dels, length;
     std::vector<int32_t>  score;
     std::vector<uint32_t> mm_begin;     // n_hits + 1, CSR into the mismatch arrays
     std::vector<uint16_t> mm_pos;
@@ -53,6 +70,8 @@ struct TextResult {
 
     size_t num_queries() const { return query_begin.empty() ? 0 : query_begin.size() - 1; }
     size_t num_hits() const { return ref_id.size(); }
+    // Best substitution distance per distinct record, sorted per query. Reject capped hits.
+    std::vector<std::vector<uint16_t>> record_distances() const;
 };
 
 // Exact k-mismatch (Hamming) search over a CONCATENATED reference text.
@@ -91,7 +110,7 @@ public:
 
     // Every query must be at least k long: seeds are only indexed where a whole k-mer fits
     // inside a record, so a shorter query could match within k-L of a record end and be
-    // missed. Raises rather than answering incompletely. threads <= 0 => hardware_concurrency.
+    // missed. Raises rather than answering incompletely. threads <= 0 => available CPU budget.
     TextResult search_batch(const std::vector<std::string>& queries, const TextQueryOpts&,
                             int threads = 0) const;
 

{seqtree-1.0.0 → seqtree-1.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "seqtree"
-version = "1.0.0"
+version = "1.1.0"
 description = "Fast fuzzy search over biological sequences (C++ core, Python bindings)"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -40,9 +40,10 @@ wheel.packages = ["python/seqtree"]
 cmake.build-type = "Release"
 # Keep the sdist lean: ship only what builds the package. Anchor with a leading "/".
 sdist.exclude = [
-    "/bench", "/tests", "/docs", "/.github", "/setup.sh", "/build",
+    "/bench", "/tests", "/docs", "/.github", "/setup.sh", "/build", "/appendix", "/examples",
     # Local agent config: machine-specific absolute paths and unrelated repo names.
-    "/.claude", "/CLAUDE.md",
+    "/.claude", "/CLAUDE.md", "/AGENTS.md", "/ROADMAP.md", "/SOURCES.md", "/skills",
+    "/tmp", "/add_bindings.py", "/add_native.py", "/appendix/immunogenicity_hooks.md",
 ]
 editable.rebuild = false
 

{seqtree-1.0.0 → seqtree-1.1.0}/python/seqtree/__init__.py

@@ -34,6 +34,7 @@ from ._core import (
 )
 from .control import load_control
 from .evalue import evalues, thetas_from_scores, threshold_for_evalue
+from .text_evalue import text_evalues, text_threshold_for_evalue
 from . import distance, gapblock, layout, pairwise, pmhc, seeds
 from .gapblock import (
     GapBlockIndex, IslandProfile, ScoreMatrix, central_prior, embed_in_frame, frame_prior,
@@ -87,4 +88,6 @@ __all__ = [
     "evalues",
     "thetas_from_scores",
     "threshold_for_evalue",
+    "text_evalues",
+    "text_threshold_for_evalue",
 ]

{seqtree-1.0.0 → seqtree-1.1.0}/python/seqtree/evalue.py

@@ -22,17 +22,31 @@ import warnings
 
 
 def _poisson_sf(k, lam):
-    """P(Poisson(lam) >= k) via the complementary CDF (stable for small k)."""
+    """P(Poisson(lam) >= k), summing the smaller tail from a log-PMF seed."""
     if k <= 0:
         return 1.0
     if lam <= 0.0:
         return 0.0
-    term = math.exp(-lam)
-    cdf = term
-    for i in range(1, k):
-        term *= lam / i
-        cdf += term
-    return min(1.0, max(0.0, 1.0 - cdf))
+    if lam == math.inf:
+        return 1.0
+    # Sum the upper tail directly when small: 1-CDF loses significant digits.
+    upper = k > lam
+    i = k if upper else k - 1
+    log_pmf = i * math.log(lam) - lam - math.lgamma(i + 1)
+    # Normalize before summation so a subnormal seed cannot erase a representable tail.
+    term = total = 1.0
+    while term > total * 1e-15:
+        if upper:
+            i += 1
+            term *= lam / i
+        else:
+            if i == 0:
+                break
+            term *= i / lam
+            i -= 1
+        total += term
+    probability = math.exp(log_pmf + math.log(total))
+    return min(1.0, max(0.0, probability if upper else 1.0 - probability))
 
 
 def evalue_result(n_target, n_control, n_ref, m_control):
@@ -61,8 +75,7 @@ def evalue_result(n_target, n_control, n_ref, m_control):
         "n_target": n_target,
         "n_control": n_control,
         "E": E,
-        # exp(-E) underflows to 0.0 well before E = 700, but guard the call anyway.
-        "p_any": 1.0 - math.exp(-E) if E < 700 else 1.0,
+        "p_any": -math.expm1(-E),
         "p_enrichment": _poisson_sf(n_target, E),
         "rule_of_three": rule3,
     }