agglovar 0.0.1.dev17__tar.gz → 0.0.1.dev18__tar.gz

{agglovar-0.0.1.dev17/src/agglovar.egg-info → agglovar-0.0.1.dev18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agglovar
-Version: 0.0.1.dev17
+Version: 0.0.1.dev18
 Summary: Toolkit for fast genomic variant transformations and intersects
 License-Expression: MIT
 Requires-Python: >=3.12
@@ -10,7 +10,7 @@ Requires-Dist: biopython>=1.85
 Requires-Dist: edlib>=1.3.9.post1
 Requires-Dist: numpy>=2.3.0
 Requires-Dist: ply>=3.11
-Requires-Dist: polars[numpy,pandas,pyarrow]>=1.35.2
+Requires-Dist: polars[numpy,pandas,pyarrow]==1.42.1
 Requires-Dist: pysam>=0.23.3
 Provides-Extra: plot
 Requires-Dist: matplotlib>=3.10.3; extra == "plot"

{agglovar-0.0.1.dev17 → agglovar-0.0.1.dev18}/pyproject.toml

@@ -15,7 +15,9 @@ dependencies = [
     "edlib>=1.3.9.post1",
     "numpy>=2.3.0",
     "ply>=3.11",
-    "polars[numpy,pandas,pyarrow]>=1.35.2",
+    # 1.41.2: merge_sorted() crashes on List(Struct) columns before this (used by the cumulative
+    # merge to carry the complex-variant "seg" column).
+    "polars[numpy,pandas,pyarrow]==1.42.1",
     "pysam>=0.23.3",
 ]
 
@@ -52,11 +54,10 @@ maintainers = [
 
 [dependency-groups]
 dev = [
-    "arrow[parquet]>=1.4.0",
     "flake8>=7.3.0",
     "flake8-bugbear>=24.12.12",
     "flake8-docstrings>=1.7.0",
-    "flake8-dunder-all>=0.5.0",
+    "flake8-dunder-all>=0.6.0",
     "flake8-mutable>=1.2.0",
     "flake8-pyproject>=1.2.3",
     "flake8-rst-docstrings>=0.3.1",
@@ -107,9 +108,17 @@ max-line-length = 120
 [tool.flake8]
 max-line-length = 120
 docstring-convention = "pep257"
+# DALL100/DALL101 (flake8-dunder-all >= 0.6.0) require a top-level PEP 562 __dir__ in every module.
+# This package has no module-level __getattr__ to justify one, and a __dir__ returning only __all__
+# would break agglovar.dev.imports.librl, which walks dir(module) to build its reload graph.
+#
+# These cannot go in extend-ignore: flake8 validates those codes against '^[A-Z]{1,3}[0-9]{0,3}$'
+# and the 4-letter DALL prefix fails. per-file-ignores skips that check, but selects the single
+# longest matching pattern rather than unioning them, so the codes must be repeated in every entry.
 per-file-ignores = [
-    "src/agglovar/merge/config/parser.py: D200, D205, D212, D300, D400, D401, D403, D415",
-    "src/agglovar/vcf/__init__.py: DALL002, F401, F403",
-    "src/agglovar/pairwise/overlap/__init__.py: DALL002, F401, F403",
+    "*: DALL100, DALL101",
+    "src/agglovar/merge/config/parser.py: DALL100, DALL101, D200, D205, D212, D300, D400, D401, D403, D415",
+    "src/agglovar/vcf/__init__.py: DALL100, DALL101, DALL002, F401, F403",
+    "src/agglovar/pairwise/overlap/__init__.py: DALL100, DALL101, DALL002, F401, F403",
 ]
 rst-roles = ["class", "meth", "const", "ref", "func", "mod", "attr", "data"]

{agglovar-0.0.1.dev17 → agglovar-0.0.1.dev18}/src/agglovar/__init__.py

@@ -1,6 +1,6 @@
 """Agglovar: A toolkit for fast genomic variant transformations and intersects."""
 
-__version__ = '0.0.1.dev17'
+__version__ = '0.0.1.dev18'
 
 __all__ = [
     'align',

{agglovar-0.0.1.dev17 → agglovar-0.0.1.dev18}/src/agglovar/bed/intersect.py

@@ -14,6 +14,7 @@ from .col import (
     get_coord_cols,
 )
 from .join import (
+    CHUNK_SIZE,
     pairwise_join,
     pairwise_join_iter,
 )
@@ -36,6 +37,7 @@ def as_bool(
         col_names_a: Optional[CoordCol | Iterable[str] | str] = None,
         col_names_b: Optional[CoordCol | Iterable[str] | str] = None,
         temp_dir: bool | str | Path = False,
+        chunk_size: int = CHUNK_SIZE,
 ) -> pl.LazyFrame:
     """Add a boolean column to df_a indicating whether each record intersects with df_b.
 
@@ -48,6 +50,8 @@ def as_bool(
     :param col_names_b: Columns in b (chromosome or query ID, pos, end).
     :param temp_dir: How to materialise the prepared tables before iterating. See
         :func:`agglovar.bed.join.pairwise_join`.
+    :param chunk_size: Chunk A by this size per chromosome. See
+        :func:`agglovar.bed.join.pairwise_join`.
 
     :return: A LazyFrame with two columns: ``_index`` and ``name``.
     """
@@ -65,12 +69,15 @@ def as_bool(
 
     hit_val = not negate
 
+    logger.debug('as_bool: name=%r distance=%d negate=%s' % (name, distance, negate))
+
     join_list = []
 
     for df_join in pairwise_join_iter(
             df_a=df_a,
             df_b=df_b,
             distance=distance,
+            chunk_size=chunk_size,
             col_names_a=col_names_a,
             col_names_b=col_names_b,
             temp_dir=temp_dir,
@@ -82,6 +89,8 @@ def as_bool(
             .lazy()
         )
 
+    logger.debug('as_bool: collected %d non-empty join chunk(s)' % len(join_list))
+
     if '_index' in df_a.collect_schema().names():
         df_a_indexed = df_a
     else:
@@ -119,6 +128,7 @@ def as_proportion(
         col_names_a: Optional[CoordCol | Iterable[str] | str] = None,
         col_names_b: Optional[CoordCol | Iterable[str] | str] = None,
         temp_dir: bool | str | Path = False,
+        chunk_size: int = CHUNK_SIZE,
 ) -> pl.LazyFrame:
     """Compute the proportion of each interval in ``df_a`` covered by intervals in ``df_b``.
 
@@ -126,12 +136,14 @@ def as_proportion(
     proportion. Zero-length intervals (``pos == end``) produce ``NaN`` (0 / 0).
 
     :param df_a: Table a.
-    :param df_b: Table b.
+    :param df_b: Table b.`
     :param name: Name of the column to add.
     :param col_names_a: Columns in a (chromosome or query ID, pos, end).
     :param col_names_b: Columns in b (chromosome or query ID, pos, end).
     :param temp_dir: How to materialise the prepared tables before iterating. See
         :func:`agglovar.bed.join.pairwise_join`.
+    :param chunk_size: Chunk A by this size per chromosome. See
+        :func:`agglovar.bed.join.pairwise_join`.
 
     :return: A LazyFrame with two columns: ``_index`` and ``name``.
     """
@@ -143,6 +155,8 @@ def as_proportion(
 
     col_expr_a = col_names_a.exprs()
 
+    logger.debug('as_proportion: name=%r' % name)
+
     if isinstance(df_a, pl.DataFrame):
         df_a = df_a.lazy()
 
@@ -165,9 +179,12 @@ def as_proportion(
         .collect()
     )
 
+    logger.debug('as_proportion: %d non-redundant B region(s) after merge_depth' % df_b_nr.height)
+
     df_join = pairwise_join(
         df_a=df_a_clean,
         df_b=df_b_nr,
+        chunk_size=chunk_size,
         col_names_a=col_names_a,
         col_names_b=col_names_b,
         temp_dir=temp_dir,

{agglovar-0.0.1.dev17 → agglovar-0.0.1.dev18}/src/agglovar/bed/join.py

@@ -7,6 +7,7 @@ __all__ = [
 
 import logging
 from pathlib import Path
+import time
 from typing import (
     Iterable,
     Iterator,
@@ -24,8 +25,14 @@ from .col import (
 
 logger = logging.getLogger(__name__)
 
-CHUNK_SIZE: int = 2_500
-"""Default size of join chunks. Breaks up tables into batches of this size or less."""
+CHUNK_SIZE: int = 25_000
+"""
+Default number of df_a records per chunk. Table A is processed one chromosome at a time and split
+into chunks of this size or less; each chunk drives one ``join_where`` (IEJoin) against the B
+records overlapping the chunk's position span. Larger chunks amortize the fixed per-chunk overhead
+(query setup and the per-chunk B pre-filter) over more rows, but widen each chunk's span so more B
+records enter each join. Tune based on available RAM.
+"""
 
 
 class _JoinResources:
@@ -194,6 +201,11 @@ def pairwise_join_iter(
         col_names_b=col_names_b,
     )
 
+    logger.debug(
+        'Starting bed join: distance=%d chunk_size=%d temp_dir=%r cols_a=%s cols_b=%s'
+        % (distance, chunk_size, temp_dir, tuple(resources.col_a), tuple(resources.col_b))
+    )
+
     with materialize_pair(
         resources.df_a, resources.df_b, temp_dir, prefix='bed_join_prep_',
     ) as (df_a_mat, df_b_mat):
@@ -273,6 +285,10 @@ def _join_chunks(
     col_a = join_resources.col_a
     col_b = join_resources.col_b
 
+    # Per-chunk timing and the df_b survivor count require extra collects, so gate them on DEBUG:
+    # an INFO-level run pays nothing and never materializes the diagnostics.
+    debug = logger.isEnabledFor(logging.DEBUG)
+
     # Restrict the chrom loop to chroms present in both tables. Chroms unique to A
     # would otherwise drive a wasted A chunk loop with no possible matches.
     for chrom, last_index_a in (
@@ -286,11 +302,30 @@ def _join_chunks(
         )
         .sort(col_a.chrom)
     ).collect().rows():
+        # Collect this chromosome's A and B once into memory. The chunk loop below re-filters
+        # df_a_chrom (per index range) and df_b_chrom (per prefilter bounds) many times; leaving
+        # them lazy re-runs the chrom filter over the full materialized df_a/df_b on every chunk,
+        # which dominates runtime (fixed per-chunk cost independent of the join size). Collecting
+        # per chrom makes the per-chunk filters operate on the small per-chrom frame instead.
+        # Mirrors agglovar.pairwise.overlap._overlap._join_iter_chunked.
         df_a_chrom = (
             df_a.filter(pl.col(col_a.chrom) == chrom)
             .with_row_index('_index_chrom_a')
+            .collect()
+            .lazy()
         )
-        df_b_chrom = df_b.filter(pl.col(col_b.chrom) == chrom)
+        df_b_chrom = (
+            df_b.filter(pl.col(col_b.chrom) == chrom)
+            .collect()
+            .lazy()
+        )
+
+        if debug:
+            chrom_t0 = time.perf_counter()
+            chrom_chunks = 0
+            chrom_pairs = 0
+            chrom_join_s = 0.0
+            chrom_b_max = 0
 
         start_index_a = 0
         while start_index_a < last_index_a:
@@ -323,12 +358,39 @@ def _join_chunks(
                 pl.col(col_b.end) + distance >= pos_min,
             )
 
-            yield _build_join_pair(
-                df_a_chunk,
-                df_b_chunk,
-                distance,
-                col_a,
-                col_b,
-            ).collect().lazy()
+            # The join itself is computed identically regardless of log level; only the diagnostics
+            # below are gated on DEBUG, so raising the log level cannot change the result.
+            if debug:
+                t0 = time.perf_counter()
+
+            df_join = _build_join_pair(df_a_chunk, df_b_chunk, distance, col_a, col_b).collect()
 
+            if debug:
+                dt = time.perf_counter() - t0
+                # n_b is the count of B records surviving the range prefilter for this chunk. When
+                # A is position-local it stays small; when it approaches the chrom's B total the
+                # prefilter is not pruning. pos_min-end_max is the chunk's position span.
+                n_b = df_b_chunk.select(pl.len()).collect().item()
+
+                logger.debug(
+                    'Join chunk: chrom %s [%d:%d] pos=%s-%s n_b=%d pairs=%d in %.3fs'
+                    % (chrom, start_index_a, end_index_a, pos_min, end_max,
+                       n_b, df_join.height, dt)
+                )
+
+                chrom_chunks += 1
+                chrom_pairs += df_join.height
+                chrom_join_s += dt
+                if n_b > chrom_b_max:
+                    chrom_b_max = n_b
+
+            yield df_join.lazy()
             start_index_a = end_index_a
+
+        if debug:
+            logger.debug(
+                'Join chrom done: chrom %s n_a=%d chunks=%d n_b_max=%d pairs=%d '
+                'join=%.3fs wall=%.3fs'
+                % (chrom, last_index_a, chrom_chunks, chrom_b_max, chrom_pairs,
+                   chrom_join_s, time.perf_counter() - chrom_t0)
+            )

{agglovar-0.0.1.dev17 → agglovar-0.0.1.dev18}/src/agglovar/bed/merge.py

@@ -26,7 +26,7 @@ def merge_depth(
         distance: int = 0,
         col_names: Optional[CoordCol | Iterable[str] | str] = None,
         col_max_depth: Optional[str] = 'max_depth'
-):
+) -> pl.LazyFrame:
     """Merge overlapping regions in a table.
 
     :param df: Table of coordinates.
@@ -48,6 +48,11 @@ def merge_depth(
     if isinstance(df, pl.DataFrame):
         df = df.lazy()
 
+    logger.debug(
+        'merge_depth: distance=%d cols=%s max_depth=%r'
+        % (distance, tuple(col_names), col_max_depth)
+    )
+
     return (
         melt_depth(
             df=df.with_columns(pl.col(col_names.end) + distance),
@@ -75,7 +80,7 @@ def melt_depth(
         col_names: CoordCol,
         col_loc: str = '_loc',
         col_depth: str = '_depth',
-):
+) -> pl.LazyFrame:
     """Get a table of depths by locations of changes.
 
     The table will have one row for each location of change with end positions having two entries (one for the
@@ -158,6 +163,8 @@ def melt_depth(
 
     assert col_loc != col_depth, 'Column names "%s" and "%s" are the same' % (col_loc, col_depth)
 
+    logger.debug('melt_depth: cols=%s loc=%s depth=%s' % (tuple(col_names), col_loc, col_depth))
+
     return (
         pl.concat(
             [

{agglovar-0.0.1.dev17 → agglovar-0.0.1.dev18}/src/agglovar/expr/variant.py

@@ -6,6 +6,7 @@ __all__ = [
     'sort_cols',
     'id_expr_noalias',
     'id_expr',
+    'id_expr_for_table',
     'lead_src_expr',
 ]
 
@@ -14,6 +15,8 @@ from typing import Optional
 
 import polars as pl
 
+import agglovar.schema as _agg_schema
+
 logger = logging.getLogger(__name__)
 
 id_snv_expr = (
@@ -64,7 +67,41 @@ id_expr_noalias = (
 """Expression for generating variant IDs."""
 
 id_expr = id_expr_noalias.alias('id')
-"""Expression for generating variant IDs including alias "id" for the result (ideal for `with_column()`)."""
+"""Expression for generating variant IDs including alias "id" for the result (ideal for `with_column()`).
+
+Requires both the SNV columns (``ref``, ``alt``) and the non-SNV column (``varlen``) to be present:
+Polars resolves both branches of the expression regardless of which one a row selects.  Use
+:func:`id_expr_for_table` for single-type tables, which carry only the columns listed in
+:data:`agglovar.schema.STANDARD_FIELDS` for their type.
+"""
+
+
+def id_expr_for_table(table: str, alias: Optional[str] = 'id') -> pl.Expr:
+    """Get a variant ID expression for a single variant-type table.
+
+    Single-type tables carry only the columns listed for their type in
+    :data:`agglovar.schema.STANDARD_FIELDS`, so :data:`id_expr` cannot be applied to them (see its
+    documentation).  This routine returns the one branch that matches the table's columns.
+
+    :param table: Table name from :data:`agglovar.schema.STANDARD_FIELDS` (e.g. "snv", "insdel").
+    :param alias: Output column name for the resulting IDs, or `None` to leave it unaliased.
+
+    :returns: An expression yielding variant IDs for *table*.
+
+    :raises ValueError: If *table* is not a known variant-type table.
+    """
+    fields = _agg_schema.STANDARD_FIELDS.get(table)
+
+    if fields is None:
+        raise ValueError(
+            f'Unknown variant-type table: {table!r}: '
+            f'expected one of {", ".join(sorted(_agg_schema.STANDARD_FIELDS))}'
+        )
+
+    # Tables without "varlen" (SNV) are identified by their changed bases, all others by length
+    expr = id_nonsnv_expr if 'varlen' in fields else id_snv_expr
+
+    return expr.alias(alias) if alias is not None else expr
 
 
 def lead_src_expr(

{agglovar-0.0.1.dev17 → agglovar-0.0.1.dev18}/src/agglovar/merge/cumulative.py

@@ -30,6 +30,7 @@ __all__ = [
 ]
 
 from collections.abc import Iterable
+from concurrent.futures import Executor
 from enum import Enum
 import logging
 from pathlib import Path
@@ -176,6 +177,7 @@ class MergeCumulative(MergeBase):
             sort: bool = True,
             add_id: bool = True,
             temp_dir: bool | str | Path = False,
+            match_threads: Optional[int] = 4,
     ) -> pl.LazyFrame:
         """
         Intersect callsets.
@@ -187,6 +189,13 @@ class MergeCumulative(MergeBase):
             last (after "_index" is set).
         :param temp_dir: Forwarded to the pairwise intersect for each cumulative step. See
             :meth:`agglovar.pairwise.base.PairwiseJoin.join_iter`.
+        :param match_threads: Number of worker processes used to score sequence matches
+            (``match_prop``) during the merge. A single ``spawn``-context process pool is created
+            once and reused across all sources. Defaults to 4. If ``None``, no pool is created and
+            match scoring runs serially (turns multiprocessing off entirely). Ignored when the
+            pairwise join does no sequence matching. In a spawn-restricted or resource-limited
+            environment the pool degrades to serial scoring with a logged warning (see
+            :meth:`agglovar.pairwise.overlap.PairwiseOverlap.make_match_pool`).
 
         :return: A merged callset table.
         """
@@ -336,13 +345,62 @@ class MergeCumulative(MergeBase):
 
         logger.debug('Merge source %d: Cumulative size %s' % (iter_idx, f'{df_cumulative.height:,}'))
 
+        # Create one match-scoring pool for the whole merge (spawn context, sized by match_threads;
+        # None or a restricted environment keeps scoring serial). The pool is owned here and passed
+        # into each per-source join; it is never retained on the pairwise object.
+        with self.pairwise_join.make_match_pool(match_threads) as match_pool:
+            df_cumulative, df_sources = self._merge_sources(
+                callsets_mat=callsets_mat,
+                df_cumulative=df_cumulative,
+                df_sources=df_sources,
+                next_mg_index=next_mg_index,
+                merge_stat_cols=merge_stat_cols,
+                stat_struct_dtype=stat_struct_dtype,
+                sources_schema=sources_schema,
+                pairwise_cols=pairwise_cols,
+                mg_sort_expr=mg_sort_expr,
+                temp_dir=temp_dir,
+                match_pool=match_pool,
+            )
+
+        return self._finalize(
+            df_sources=df_sources,
+            callsets_mat=callsets_mat,
+            all_col_dict=all_col_dict,
+            sort=sort,
+            add_id=add_id,
+        )
+
+    def _merge_sources(
+            self,
+            callsets_mat: list[dict[str, Any]],
+            df_cumulative: pl.DataFrame,
+            df_sources: pl.DataFrame,
+            next_mg_index: int,
+            merge_stat_cols: dict[str, pl.DataType],
+            stat_struct_dtype: pl.DataType,
+            sources_schema: pl.Schema,
+            pairwise_cols: list[str],
+            mg_sort_expr: pl.Expr,
+            temp_dir: bool | str | Path,
+            match_pool: Optional[Executor] = None,
+    ) -> tuple[pl.DataFrame, pl.DataFrame]:
+        """Merge each remaining callset into the cumulative table, one source at a time.
+
+        The ``match_pool`` created by :meth:`__call__` is passed to each per-source pairwise join so
+        sequence matches score on the shared process pool (or serially when it is ``None``). Returns
+        the updated cumulative variant table and the long-form sources sidecar.
+        """
         # Iterate remaining callsets.
         for iter_idx, callset_mat in enumerate(callsets_mat[1:], start=1):
             logger.debug('Merge source %d: Merging "%s"' % (iter_idx, callset_mat['name']))
 
             df_join = (
-                self.pairwise_join.join(df_cumulative.lazy(), callset_mat['table'].lazy(), temp_dir=temp_dir)
-                .sort('weight', descending=True)
+                self.pairwise_join.join(
+                    df_cumulative.lazy(), callset_mat['table'].lazy(),
+                    temp_dir=temp_dir, match_pool=match_pool,
+                )
+                .sort('weight', descending=True, maintain_order=True)
                 .unique('index_a', keep='first')
                 .unique('index_b', keep='first')
                 .collect()
@@ -460,13 +518,7 @@ class MergeCumulative(MergeBase):
 
             logger.debug('Merge source %d: Cumulative size %s' % (iter_idx, f'{df_cumulative.height:,}'))
 
-        return self._finalize(
-            df_sources=df_sources,
-            callsets_mat=callsets_mat,
-            all_col_dict=all_col_dict,
-            sort=sort,
-            add_id=add_id,
-        )
+        return df_cumulative, df_sources
 
     def _finalize(
             self,

{agglovar-0.0.1.dev17 → agglovar-0.0.1.dev18}/src/agglovar/pairwise/base.py

@@ -15,8 +15,11 @@ from collections.abc import (
     Iterator,
     Iterable,
 )
+from concurrent.futures import Executor
+from contextlib import contextmanager
 import logging
 from pathlib import Path
+from typing import Optional
 
 import polars as pl
 
@@ -54,6 +57,7 @@ class PairwiseJoin(ABC):
             df_b: pl.DataFrame | pl.LazyFrame,
             retain_index: bool = False,
             temp_dir: bool | str | Path = False,
+            match_pool: Optional[Executor] = None,
     ) -> Iterator[pl.LazyFrame]:
         """Find all pairs of variants in two sources that meet a set of criteria.
 
@@ -65,6 +69,9 @@ class PairwiseJoin(ABC):
             ``False`` (default) collects both into memory; ``True`` writes them to the
             system temp directory as parquet files; a ``str``/``Path`` writes them to
             that directory. Temp files are always removed on exit.
+        :param match_pool: Optional worker pool for parallel sequence matching (from
+            :meth:`make_match_pool`), or ``None`` to score serially. Ignored by strategies that do
+            no sequence matching.
 
         :yields: A LazyFrame for each chunk.
         """
@@ -76,6 +83,7 @@ class PairwiseJoin(ABC):
             df_b: pl.DataFrame | pl.LazyFrame,
             retain_index: bool = False,
             temp_dir: bool | str | Path = False,
+            match_pool: Optional[Executor] = None,
     ) -> pl.LazyFrame:
         """Find all pairs of variants in two sources that meet a set of criteria.
 
@@ -86,13 +94,32 @@ class PairwiseJoin(ABC):
         :param retain_index: If True, do not drop an existing "_index" column in callset tables
             if they exist.
         :param temp_dir: See :meth:`join_iter`.
+        :param match_pool: See :meth:`join_iter`.
 
         :returns: A join table.
         """
         return pl.concat(
-            self.join_iter(df_a, df_b, retain_index=retain_index, temp_dir=temp_dir)
+            self.join_iter(
+                df_a, df_b, retain_index=retain_index, temp_dir=temp_dir, match_pool=match_pool,
+            )
         )
 
+    @contextmanager
+    def make_match_pool(self, match_threads: Optional[int] = None) -> Iterator[Optional[Executor]]:
+        """Context manager yielding a worker pool for parallel sequence matching, or ``None``.
+
+        The default is a no-op that always yields ``None`` (serial scoring): strategies that do no
+        sequence matching (or do not parallelize it) ignore ``match_threads``. Subclasses that
+        parallelize sequence matching override this to create and tear down a worker pool the caller
+        passes to :meth:`join`/:meth:`join_iter` via ``match_pool``. See
+        :meth:`agglovar.pairwise.overlap.PairwiseOverlap.make_match_pool`.
+
+        :param match_threads: Requested worker count, or ``None`` to keep scoring serial.
+
+        :yields: A worker pool, or ``None`` when scoring serially.
+        """
+        yield None
+
     @property
     @abstractmethod
     def required_cols(self) -> set[str]:

{agglovar-0.0.1.dev17 → agglovar-0.0.1.dev18}/src/agglovar/pairwise/overlap/__init__.py

@@ -219,6 +219,42 @@ but they are different and both have merits. While Agglovar compares *only* vari
 if they are shifted, Truvari includes the shift in the sequence match (i.e. the compared sequences
 include the variant and flanking reference bases).
 
+Minimum segment reciprocal overlap (seg_ro_min)
+-----------------------------------------------
+
+**seg_ro_min**: Minimum segment reciprocal overlap (segment RO) for complex variants.
+
+A complex variant is not one contiguous change but a set of *segments*, some aligned to a reference
+locus and some not aligned at all. Comparing two complex variants by their reference span alone is
+crude: two variants may cover the same span while being built from entirely different pieces.
+Segment RO instead compares the pieces.
+
+Segment RO is the size of the overlap between the two variants' aligned segments, treating segments
+as a multiset. Both variants' segments are broken into ranges at every segment boundary, and each
+range contributes its length times the *smaller* of the two variants' segment depths over it, where
+depth is the number of segments covering the range. Taking the smaller depth is what keeps a segment
+from being counted more than once when it overlaps several segments on the other side, and it makes
+a duplicated segment count only as many times as both variants duplicate it. Segments overlap only
+within the same chromosome and orientation.
+
+Unaligned bases have no reference locus, so there is nothing to intersect them against. The smaller
+of the two variants' unaligned lengths is credited as overlapping, which treats unaligned bases as
+maximally agreeing:
+
+.. code-block:: python
+
+    seg_ro = (overlap + min(unaligned_a, unaligned_b)) / max(total_len_a, total_len_b)
+
+Where `total_len` is the length of all of a variant's segments, aligned and unaligned. Segment RO is
+symmetric and is exactly 1.0 for a variant compared against itself.
+
+Segment RO requires "seg", "qry_pos", and "qry_end" columns on both input tables. The "seg" column
+is a list of segment structs with "chrom", "pos", "end", "qry_pos", "qry_end", and "is_rev" fields.
+
+.. note::
+   `seg_ro_min` is applied after the join, so it does not narrow the candidate pairs on its own.
+   Pair it with a criterion that does, such as `ro_min`.
+
 Reasonable join parameters
 --------------------------
 

{agglovar-0.0.1.dev17 → agglovar-0.0.1.dev18}/src/agglovar/pairwise/overlap/_const.py

@@ -4,6 +4,7 @@ __all__ = [
     'RESERVED_COLS',
     'AUTOGEN_COLS',
     'DEFAULT_CHUNK_SIZE',
+    'MATCH_CHUNK_SIZE',
     'EXPR_OVERLAP_RO',
     'EXPR_SZRO',
     'EXPR_OFFSET_DIST',
@@ -22,6 +23,8 @@ import polars as pl
 RESERVED_COLS: frozenset[str] = frozenset({
     '_index',
     '_end_ro',
+    '_seg_unaligned',
+    '_seg_total_len',
 })
 """Reserved columns are added automatically to input tables."""
 
@@ -38,6 +41,14 @@ size presented to the sequence-alignment thread pool — larger values amortize
 better but use more memory. Tune alongside ``n_threads`` based on available RAM and CPU count.
 """
 
+MATCH_CHUNK_SIZE: int = 128
+"""
+Number of ``(seq_a, seq_b)`` pairs scored per worker task when ``match_prop`` is computed on a
+process pool. A small fixed chunk balances the heavy-tailed sequence-length distribution across
+workers; larger chunks starve workers on the long tail. Batches smaller than ``2 * MATCH_CHUNK_SIZE``
+are scored serially because the pool's per-batch overhead outweighs the parallel gain (benchmarked).
+"""
+
 EXPR_OVERLAP_RO = (
     (
         (