agglovar 0.0.1.dev16__tar.gz → 0.0.1.dev18__tar.gz

{agglovar-0.0.1.dev16/src/agglovar.egg-info → agglovar-0.0.1.dev18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agglovar
-Version: 0.0.1.dev16
+Version: 0.0.1.dev18
 Summary: Toolkit for fast genomic variant transformations and intersects
 License-Expression: MIT
 Requires-Python: >=3.12
@@ -10,7 +10,7 @@ Requires-Dist: biopython>=1.85
 Requires-Dist: edlib>=1.3.9.post1
 Requires-Dist: numpy>=2.3.0
 Requires-Dist: ply>=3.11
-Requires-Dist: polars[numpy,pandas,pyarrow]>=1.35.2
+Requires-Dist: polars[numpy,pandas,pyarrow]==1.42.1
 Requires-Dist: pysam>=0.23.3
 Provides-Extra: plot
 Requires-Dist: matplotlib>=3.10.3; extra == "plot"

{agglovar-0.0.1.dev16 → agglovar-0.0.1.dev18}/pyproject.toml

@@ -15,7 +15,9 @@ dependencies = [
     "edlib>=1.3.9.post1",
     "numpy>=2.3.0",
     "ply>=3.11",
-    "polars[numpy,pandas,pyarrow]>=1.35.2",
+    # 1.41.2: merge_sorted() crashes on List(Struct) columns before this (used by the cumulative
+    # merge to carry the complex-variant "seg" column).
+    "polars[numpy,pandas,pyarrow]==1.42.1",
     "pysam>=0.23.3",
 ]
 
@@ -52,11 +54,10 @@ maintainers = [
 
 [dependency-groups]
 dev = [
-    "arrow[parquet]>=1.4.0",
     "flake8>=7.3.0",
     "flake8-bugbear>=24.12.12",
     "flake8-docstrings>=1.7.0",
-    "flake8-dunder-all>=0.5.0",
+    "flake8-dunder-all>=0.6.0",
     "flake8-mutable>=1.2.0",
     "flake8-pyproject>=1.2.3",
     "flake8-rst-docstrings>=0.3.1",
@@ -107,9 +108,17 @@ max-line-length = 120
 [tool.flake8]
 max-line-length = 120
 docstring-convention = "pep257"
+# DALL100/DALL101 (flake8-dunder-all >= 0.6.0) require a top-level PEP 562 __dir__ in every module.
+# This package has no module-level __getattr__ to justify one, and a __dir__ returning only __all__
+# would break agglovar.dev.imports.librl, which walks dir(module) to build its reload graph.
+#
+# These cannot go in extend-ignore: flake8 validates those codes against '^[A-Z]{1,3}[0-9]{0,3}$'
+# and the 4-letter DALL prefix fails. per-file-ignores skips that check, but selects the single
+# longest matching pattern rather than unioning them, so the codes must be repeated in every entry.
 per-file-ignores = [
-    "src/agglovar/merge/config/parser.py: D200, D205, D212, D300, D400, D401, D403, D415",
-    "src/agglovar/vcf/__init__.py: DALL002, F401, F403",
-    "src/agglovar/pairwise/overlap/__init__.py: DALL002, F401, F403",
+    "*: DALL100, DALL101",
+    "src/agglovar/merge/config/parser.py: DALL100, DALL101, D200, D205, D212, D300, D400, D401, D403, D415",
+    "src/agglovar/vcf/__init__.py: DALL100, DALL101, DALL002, F401, F403",
+    "src/agglovar/pairwise/overlap/__init__.py: DALL100, DALL101, DALL002, F401, F403",
 ]
 rst-roles = ["class", "meth", "const", "ref", "func", "mod", "attr", "data"]

{agglovar-0.0.1.dev16 → agglovar-0.0.1.dev18}/src/agglovar/__init__.py

@@ -1,6 +1,6 @@
 """Agglovar: A toolkit for fast genomic variant transformations and intersects."""
 
-__version__ = '0.0.1.dev16'
+__version__ = '0.0.1.dev18'
 
 __all__ = [
     'align',
@@ -29,3 +29,9 @@ from . import schema
 from . import seqmatch
 from . import vcf
 from . import util
+
+
+# Configure logging
+
+import logging
+logging.getLogger(__name__).addHandler(logging.NullHandler())

{agglovar-0.0.1.dev16 → agglovar-0.0.1.dev18}/src/agglovar/align/op.py

@@ -42,10 +42,14 @@ __all__ = [
 ]
 
 
-import numpy as np
+import logging
 from types import MappingProxyType
 from typing import Mapping
 
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
 INT_STR_SET: frozenset[str] = frozenset({'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', })
 """Set of valid integer character strings representing operation codes."""
 

{agglovar-0.0.1.dev16 → agglovar-0.0.1.dev18}/src/agglovar/align/score.py

@@ -12,13 +12,24 @@ __all__ = [
     'get_affine_by_params',
 ]
 
-from abc import ABC, abstractmethod
-import numpy as np
+from abc import (
+    ABC,
+    abstractmethod,
+)
+import logging
 import re
-from typing import Any, Iterable, Self
+from typing import (
+    Any,
+    Iterable,
+    Self,
+)
+
+import numpy as np
 
 from . import op
 
+logger = logging.getLogger(__name__)
+
 AFFINE_SCORE_MATCH = 2.0
 """Default match score."""
 

{agglovar-0.0.1.dev16 → agglovar-0.0.1.dev18}/src/agglovar/bed/col.py

@@ -7,13 +7,23 @@ __all__ = [
     'standardize',
 ]
 
-from collections.abc import Iterable, Container
+from collections.abc import (
+    Iterable,
+    Container,
+)
 from dataclasses import dataclass
+import logging
 import re
-from typing import Optional, Self, Any
+from typing import (
+    Optional,
+    Self,
+    Any,
+)
 
 import polars as pl
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass(frozen=True)
 class CoordColExpr(Iterable[pl.Expr]):

{agglovar-0.0.1.dev16 → agglovar-0.0.1.dev18}/src/agglovar/bed/intersect.py

@@ -1,19 +1,32 @@
 """Table intersects."""
 
+import logging
 from pathlib import Path
-from typing import Iterable, Optional
+from typing import (
+    Iterable,
+    Optional,
+)
 
 import polars as pl
 
-from .join import pairwise_join, pairwise_join_iter
+from .col import (
+    CoordCol,
+    get_coord_cols,
+)
+from .join import (
+    CHUNK_SIZE,
+    pairwise_join,
+    pairwise_join_iter,
+)
 from .merge import merge_depth
-from .col import CoordCol, get_coord_cols
 
 __all__ = [
     'as_bool',
     'as_proportion',
 ]
 
+logger = logging.getLogger(__name__)
+
 
 def as_bool(
         df_a: pl.LazyFrame | pl.DataFrame,
@@ -24,6 +37,7 @@ def as_bool(
         col_names_a: Optional[CoordCol | Iterable[str] | str] = None,
         col_names_b: Optional[CoordCol | Iterable[str] | str] = None,
         temp_dir: bool | str | Path = False,
+        chunk_size: int = CHUNK_SIZE,
 ) -> pl.LazyFrame:
     """Add a boolean column to df_a indicating whether each record intersects with df_b.
 
@@ -36,6 +50,8 @@ def as_bool(
     :param col_names_b: Columns in b (chromosome or query ID, pos, end).
     :param temp_dir: How to materialise the prepared tables before iterating. See
         :func:`agglovar.bed.join.pairwise_join`.
+    :param chunk_size: Chunk A by this size per chromosome. See
+        :func:`agglovar.bed.join.pairwise_join`.
 
     :return: A LazyFrame with two columns: ``_index`` and ``name``.
     """
@@ -53,12 +69,15 @@ def as_bool(
 
     hit_val = not negate
 
+    logger.debug('as_bool: name=%r distance=%d negate=%s' % (name, distance, negate))
+
     join_list = []
 
     for df_join in pairwise_join_iter(
             df_a=df_a,
             df_b=df_b,
             distance=distance,
+            chunk_size=chunk_size,
             col_names_a=col_names_a,
             col_names_b=col_names_b,
             temp_dir=temp_dir,
@@ -70,6 +89,8 @@ def as_bool(
             .lazy()
         )
 
+    logger.debug('as_bool: collected %d non-empty join chunk(s)' % len(join_list))
+
     if '_index' in df_a.collect_schema().names():
         df_a_indexed = df_a
     else:
@@ -107,6 +128,7 @@ def as_proportion(
         col_names_a: Optional[CoordCol | Iterable[str] | str] = None,
         col_names_b: Optional[CoordCol | Iterable[str] | str] = None,
         temp_dir: bool | str | Path = False,
+        chunk_size: int = CHUNK_SIZE,
 ) -> pl.LazyFrame:
     """Compute the proportion of each interval in ``df_a`` covered by intervals in ``df_b``.
 
@@ -114,12 +136,14 @@ def as_proportion(
     proportion. Zero-length intervals (``pos == end``) produce ``NaN`` (0 / 0).
 
     :param df_a: Table a.
-    :param df_b: Table b.
+    :param df_b: Table b.`
     :param name: Name of the column to add.
     :param col_names_a: Columns in a (chromosome or query ID, pos, end).
     :param col_names_b: Columns in b (chromosome or query ID, pos, end).
     :param temp_dir: How to materialise the prepared tables before iterating. See
         :func:`agglovar.bed.join.pairwise_join`.
+    :param chunk_size: Chunk A by this size per chromosome. See
+        :func:`agglovar.bed.join.pairwise_join`.
 
     :return: A LazyFrame with two columns: ``_index`` and ``name``.
     """
@@ -131,6 +155,8 @@ def as_proportion(
 
     col_expr_a = col_names_a.exprs()
 
+    logger.debug('as_proportion: name=%r' % name)
+
     if isinstance(df_a, pl.DataFrame):
         df_a = df_a.lazy()
 
@@ -153,9 +179,12 @@ def as_proportion(
         .collect()
     )
 
+    logger.debug('as_proportion: %d non-redundant B region(s) after merge_depth' % df_b_nr.height)
+
     df_join = pairwise_join(
         df_a=df_a_clean,
         df_b=df_b_nr,
+        chunk_size=chunk_size,
         col_names_a=col_names_a,
         col_names_b=col_names_b,
         temp_dir=temp_dir,

{agglovar-0.0.1.dev16 → agglovar-0.0.1.dev18}/src/agglovar/bed/join.py

@@ -5,16 +5,34 @@ __all__ = [
     'pairwise_join_iter',
 ]
 
+import logging
 from pathlib import Path
-from typing import Iterable, Iterator, Optional
+import time
+from typing import (
+    Iterable,
+    Iterator,
+    Optional,
+)
 
 import polars as pl
 
-from .col import CoordCol, get_coord_cols
 from ..util.lazy import materialize_pair
 
-CHUNK_SIZE: int = 2_500
-"""Default size of join chunks. Breaks up tables into batches of this size or less."""
+from .col import (
+    CoordCol,
+    get_coord_cols,
+)
+
+logger = logging.getLogger(__name__)
+
+CHUNK_SIZE: int = 25_000
+"""
+Default number of df_a records per chunk. Table A is processed one chromosome at a time and split
+into chunks of this size or less; each chunk drives one ``join_where`` (IEJoin) against the B
+records overlapping the chunk's position span. Larger chunks amortize the fixed per-chunk overhead
+(query setup and the per-chunk B pre-filter) over more rows, but widen each chunk's span so more B
+records enter each join. Tune based on available RAM.
+"""
 
 
 class _JoinResources:
@@ -183,6 +201,11 @@ def pairwise_join_iter(
         col_names_b=col_names_b,
     )
 
+    logger.debug(
+        'Starting bed join: distance=%d chunk_size=%d temp_dir=%r cols_a=%s cols_b=%s'
+        % (distance, chunk_size, temp_dir, tuple(resources.col_a), tuple(resources.col_b))
+    )
+
     with materialize_pair(
         resources.df_a, resources.df_b, temp_dir, prefix='bed_join_prep_',
     ) as (df_a_mat, df_b_mat):
@@ -262,6 +285,10 @@ def _join_chunks(
     col_a = join_resources.col_a
     col_b = join_resources.col_b
 
+    # Per-chunk timing and the df_b survivor count require extra collects, so gate them on DEBUG:
+    # an INFO-level run pays nothing and never materializes the diagnostics.
+    debug = logger.isEnabledFor(logging.DEBUG)
+
     # Restrict the chrom loop to chroms present in both tables. Chroms unique to A
     # would otherwise drive a wasted A chunk loop with no possible matches.
     for chrom, last_index_a in (
@@ -275,11 +302,30 @@ def _join_chunks(
         )
         .sort(col_a.chrom)
     ).collect().rows():
+        # Collect this chromosome's A and B once into memory. The chunk loop below re-filters
+        # df_a_chrom (per index range) and df_b_chrom (per prefilter bounds) many times; leaving
+        # them lazy re-runs the chrom filter over the full materialized df_a/df_b on every chunk,
+        # which dominates runtime (fixed per-chunk cost independent of the join size). Collecting
+        # per chrom makes the per-chunk filters operate on the small per-chrom frame instead.
+        # Mirrors agglovar.pairwise.overlap._overlap._join_iter_chunked.
         df_a_chrom = (
             df_a.filter(pl.col(col_a.chrom) == chrom)
             .with_row_index('_index_chrom_a')
+            .collect()
+            .lazy()
+        )
+        df_b_chrom = (
+            df_b.filter(pl.col(col_b.chrom) == chrom)
+            .collect()
+            .lazy()
         )
-        df_b_chrom = df_b.filter(pl.col(col_b.chrom) == chrom)
+
+        if debug:
+            chrom_t0 = time.perf_counter()
+            chrom_chunks = 0
+            chrom_pairs = 0
+            chrom_join_s = 0.0
+            chrom_b_max = 0
 
         start_index_a = 0
         while start_index_a < last_index_a:
@@ -312,12 +358,39 @@ def _join_chunks(
                 pl.col(col_b.end) + distance >= pos_min,
             )
 
-            yield _build_join_pair(
-                df_a_chunk,
-                df_b_chunk,
-                distance,
-                col_a,
-                col_b,
-            ).collect().lazy()
+            # The join itself is computed identically regardless of log level; only the diagnostics
+            # below are gated on DEBUG, so raising the log level cannot change the result.
+            if debug:
+                t0 = time.perf_counter()
 
+            df_join = _build_join_pair(df_a_chunk, df_b_chunk, distance, col_a, col_b).collect()
+
+            if debug:
+                dt = time.perf_counter() - t0
+                # n_b is the count of B records surviving the range prefilter for this chunk. When
+                # A is position-local it stays small; when it approaches the chrom's B total the
+                # prefilter is not pruning. pos_min-end_max is the chunk's position span.
+                n_b = df_b_chunk.select(pl.len()).collect().item()
+
+                logger.debug(
+                    'Join chunk: chrom %s [%d:%d] pos=%s-%s n_b=%d pairs=%d in %.3fs'
+                    % (chrom, start_index_a, end_index_a, pos_min, end_max,
+                       n_b, df_join.height, dt)
+                )
+
+                chrom_chunks += 1
+                chrom_pairs += df_join.height
+                chrom_join_s += dt
+                if n_b > chrom_b_max:
+                    chrom_b_max = n_b
+
+            yield df_join.lazy()
             start_index_a = end_index_a
+
+        if debug:
+            logger.debug(
+                'Join chrom done: chrom %s n_a=%d chunks=%d n_b_max=%d pairs=%d '
+                'join=%.3fs wall=%.3fs'
+                % (chrom, last_index_a, chrom_chunks, chrom_b_max, chrom_pairs,
+                   chrom_join_s, time.perf_counter() - chrom_t0)
+            )

{agglovar-0.0.1.dev16 → agglovar-0.0.1.dev18}/src/agglovar/bed/merge.py

@@ -4,11 +4,21 @@ __all__ = [
     'merge_depth',
 ]
 
-from typing import Optional, Iterable
+import logging
+from typing import (
+    Optional,
+    Iterable,
+)
 
 import polars as pl
 
-from .col import CoordCol, get_coord_cols, make_unique_col
+from .col import (
+    CoordCol,
+    get_coord_cols,
+    make_unique_col,
+)
+
+logger = logging.getLogger(__name__)
 
 
 def merge_depth(
@@ -16,7 +26,7 @@ def merge_depth(
         distance: int = 0,
         col_names: Optional[CoordCol | Iterable[str] | str] = None,
         col_max_depth: Optional[str] = 'max_depth'
-):
+) -> pl.LazyFrame:
     """Merge overlapping regions in a table.
 
     :param df: Table of coordinates.
@@ -38,6 +48,11 @@ def merge_depth(
     if isinstance(df, pl.DataFrame):
         df = df.lazy()
 
+    logger.debug(
+        'merge_depth: distance=%d cols=%s max_depth=%r'
+        % (distance, tuple(col_names), col_max_depth)
+    )
+
     return (
         melt_depth(
             df=df.with_columns(pl.col(col_names.end) + distance),
@@ -65,7 +80,7 @@ def melt_depth(
         col_names: CoordCol,
         col_loc: str = '_loc',
         col_depth: str = '_depth',
-):
+) -> pl.LazyFrame:
     """Get a table of depths by locations of changes.
 
     The table will have one row for each location of change with end positions having two entries (one for the
@@ -148,6 +163,8 @@ def melt_depth(
 
     assert col_loc != col_depth, 'Column names "%s" and "%s" are the same' % (col_loc, col_depth)
 
+    logger.debug('melt_depth: cols=%s loc=%s depth=%s' % (tuple(col_names), col_loc, col_depth))
+
     return (
         pl.concat(
             [

{agglovar-0.0.1.dev16 → agglovar-0.0.1.dev18}/src/agglovar/dev/imports.py

@@ -11,14 +11,24 @@ __all__ = [
 
 import ast
 from collections.abc import Iterable
-from dataclasses import dataclass, field
+from dataclasses import (
+    dataclass,
+    field,
+)
 from graphlib import TopologicalSorter
 import importlib
 import inspect
+import logging
 import re
 import sys
 from types import ModuleType
-from typing import Any, Generator, Optional
+from typing import (
+    Any,
+    Generator,
+    Optional,
+)
+
+logger = logging.getLogger(__name__)
 
 
 @dataclass(frozen=True)

{agglovar-0.0.1.dev16 → agglovar-0.0.1.dev18}/src/agglovar/expr/variant.py

@@ -6,13 +6,19 @@ __all__ = [
     'sort_cols',
     'id_expr_noalias',
     'id_expr',
+    'id_expr_for_table',
     'lead_src_expr',
 ]
 
+import logging
 from typing import Optional
 
 import polars as pl
 
+import agglovar.schema as _agg_schema
+
+logger = logging.getLogger(__name__)
+
 id_snv_expr = (
     pl.concat_str(
         pl.col('chrom'),
@@ -61,7 +67,41 @@ id_expr_noalias = (
 """Expression for generating variant IDs."""
 
 id_expr = id_expr_noalias.alias('id')
-"""Expression for generating variant IDs including alias "id" for the result (ideal for `with_column()`)."""
+"""Expression for generating variant IDs including alias "id" for the result (ideal for `with_column()`).
+
+Requires both the SNV columns (``ref``, ``alt``) and the non-SNV column (``varlen``) to be present:
+Polars resolves both branches of the expression regardless of which one a row selects.  Use
+:func:`id_expr_for_table` for single-type tables, which carry only the columns listed in
+:data:`agglovar.schema.STANDARD_FIELDS` for their type.
+"""
+
+
+def id_expr_for_table(table: str, alias: Optional[str] = 'id') -> pl.Expr:
+    """Get a variant ID expression for a single variant-type table.
+
+    Single-type tables carry only the columns listed for their type in
+    :data:`agglovar.schema.STANDARD_FIELDS`, so :data:`id_expr` cannot be applied to them (see its
+    documentation).  This routine returns the one branch that matches the table's columns.
+
+    :param table: Table name from :data:`agglovar.schema.STANDARD_FIELDS` (e.g. "snv", "insdel").
+    :param alias: Output column name for the resulting IDs, or `None` to leave it unaliased.
+
+    :returns: An expression yielding variant IDs for *table*.
+
+    :raises ValueError: If *table* is not a known variant-type table.
+    """
+    fields = _agg_schema.STANDARD_FIELDS.get(table)
+
+    if fields is None:
+        raise ValueError(
+            f'Unknown variant-type table: {table!r}: '
+            f'expected one of {", ".join(sorted(_agg_schema.STANDARD_FIELDS))}'
+        )
+
+    # Tables without "varlen" (SNV) are identified by their changed bases, all others by length
+    expr = id_nonsnv_expr if 'varlen' in fields else id_snv_expr
+
+    return expr.alias(alias) if alias is not None else expr
 
 
 def lead_src_expr(

{agglovar-0.0.1.dev16 → agglovar-0.0.1.dev18}/src/agglovar/fa.py

@@ -6,14 +6,20 @@ __all__ = [
 ]
 
 import hashlib
+import logging
 from pathlib import Path
-from typing import Iterable, Optional
+from typing import (
+    Iterable,
+    Optional,
+)
 
 import Bio.SeqIO
 import polars as pl
 
 from . import io
 
+logger = logging.getLogger(__name__)
+
 
 def read_fai(
         fai_file_name: str | Path,

{agglovar-0.0.1.dev16 → agglovar-0.0.1.dev18}/src/agglovar/io.py

@@ -4,10 +4,19 @@ __all__ = [
     'PlainOrGzReader',
 ]
 
-from dataclasses import dataclass, field
+from dataclasses import (
+    dataclass,
+    field,
+)
 import gzip
+import logging
 from pathlib import Path
-from typing import Any, Optional
+from typing import (
+    Any,
+    Optional,
+)
+
+logger = logging.getLogger(__name__)
 
 
 @dataclass

{agglovar-0.0.1.dev16 → agglovar-0.0.1.dev18}/src/agglovar/kmer/plot.py

@@ -5,9 +5,11 @@ __all__ = [
 ]
 
 import collections
-import numpy as np
+import logging
 from typing import Any
 
+import numpy as np
+
 from . import util as kmer_util
 
 # Get type hints for optional matplotlib objects
@@ -22,6 +24,8 @@ try:
 except ImportError:
     pass
 
+logger = logging.getLogger(__name__)
+
 
 def dotplot(
         seq_x: str,

{agglovar-0.0.1.dev16 → agglovar-0.0.1.dev18}/src/agglovar/kmer/util.py

@@ -12,12 +12,20 @@ __all__ = [
     'stream_index',
 ]
 
+import logging
 import math
 from types import MappingProxyType
-from typing import Self, Optional, Iterator, Mapping
+from typing import (
+    Self,
+    Optional,
+    Iterator,
+    Mapping,
+)
 
 import numpy as np
 
+logger = logging.getLogger(__name__)
+
 # Integer to base
 INT_TO_BASE: list[str] = ['A', 'C', 'G', 'T']
 """Maps 2-bit integer to base string."""