agglovar 0.0.1.dev13__tar.gz → 0.0.1.dev15__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agglovar-0.0.1.dev13/src/agglovar.egg-info → agglovar-0.0.1.dev15}/PKG-INFO +1 -2
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/pyproject.toml +5 -3
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/__init__.py +1 -1
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/bed/col.py +16 -3
- agglovar-0.0.1.dev15/src/agglovar/bed/intersect.py +186 -0
- agglovar-0.0.1.dev15/src/agglovar/bed/join.py +323 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/dev/imports.py +7 -5
- agglovar-0.0.1.dev15/src/agglovar/expr/variant.py +95 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/kmer/plot.py +5 -2
- agglovar-0.0.1.dev15/src/agglovar/merge/base.py +297 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/merge/config/parser.py +1 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/merge/config/stage.py +1 -0
- agglovar-0.0.1.dev15/src/agglovar/merge/cumulative.py +484 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/meta/decorators.py +8 -10
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/meta/descriptors.py +48 -11
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/pairwise/base.py +11 -3
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/pairwise/overlap/_overlap.py +29 -59
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/pairwise/overlap/_stage.py +11 -3
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/pairwise/weights.py +12 -2
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/util/__init__.py +2 -0
- agglovar-0.0.1.dev15/src/agglovar/util/lazy.py +104 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/util/str.py +8 -2
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/util/var.py +1 -3
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/vcf/__init__.py +4 -4
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/vcf/_vcf_const.py +5 -2
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/vcf/_vcf_header.py +22 -11
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/vcf/_vcf_read.py +8 -5
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/vcf/_vcf_write.py +1 -1
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15/src/agglovar.egg-info}/PKG-INFO +1 -2
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar.egg-info/SOURCES.txt +1 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar.egg-info/requires.txt +0 -1
- agglovar-0.0.1.dev13/src/agglovar/bed/intersect.py +0 -193
- agglovar-0.0.1.dev13/src/agglovar/bed/join.py +0 -408
- agglovar-0.0.1.dev13/src/agglovar/expr/variant.py +0 -61
- agglovar-0.0.1.dev13/src/agglovar/merge/base.py +0 -195
- agglovar-0.0.1.dev13/src/agglovar/merge/cumulative.py +0 -383
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/LICENSE +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/README.md +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/setup.cfg +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/align/__init__.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/align/op.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/align/score.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/bed/__init__.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/bed/merge.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/dev/__init__.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/expr/__init__.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/fa.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/io.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/kmer/__init__.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/kmer/util.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/merge/__init__.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/merge/config/__init__.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/merge/config/strategy.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/merge/om/__init__.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/merge/om/_executor_base.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/merge/om/_executor_nr.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/meta/__init__.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/pairwise/__init__.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/pairwise/overlap/__init__.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/pairwise/overlap/_const.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/schema.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar/seqmatch.py +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar.egg-info/dependency_links.txt +0 -0
- {agglovar-0.0.1.dev13 → agglovar-0.0.1.dev15}/src/agglovar.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agglovar
|
|
3
|
-
Version: 0.0.1.
|
|
3
|
+
Version: 0.0.1.dev15
|
|
4
4
|
Summary: Toolkit for fast genomic variant transformations and intersects
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Requires-Python: >=3.12
|
|
@@ -8,7 +8,6 @@ Description-Content-Type: text/markdown
|
|
|
8
8
|
License-File: LICENSE
|
|
9
9
|
Requires-Dist: biopython>=1.85
|
|
10
10
|
Requires-Dist: edlib>=1.3.9.post1
|
|
11
|
-
Requires-Dist: intervaltree>=3.1.0
|
|
12
11
|
Requires-Dist: numpy>=2.3.0
|
|
13
12
|
Requires-Dist: ply>=3.11
|
|
14
13
|
Requires-Dist: polars[numpy,pandas,pyarrow]>=1.35.2
|
|
@@ -13,7 +13,6 @@ requires-python = ">=3.12"
|
|
|
13
13
|
dependencies = [
|
|
14
14
|
"biopython>=1.85",
|
|
15
15
|
"edlib>=1.3.9.post1",
|
|
16
|
-
"intervaltree>=3.1.0",
|
|
17
16
|
"numpy>=2.3.0",
|
|
18
17
|
"ply>=3.11",
|
|
19
18
|
"polars[numpy,pandas,pyarrow]>=1.35.2",
|
|
@@ -62,6 +61,7 @@ dev = [
|
|
|
62
61
|
"flake8-pyproject>=1.2.3",
|
|
63
62
|
"flake8-rst-docstrings>=0.3.1",
|
|
64
63
|
"ipython>=9.4.0",
|
|
64
|
+
"psutil>=7.2.2",
|
|
65
65
|
"pyarrow>=22.0.0",
|
|
66
66
|
"pylint>=3.3.7",
|
|
67
67
|
"pytest>=8.4.1",
|
|
@@ -108,6 +108,8 @@ max-line-length = 120
|
|
|
108
108
|
max-line-length = 120
|
|
109
109
|
docstring-convention = "pep257"
|
|
110
110
|
per-file-ignores = [
|
|
111
|
-
"src/agglovar/
|
|
111
|
+
"src/agglovar/merge/config/parser.py: D200, D205, D212, D300, D400, D401, D403, D415",
|
|
112
|
+
"src/agglovar/vcf/__init__.py: DALL002, F401, F403",
|
|
113
|
+
"src/agglovar/pairwise/overlap/__init__.py: DALL002, F401, F403",
|
|
112
114
|
]
|
|
113
|
-
rst-roles = ["class", "meth", "const", "ref", "func", "mod"]
|
|
115
|
+
rst-roles = ["class", "meth", "const", "ref", "func", "mod", "attr", "data"]
|
|
@@ -14,6 +14,7 @@ from typing import Optional, Self, Any
|
|
|
14
14
|
|
|
15
15
|
import polars as pl
|
|
16
16
|
|
|
17
|
+
|
|
17
18
|
@dataclass(frozen=True)
|
|
18
19
|
class CoordColExpr(Iterable[pl.Expr]):
|
|
19
20
|
"""Represents coordinate column expressions.
|
|
@@ -22,6 +23,7 @@ class CoordColExpr(Iterable[pl.Expr]):
|
|
|
22
23
|
:ivar pos: Position column expression.
|
|
23
24
|
:ivar end: End column expression.
|
|
24
25
|
"""
|
|
26
|
+
|
|
25
27
|
chrom: pl.Expr
|
|
26
28
|
pos: pl.Expr
|
|
27
29
|
end: pl.Expr
|
|
@@ -49,11 +51,14 @@ class CoordColExpr(Iterable[pl.Expr]):
|
|
|
49
51
|
return self.end.meta.output_name()
|
|
50
52
|
|
|
51
53
|
def __iter__(self):
|
|
54
|
+
"""Iterate over chrom, pos, and end expressions."""
|
|
52
55
|
return iter((self.chrom, self.pos, self.end))
|
|
53
56
|
|
|
54
57
|
def __repr__(self):
|
|
58
|
+
"""Return a string representation."""
|
|
55
59
|
return f"CoordColExpr({self.chrom!r}, {self.pos!r}, {self.end!r})"
|
|
56
60
|
|
|
61
|
+
|
|
57
62
|
@dataclass(frozen=True, order=True)
|
|
58
63
|
class CoordCol(Iterable[str], Container[str]):
|
|
59
64
|
"""Represents coordinate columns to use.
|
|
@@ -62,6 +67,7 @@ class CoordCol(Iterable[str], Container[str]):
|
|
|
62
67
|
:ivar pos: Position column name.
|
|
63
68
|
:ivar end: End column name.
|
|
64
69
|
"""
|
|
70
|
+
|
|
65
71
|
chrom: str
|
|
66
72
|
pos: str
|
|
67
73
|
end: str
|
|
@@ -89,25 +95,30 @@ class CoordCol(Iterable[str], Container[str]):
|
|
|
89
95
|
return CoordColExpr(
|
|
90
96
|
*(
|
|
91
97
|
pl.col(col).alias(alias + suffix)
|
|
92
|
-
|
|
98
|
+
for col, alias in zip(self, alias)
|
|
93
99
|
)
|
|
94
100
|
)
|
|
95
101
|
|
|
96
102
|
def __iter__(self):
|
|
103
|
+
"""Iterate over chrom, pos, and end column names."""
|
|
97
104
|
return iter((self.chrom, self.pos, self.end))
|
|
98
105
|
|
|
99
106
|
def __contains__(self, o: Optional[Any]):
|
|
107
|
+
"""Test whether ``o`` is one of the coordinate column names."""
|
|
100
108
|
return o in (self.chrom, self.pos, self.end)
|
|
101
109
|
|
|
102
110
|
def __repr__(self):
|
|
111
|
+
"""Return a string representation."""
|
|
103
112
|
return f"CoordCol({self.chrom!r}, {self.pos!r}, {self.end!r})"
|
|
104
113
|
|
|
114
|
+
|
|
105
115
|
COL_CHROM: CoordCol = CoordCol('chrom', 'pos', 'end')
|
|
106
116
|
"""Standard chromosome columns."""
|
|
107
117
|
|
|
108
118
|
COL_QRY: CoordCol = CoordCol('qry_id', 'qry_pos', 'qry_end')
|
|
109
119
|
"""Standard query columns."""
|
|
110
120
|
|
|
121
|
+
|
|
111
122
|
def get_coord_cols(
|
|
112
123
|
col_names: Optional[CoordCol | str | Iterable[str]] = None,
|
|
113
124
|
) -> CoordCol:
|
|
@@ -140,10 +151,12 @@ def get_coord_cols(
|
|
|
140
151
|
))
|
|
141
152
|
|
|
142
153
|
if len(col_names_tuple) != 3:
|
|
143
|
-
raise ValueError(
|
|
154
|
+
raise ValueError(
|
|
155
|
+
f'Invalid cols: Expected 3 elements, found {len(col_names_tuple)}: {col_names!r}'
|
|
156
|
+
)
|
|
144
157
|
|
|
145
158
|
if any(col is None or col == '' for col in col_names_tuple):
|
|
146
|
-
raise ValueError(f'Columns is missing values: {col_names
|
|
159
|
+
raise ValueError(f'Columns is missing values: {col_names!r}')
|
|
147
160
|
|
|
148
161
|
return CoordCol(*col_names_tuple)
|
|
149
162
|
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""Table intersects."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Iterable, Optional
|
|
5
|
+
|
|
6
|
+
import polars as pl
|
|
7
|
+
|
|
8
|
+
from .join import pairwise_join, pairwise_join_iter
|
|
9
|
+
from .merge import merge_depth
|
|
10
|
+
from .col import CoordCol, get_coord_cols
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
'as_bool',
|
|
14
|
+
'as_proportion',
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def as_bool(
|
|
19
|
+
df_a: pl.LazyFrame | pl.DataFrame,
|
|
20
|
+
df_b: pl.LazyFrame | pl.DataFrame,
|
|
21
|
+
name: str,
|
|
22
|
+
distance: int = 0,
|
|
23
|
+
negate: bool = False,
|
|
24
|
+
col_names_a: Optional[CoordCol | Iterable[str] | str] = None,
|
|
25
|
+
col_names_b: Optional[CoordCol | Iterable[str] | str] = None,
|
|
26
|
+
temp_dir: bool | str | Path = False,
|
|
27
|
+
) -> pl.LazyFrame:
|
|
28
|
+
"""Add a boolean column to df_a indicating whether each record intersects with df_b.
|
|
29
|
+
|
|
30
|
+
:param df_a: Table a.
|
|
31
|
+
:param df_b: Table b.
|
|
32
|
+
:param name: Name of the column to add.
|
|
33
|
+
:param distance: Maximum distance between two records. May be negative to require overlap.
|
|
34
|
+
:param negate: If True, negate the boolean column to annotate misses instead of hits.
|
|
35
|
+
:param col_names_a: Columns in a (chromosome or query ID, pos, end).
|
|
36
|
+
:param col_names_b: Columns in b (chromosome or query ID, pos, end).
|
|
37
|
+
:param temp_dir: How to materialise the prepared tables before iterating. See
|
|
38
|
+
:func:`agglovar.bed.join.pairwise_join`.
|
|
39
|
+
|
|
40
|
+
:return: A LazyFrame with two columns: ``_index`` and ``name``.
|
|
41
|
+
"""
|
|
42
|
+
if name is None or not (name := name.strip()):
|
|
43
|
+
raise ValueError('Name must be a non-empty string')
|
|
44
|
+
|
|
45
|
+
col_names_a = get_coord_cols(col_names_a)
|
|
46
|
+
col_names_b = get_coord_cols(col_names_b)
|
|
47
|
+
|
|
48
|
+
if isinstance(df_a, pl.DataFrame):
|
|
49
|
+
df_a = df_a.lazy()
|
|
50
|
+
|
|
51
|
+
if isinstance(df_b, pl.DataFrame):
|
|
52
|
+
df_b = df_b.lazy()
|
|
53
|
+
|
|
54
|
+
hit_val = not negate
|
|
55
|
+
|
|
56
|
+
join_list = []
|
|
57
|
+
|
|
58
|
+
for df_join in pairwise_join_iter(
|
|
59
|
+
df_a=df_a,
|
|
60
|
+
df_b=df_b,
|
|
61
|
+
distance=distance,
|
|
62
|
+
col_names_a=col_names_a,
|
|
63
|
+
col_names_b=col_names_b,
|
|
64
|
+
temp_dir=temp_dir,
|
|
65
|
+
):
|
|
66
|
+
join_list.append(
|
|
67
|
+
df_join
|
|
68
|
+
.select(pl.col('index_a').alias('_index'))
|
|
69
|
+
.collect()
|
|
70
|
+
.lazy()
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
if '_index' in df_a.collect_schema().names():
|
|
74
|
+
df_a_indexed = df_a
|
|
75
|
+
else:
|
|
76
|
+
df_a_indexed = df_a.with_row_index('_index').with_columns(pl.col('_index').cast(pl.UInt64))
|
|
77
|
+
|
|
78
|
+
if not join_list:
|
|
79
|
+
return (
|
|
80
|
+
df_a_indexed
|
|
81
|
+
.select(
|
|
82
|
+
'_index',
|
|
83
|
+
pl.lit(not hit_val).alias(name),
|
|
84
|
+
)
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
hits = pl.concat(join_list).unique('_index')
|
|
88
|
+
|
|
89
|
+
return (
|
|
90
|
+
df_a_indexed
|
|
91
|
+
.select('_index')
|
|
92
|
+
.join(
|
|
93
|
+
hits.with_columns(pl.lit(hit_val).alias(name)),
|
|
94
|
+
on='_index', how='left',
|
|
95
|
+
)
|
|
96
|
+
.select(
|
|
97
|
+
'_index',
|
|
98
|
+
pl.col(name).fill_null(not hit_val),
|
|
99
|
+
)
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def as_proportion(
|
|
104
|
+
df_a: pl.LazyFrame | pl.DataFrame,
|
|
105
|
+
df_b: pl.LazyFrame | pl.DataFrame,
|
|
106
|
+
name: str,
|
|
107
|
+
col_names_a: Optional[CoordCol | Iterable[str] | str] = None,
|
|
108
|
+
col_names_b: Optional[CoordCol | Iterable[str] | str] = None,
|
|
109
|
+
temp_dir: bool | str | Path = False,
|
|
110
|
+
) -> pl.LazyFrame:
|
|
111
|
+
"""Compute the proportion of each interval in ``df_a`` covered by intervals in ``df_b``.
|
|
112
|
+
|
|
113
|
+
Rows in ``df_a`` with null ``pos`` or ``end`` are preserved in the output with a null
|
|
114
|
+
proportion. Zero-length intervals (``pos == end``) produce ``NaN`` (0 / 0).
|
|
115
|
+
|
|
116
|
+
:param df_a: Table a.
|
|
117
|
+
:param df_b: Table b.
|
|
118
|
+
:param name: Name of the column to add.
|
|
119
|
+
:param col_names_a: Columns in a (chromosome or query ID, pos, end).
|
|
120
|
+
:param col_names_b: Columns in b (chromosome or query ID, pos, end).
|
|
121
|
+
:param temp_dir: How to materialise the prepared tables before iterating. See
|
|
122
|
+
:func:`agglovar.bed.join.pairwise_join`.
|
|
123
|
+
|
|
124
|
+
:return: A LazyFrame with two columns: ``_index`` and ``name``.
|
|
125
|
+
"""
|
|
126
|
+
if name is None or not (name := name.strip()):
|
|
127
|
+
raise ValueError('Name must be a non-empty string')
|
|
128
|
+
|
|
129
|
+
col_names_a = get_coord_cols(col_names_a)
|
|
130
|
+
col_names_b = get_coord_cols(col_names_b)
|
|
131
|
+
|
|
132
|
+
col_expr_a = col_names_a.exprs()
|
|
133
|
+
|
|
134
|
+
if isinstance(df_a, pl.DataFrame):
|
|
135
|
+
df_a = df_a.lazy()
|
|
136
|
+
|
|
137
|
+
if isinstance(df_b, pl.DataFrame):
|
|
138
|
+
df_b = df_b.lazy()
|
|
139
|
+
|
|
140
|
+
if '_index' not in df_a.collect_schema().names():
|
|
141
|
+
df_a = df_a.with_row_index('_index').with_columns(pl.col('_index').cast(pl.UInt64))
|
|
142
|
+
|
|
143
|
+
# Records with null pos/end are kept and assigned a null proportion at the end.
|
|
144
|
+
df_a_clean = df_a.filter(
|
|
145
|
+
pl.col(col_names_a.pos).is_not_null(),
|
|
146
|
+
pl.col(col_names_a.end).is_not_null(),
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
# Collapse overlapping b intervals so coverage is not double-counted.
|
|
150
|
+
df_b_nr = (
|
|
151
|
+
merge_depth(df_b, 0, col_names_b)
|
|
152
|
+
.select(*col_names_b.exprs())
|
|
153
|
+
.collect()
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
df_join = pairwise_join(
|
|
157
|
+
df_a=df_a_clean,
|
|
158
|
+
df_b=df_b_nr,
|
|
159
|
+
col_names_a=col_names_a,
|
|
160
|
+
col_names_b=col_names_b,
|
|
161
|
+
temp_dir=temp_dir,
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
df_overlap = (
|
|
165
|
+
df_join
|
|
166
|
+
.filter(pl.col('end') > pl.col('pos'))
|
|
167
|
+
.select(
|
|
168
|
+
pl.col('index_a').alias('_index'),
|
|
169
|
+
(pl.col('end') - pl.col('pos')).alias('_overlap'),
|
|
170
|
+
)
|
|
171
|
+
.group_by('_index')
|
|
172
|
+
.agg(pl.col('_overlap').sum())
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
return (
|
|
176
|
+
df_a
|
|
177
|
+
.select(
|
|
178
|
+
'_index',
|
|
179
|
+
(col_expr_a.end - col_expr_a.pos).alias('_len'),
|
|
180
|
+
)
|
|
181
|
+
.join(df_overlap.lazy(), on='_index', how='left')
|
|
182
|
+
.select(
|
|
183
|
+
'_index',
|
|
184
|
+
(pl.col('_overlap').fill_null(0.0) / pl.col('_len')).alias(name),
|
|
185
|
+
)
|
|
186
|
+
)
|
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
"""Base join operations for intersects."""
|
|
2
|
+
|
|
3
|
+
__all__ = [
|
|
4
|
+
'pairwise_join',
|
|
5
|
+
'pairwise_join_iter',
|
|
6
|
+
]
|
|
7
|
+
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Iterable, Iterator, Optional
|
|
10
|
+
|
|
11
|
+
import polars as pl
|
|
12
|
+
|
|
13
|
+
from .col import CoordCol, get_coord_cols
|
|
14
|
+
from ..util.lazy import materialize_pair
|
|
15
|
+
|
|
16
|
+
CHUNK_SIZE: int = 2_500
|
|
17
|
+
"""Default size of join chunks. Breaks up tables into batches of this size or less."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class _JoinResources:
|
|
21
|
+
"""Resources for joining tables."""
|
|
22
|
+
|
|
23
|
+
df_a: pl.LazyFrame
|
|
24
|
+
df_b: pl.LazyFrame
|
|
25
|
+
distance: int
|
|
26
|
+
chunk_size: int
|
|
27
|
+
col_a: CoordCol
|
|
28
|
+
col_b: CoordCol
|
|
29
|
+
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
df_a: pl.LazyFrame,
|
|
33
|
+
df_b: pl.LazyFrame,
|
|
34
|
+
distance: int = 0,
|
|
35
|
+
chunk_size: int = CHUNK_SIZE,
|
|
36
|
+
col_names_a: Optional[CoordCol | Iterable[str] | str] = None,
|
|
37
|
+
col_names_b: Optional[CoordCol | Iterable[str] | str] = None,
|
|
38
|
+
):
|
|
39
|
+
if chunk_size < 1:
|
|
40
|
+
raise ValueError('chunk_size must be greater than 0')
|
|
41
|
+
|
|
42
|
+
if isinstance(df_a, pl.DataFrame):
|
|
43
|
+
df_a = df_a.lazy()
|
|
44
|
+
|
|
45
|
+
if isinstance(df_b, pl.DataFrame):
|
|
46
|
+
df_b = df_b.lazy()
|
|
47
|
+
|
|
48
|
+
cols_a = set(df_a.collect_schema().keys())
|
|
49
|
+
cols_b = set(df_b.collect_schema().keys())
|
|
50
|
+
|
|
51
|
+
if '_index' not in cols_a:
|
|
52
|
+
df_a = df_a.with_row_index('_index').with_columns(pl.col('_index').cast(pl.UInt64))
|
|
53
|
+
|
|
54
|
+
if '_index' not in cols_b:
|
|
55
|
+
df_b = df_b.with_row_index('_index').with_columns(pl.col('_index').cast(pl.UInt64))
|
|
56
|
+
|
|
57
|
+
# Set column names
|
|
58
|
+
ref_cols = get_coord_cols('ref')
|
|
59
|
+
|
|
60
|
+
try:
|
|
61
|
+
col_select_a = get_coord_cols(col_names_a).exprs(alias=ref_cols, suffix='_a')
|
|
62
|
+
except (ValueError, TypeError) as e:
|
|
63
|
+
raise ValueError(f'col_names_a: {e}')
|
|
64
|
+
|
|
65
|
+
try:
|
|
66
|
+
col_select_b = get_coord_cols(col_names_b).exprs(alias=ref_cols, suffix='_b')
|
|
67
|
+
except (ValueError, TypeError) as e:
|
|
68
|
+
raise ValueError(f'col_names_b: {e}')
|
|
69
|
+
|
|
70
|
+
col_a = col_select_a.col_names()
|
|
71
|
+
col_b = col_select_b.col_names()
|
|
72
|
+
|
|
73
|
+
# Prepare tables
|
|
74
|
+
df_a = (
|
|
75
|
+
df_a
|
|
76
|
+
.select(
|
|
77
|
+
pl.col('_index').alias('_index_a'),
|
|
78
|
+
*col_select_a
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
df_b = (
|
|
83
|
+
df_b
|
|
84
|
+
.select(
|
|
85
|
+
pl.col('_index').alias('_index_b'),
|
|
86
|
+
*col_select_b
|
|
87
|
+
)
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
self.df_a = df_a
|
|
91
|
+
self.df_b = df_b
|
|
92
|
+
self.distance = distance
|
|
93
|
+
self.chunk_size = chunk_size
|
|
94
|
+
self.col_a = col_a
|
|
95
|
+
self.col_b = col_b
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def pairwise_join(
|
|
99
|
+
df_a: pl.LazyFrame | pl.DataFrame,
|
|
100
|
+
df_b: pl.LazyFrame | pl.DataFrame,
|
|
101
|
+
distance: int = 0,
|
|
102
|
+
chunk_size: int = CHUNK_SIZE,
|
|
103
|
+
col_names_a: Optional[CoordCol | Iterable[str] | str] = None,
|
|
104
|
+
col_names_b: Optional[CoordCol | Iterable[str] | str] = None,
|
|
105
|
+
temp_dir: bool | str | Path = False,
|
|
106
|
+
) -> pl.LazyFrame:
|
|
107
|
+
"""Join two tables.
|
|
108
|
+
|
|
109
|
+
Thin wrapper around :func:`pairwise_join_iter` that concatenates all yielded
|
|
110
|
+
chunks into a single table.
|
|
111
|
+
|
|
112
|
+
Returns a table with columns:
|
|
113
|
+
|
|
114
|
+
* index_a: Index in table a.
|
|
115
|
+
* index_b: Index in table b.
|
|
116
|
+
* chrom: Chromosome matched.
|
|
117
|
+
* pos: Start position of intersection.
|
|
118
|
+
* end: End position of intersection.
|
|
119
|
+
* distance: Distance between the two intervals with negative values representing overlapping intervals.
|
|
120
|
+
|
|
121
|
+
Note that if padding is greater than 0, the "pos" and "end" will have been modified to include padding.
|
|
122
|
+
|
|
123
|
+
:param df_a: Table a.
|
|
124
|
+
:param df_b: Table b.
|
|
125
|
+
:param distance: Maximum distance between two records. May be negative to force overlapping.
|
|
126
|
+
:param chunk_size: Chunk A by this size per chromosome to bound the IEJoin working set.
|
|
127
|
+
:param col_names_a: Columns to select from `df_a` if not None, otherwise, use object defaults.
|
|
128
|
+
:param col_names_b: Columns to select from `df_b` if not None, otherwise, use object defaults.
|
|
129
|
+
:param temp_dir: How to materialise the prepared tables before the chunked loop.
|
|
130
|
+
``False`` (default) collects both into memory; ``True`` writes them to the
|
|
131
|
+
system temp directory as parquet files; a ``str``/``Path`` writes them to
|
|
132
|
+
that directory. Temp files are always removed on exit.
|
|
133
|
+
|
|
134
|
+
:return: A LazyFrame with the joined tables.
|
|
135
|
+
"""
|
|
136
|
+
return pl.concat(list(pairwise_join_iter(
|
|
137
|
+
df_a=df_a,
|
|
138
|
+
df_b=df_b,
|
|
139
|
+
distance=distance,
|
|
140
|
+
chunk_size=chunk_size,
|
|
141
|
+
col_names_a=col_names_a,
|
|
142
|
+
col_names_b=col_names_b,
|
|
143
|
+
temp_dir=temp_dir,
|
|
144
|
+
)))
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def pairwise_join_iter(
|
|
148
|
+
df_a: pl.LazyFrame | pl.DataFrame,
|
|
149
|
+
df_b: pl.LazyFrame | pl.DataFrame,
|
|
150
|
+
distance: int = 0,
|
|
151
|
+
chunk_size: int = CHUNK_SIZE,
|
|
152
|
+
col_names_a: Optional[CoordCol | Iterable[str] | str] = None,
|
|
153
|
+
col_names_b: Optional[CoordCol | Iterable[str] | str] = None,
|
|
154
|
+
temp_dir: bool | str | Path = False,
|
|
155
|
+
) -> Iterator[pl.LazyFrame]:
|
|
156
|
+
"""Join two tables, yielding one LazyFrame per chunk.
|
|
157
|
+
|
|
158
|
+
Returns chunks with the same columns as :func:`pairwise_join`. At least one chunk
|
|
159
|
+
is always yielded; an empty schema-only frame is yielded when no chunk would otherwise
|
|
160
|
+
have been produced (so callers can safely call ``pl.concat`` on the result).
|
|
161
|
+
|
|
162
|
+
Each chunk is the result of a per-chromosome ``join_where`` (Polars IEJoin) over a
|
|
163
|
+
bounded slice of A and the chrom-matched B records pre-filtered to that slice's
|
|
164
|
+
range.
|
|
165
|
+
|
|
166
|
+
:param df_a: Table a.
|
|
167
|
+
:param df_b: Table b.
|
|
168
|
+
:param distance: Maximum distance between two records. May be negative to force overlapping.
|
|
169
|
+
:param chunk_size: Chunk A by this size per chromosome to bound the IEJoin working set.
|
|
170
|
+
:param col_names_a: Columns to select from `df_a` if not None, otherwise, use object defaults.
|
|
171
|
+
:param col_names_b: Columns to select from `df_b` if not None, otherwise, use object defaults.
|
|
172
|
+
:param temp_dir: How to materialise the prepared tables before the chunked loop.
|
|
173
|
+
See :func:`pairwise_join`.
|
|
174
|
+
|
|
175
|
+
:return: An iterator of LazyFrames.
|
|
176
|
+
"""
|
|
177
|
+
resources = _JoinResources(
|
|
178
|
+
df_a=df_a,
|
|
179
|
+
df_b=df_b,
|
|
180
|
+
distance=distance,
|
|
181
|
+
chunk_size=chunk_size,
|
|
182
|
+
col_names_a=col_names_a,
|
|
183
|
+
col_names_b=col_names_b,
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
with materialize_pair(
|
|
187
|
+
resources.df_a, resources.df_b, temp_dir, prefix='bed_join_prep_',
|
|
188
|
+
) as (df_a_mat, df_b_mat):
|
|
189
|
+
resources.df_a = df_a_mat
|
|
190
|
+
resources.df_b = df_b_mat
|
|
191
|
+
|
|
192
|
+
yielded = False
|
|
193
|
+
|
|
194
|
+
for chunk in _join_chunks(resources):
|
|
195
|
+
yielded = True
|
|
196
|
+
yield chunk
|
|
197
|
+
|
|
198
|
+
if not yielded:
|
|
199
|
+
yield _empty_join(resources)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _build_join_pair(
|
|
203
|
+
df_a: pl.LazyFrame,
|
|
204
|
+
df_b: pl.LazyFrame,
|
|
205
|
+
distance: int,
|
|
206
|
+
col_a: CoordCol,
|
|
207
|
+
col_b: CoordCol,
|
|
208
|
+
) -> pl.LazyFrame:
|
|
209
|
+
"""Apply ``join_where`` (IEJoin) and the standard output projection.
|
|
210
|
+
|
|
211
|
+
Both ``df_a`` and ``df_b`` must already have ``_index_a``/``_index_b`` columns
|
|
212
|
+
and the coordinate columns named per ``col_a``/``col_b``. The chrom equality is
|
|
213
|
+
enforced by the caller filtering both inputs to the same chromosome upstream;
|
|
214
|
+
only the inequality predicates are passed to ``join_where`` (the shape Polars's
|
|
215
|
+
IEJoin path is optimised for).
|
|
216
|
+
"""
|
|
217
|
+
return (
|
|
218
|
+
df_a
|
|
219
|
+
.join_where(
|
|
220
|
+
df_b,
|
|
221
|
+
pl.col(col_b.pos) - distance <= pl.col(col_a.end),
|
|
222
|
+
pl.col(col_b.end) + distance >= pl.col(col_a.pos),
|
|
223
|
+
)
|
|
224
|
+
.select(
|
|
225
|
+
pl.col('_index_a').alias('index_a'),
|
|
226
|
+
pl.col('_index_b').alias('index_b'),
|
|
227
|
+
pl.col(col_a.chrom).alias('chrom'),
|
|
228
|
+
pl.max_horizontal(col_a.pos, col_b.pos).alias('pos'),
|
|
229
|
+
pl.min_horizontal(col_a.end, col_b.end).alias('end'),
|
|
230
|
+
)
|
|
231
|
+
.with_columns(
|
|
232
|
+
pl.min_horizontal('pos', 'end').alias('pos'),
|
|
233
|
+
pl.max_horizontal('pos', 'end').alias('end'),
|
|
234
|
+
(pl.col('pos') - pl.col('end')).alias('distance'),
|
|
235
|
+
)
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _empty_join(join_resources: _JoinResources) -> pl.LazyFrame:
|
|
240
|
+
"""Schema-only LazyFrame for the empty-input case."""
|
|
241
|
+
return _build_join_pair(
|
|
242
|
+
join_resources.df_a.head(0),
|
|
243
|
+
join_resources.df_b.head(0),
|
|
244
|
+
join_resources.distance,
|
|
245
|
+
join_resources.col_a,
|
|
246
|
+
join_resources.col_b,
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _join_chunks(
|
|
251
|
+
join_resources: _JoinResources
|
|
252
|
+
) -> Iterator[pl.LazyFrame]:
|
|
253
|
+
"""Iterate over join results by chrom + A chunk via IEJoin.
|
|
254
|
+
|
|
255
|
+
Yields nothing if no chunks have rows; callers (``pairwise_join_iter``) handle
|
|
256
|
+
the empty-case schema frame.
|
|
257
|
+
"""
|
|
258
|
+
df_a = join_resources.df_a
|
|
259
|
+
df_b = join_resources.df_b
|
|
260
|
+
distance = join_resources.distance
|
|
261
|
+
chunk_size = join_resources.chunk_size
|
|
262
|
+
col_a = join_resources.col_a
|
|
263
|
+
col_b = join_resources.col_b
|
|
264
|
+
|
|
265
|
+
# Restrict the chrom loop to chroms present in both tables. Chroms unique to A
|
|
266
|
+
# would otherwise drive a wasted A chunk loop with no possible matches.
|
|
267
|
+
for chrom, last_index_a in (
|
|
268
|
+
df_a
|
|
269
|
+
.group_by(col_a.chrom)
|
|
270
|
+
.agg(pl.len().alias('last_index'))
|
|
271
|
+
.join(
|
|
272
|
+
df_b.select(pl.col(col_b.chrom)).unique(),
|
|
273
|
+
left_on=col_a.chrom, right_on=col_b.chrom,
|
|
274
|
+
how='inner',
|
|
275
|
+
)
|
|
276
|
+
.sort(col_a.chrom)
|
|
277
|
+
).collect().rows():
|
|
278
|
+
df_a_chrom = (
|
|
279
|
+
df_a.filter(pl.col(col_a.chrom) == chrom)
|
|
280
|
+
.with_row_index('_index_chrom_a')
|
|
281
|
+
)
|
|
282
|
+
df_b_chrom = df_b.filter(pl.col(col_b.chrom) == chrom)
|
|
283
|
+
|
|
284
|
+
start_index_a = 0
|
|
285
|
+
while start_index_a < last_index_a:
|
|
286
|
+
end_index_a = start_index_a + chunk_size
|
|
287
|
+
|
|
288
|
+
df_a_chunk = df_a_chrom.filter(
|
|
289
|
+
pl.col('_index_chrom_a') >= start_index_a,
|
|
290
|
+
pl.col('_index_chrom_a') < end_index_a,
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
end_max, pos_min = (
|
|
294
|
+
df_a_chunk
|
|
295
|
+
.select(
|
|
296
|
+
pl.col(col_a.end).max().alias('end_max'),
|
|
297
|
+
pl.col(col_a.pos).min().alias('pos_min'),
|
|
298
|
+
)
|
|
299
|
+
.collect()
|
|
300
|
+
.row(0)
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
if end_max is None or pos_min is None:
|
|
304
|
+
start_index_a = end_index_a
|
|
305
|
+
continue
|
|
306
|
+
|
|
307
|
+
# Pre-filter B to records that could possibly match any A record in this
|
|
308
|
+
# chunk. Bounds use <= and >= to match the inner predicate (touching cases
|
|
309
|
+
# at distance == 0 must survive the prefilter).
|
|
310
|
+
df_b_chunk = df_b_chrom.filter(
|
|
311
|
+
pl.col(col_b.pos) - distance <= end_max,
|
|
312
|
+
pl.col(col_b.end) + distance >= pos_min,
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
yield _build_join_pair(
|
|
316
|
+
df_a_chunk,
|
|
317
|
+
df_b_chunk,
|
|
318
|
+
distance,
|
|
319
|
+
col_a,
|
|
320
|
+
col_b,
|
|
321
|
+
).collect().lazy()
|
|
322
|
+
|
|
323
|
+
start_index_a = end_index_a
|
|
@@ -209,7 +209,7 @@ def alias_to_obj_dict(
|
|
|
209
209
|
)
|
|
210
210
|
) + (
|
|
211
211
|
('.' + module_path)
|
|
212
|
-
|
|
212
|
+
if module_path is not None else ''
|
|
213
213
|
)
|
|
214
214
|
)
|
|
215
215
|
|
|
@@ -350,7 +350,6 @@ def get_defined_names(
|
|
|
350
350
|
imports (None for others) and is the level of the import with 0 being the top-level and 1 or more being a
|
|
351
351
|
relative import.
|
|
352
352
|
"""
|
|
353
|
-
|
|
354
353
|
tree = ast.parse(inspect.getsource(mod))
|
|
355
354
|
|
|
356
355
|
for node in tree.body:
|
|
@@ -371,8 +370,10 @@ def get_defined_names(
|
|
|
371
370
|
elif isinstance(node, ast.ImportFrom):
|
|
372
371
|
|
|
373
372
|
module_name = (
|
|
374
|
-
'.'.join(
|
|
375
|
-
|
|
373
|
+
'.'.join(
|
|
374
|
+
mod.__name__.split('.')[:-node.level]
|
|
375
|
+
+ (node.module.split('.') if node.module else [])
|
|
376
|
+
)
|
|
376
377
|
)
|
|
377
378
|
|
|
378
379
|
for node_name in node.names:
|
|
@@ -443,6 +444,7 @@ def get_module_definitions(
|
|
|
443
444
|
if include_dunder or not target.id.startswith('__'):
|
|
444
445
|
yield target.id
|
|
445
446
|
|
|
447
|
+
|
|
446
448
|
def librl(
|
|
447
449
|
packages: Iterable[str] | str = 'agglovar',
|
|
448
450
|
) -> None:
|
|
@@ -455,7 +457,7 @@ def librl(
|
|
|
455
457
|
before the first dot).
|
|
456
458
|
"""
|
|
457
459
|
if isinstance(packages, str):
|
|
458
|
-
packages = {packages,}
|
|
460
|
+
packages = {packages, }
|
|
459
461
|
else:
|
|
460
462
|
packages = set(packages)
|
|
461
463
|
|