ag2kit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ag2kit/io.py ADDED
@@ -0,0 +1,248 @@
1
+ """Variant input adapters that preserve source order and 1-based coordinates."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ import numpy as np
10
+ import pandas as pd
11
+ from pandas.api.types import is_bool_dtype, is_integer_dtype, is_unsigned_integer_dtype
12
+
13
+ OUTPUT_COLUMNS = (
14
+ "variant_id",
15
+ "chrom",
16
+ "pos",
17
+ "ref",
18
+ "alt",
19
+ "source_id",
20
+ "assembly",
21
+ "record_index",
22
+ "alt_index",
23
+ "input_status",
24
+ )
25
+ _ALIASES = {
26
+ "chrom": ("chrom", "chromosome", "chr", "#chrom"),
27
+ "pos": ("pos", "position", "bp"),
28
+ "ref": ("ref", "reference", "ref_allele"),
29
+ "alt": ("alt", "alternate", "alt_allele"),
30
+ }
31
+ _ID_COLUMNS = ("source_id", "id", "variant_id", "rsid")
32
+ _VCF_SUFFIXES = (".vcf", ".vcf.gz", ".vcf.bgz", ".bcf")
33
+ _ALLELE = re.compile(r"^[ACGTN]+$")
34
+ _MISSING = frozenset({"", "."})
35
+ _BUILDS = {
36
+ "GRCh37": "grch37|hg19|b37|hs37d5|human_g1k_v37|g1k_v37|assembly19",
37
+ "GRCh38": "grch38|hg38|b38|hs38dh|assembly38",
38
+ }
39
+ # Chromosome 1 length identifies the build when the contig lacks an assembly label.
40
+ _CHR1_LENGTH = {249250621: "GRCh37", 248956422: "GRCh38"}
41
+
42
+
43
+ def _pysam() -> Any:
44
+ try:
45
+ import pysam
46
+ except ImportError as exc:
47
+ raise ImportError(
48
+ 'Reading VCF/BCF or FASTA needs pysam: pip install "ag2kit[vcf]"'
49
+ ) from exc
50
+ return pysam
51
+
52
+
53
+ def _chrom(chrom: object) -> str:
54
+ chrom = "chr" + str(chrom).removeprefix("chr")
55
+ return "chrM" if chrom == "chrMT" else chrom
56
+
57
+
58
+ def _alleles(column: pd.Series) -> pd.Series:
59
+ normalized = column.map(lambda allele: str(allele).strip().upper(), na_action="ignore")
60
+ return normalized.astype(object).fillna("")
61
+
62
+
63
+ def _builds(text: str) -> set[str]:
64
+ text = text.lower()
65
+ return {b for b, p in _BUILDS.items() if re.search(rf"(?<![a-z0-9])(?:{p})(?![a-z0-9])", text)}
66
+
67
+
68
+ def _check_header(header: Any, assembly: str) -> None:
69
+ declared = _builds(assembly) or {assembly}
70
+ for record in header.records:
71
+ if record.key == "reference":
72
+ value = record.value
73
+ elif record.type == "CONTIG":
74
+ value = record.get("assembly")
75
+ length = _CHR1_LENGTH.get(int(record.get("length") or 0))
76
+ if record.get("ID") in ("1", "chr1") and length and not value:
77
+ value = f"{length} (chr1 length {record.get('length')})"
78
+ else:
79
+ continue
80
+ found = _builds(str(value or ""))
81
+ if found and not found & declared:
82
+ raise ValueError(
83
+ f"VCF header names build {sorted(found)[0]} ({value!r}) but assembly={assembly!r};"
84
+ " ag2kit never converts coordinates between assemblies"
85
+ )
86
+
87
+
88
+ def _from_vcf(path: Path, assembly: str) -> pd.DataFrame:
89
+ pysam = _pysam()
90
+ rows = []
91
+ with pysam.VariantFile(str(path)) as vcf:
92
+ _check_header(vcf.header, assembly)
93
+ for i, rec in enumerate(vcf):
94
+ for j, alt in enumerate(rec.alts or (".",), start=1):
95
+ rows.append((rec.chrom, rec.pos, rec.ref or ".", alt, rec.id or "", i, j))
96
+ columns = ["chrom", "pos", "ref", "alt", "source_id", "record_index", "alt_index"]
97
+ return pd.DataFrame(rows, columns=columns)
98
+
99
+
100
+ def _from_table(table: pd.DataFrame, assembly: str) -> tuple[pd.DataFrame, pd.DataFrame]:
101
+ table = table.reset_index(drop=True)
102
+ lower: dict[str, Any] = {}
103
+ for column in table.columns:
104
+ lower.setdefault(str(column).lower(), column)
105
+ found = {k: next((lower[a] for a in v if a in lower), None) for k, v in _ALIASES.items()}
106
+ missing = [k for k, v in found.items() if v is None]
107
+ if missing:
108
+ raise ValueError(f"variant table is missing required columns {missing}")
109
+ declared = _builds(assembly) or {assembly}
110
+ if "assembly" in table.columns and any(
111
+ (_builds(str(a)) or {str(a)}) != declared for a in table["assembly"].unique()
112
+ ):
113
+ raise ValueError(f"table has an assembly column that differs from assembly={assembly!r}")
114
+ raw_pos = table[found["pos"]]
115
+ pos = pd.to_numeric(raw_pos, errors="coerce")
116
+ bad = pos.isna() | (pos < 1)
117
+ if is_unsigned_integer_dtype(pos.dtype):
118
+ bad |= pos > np.uint64(2**63 - 1)
119
+ elif not (is_integer_dtype(pos.dtype) or is_bool_dtype(pos.dtype)):
120
+ # The largest int64 rounds up to 2**63 as a float.
121
+ bad |= (pos >= 2.0**63) | (pos != np.floor(pos))
122
+ if bad.any():
123
+ i = int(np.argmax(bad.to_numpy()))
124
+ raise ValueError(f"row {i}: pos {raw_pos.iloc[i]!r} is not a positive 64-bit integer")
125
+ id_column = next((lower[a] for a in _ID_COLUMNS if a in lower), None)
126
+ source_id = (
127
+ table[id_column].map(lambda v: "" if pd.isna(v) or str(v) == "." else str(v))
128
+ if id_column is not None
129
+ else pd.Series("", index=table.index, dtype=object)
130
+ )
131
+ core = pd.DataFrame(
132
+ {
133
+ "chrom": table[found["chrom"]],
134
+ "pos": pos,
135
+ "ref": table[found["ref"]],
136
+ "alt": table[found["alt"]],
137
+ "source_id": source_id,
138
+ "record_index": table.get("record_index", pd.Series(table.index)),
139
+ "alt_index": table.get("alt_index", pd.Series(1, index=table.index)),
140
+ }
141
+ )
142
+ used = {*found.values(), id_column, *OUTPUT_COLUMNS}
143
+ return core, table[[c for c in table.columns if c not in used]]
144
+
145
+
146
+ def _contig(chrom: str, references: set[str]) -> str | None:
147
+ bare = chrom.removeprefix("chr")
148
+ candidates = [chrom, bare, *(["MT", "chrMT"] if bare == "M" else [])]
149
+ return next((c for c in candidates if c in references), None)
150
+
151
+
152
+ def _status(chrom: str, pos: int, ref: str, alt: str, fasta: Any, references: set[str]) -> str:
153
+ if ref in _MISSING or alt in _MISSING:
154
+ return "missing_allele"
155
+ if alt == "*" or alt.startswith("<") or "[" in alt or "]" in alt:
156
+ return "symbolic_allele"
157
+ if not (_ALLELE.match(ref) and _ALLELE.match(alt)):
158
+ return "invalid_allele"
159
+ if ref == alt:
160
+ return "ref_equals_alt"
161
+ if fasta is None:
162
+ return "ok"
163
+ contig = _contig(chrom, references)
164
+ if contig is None:
165
+ return "contig_not_in_reference"
166
+ end = pos - 1 + len(ref)
167
+ if end > fasta.get_reference_length(contig):
168
+ return "position_out_of_range"
169
+ if fasta.fetch(contig, pos - 1, end).upper() != ref:
170
+ return "ref_mismatch"
171
+ return "ok"
172
+
173
+
174
+ def read_variants(
175
+ source: str | Path | pd.DataFrame,
176
+ *,
177
+ assembly: str = "GRCh38",
178
+ fasta: str | Path | None = None,
179
+ ) -> pd.DataFrame:
180
+ """Loads a canonical variant table with one row per record and ALT allele.
181
+
182
+ Coordinates stay on the declared assembly. Indels retain their source
183
+ representation, so normalize upstream if downstream matching requires it.
184
+
185
+ Args:
186
+ source: VCF/BCF, CSV, TSV, TXT, Parquet, or DataFrame. Compressed VCF
187
+ and gzip-compressed text are supported. Tabular input needs
188
+ chromosome, position, REF, and ALT columns. Column aliases are
189
+ case-insensitive, and unused columns pass through.
190
+ assembly: Genome assembly that the input coordinates use.
191
+ fasta: Optional indexed FASTA on that assembly, used to check the
192
+ contig, bounds, and full REF sequence.
193
+
194
+ Returns:
195
+ A table in source order with chr-prefixed chromosomes (MT becomes
196
+ chrM), 1-based positions, uppercase alleles, canonical variant IDs,
197
+ and source IDs when available. record_index is zero-based and shared
198
+ by split records. alt_index is one-based. input_status records the
199
+ first allele or reference check that fails, or "ok".
200
+
201
+ Raises:
202
+ ValueError: If the format is unsupported, required columns are missing,
203
+ table positions are invalid, or the assembly is blank or conflicts
204
+ with input metadata.
205
+ ImportError: If VCF/BCF or FASTA input is used without pysam.
206
+ OSError: If an input file cannot be read.
207
+ """
208
+ if not isinstance(assembly, str) or not assembly.strip():
209
+ raise ValueError(f"assembly must be a non-empty string, got {assembly!r}")
210
+ extra = None
211
+ if isinstance(source, pd.DataFrame):
212
+ core, extra = _from_table(source, assembly)
213
+ else:
214
+ path = Path(source)
215
+ name = path.name.lower()
216
+ if name.endswith(_VCF_SUFFIXES):
217
+ core = _from_vcf(path, assembly)
218
+ elif name.endswith(".parquet"):
219
+ core, extra = _from_table(pd.read_parquet(path), assembly)
220
+ elif name.endswith((".csv", ".csv.gz")):
221
+ core, extra = _from_table(pd.read_csv(path), assembly)
222
+ elif name.endswith((".tsv", ".txt", ".tsv.gz", ".txt.gz")):
223
+ core, extra = _from_table(pd.read_csv(path, sep="\t"), assembly)
224
+ else:
225
+ raise ValueError(f"unrecognised variant file type: {path.name}")
226
+
227
+ out = pd.DataFrame(index=core.index)
228
+ out["chrom"] = core["chrom"].map(_chrom).astype(object)
229
+ out["pos"] = core["pos"].astype("int64")
230
+ out["ref"] = _alleles(core["ref"])
231
+ out["alt"] = _alleles(core["alt"])
232
+ out["source_id"] = core["source_id"].astype(str).astype(object)
233
+ out["assembly"] = pd.Series(assembly, index=core.index, dtype=object)
234
+ out["record_index"] = core["record_index"].astype("int64")
235
+ out["alt_index"] = core["alt_index"].astype("int64")
236
+ out["variant_id"] = (
237
+ out["chrom"] + ":" + out["pos"].astype(str) + ":" + out["ref"] + ">" + out["alt"]
238
+ ).astype(object)
239
+ rows = zip(out["chrom"], out["pos"], out["ref"], out["alt"], strict=True)
240
+ if fasta is None:
241
+ status = [_status(c, int(p), r, a, None, set()) for c, p, r, a in rows]
242
+ else:
243
+ with _pysam().FastaFile(str(fasta)) as handle:
244
+ refs = set(handle.references)
245
+ status = [_status(c, int(p), r, a, handle, refs) for c, p, r, a in rows]
246
+ out["input_status"] = pd.Series(status, index=core.index, dtype=object)
247
+ out = out[list(OUTPUT_COLUMNS)]
248
+ return out if extra is None else pd.concat([out, extra], axis=1)
ag2kit/provenance.py ADDED
@@ -0,0 +1,316 @@
1
+ """Run identifiers, typed statuses, lossless TSV tables and run manifests."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import importlib.metadata
7
+ import json
8
+ import logging
9
+ import os
10
+ import platform
11
+ import subprocess
12
+ import sys
13
+ from dataclasses import dataclass, field
14
+ from datetime import UTC, datetime
15
+ from pathlib import Path
16
+ from typing import Any
17
+
18
+ import pandas as pd
19
+
20
+ log = logging.getLogger("ag2kit")
21
+
22
+ STATUSES = (
23
+ "completed",
24
+ "completed_with_warning",
25
+ "not_estimable",
26
+ "blocked",
27
+ "failed",
28
+ "not_run",
29
+ )
30
+
31
+ _VERSIONED_PACKAGES = ("ag2kit", "alphagenome", "numpy", "pandas", "scipy", "anndata")
32
+
33
+
34
+ def cache_dir() -> Path:
35
+ """Resolves the cache directory from the current environment.
36
+
37
+ Returns:
38
+ AG2KIT_CACHE if set, otherwise the ag2kit subdirectory under
39
+ XDG_CACHE_HOME or ~/.cache.
40
+ """
41
+ if env := os.environ.get("AG2KIT_CACHE"):
42
+ return Path(env)
43
+ return Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") / "ag2kit"
44
+
45
+
46
+ def utc_now() -> str:
47
+ """Returns the current UTC timestamp with fractional seconds omitted."""
48
+ return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
49
+
50
+
51
+ def _git(root: Path, *args: str) -> str:
52
+ try:
53
+ return subprocess.run(
54
+ ("git", "-C", str(root), *args), capture_output=True, text=True, check=True
55
+ ).stdout.strip()
56
+ except (subprocess.CalledProcessError, FileNotFoundError):
57
+ return ""
58
+
59
+
60
+ def git_sha(root: str | Path = ".") -> str:
61
+ """Identifies the current commit of a Git checkout.
62
+
63
+ Args:
64
+ root: Directory from which to locate the checkout.
65
+
66
+ Returns:
67
+ The short HEAD hash, or "nogit" if Git cannot resolve it.
68
+ """
69
+ return _git(Path(root), "rev-parse", "--short", "HEAD") or "nogit"
70
+
71
+
72
+ def worktree_dirty(root: str | Path = ".") -> bool:
73
+ """Checks for uncommitted changes, including untracked files.
74
+
75
+ Args:
76
+ root: Directory from which to locate the checkout.
77
+
78
+ Returns:
79
+ Whether Git reports changes. False if Git status is unavailable.
80
+ """
81
+ return bool(_git(Path(root), "status", "--porcelain"))
82
+
83
+
84
+ def make_run_id(root: str | Path = ".") -> str:
85
+ """Builds a timestamped run identifier.
86
+
87
+ Args:
88
+ root: Directory from which to locate the Git checkout.
89
+
90
+ Returns:
91
+ UTC timestamp to milliseconds, short commit hash or "nogit",
92
+ and a -dirty suffix if Git reports uncommitted changes.
93
+ """
94
+ now = datetime.now(UTC)
95
+ stamp = f"{now:%Y%m%dT%H%M%S}{now.microsecond // 1000:03d}Z"
96
+ suffix = "-dirty" if worktree_dirty(root) else ""
97
+ return f"{stamp}-{git_sha(root)}{suffix}"
98
+
99
+
100
+ def sha256_file(path: str | Path, chunk: int = 1 << 20) -> str:
101
+ """Hashes a file without loading it into memory.
102
+
103
+ Args:
104
+ path: File to hash.
105
+ chunk: Number of bytes to read at a time. Use a positive value.
106
+
107
+ Returns:
108
+ Hexadecimal SHA-256 digest.
109
+
110
+ Raises:
111
+ OSError: If the file cannot be opened or read.
112
+ """
113
+ h = hashlib.sha256()
114
+ with open(path, "rb") as fh:
115
+ while block := fh.read(chunk):
116
+ h.update(block)
117
+ return h.hexdigest()
118
+
119
+
120
+ def stable_hash(obj: Any) -> str:
121
+ """Hashes canonical JSON with sorted keys.
122
+
123
+ Args:
124
+ obj: JSON-serializable value.
125
+
126
+ Returns:
127
+ Hexadecimal SHA-256 digest of the encoded JSON.
128
+
129
+ Raises:
130
+ ValueError: If the value contains NaN or infinity, or is circular.
131
+ TypeError: If the value cannot be encoded or its keys cannot be sorted.
132
+ """
133
+ payload = json.dumps(obj, sort_keys=True, separators=(",", ":"), allow_nan=False)
134
+ return hashlib.sha256(payload.encode()).hexdigest()
135
+
136
+
137
+ def software_versions() -> dict[str, str]:
138
+ """Collects software versions for run provenance.
139
+
140
+ Returns:
141
+ Python, platform, and package version strings, with "not_installed"
142
+ for missing packages.
143
+ """
144
+ versions = {"python": sys.version.split()[0], "platform": platform.platform()}
145
+ for name in _VERSIONED_PACKAGES:
146
+ try:
147
+ versions[name] = importlib.metadata.version(name)
148
+ except importlib.metadata.PackageNotFoundError:
149
+ versions[name] = "not_installed"
150
+ return versions
151
+
152
+
153
+ def write_table(df: pd.DataFrame, path: str | Path, run_id: str) -> Path:
154
+ """Writes a TSV for use with read_table, which preserves float precision.
155
+
156
+ Args:
157
+ df: Data to write, with any existing run_id column replaced in the output.
158
+ path: Destination file. Parent directories are created as needed.
159
+ run_id: Run identifier to store on every row.
160
+
161
+ Returns:
162
+ Path to the written TSV.
163
+
164
+ Raises:
165
+ OSError: If the destination cannot be created or written.
166
+ """
167
+ path = Path(path)
168
+ path.parent.mkdir(parents=True, exist_ok=True)
169
+ out = df.copy()
170
+ out["run_id"] = run_id
171
+ out.to_csv(path, sep="\t", index=False, na_rep="")
172
+ return path
173
+
174
+
175
+ def read_table(path: str | Path) -> pd.DataFrame:
176
+ """Reads a TSV with round-trip float precision.
177
+
178
+ Args:
179
+ path: Path to the TSV.
180
+
181
+ Returns:
182
+ Parsed table, with empty fields and pandas default NA tokens treated
183
+ as missing.
184
+
185
+ Raises:
186
+ OSError: If the file cannot be read.
187
+ """
188
+ # pandas 3 parses floats exactly by default and deprecates the option.
189
+ if int(pd.__version__.split(".")[0]) < 3:
190
+ return pd.read_csv(path, sep="\t", float_precision="round_trip")
191
+ return pd.read_csv(path, sep="\t")
192
+
193
+
194
+ @dataclass
195
+ class Run:
196
+ """Collects output tables, checks, and provenance for one analysis stage.
197
+
198
+ Attributes:
199
+ stage: Stage name used in the output path and manifest.
200
+ root: Project directory whose Git state is recorded.
201
+ run_id: Output directory name, generated from root when omitted.
202
+ started_utc: UTC start timestamp recorded in the manifest.
203
+ outdir: Run output directory, created on access. AG2KIT_RESULTS_ROOT
204
+ overrides the default results directory under root.
205
+ failed_checks: Recorded verification rows whose pass flag is false.
206
+ """
207
+
208
+ stage: str
209
+ root: Path
210
+ run_id: str = ""
211
+ started_utc: str = field(default_factory=utc_now)
212
+ _status: list[dict] = field(default_factory=list, init=False, repr=False)
213
+ _checks: list[dict] = field(default_factory=list, init=False, repr=False)
214
+ _manifest: dict[str, str] = field(default_factory=dict, init=False, repr=False)
215
+
216
+ def __post_init__(self) -> None:
217
+ self.root = Path(self.root)
218
+ self.run_id = self.run_id or make_run_id(self.root)
219
+
220
+ @property
221
+ def outdir(self) -> Path:
222
+ """Output directory for this stage and run ID, created on access."""
223
+ results = os.environ.get("AG2KIT_RESULTS_ROOT") or self.root / "results"
224
+ d = Path(results) / self.stage / self.run_id
225
+ d.mkdir(parents=True, exist_ok=True)
226
+ return d
227
+
228
+ def status(self, component: str, status: str, reason: str = "") -> None:
229
+ """Records and logs a component status.
230
+
231
+ Args:
232
+ component: Name of the component reporting its status.
233
+ status: One of the values in STATUSES.
234
+ reason: Optional explanation recorded alongside the status.
235
+
236
+ Raises:
237
+ ValueError: If status is unknown.
238
+ """
239
+ if status not in STATUSES:
240
+ raise ValueError(f"unknown status {status!r}; expected one of {STATUSES}")
241
+ self._status.append({"component": component, "status": status, "reason": reason})
242
+ log.info("[%s] %s%s", status, component, f" - {reason}" if reason else "")
243
+
244
+ def check(self, area: str, name: str, passed: bool, value: Any = "", note: str = "") -> bool:
245
+ """Records a verification result.
246
+
247
+ Args:
248
+ area: Category for the check.
249
+ name: Check name.
250
+ passed: Whether the check passed.
251
+ value: Optional observed value.
252
+ note: Optional context for the result.
253
+
254
+ Returns:
255
+ The recorded pass flag.
256
+ """
257
+ self._checks.append(
258
+ {"area": area, "check": name, "pass": bool(passed), "value": value, "note": note}
259
+ )
260
+ return bool(passed)
261
+
262
+ def note(self, key: str, value: Any) -> None:
263
+ """Adds or replaces a manifest field.
264
+
265
+ Args:
266
+ key: Manifest key, which can also override a built-in field.
267
+ value: Value to store as text.
268
+ """
269
+ self._manifest[key] = str(value)
270
+
271
+ def table(self, df: pd.DataFrame, name: str) -> Path:
272
+ """Writes a table under the run output directory.
273
+
274
+ Args:
275
+ df: Data to export.
276
+ name: Relative filename for the TSV.
277
+
278
+ Returns:
279
+ Path to the written table.
280
+
281
+ Raises:
282
+ OSError: If the output cannot be written.
283
+ """
284
+ return write_table(df, self.outdir / name, self.run_id)
285
+
286
+ def finish(self) -> Path:
287
+ """Writes the manifest and any recorded statuses and checks.
288
+
289
+ Returns:
290
+ Path to run_manifest.tsv.
291
+
292
+ Raises:
293
+ OSError: If an output table cannot be written.
294
+ """
295
+ if self._status:
296
+ self.table(pd.DataFrame(self._status), "status.tsv")
297
+ if self._checks:
298
+ self.table(pd.DataFrame(self._checks), "verification.tsv")
299
+ items = {
300
+ "run_id": self.run_id,
301
+ "stage": self.stage,
302
+ "started_utc": self.started_utc,
303
+ "finished_utc": utc_now(),
304
+ "git_sha": git_sha(self.root),
305
+ "worktree_dirty": str(worktree_dirty(self.root)).lower(),
306
+ }
307
+ items |= {f"version.{k}": v for k, v in software_versions().items()}
308
+ items |= self._manifest
309
+ return self.table(
310
+ pd.DataFrame({"item": list(items), "value": list(items.values())}), "run_manifest.tsv"
311
+ )
312
+
313
+ @property
314
+ def failed_checks(self) -> list[dict]:
315
+ """Verification rows whose pass flag is false."""
316
+ return [c for c in self._checks if not c["pass"]]
ag2kit/py.typed ADDED
File without changes