zipoop 0.1.0__tar.gz

zipoop-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 zatumout
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

zipoop-0.1.0/PKG-INFO

@@ -0,0 +1,51 @@
+Metadata-Version: 2.4
+Name: zipoop
+Version: 0.1.0
+Summary: Decoder and reader for Zipoop teacher shards (per-window neural compression of ENCODE signal tracks)
+Author: zatumout
+License-Expression: MIT
+Project-URL: Dataset, https://huggingface.co/datasets/zatumout/zipoop-teacher-shards
+Keywords: genomics,ENCODE,neural compression,distillation
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+Requires-Dist: torch>=2.2
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == "test"
+Dynamic: license-file
+
+# zipoop
+
+Decoder and reader for the Zipoop teacher shards (dataset: https://huggingface.co/datasets/zatumout/zipoop-teacher-shards): 89,438 windows of 32 kb, 1,885 ENCODE tracks, stored as one `<chrom>.npz` per chromosome (`weights_npz`, with `MANIFEST.tsv`).
+
+## Data layout
+
+`<root>/<chrom>.npz` (25 files) plus `MANIFEST.tsv` (chrom, windows, members, bytes, sha256). Each file is a deflate zip with members `<window>/<key>.npy` (161 arrays per window), windows in genome order. Window names are `<chrom>_<start>_<end>`.
+
+## Usage
+
+```python
+from zipoop import Dataset
+with Dataset("/path/to/weights_npz") as ds:
+    w = ds.load_window(ds.windows("chr21")[0])   # TeacherWindow: key, dna (L,), signal (L, T), valid (L, T)
+```
+
+```
+zipoop verify /path/to/weights_npz      # size + sha256 against MANIFEST.tsv
+zipoop show   /path/to/weights_npz chr21_10027008_10059776
+```
+
+Lower level: `Dataset.read(window)` gives the parsed `Shard`, `Decoder(shard).decode(start, end, tracks)` decodes any slice or track subset. Evaluation helpers (`window_pearson`, `summarize`, `calibrate`, `restore_real`) are in `zipoop.evaluation`.
+
+Rules baked into `load_window` (see `zipoop/teacher.py`): RNA-seq tracks are 0 and valid where unrecorded; other tracks stay missing there. Opening a chromosome file parses its whole zip directory (chr1 about 3.6 s and 0.9 GB), so keep the `Dataset` open; forked DataLoader workers reopen their own handles.
+
+Only the shard variant in this dataset is supported (hybrid `pooper-dna-v3`, max-pool, sinusoidal positions, mask `track-rle-lzma-v2`); other variants raise a clear error.
+
+Tests: `ZIPOOP_DATA=<weights_npz> pytest tests` (the pooper-equivalence test also needs the original shards and the pooper package).
+
+## License and data terms
+
+The code is MIT-licensed (see `LICENSE`; keep the copyright notice "zatumout"). The shard dataset is released separately under CC BY 4.0 (credit: zatumout), commercial use and model distillation allowed. The shards and `zipoop/data/encode-panel_1885.json` are derived from public ENCODE tracks: when you use them, also cite and acknowledge the ENCODE Consortium as described at https://www.encodeproject.org/help/citing-encode.

zipoop-0.1.0/README.md

@@ -0,0 +1,32 @@
+# zipoop
+
+Decoder and reader for the Zipoop teacher shards (dataset: https://huggingface.co/datasets/zatumout/zipoop-teacher-shards): 89,438 windows of 32 kb, 1,885 ENCODE tracks, stored as one `<chrom>.npz` per chromosome (`weights_npz`, with `MANIFEST.tsv`).
+
+## Data layout
+
+`<root>/<chrom>.npz` (25 files) plus `MANIFEST.tsv` (chrom, windows, members, bytes, sha256). Each file is a deflate zip with members `<window>/<key>.npy` (161 arrays per window), windows in genome order. Window names are `<chrom>_<start>_<end>`.
+
+## Usage
+
+```python
+from zipoop import Dataset
+with Dataset("/path/to/weights_npz") as ds:
+    w = ds.load_window(ds.windows("chr21")[0])   # TeacherWindow: key, dna (L,), signal (L, T), valid (L, T)
+```
+
+```
+zipoop verify /path/to/weights_npz      # size + sha256 against MANIFEST.tsv
+zipoop show   /path/to/weights_npz chr21_10027008_10059776
+```
+
+Lower level: `Dataset.read(window)` gives the parsed `Shard`, `Decoder(shard).decode(start, end, tracks)` decodes any slice or track subset. Evaluation helpers (`window_pearson`, `summarize`, `calibrate`, `restore_real`) are in `zipoop.evaluation`.
+
+Rules baked into `load_window` (see `zipoop/teacher.py`): RNA-seq tracks are 0 and valid where unrecorded; other tracks stay missing there. Opening a chromosome file parses its whole zip directory (chr1 about 3.6 s and 0.9 GB), so keep the `Dataset` open; forked DataLoader workers reopen their own handles.
+
+Only the shard variant in this dataset is supported (hybrid `pooper-dna-v3`, max-pool, sinusoidal positions, mask `track-rle-lzma-v2`); other variants raise a clear error.
+
+Tests: `ZIPOOP_DATA=<weights_npz> pytest tests` (the pooper-equivalence test also needs the original shards and the pooper package).
+
+## License and data terms
+
+The code is MIT-licensed (see `LICENSE`; keep the copyright notice "zatumout"). The shard dataset is released separately under CC BY 4.0 (credit: zatumout), commercial use and model distillation allowed. The shards and `zipoop/data/encode-panel_1885.json` are derived from public ENCODE tracks: when you use them, also cite and acknowledge the ENCODE Consortium as described at https://www.encodeproject.org/help/citing-encode.

zipoop-0.1.0/pyproject.toml

@@ -0,0 +1,35 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "zipoop"
+version = "0.1.0"
+description = "Decoder and reader for Zipoop teacher shards (per-window neural compression of ENCODE signal tracks)"
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{name = "zatumout"}]
+keywords = ["genomics", "ENCODE", "neural compression", "distillation"]
+classifiers = ["Programming Language :: Python :: 3", "Topic :: Scientific/Engineering :: Bio-Informatics"]
+dependencies = ["numpy>=1.26", "torch>=2.2"]
+
+[project.optional-dependencies]
+test = ["pytest>=8"]
+
+[project.urls]
+Dataset = "https://huggingface.co/datasets/zatumout/zipoop-teacher-shards"
+
+[project.scripts]
+zipoop = "zipoop.cli:main"
+
+[tool.setuptools.packages.find]
+include = ["zipoop*"]
+
+[tool.setuptools.package-data]
+zipoop = ["data/*.json"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-q"

zipoop-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

zipoop-0.1.0/tests/test_zipoop.py

@@ -0,0 +1,123 @@
+"""Needs the real dataset: set ZIPOOP_DATA to the merged weights_npz directory (tests skip otherwise).
+The equivalence tests additionally need the original per-window shards (ZIPOOP_SINGLE, default
+/mnt/e/zipoop/weights) and the pooper package on the path."""
+import os
+import random
+from pathlib import Path
+
+import numpy as np
+import pytest
+import torch
+
+import zipoop
+from zipoop.masks import decode_mask
+from zipoop.network import ModelConfig
+
+DATA = os.environ.get("ZIPOOP_DATA")
+SINGLE = Path(os.environ.get("ZIPOOP_SINGLE", "/mnt/e/zipoop/weights"))
+needs_data = pytest.mark.skipif(not DATA, reason="ZIPOOP_DATA not set")
+
+
+@pytest.fixture(scope="module")
+def ds():
+    with zipoop.Dataset(DATA) as d:
+        yield d
+
+
+@pytest.fixture(scope="module")
+def windows(ds):
+    ws = ds.windows("chr21")
+    return random.Random(0).sample(ws, 3)
+
+
+@needs_data
+def test_manifest_verifies_small_chromosomes():
+    assert zipoop.verify(DATA, {"chrM", "chr22"}) == []
+
+
+@needs_data
+def test_window_listing_is_genome_ordered(ds):
+    starts = [int(w.split("_")[1]) for w in ds.windows("chr21")]
+    assert starts == sorted(starts)
+
+
+@needs_data
+def test_load_window_shapes_and_rna_rule(ds, windows):
+    w = ds.load_window(windows[0])
+    L, T = w.signal.shape
+    assert (L, T) == (32768, 1885) and w.dna.shape == (L,) and w.valid.shape == (L, T)
+    zero = torch.from_numpy(zipoop.zero_tracks())
+    assert w.valid[:, zero].all()
+    assert torch.isfinite(w.signal).all()
+
+
+@needs_data
+def test_slice_and_track_subset_match_full_decode(ds, windows):
+    dec = zipoop.Decoder(ds.read(windows[0]))
+    full = dec.decode()
+    part = dec.decode(1000, 5000, tracks=[3, 1884, 7])
+    expected = full[1000:5000][:, [3, 1884, 7]]
+    assert torch.allclose(part, expected, atol=1e-5, equal_nan=True)
+
+
+@needs_data
+def test_region_is_concatenation(ds):
+    ws = ds.windows("chr22")[:2]
+    r = ds.load_region(ws)
+    assert r.signal.shape[0] == 2 * 32768 and r.key == "+".join(ws)
+
+
+@needs_data
+def test_unknown_window_and_chrom(ds):
+    with pytest.raises(KeyError):
+        ds.read("chr21_1_2")
+    with pytest.raises(FileNotFoundError):
+        ds.windows("chrZ")
+
+
+@needs_data
+def test_bitwise_identical_to_pooper(ds, windows):
+    ref = pytest.importorskip("pooper.teacher_io")
+    from pooper.hybrid import archive_decoder
+    for w in windows:
+        path = SINGLE / "chr21" / f"{w}.npz"
+        if not path.exists():
+            pytest.skip("original shards not available")
+        a, b = ds.load_window(w), ref.load_window(path)
+        assert a.key == b.key
+        for name in ("dna", "signal", "valid"):
+            assert torch.equal(getattr(a, name), getattr(b, name)), name
+    old = archive_decoder(str(SINGLE / "chr21" / f"{windows[0]}.npz"))
+    new = zipoop.Decoder(ds.read(windows[0]))
+    for args in ({}, {"start": 4096, "end": 9000, "tracks": [0, 5, 1884]}, {"missing_nan": False}):
+        x, y = new.decode(**args), old.decode(**args)
+        assert torch.equal(torch.isnan(x), torch.isnan(y))
+        assert torch.equal(x.nan_to_num(), y.nan_to_num())
+
+
+def test_model_config_rejects_unsupported_variants():
+    ok = {"width": 48, "rank": 48, "transformer_layers": 2, "attention_heads": 4, "downsample_levels": 7,
+          "pooling": "max", "position_encoding": "sinusoidal", "pair_channels": 0, "head_nonlinear_rank": 0}
+    assert ModelConfig.from_json(ok).width == 48
+    for bad in ({"pooling": "avg"}, {"position_encoding": "rope"}, {"pair_channels": 4}, {"head_nonlinear_rank": 8}):
+        with pytest.raises(ValueError):
+            ModelConfig.from_json({**ok, **bad})
+    legacy = {k: v for k, v in ok.items() if k != "pooling"}
+    with pytest.raises(ValueError):
+        ModelConfig.from_json(legacy)
+    with pytest.raises(ValueError):
+        ModelConfig.from_json({**ok, "width": 50})
+
+
+def test_decode_mask_rejects_garbage():
+    with pytest.raises(ValueError):
+        decode_mask(b"\x05\x00", (10, 3))
+    with pytest.raises(ValueError):
+        decode_mask(b"\x80", (10, 1))
+
+
+@needs_data
+def test_dataset_is_picklable_for_spawn_workers(ds):
+    import pickle
+    clone = pickle.loads(pickle.dumps(ds))
+    assert clone.chroms() == ds.chroms()

zipoop-0.1.0/zipoop/__init__.py

@@ -0,0 +1,11 @@
+"""Zipoop: decoder and reader for the per-window teacher shards (merged per-chromosome npz)."""
+from .dataset import Dataset, verify
+from .decoder import Decoder
+from .evaluation import calibrate, restore_real, summarize, window_pearson
+from .panel import panel_tracks, zero_tracks
+from .shard import Shard, read_shard
+from .teacher import TeacherWindow, load_window
+
+__version__ = "0.1.0"
+__all__ = ["Dataset", "verify", "Decoder", "Shard", "read_shard", "TeacherWindow", "load_window",
+           "panel_tracks", "zero_tracks", "calibrate", "restore_real", "summarize", "window_pearson"]

zipoop-0.1.0/zipoop/cli.py

@@ -0,0 +1,30 @@
+import argparse
+import sys
+
+from .dataset import Dataset, verify
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="zipoop", description="Zipoop teacher-shard dataset tools")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    v = sub.add_parser("verify", help="check <root>/*.npz against MANIFEST.tsv (size + sha256)")
+    v.add_argument("root")
+    v.add_argument("--chroms", default="", help="comma-separated subset")
+    v.add_argument("--workers", type=int, default=4, help="files hashed in parallel")
+    s = sub.add_parser("show", help="decode one window and print its shapes")
+    s.add_argument("root")
+    s.add_argument("window")
+    args = parser.parse_args(argv)
+    if args.cmd == "verify":
+        chroms = set(args.chroms.split(",")) if args.chroms else None
+        bad = verify(args.root, chroms, workers=args.workers, progress=lambda c, problem: print(f"{c}: {problem or 'ok'}", flush=True))
+        print(f"{len(bad)} problem(s)" if bad else "all files verified")
+        return 1 if bad else 0
+    with Dataset(args.root) as ds:
+        w = ds.load_window(args.window)
+    print(f"{w.key}: dna {tuple(w.dna.shape)}, signal {tuple(w.signal.shape)}, valid fraction {w.valid.float().mean():.4f}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())