tftokenizer 0.1.0__tar.gz

tftokenizer-0.1.0/Cargo.lock

@@ -0,0 +1,116 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "adler2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
+
+[[package]]
+name = "crc32fast"
+version = "1.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
+
+[[package]]
+name = "either"
+version = "1.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
+
+[[package]]
+name = "flate2"
+version = "1.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+ "zlib-rs",
+]
+
+[[package]]
+name = "miniz_oxide"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
+dependencies = [
+ "adler2",
+ "simd-adler32",
+]
+
+[[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "simd-adler32"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
+
+[[package]]
+name = "tftokenizer"
+version = "0.1.0"
+dependencies = [
+ "flate2",
+ "rayon",
+]
+
+[[package]]
+name = "zlib-rs"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"

tftokenizer-0.1.0/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "tftokenizer"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+path = "src/lib.rs"
+
+[dependencies]
+flate2 = "1.1"
+rayon = "1.12"
+
+[profile.release]
+lto = true
+codegen-units = 1

tftokenizer-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 SugarfreeSoda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tftokenizer-0.1.0/MANIFEST.in

@@ -0,0 +1,2 @@
+include Cargo.toml Cargo.lock
+recursive-include src *.rs

tftokenizer-0.1.0/PKG-INFO

@@ -0,0 +1,98 @@
+Metadata-Version: 2.4
+Name: tftokenizer
+Version: 0.1.0
+Summary: Tokenize the genome into bins carrying per-motif transcription-factor occupancy
+License-Expression: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+Requires-Dist: scipy>=1.11
+Requires-Dist: zarr>=3
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Dynamic: license-file
+
+# TFTokenizer
+
+TFTokenizer cuts a chromosome into variable-length tokens from transcription-factor motif hits. Each token
+records the expected occupancy of every motif inside it.
+
+It builds on the interval tokenizer of *Motif-aware Tokenization of the Genome: Towards Interpretable Modeling of
+Gene Regulation*. That tokenizer marks each motif hit as 0/1. TFTokenizer instead weights each hit by its binding
+strength and merges overlapping hits of a motif into occupancy sites.
+
+## Method
+
+1. **Binding strength.** A hit of motif *m* with JASPAR score *s* gets strength a = exp(β (S − μ<sub>m</sub>)), where
+   S = s / 100 · ln 10 (= −ln p). μ<sub>m</sub> is the 90th percentile of motif *m*'s scores on the training chromosomes,
+   and β = 0.5.
+2. **Occupancy sites.** Overlapping hits of the same motif (both strands) are merged into one site, with occupancy
+   θ = Σa / (1 + Σa) (capped at 0.99) and load H = −ln(1 − θ). H is spread evenly over the site, so each base carries H / site length.
+3. **Bins.** The per-base load of all motifs is summed and smoothed (Gaussian, σ = 3). Every peak contributes its two
+   crossings at half its prominence as bin edges. Stretches longer than 50 bp are cut every 25 bp.
+4. **Tokens.** For each bin and motif, the token value is Σ site load × fraction of the site inside the bin.
+
+## Install
+
+```bash
+pip install tftokenizer
+```
+
+Requires Python ≥ 3.11. Prebuilt wheels are published for Linux (x86_64, aarch64), macOS (Apple silicon, Intel) and
+Windows (x64), so no Rust toolchain is needed. On other platforms pip builds from source, which needs
+[Rust](https://rustup.rs).
+
+## Input
+
+The motif hits of the UCSC JASPAR track, one table per chromosome. TFTokenizer downloads them itself:
+
+```bash
+tftokenizer download --chroms chr19 chr22      # writes hits/chr19.tsv.gz and hits/chr22.tsv.gz
+```
+
+The track file is large (196 GB for hg38 JASPAR2026), so only the chosen chromosomes are fetched: the records of a
+chromosome lie together in the file, are located through its index and downloaded in one range request (2.0 GiB for
+chr22). An interrupted download resumes where it stopped, and tables already there are skipped. Each table is what
+`bigBedToBed -chrom=<chrom> -tsv` writes; `<chrom>.json` next to it records the source, row count and SHA-256.
+
+Options: `--data` (folder, default `hits`), `--assembly` (default `hg38`), `--release` (default `JASPAR2026`) and
+`--mirror` (default `https://hgdownload.soe.ucsc.edu`; the Europe mirror `https://hgdownload-euro.soe.ucsc.edu` can be
+faster, depending on the network).
+
+Tables of your own are passed with `--hits`: tab-separated, optionally gzipped, with a header, sorted by `chromStart`.
+Columns used: `chrom chromStart chromEnd name score`.
+
+## Usage
+
+Command line. Without `--hits`, `fit` and `tokenize` first download the missing tables into `--data`; `tokenize`
+downloads the assembly and release the model was fitted on:
+
+```bash
+tftokenizer fit      --chroms chr19 --model model.npz
+tftokenizer tokenize --chroms chr19 chr22 --model model.npz --out tokens
+```
+
+With tables of your own (`{chrom}` is replaced by each chromosome name):
+
+```bash
+tftokenizer fit --chroms chr19 --model model.npz --hits 'my_hits/{chrom}.tsv.gz'
+```
+
+Python:
+
+```python
+from tftokenizer import TFTokenizer, download
+
+chr19, chr22 = download(["chr19", "chr22"], "hits")
+tok = TFTokenizer().fit([(chr19, "chr19")])
+tok.save("model.npz")
+tok.tokenize(chr22, "chr22", "tokens/chr22")
+```
+
+All tunable values live in `Params` (`tftokenizer/tokenizer.py`).
+[`examples/quickstart.ipynb`](examples/quickstart.ipynb) reproduces the whole run: download, fit on chr19,
+tokenize chr19 and chr22, and load the result.
+
+
+

tftokenizer-0.1.0/README.md

@@ -0,0 +1,83 @@
+# TFTokenizer
+
+TFTokenizer cuts a chromosome into variable-length tokens from transcription-factor motif hits. Each token
+records the expected occupancy of every motif inside it.
+
+It builds on the interval tokenizer of *Motif-aware Tokenization of the Genome: Towards Interpretable Modeling of
+Gene Regulation*. That tokenizer marks each motif hit as 0/1. TFTokenizer instead weights each hit by its binding
+strength and merges overlapping hits of a motif into occupancy sites.
+
+## Method
+
+1. **Binding strength.** A hit of motif *m* with JASPAR score *s* gets strength a = exp(β (S − μ<sub>m</sub>)), where
+   S = s / 100 · ln 10 (= −ln p). μ<sub>m</sub> is the 90th percentile of motif *m*'s scores on the training chromosomes,
+   and β = 0.5.
+2. **Occupancy sites.** Overlapping hits of the same motif (both strands) are merged into one site, with occupancy
+   θ = Σa / (1 + Σa) (capped at 0.99) and load H = −ln(1 − θ). H is spread evenly over the site, so each base carries H / site length.
+3. **Bins.** The per-base load of all motifs is summed and smoothed (Gaussian, σ = 3). Every peak contributes its two
+   crossings at half its prominence as bin edges. Stretches longer than 50 bp are cut every 25 bp.
+4. **Tokens.** For each bin and motif, the token value is Σ site load × fraction of the site inside the bin.
+
+## Install
+
+```bash
+pip install tftokenizer
+```
+
+Requires Python ≥ 3.11. Prebuilt wheels are published for Linux (x86_64, aarch64), macOS (Apple silicon, Intel) and
+Windows (x64), so no Rust toolchain is needed. On other platforms pip builds from source, which needs
+[Rust](https://rustup.rs).
+
+## Input
+
+The motif hits of the UCSC JASPAR track, one table per chromosome. TFTokenizer downloads them itself:
+
+```bash
+tftokenizer download --chroms chr19 chr22      # writes hits/chr19.tsv.gz and hits/chr22.tsv.gz
+```
+
+The track file is large (196 GB for hg38 JASPAR2026), so only the chosen chromosomes are fetched: the records of a
+chromosome lie together in the file, are located through its index and downloaded in one range request (2.0 GiB for
+chr22). An interrupted download resumes where it stopped, and tables already there are skipped. Each table is what
+`bigBedToBed -chrom=<chrom> -tsv` writes; `<chrom>.json` next to it records the source, row count and SHA-256.
+
+Options: `--data` (folder, default `hits`), `--assembly` (default `hg38`), `--release` (default `JASPAR2026`) and
+`--mirror` (default `https://hgdownload.soe.ucsc.edu`; the Europe mirror `https://hgdownload-euro.soe.ucsc.edu` can be
+faster, depending on the network).
+
+Tables of your own are passed with `--hits`: tab-separated, optionally gzipped, with a header, sorted by `chromStart`.
+Columns used: `chrom chromStart chromEnd name score`.
+
+## Usage
+
+Command line. Without `--hits`, `fit` and `tokenize` first download the missing tables into `--data`; `tokenize`
+downloads the assembly and release the model was fitted on:
+
+```bash
+tftokenizer fit      --chroms chr19 --model model.npz
+tftokenizer tokenize --chroms chr19 chr22 --model model.npz --out tokens
+```
+
+With tables of your own (`{chrom}` is replaced by each chromosome name):
+
+```bash
+tftokenizer fit --chroms chr19 --model model.npz --hits 'my_hits/{chrom}.tsv.gz'
+```
+
+Python:
+
+```python
+from tftokenizer import TFTokenizer, download
+
+chr19, chr22 = download(["chr19", "chr22"], "hits")
+tok = TFTokenizer().fit([(chr19, "chr19")])
+tok.save("model.npz")
+tok.tokenize(chr22, "chr22", "tokens/chr22")
+```
+
+All tunable values live in `Params` (`tftokenizer/tokenizer.py`).
+[`examples/quickstart.ipynb`](examples/quickstart.ipynb) reproduces the whole run: download, fit on chr19,
+tokenize chr19 and chr22, and load the result.
+
+
+

tftokenizer-0.1.0/pyproject.toml

@@ -0,0 +1,46 @@
+[build-system]
+requires = ["setuptools>=77", "setuptools-rust>=1.9"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tftokenizer"
+version = "0.1.0"
+description = "Tokenize the genome into bins carrying per-motif transcription-factor occupancy"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.11"
+dependencies = ["numpy>=1.26", "scipy>=1.11", "zarr>=3"]
+
+[project.optional-dependencies]
+test = ["pytest"]
+
+[project.scripts]
+tftokenizer = "tftokenizer.cli:main"
+
+[tool.setuptools]
+packages = ["tftokenizer"]
+
+[[tool.setuptools-rust.ext-modules]]
+target = "tftokenizer._native"
+binding = "NoBinding"
+py-limited-api = true
+debug = false  # release build also for editable installs (pip install -e .)
+
+[tool.distutils.bdist_wheel]
+py-limited-api = "cp311"
+
+[tool.cibuildwheel]
+build = "cp311-*"
+skip = "*-musllinux_*"
+archs = ["native"]
+test-requires = "pytest"
+test-command = "pytest {project}/tests"
+
+[tool.cibuildwheel.linux]
+before-all = "curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal"
+environment = { PATH = "$PATH:$HOME/.cargo/bin" }
+
+[[tool.cibuildwheel.overrides]]
+select = "*-macosx_x86_64"
+environment = { MACOSX_DEPLOYMENT_TARGET = "10.12" }  # Rust's standard library needs macOS ≥ 10.12 on Intel Macs

tftokenizer-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+