smftools 0.2.4__tar.gz → 0.2.5__tar.gz

smftools-0.2.5/.github/workflows/ci.yml

@@ -0,0 +1,115 @@
+name: CI
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+
+jobs:
+  format:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install ruff
+
+      - name: Check formatting with ruff
+        run: ruff format --check .
+
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install ruff
+
+      - name: Lint with ruff
+        run: ruff check .
+
+  smoke:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install system deps for pysam
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            build-essential \
+            zlib1g-dev \
+            libbz2-dev \
+            liblzma-dev \
+            libcurl4-openssl-dev \
+            pkg-config
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install .[dev]
+
+      - name: Run smoke tests
+        run: pytest -m smoke -q
+
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install build tooling
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install build
+
+      - name: Build package
+        run: python -m build

{smftools-0.2.4 → smftools-0.2.5}/.gitignore

@@ -20,8 +20,10 @@ venvs/
 /environment.yml
 
 # Tests
-/tests/_test_inputs/
+/tests/_test_inputs/dorado_models
 /tests/_test_outputs/
+/tests/_test_outputs*/
+
 
 # OS
 .DS_Store

smftools-0.2.5/.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.8
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+      - id: ruff-format

{smftools-0.2.4 → smftools-0.2.5}/.readthedocs.yaml

@@ -1,6 +1,6 @@
 version: 2
 build:
-  os: ubuntu-20.04
+  os: ubuntu-22.04
   tools:
     python: "3.12"
 sphinx:
@@ -10,8 +10,7 @@ python:
   install:
     - method: pip
       path: .
-      extra_requirements:
-      - docs
+    - requirements: docs/source/requirements.txt
 submodules:
   include: all
   recursive: true

smftools-0.2.5/AGENTS.md

@@ -0,0 +1,59 @@
+# AGENTS.md
+
+This file tells coding agents (including OpenAI Codex) how to work in this repo.
+
+## Goals
+- Make minimal, correct changes.
+- Prefer small PRs / diffs.
+- Keep behavior stable unless the task explicitly requests changes.
+
+## Repo orientation
+- Read existing patterns before inventing new ones.
+- Don’t refactor broadly unless asked.
+- If you’re unsure about intended behavior, look for tests/docs first.
+
+## Setup
+- Create env (pick one):
+  - `python -m venv .venv && source .venv/bin/activate`
+  - or `conda env create -f environment.yml && conda activate <env>`
+- Install:
+  - `pip install -e ".[dev]"`
+
+## How to run checks
+- Smoke tests: `pytest -m smoke -q`
+- Unit tests: `pytest -m unit -q`
+- Integration tests: `pytest -m integration -q`
+- E2E tests: `pytest -m e2e -q`
+- Coverage (if configured): `pytest --cov`
+- Lint: `ruff check .`
+- Format: `ruff format .`
+- Type-check (if configured): `mypy .`
+
+## Coding conventions
+- Follow existing style and module layout.
+- Prefer clear, explicit code over cleverness.
+- Add/adjust tests for bug fixes and new behavior.
+- Keep public APIs backward compatible unless explicitly changing them.
+- Python:
+  - Use type hints for new/modified functions where reasonable.
+  - Use Google style docstring format.
+  - Avoid heavy dependencies unless necessary.
+
+## Testing expectations
+- New functionality must include tests.
+- Bug fix PRs should include a regression test.
+- If tests are flaky or slow, note it and scope the change.
+
+## Logging & secrets
+- Don’t log secrets, tokens, or PII.
+- Never hardcode credentials.
+- If sample keys are needed, use obvious placeholders like `YOUR_API_KEY_HERE`.
+
+## Git / PR hygiene
+- Keep commits focused.
+- Update docs/changelog if behavior or user-facing CLI changes.
+- If you change a CLI flag or config schema, add a migration note.
+
+## If something fails
+- If a command fails, paste the full error and summarize likely causes.
+- Don’t “fix” by deleting tests or weakening assertions unless explicitly instructed.

{smftools-0.2.4 → smftools-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: smftools
-Version: 0.2.4
+Version: 0.2.5
 Summary: Single Molecule Footprinting Analysis in Python.
 Project-URL: Source, https://github.com/jkmckenna/smftools
 Project-URL: Documentation, https://smftools.readthedocs.io/
@@ -37,13 +37,14 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Natural Language :: English
 Classifier: Operating System :: MacOS :: MacOS X
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Classifier: Topic :: Scientific/Engineering :: Visualization
-Requires-Python: <3.13,>=3.9
+Requires-Python: >=3.10
 Requires-Dist: anndata>=0.10.0
 Requires-Dist: biopython>=1.79
 Requires-Dist: captum
@@ -72,24 +73,28 @@ Requires-Dist: torch>=1.9.0
 Requires-Dist: tqdm
 Requires-Dist: upsetplot
 Requires-Dist: wandb
+Provides-Extra: dev
+Requires-Dist: pre-commit; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
 Provides-Extra: docs
 Requires-Dist: ipython>=7.20; extra == 'docs'
 Requires-Dist: matplotlib!=3.6.1; extra == 'docs'
 Requires-Dist: myst-nb>=1; extra == 'docs'
 Requires-Dist: myst-parser>=2; extra == 'docs'
 Requires-Dist: nbsphinx>=0.9; extra == 'docs'
+Requires-Dist: pyyaml; extra == 'docs'
 Requires-Dist: readthedocs-sphinx-search; extra == 'docs'
 Requires-Dist: setuptools; extra == 'docs'
 Requires-Dist: sphinx-autodoc-typehints>=1.25.2; extra == 'docs'
 Requires-Dist: sphinx-book-theme>=1.1.0; extra == 'docs'
+Requires-Dist: sphinx-click; extra == 'docs'
 Requires-Dist: sphinx-copybutton; extra == 'docs'
 Requires-Dist: sphinx-design; extra == 'docs'
 Requires-Dist: sphinx>=7; extra == 'docs'
 Requires-Dist: sphinxcontrib-bibtex; extra == 'docs'
 Requires-Dist: sphinxext-opengraph; extra == 'docs'
-Provides-Extra: tests
-Requires-Dist: pytest; extra == 'tests'
-Requires-Dist: pytest-cov; extra == 'tests'
 Description-Content-Type: text/markdown
 
 [![PyPI](https://img.shields.io/pypi/v/smftools.svg)](https://pypi.org/project/smftools)
@@ -102,40 +107,7 @@ A Python tool for processing raw sequencing data derived from single molecule fo
 While genomic data structures (SAM/BAM) were built to handle low-coverage data (<1000X) along large references, smftools prioritizes high-coverage data (scalable to >1,000,000X coverage) of a few genomic loci at a time. This enables efficient data storage, rapid data operations, hierarchical metadata handling, seamless integration with various machine-learning packages, and ease of visualization. Furthermore, functionality is modularized, enabling analysis sessions to be saved, reloaded, and easily shared with collaborators. Analyses are centered around the [anndata](https://anndata.readthedocs.io/en/latest/) object, and are heavily inspired by the work conducted within the single-cell genomics community.
 
 ## Dependencies
-The following CLI tools need to be installed and configured before using the informatics (smftools.inform) module of smftools:
-1) [Dorado](https://github.com/nanoporetech/dorado) -> Basecalling, alignment, demultiplexing.
-2) [Minimap2](https://github.com/lh3/minimap2) -> Alignment if not using dorado.
-3) [Modkit](https://github.com/nanoporetech/modkit) -> Extracting read level methylation metrics from modified BAM files. Only required for direct modification detection (ie methylation).
-
-## Main Commands
-### smftools load: Processes raw Nanopore/Illumina data from SMF experiments into an AnnData object.
-![](docs/source/_static/smftools_informatics_diagram.png)
-### smftools preprocess: Appends QC metrics to the AnnData object and performs filtering.
-![](docs/source/_static/smftools_preprocessing_diagram.png)
-### smftools spatial: Appends spatial analyses to the AnnData object.
-- Currently Includes: Position X Position correlation matrices, clustering, dimensionality reduction, spatial autocorrelation. 
-### smftools hmm: Fits a basic HMM to each sample and appends HMM feature layers
-- Main outputs wills be stored in adata.layers
-### smftools batch <command>: Performs batch processing on a csv of config file pathes for any of the above commands.
-- Nice when analyzing multiple experiments
-### smftools concatenate: Concatenates a list or directory of anndata objects.
-- Mainly used for combining multiple experiments into a single anndata object.
-
-## Announcements
-
-### 12/02/25 - Version 0.2.3 is available through PyPI
-Version 0.2.3 provides the core smftools functionality through several command line commands (load, preprocess, spatial, hmm).
-
-### 11/05/25 - Version 0.2.1 is available through PyPI
-Version 0.2.1 makes the core workflow (smftools load) a command line tool that takes in an experiment_config.csv file for input/output and parameter management.
-
-### 05/29/25 - Version 0.1.6 is available through PyPI.
-Informatics, preprocessing, tools, plotting modules have core functionality that is approaching stability on MacOS(Intel/Silicon) and Linux(Ubuntu). I will work on improving documentation/tutorials shortly. The base PyTorch/Scikit-Learn ML-infrastructure is going through some organizational changes to work with PyTorch Lightning, Hydra, and WanDB to facilitate organizational scaling, multi-device usage, and logging.
-
-### 10/01/24 - More recent versions are being updated frequently. Installation from source over PyPI is recommended!
-
-### 09/09/24 - The version 0.1.1 package ([smftools-0.1.1](https://pypi.org/project/smftools/)) is installable through pypi!
-The informatics module has been bumped to alpha-phase status. This module can deal with POD5s and unaligned BAMS from nanopore conversion and direct SMF experiments, as well as FASTQs from Illumina conversion SMF experiments. Primary output from this module is an AnnData object containing all relevant SMF data, which is compatible with all downstream smftools modules. The other modules are still in pre-alpha phase. Preprocessing, Tools, and Plotting modules should be promoted to alpha-phase within the next month or so.
-
-### 08/30/24 - The version 0.1.0 package ([smftools-0.1.0](https://pypi.org/project/smftools/)) is installable through pypi!
-Currently, this package (smftools-0.1.0) is going through rapid improvement (dependency handling accross Linux and Mac OS, testing, documentation, debugging) and is still too early in development for widespread use. The underlying functionality was originally developed as a collection of scripts for single molecule footprinting (SMF) experiments in our lab, but is being packaged/developed to facilitate the expansion of SMF to any lab that is interested in performing these styles of experiments/analyses. The alpha-phase package is expected to be available within a couple months, so stay tuned!
+The following CLI tools need to be installed and configured before using the informatics (smftools.inform) module of smftools, which is used by the smftools load CLI command:
+1) [Dorado](https://github.com/nanoporetech/dorado) -> Basecalling, alignment, demultiplexing. Required for Nanopore SMF experiments, but not Illumina SMF experiments.
+2) [Minimap2](https://github.com/lh3/minimap2) -> Aligner if not using dorado. Support for other aligners could eventually be added if needed.
+3) [Modkit](https://github.com/nanoporetech/modkit) -> Extracting read level methylation metrics from the MM/ML tags in BAM files. Only required for direct modification detection SMF protocols.

smftools-0.2.5/README.md

@@ -0,0 +1,14 @@
+[![PyPI](https://img.shields.io/pypi/v/smftools.svg)](https://pypi.org/project/smftools)
+[![Docs](https://readthedocs.org/projects/smftools/badge/?version=latest)](https://smftools.readthedocs.io/en/latest/?badge=latest)
+
+# smftools
+A Python tool for processing raw sequencing data derived from single molecule footprinting experiments into [anndata](https://anndata.readthedocs.io/en/latest/) objects. Additional functionality for preprocessing, spatial analyses, and HMM based feature annotation.
+
+## Philosophy
+While genomic data structures (SAM/BAM) were built to handle low-coverage data (<1000X) along large references, smftools prioritizes high-coverage data (scalable to >1,000,000X coverage) of a few genomic loci at a time. This enables efficient data storage, rapid data operations, hierarchical metadata handling, seamless integration with various machine-learning packages, and ease of visualization. Furthermore, functionality is modularized, enabling analysis sessions to be saved, reloaded, and easily shared with collaborators. Analyses are centered around the [anndata](https://anndata.readthedocs.io/en/latest/) object, and are heavily inspired by the work conducted within the single-cell genomics community.
+
+## Dependencies
+The following CLI tools need to be installed and configured before using the informatics (smftools.inform) module of smftools, which is used by the smftools load CLI command:
+1) [Dorado](https://github.com/nanoporetech/dorado) -> Basecalling, alignment, demultiplexing. Required for Nanopore SMF experiments, but not Illumina SMF experiments.
+2) [Minimap2](https://github.com/lh3/minimap2) -> Aligner if not using dorado. Support for other aligners could eventually be added if needed.
+3) [Modkit](https://github.com/nanoporetech/modkit) -> Extracting read level methylation metrics from the MM/ML tags in BAM files. Only required for direct modification detection SMF protocols.

smftools-0.2.5/docs/source/api/datasets.md

@@ -0,0 +1,14 @@
+## Datasets:
+
+```{eval-rst}
+.. autosummary::
+   :toctree: generated/datasets
+
+   smftools.datasets.datasets
+```
+
+```{eval-rst}
+.. automodule:: smftools.datasets
+   :no-members:
+   :show-inheritance:
+```

smftools-0.2.5/docs/source/api/informatics.md

@@ -0,0 +1,44 @@
+## Informatics: `inform`
+
+## Informatics module diagram
+```{image} ../_static/smftools_informatics_diagram.png
+:width: 1000px
+```
+
+Processes raw sequencing data to load an adata object.
+
+```{eval-rst}
+.. autosummary::
+   :toctree: generated/informatics
+   
+   smftools.informatics.bam_functions
+   smftools.informatics.basecalling
+   smftools.informatics.bed_functions
+   smftools.informatics.binarize_converted_base_identities
+   smftools.informatics.complement_base_list
+   smftools.informatics.converted_BAM_to_adata
+   smftools.informatics.fasta_functions
+   smftools.informatics.h5ad_functions
+   smftools.informatics.modkit_extract_to_adata
+   smftools.informatics.modkit_functions
+   smftools.informatics.ohe
+   smftools.informatics.pod5_functions
+   smftools.informatics.run_multiqc
+```
+
+```{eval-rst}
+.. automodule:: smftools.informatics
+   :no-members:
+   :show-inheritance:
+```
+
+
+### Diagram of final steps of Direct SMF workflow
+```{image} ../_static/modkit_extract_to_adata.png
+:width: 1000px
+```
+
+### Diagram of final steps of Conversion SMF workflow
+```{image} ../_static/converted_BAM_to_adata.png
+:width: 1000px
+```

smftools-0.2.5/docs/source/api/preprocessing.md

@@ -0,0 +1,43 @@
+## Preprocessing: `pp`
+
+## Preprocessing module diagram
+```{image} ../_static/smftools_preprocessing_diagram.png
+:width: 1000px
+```
+
+```{eval-rst}
+.. autosummary::
+   :toctree: generated/preprocessing
+   
+   smftools.preprocessing.append_base_context
+   smftools.preprocessing.append_binary_layer_by_base_context
+   smftools.preprocessing.binarize
+   smftools.preprocessing.binarize_on_Youden
+   smftools.preprocessing.binary_layers_to_ohe
+   smftools.preprocessing.calculate_complexity_II
+   smftools.preprocessing.calculate_consensus
+   smftools.preprocessing.calculate_coverage
+   smftools.preprocessing.calculate_pairwise_differences
+   smftools.preprocessing.calculate_pairwise_hamming_distances
+   smftools.preprocessing.calculate_position_Youden
+   smftools.preprocessing.calculate_read_length_stats
+   smftools.preprocessing.calculate_read_modification_stats
+   smftools.preprocessing.clean_NaN
+   smftools.preprocessing.filter_adata_by_nan_proportion
+   smftools.preprocessing.filter_reads_on_length_quality_mapping
+   smftools.preprocessing.filter_reads_on_modification_thresholds
+   smftools.preprocessing.flag_duplicate_reads
+   smftools.preprocessing.invert_adata
+   smftools.preprocessing.load_sample_sheet
+   smftools.preprocessing.make_dirs
+   smftools.preprocessing.min_non_diagonal
+   smftools.preprocessing.recipes
+   smftools.preprocessing.reindex_references_adata
+   smftools.preprocessing.subsample_adata
+```
+
+```{eval-rst}
+.. automodule:: smftools.preprocessing
+   :no-members:
+   :show-inheritance:
+```

smftools-0.2.5/docs/source/api/tools.md

@@ -0,0 +1,20 @@
+## Tools: `tl`
+
+```{eval-rst}
+.. autosummary::
+   :toctree: generated/tools
+   
+   smftools.tools.calculate_umap
+   smftools.tools.cluster_adata_on_methylation
+   smftools.tools.general_tools
+   smftools.tools.position_stats
+   smftools.tools.read_stats
+   smftools.tools.spatial_autocorrelation
+   smftools.tools.subset_adata
+```
+
+```{eval-rst}
+.. automodule:: smftools.tools
+   :no-members:
+   :show-inheritance:
+```

{smftools-0.2.4 → smftools-0.2.5}/docs/source/basic_usage.md

@@ -13,6 +13,8 @@ This command takes a user passed config file handling:
     - Experiment info (SMF modality, sequencer type, barcoding kit if nanopore, sample sheet with metadata mapping)
     - Options to override default workflow parameters from smftools/config. Params are handled from default.yaml -> modality_type.yaml -> user passed config.csv.
 
+![](_static/smftools_informatics_diagram.png)
+
 ## Preprocess Usage
 
 This command performs preprocessing on the anndata object. It automatically runs the load command under the hood if starting from raw data.
@@ -21,6 +23,8 @@ This command performs preprocessing on the anndata object. It automatically runs
 smftools preprocess "/Path_to_experiment_config.csv"
 ```
 
+![](_static/smftools_preprocessing_diagram.png)
+
 ## Spatial Usage
 
 This command performs spatial analysis on the anndata object. It automatically runs the load command and preprocessing under the hood if they have not been already run.
@@ -29,6 +33,8 @@ This command performs spatial analysis on the anndata object. It automatically r
 smftools spatial "/Path_to_experiment_config.csv"
 ```
 
+- Currently Includes: Position X Position correlation matrices, clustering, dimensionality reduction, spatial autocorrelation. 
+
 ## HMM Usage
 
 This command performs hmm based feature annotation on the anndata object. It automatically runs the load command and preprocessing under the hood if they have not been already run.
@@ -37,6 +43,8 @@ This command performs hmm based feature annotation on the anndata object. It aut
 smftools hmm "/Path_to_experiment_config.csv"
 ```
 
+- Main outputs wills be stored in adata.layers
+
 ## Batch Usage
 
 This command performs batch processing of any of the above commands across multiple experiments. It takes in a tsv, txt, or csv of experiment specific config csvs.
@@ -44,11 +52,38 @@ This command performs batch processing of any of the above commands across multi
 smftools batch preprocess "/Path_to_experiment_config_path_list.csv"
 ```
 
+- Nice when analyzing multiple experiments
+
 ## Concatenate Usage
 
 This command concatenates multiple h5ad files and saves them to a new output. The h5ads to concatenate are provided as a txt, tsv, or h5ad file of paths.
 ```shell
-smftools concatenate output.h5ad "/Path_to_h5ad_path_list.csv"
+smftools concatenate output.h5ad -c "/Path_to_h5ad_path_list.csv"
+```
+
+Alternatively, you can just concatenate all h5ads within a given directory.
+```shell
+smftools concatenate output.h5ad -d "/Path_to_h5ad_file_dir/"
+```
+
+- Mainly used for combining multiple experiments into a single anndata object.
+
+## Subsample POD5 Usage
+
+This command subsamples a POD5 file or a directory of POD5 files. It can be done by passing a txt file of read names to use, or an integer number of reads.
+```shell
+smftools subsample-pod5 -r "/Path_to_read_name_list.txt" -o "/Path_to_output_directory" "/Path_to_input_POD5_dir_or_file"
+```
+
+```shell
+smftools subsample-pod5 -n 1000 -o "/Path_to_output_directory" "/Path_to_input_POD5_dir_or_file"
+```
+
+## Optional run logging
+
+If you want to maintain run log files of CLI processes, you can use the following syntax to any of the CLI commands. Here is an example using smftools load with logging performed on INFO level logging outputs and above.
+```shell
+smftools --log-file "/Path_to_output_log_file.log" --log-level INFO load "/Path_to_input_config.csv"
 ```
 
 ## Reading AnnData objects created by smftools

smftools-0.2.5/docs/source/cli.md

@@ -0,0 +1,6 @@
+# Command-line interface
+
+```{click} smftools.cli_entry:cli
+:prog: smftools
+:nested: full
+```