smftools 0.3.2__tar.gz → 2.16.0__tar.gz

smftools-2.16.0/.devcontainer/Dockerfile

@@ -0,0 +1,38 @@
+FROM python:3.12-slim-bookworm
+
+# System deps needed by pysam, pybedtools, htslib, etc.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    git \
+    zlib1g-dev \
+    libbz2-dev \
+    liblzma-dev \
+    libcurl4-openssl-dev \
+    libssl-dev \
+    libdeflate-dev \
+    samtools \
+    bedtools \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user
+ARG USERNAME=vscode
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+RUN groupadd --gid $USER_GID $USERNAME \
+    && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME
+
+# Set up a venv so the editable install is isolated
+RUN python -m venv /opt/venv \
+    && /opt/venv/bin/pip install --upgrade pip
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Pre-install heavy deps so rebuilds are faster
+COPY pyproject.toml README.md LICENSE /tmp/smftools/
+RUN pip install --no-cache-dir \
+    "numpy>=1.22.0,<2" \
+    "scipy>=1.7.3" \
+    "torch>=1.9.0" \
+    "anndata>=0.10.0" \
+    "pysam>=0.19.1"
+
+USER $USERNAME

smftools-2.16.0/.devcontainer/devcontainer.json

@@ -0,0 +1,32 @@
+{
+  "name": "smftools",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "context": ".."
+  },
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "python.defaultInterpreterPath": "/opt/venv/bin/python",
+        "python.testing.pytestEnabled": true,
+        "python.testing.pytestArgs": ["tests"],
+        "editor.formatOnSave": true,
+        "[python]": {
+          "editor.defaultFormatter": "charliermarsh.ruff",
+          "editor.codeActionsOnSave": {
+            "source.fixAll": "explicit",
+            "source.organizeImports": "explicit"
+          }
+        }
+      },
+      "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance",
+        "charliermarsh.ruff",
+        "ms-toolsai.jupyter"
+      ]
+    }
+  },
+  "postCreateCommand": "pip install -e '.[all,dev]'",
+  "remoteUser": "vscode"
+}

{smftools-0.3.2 → smftools-2.16.0}/.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install "ruff==0.8.6"
+          python -m pip install "ruff==0.15.0"
       - name: Check formatting with ruff
         run: ruff format --check .
 
@@ -45,38 +45,62 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install "ruff==0.8.6"
+          python -m pip install "ruff==0.15.0"
       - name: Lint with ruff
         run: ruff check --output-format=github .
 
   pytest:
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.12"]
+        python-version: ["3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: pip
-      - name: Install system deps for pysam
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y \
-            build-essential \
-            zlib1g-dev \
-            libbz2-dev \
-            liblzma-dev \
-            libcurl4-openssl-dev \
-            pkg-config
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install .[dev,torch,plotting]
-      - name: Run pytest
-        run: pytest -m "smoke" -q
+          python -m pip install ".[all]"
+          python -m pip install --group test
+      - name: Run Python 3.11 smoke compatibility tests
+        if: matrix.python-version == '3.11'
+        run: pytest -m smoke -q
+      - name: Run Python 3.12 smoke and unit tests
+        if: matrix.python-version == '3.12'
+        run: pytest -m "smoke or unit" -vv --durations=30
+
+  storage-minimums:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - name: Install minimum supported storage stack
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install \
+            "anndata==0.12.0" \
+            "biopython>=1.79" \
+            "zarr==3.1.0" \
+            "pyarrow==15.0.0" \
+            "pandas==2.1.0" \
+            "psutil>=5.9.0"
+          python -m pip install --group test
+          python -m pip install --no-deps .
+      - name: Test minimum supported storage stack
+        run: |
+          pytest -q \
+            tests/unit/informatics/test_incremental_zarr.py \
+            tests/unit/informatics/test_partition_read.py \
+            tests/unit/informatics/test_raw_store.py \
+            tests/unit/test_partitioned_pipeline_fixture.py
 
   docs:
     runs-on: ubuntu-latest
@@ -92,7 +116,8 @@ jobs:
       - name: Install docs deps
         run: |
           python -m pip install --upgrade pip
-          python -m pip install .[docs]
+          python -m pip install --group docs
+          python -m pip install --no-deps .
       - name: Build docs
         run: sphinx-build -W -b html docs/source docs/_build/html
 
@@ -101,7 +126,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5

smftools-2.16.0/.github/workflows/extended-ci.yml

@@ -0,0 +1,48 @@
+name: Extended CI
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "17 9 * * 1"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  minimum-python-unit:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install ".[all]"
+          python -m pip install --group test
+      - name: Run smoke and unit tests
+        run: pytest -m "smoke or unit" -q
+
+  integration:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install ".[all]"
+          python -m pip install --group test
+      - name: Run integration tests
+        run: pytest -m integration -q

smftools-2.16.0/.github/workflows/partitioned-acceptance.yml

@@ -0,0 +1,67 @@
+name: Partitioned Pipeline Acceptance
+
+on:
+  pull_request:
+    branches: ["main"]
+    paths:
+      - ".github/workflows/partitioned-acceptance.yml"
+      - "src/smftools/config/**"
+      - "src/smftools/informatics/**"
+      - "src/smftools/memory_guard.py"
+      - "src/smftools/project/**"
+      - "tests/acceptance/**"
+      - "tests/fixtures/partitioned_pipeline.py"
+      - "tests/integration/test_resource_runtime.py"
+      - "tests/unit/test_partitioned_pipeline_acceptance_catalog.py"
+      - "tests/unit/test_partitioned_pipeline_fixture.py"
+      - "tests/unit/test_resource_envelope.py"
+      - "tests/unit/config/test_LoadExperimentConfig.py"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  portable-storage-config-query:
+    name: Portable storage/config/query (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - name: Install portable storage test stack
+        run: python -m pip install --upgrade pip && python -m pip install "anndata==0.12.0" "biopython>=1.79" "zarr==3.1.0" "pyarrow==15.0.0" "pandas==2.1.0" "psutil>=5.9.0" "dask[array]" click pyyaml xarray && python -m pip install --group test && python -m pip install --no-deps .
+      - name: Run portable acceptance checks
+        run: python -m pytest -q tests/unit/test_partitioned_pipeline_acceptance_catalog.py tests/unit/test_partitioned_pipeline_fixture.py tests/unit/informatics/test_partition_query.py tests/unit/test_resource_envelope.py tests/integration/test_resource_runtime.py tests/unit/config/test_LoadExperimentConfig.py -k "var_dict or region_catalog_config_defaults_are_independent or invalid_resource_values or partitioned_pipeline or partition_query or resource_envelope or resource_runtime"
+
+  linux-resource-runtime:
+    name: Linux cgroup capability and fallback
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - name: Install resource test stack
+        run: python -m pip install --upgrade pip && python -m pip install "anndata==0.12.0" "biopython>=1.79" "zarr==3.1.0" "pyarrow==15.0.0" "pandas==2.1.0" "psutil>=5.9.0" && python -m pip install --group test && python -m pip install --no-deps .
+      - name: Record cgroup mount and delegation
+        shell: bash
+        run: |
+          test -r /proc/self/cgroup && sed -n '1,20p' /proc/self/cgroup
+          test -r /sys/fs/cgroup/cgroup.controllers && sed -n '1,20p' /sys/fs/cgroup/cgroup.controllers || true
+          test -r /sys/fs/cgroup/cgroup.subtree_control && sed -n '1,20p' /sys/fs/cgroup/cgroup.subtree_control || true
+      - name: Test cgroup activation or explicit fallback
+        run: python -m pytest -q tests/integration/test_resource_runtime.py tests/unit/test_resource_envelope.py

{smftools-0.3.2 → smftools-2.16.0}/.gitignore

@@ -1,3 +1,6 @@
+# Claude Code local state
+.claude/
+
 # Python
 __pycache__/
 
@@ -54,3 +57,4 @@ Thumbs.db
 htmlcov/
 
 # Docs
+.aider*

smftools-2.16.0/.readthedocs.yaml

@@ -0,0 +1,16 @@
+version: 2
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+  jobs:
+    post_install:
+      - python -m pip install --upgrade pip
+      - python -m pip install --group docs
+      - python -m pip install --no-deps .
+sphinx:
+  configuration: docs/source/conf.py
+  fail_on_warning: true
+submodules:
+  include: all
+  recursive: true

smftools-2.16.0/AGENTS.md

@@ -0,0 +1,162 @@
+# AGENTS.md - root
+
+You are the implementation agent defined in smftools/AGENTS.md
+
+- For AGENTS.md or CLAUDE.md files (this one, and any nested ones under `src/`, `docs/`, `tests/`):
+  - Agents can read from these files.
+  - Agents can never edit these files.
+
+## Goals
+- Make minimal, correct changes.
+- Prefer small PRs / diffs.
+- Keep behavior stable unless the task explicitly requests changes.
+- Generate production grade, scalable code.
+
+## Prompt interface
+- When asked about a problem or task, first read all files relevent to the task's scope.
+- Describe the problem given the context.
+- Formulate a plan to address the problem within scope.
+- Refine the plan with user input.
+- Implement code after being told to proceed.
+
+## Repo orientation
+- Read existing patterns before inventing new ones.
+- Don't refactor broadly unless asked.
+- If you're unsure about intended behavior, look for tests or docs first.
+- Ignore all files in any directory named "archived".
+- User defined parameters exist within src/smftools/config.
+- Parameters are inherited from default.yaml -> MODALITY.yaml -> user_defined_config.csv
+- Frequently used non user defined variables should exist within src/smftools/constants.py
+- Logging functionality is defined within src/smftools/logging_utils.py
+- Optional dependency handling is defined within src/smftools/optional_imports.py
+- Frequently used I/O functionality is defined within src/smftools/readwrite.py
+- CLI functionality is provided through click; see src/smftools/cli/AGENTS.md for the command
+  map and conventions before editing anything under src/smftools/cli/.
+- RTD documentation organization through smftools/docs; see docs/source/AGENTS.md before editing
+  docstrings or anything under docs/.
+- Pytest testing within smftools/tests; see tests/AGENTS.md for markers and known gotchas before
+  running or writing tests.
+- smftools.analysis (downstream analysis library) has its own design contract at
+  src/smftools/analysis/AGENTS.md — read it before adding to compute/plot/filters/config.
+- Nested AGENTS.md/CLAUDE.md files exist to keep this file from re-growing per-subpackage detail.
+  If you find yourself wanting to document something specific to one subpackage here, it
+  probably belongs in that subpackage's own file instead (create one if it doesn't exist, and
+  flag it to the user since agents can't create AGENTS.md/CLAUDE.md files themselves).
+
+## Project dependencies
+- A core set of dependencies is required for the project.
+- Various optional dependencies are provided for:
+    - Optional functional modules of the package (ont, plotting, ml-base, ml-extended, umap, qc,
+      pysam, catalog, cluster, ...) — see pyproject.toml's [project.optional-dependencies] for
+      the full, current list; do not enumerate it here, it changes often.
+    - If available, a Python version of a CLI tool is preferred (Such as for Samtools, Bedtools,
+      BedGraphToBigWig).
+    - torch is listed as an extra dependency, but is currently required.
+    - All functional extras can be installed with `pip install -e ".[all_2]"` (the more complete
+      of two overlapping "everything" extras — `all` predates `pybedtools`/`pyBigWig` being added
+      and is missing them; prefer `all_2`). See Setup below for the canonical dev venv this
+      produces.- Certain command line tools are currently needed for certain functionalities within smftools load:
+  - dorado: Used for nanopore basecalling from POD5/FAST5 files to BAM.
+  - dorado/minimap2: Used for alignment of reads to reference.
+  - dorado: Used for demultiplexing of nanopore derived BAMs.
+  - modkit: Used for extracting modification probabilities from MM/ML BAM tags for native smf modality.
+
+## Setup
+
+Which interpreter to use, in priority order:
+
+1. **User- or task-specified venv/interpreter** — if one is given, use it, full stop.
+2. **The currently active environment**, if it already satisfies what the task needs (e.g. it's
+   already running with the right packages importable) — don't switch environments just because
+   a canonical one exists.
+3. **`venvs/venv-all`** — the canonical, fully-provisioned dev venv (editable install, every
+   functional extra: `pip install -e ".[all_2,dev,docs]"`). This is the default when neither of
+   the above applies. Being editable, it always reflects whatever branch is currently checked
+   out — it does not need to be recreated per branch, only re-installed
+   (`venvs/venv-all/bin/pip install -e ".[all_2,dev,docs]"`) if `pyproject.toml`'s dependencies
+   changed since it was last built.
+
+If `venvs/venv-all` doesn't exist yet or a narrower environment is wanted, create one:
+- `python3 -m venv venvs/<name> && venvs/<name>/bin/pip install -e ".[dev,torch]"` (core +
+  dev/test tooling), then add extras as needed: `venvs/<name>/bin/pip install -e ".[EXTRA_NAME]"`.
+
+**Common trap**: a venv/interpreter that's missing an optional extra (e.g. `pod5`, `pysam`,
+`umap`) will fail *test collection*, not just individual tests, for any file that imports it
+at module level — this looks like a code regression but usually isn't. Before debugging, check
+which interpreter you're actually running and whether it has the extras the failing files need.
+`venvs/venv-all` exists specifically to make this class of bug not happen in the first place.
+
+## How to run checks
+- Smoke tests: `pytest -m smoke -q`
+- Unit tests: `pytest -m unit -q`
+- Integration tests: `pytest -m integration -q`
+- E2E tests: `pytest -m e2e -q`
+- Coverage (if configured): `pytest --cov`
+- Lint: `ruff check .`
+- Format: `ruff format .`
+- Type-check (if configured): `mypy .`
+- **Docs build** (before committing anything that touches a docstring or `docs/`):
+  `sphinx-build -W -b html docs/source docs/_build/html`. `-W` treats warnings as errors, matching
+  CI's `docs` job and Read the Docs' `fail_on_warning: true` — a docstring that imports fine can
+  still fail this. See docs/source/AGENTS.md for the specific pitfalls that trip this up.
+
+## Coding conventions
+- Follow existing style and module layout.
+- Prefer clear, explicit code over cleverness.
+- Prefer modular functionality to facilitate testing and future development.
+- Do not over-parametize functions when possible.
+- For function parameters that a user may want to tune, use the config management strategy.
+- Use constants.py when appropriate.
+- Annotate code blocks to describe functionality.
+- Add/adjust tests for bug fixes and new behavior.
+- Keep public APIs backward compatible unless explicitly changing them.
+- Python:
+  - Use type hints for new/modified functions where reasonable.
+  - Use Google style docstring format.
+  - Avoid heavy dependencies unless necessary.
+  - Use typing.TYPE_CHECKING and annotations.
+  - In docstring of new functions, define the purpose of the function and what it does.
+  - If a function's return-type annotation (or any forward-referenced type) names a symbol that
+    is only ever imported inside a `TYPE_CHECKING` block, that symbol's *top-level package* must
+    also be in `docs/source/conf.py`'s `autodoc_mock_imports` — otherwise the docs build breaks
+    even though the code runs fine. See docs/source/AGENTS.md.
+
+## Testing expectations
+- New functionality must include tests.
+- If tests are flaky or slow, note it and scope the change.
+- There is currently no "regression" test marker/category, despite the concept coming up in
+  practice — if you need one, propose it explicitly rather than assuming a convention exists.
+
+## Logging & secrets
+- Don't log secrets, tokens, or PII.
+- Never hardcode credentials.
+- If sample keys are needed, use obvious placeholders like `YOUR_API_KEY_HERE`.
+
+## Git / PR hygiene
+- Keep commits focused.
+- Update docs/changelog if behavior or user-facing CLI changes.
+- If you change a CLI flag or config schema, add a migration note.
+- Feature/fix identity is decoupled from release versions. Branch names describe the work, not a
+  planned version slot: `feature/<description>` or `fix/<description>`, cut from `main`. Do not
+  bump `src/smftools/_version.py` on these branches — that happens only on a release branch (see
+  below), which avoids merge conflicts when multiple feature branches are in flight and lets them
+  merge into `main` in whatever order they're ready, not a predetermined sequence.
+- `main`'s `_version.py` carries a `.devN` suffix between releases (e.g. `2.16.0.dev0`) so its
+  version string is always distinguishable from an actual release. Hatch reads this directly, so
+  `.devN` is a valid build version, not just a convention.
+- To cut a release: create `release/<version>` from `main`, set `_version.py` to the plain
+  `MAJOR.MINOR.PATCH` (no `.devN`), add `docs/source/release-notes/<version>.md` (summarize
+  `git log <prev-tag>..HEAD` — one to a few sentences, matching the terse style of existing
+  entries in that directory) and add its `{include}` block to `docs/source/release-notes/index.md`,
+  then run `python -m build && twine check dist/*` as a local sanity check. After that branch
+  merges to `main`, tag the resulting merge commit locally (`git tag -a vMAJOR.MINOR.PATCH -m
+  "..."`), then bump `main`'s `_version.py` to the next `MAJOR.MINOR.PATCH.dev0`.
+- A minor release does not need to correspond to exactly one feature — bundle whatever's ready
+  when the release branch is cut. Patch releases (`MAJOR.MINOR.PATCH+1`) are for bugfix-only
+  follow-ups to an already-tagged release; breaking changes bump `MAJOR`.
+- Do not push tags, delete branches, merge PRs, or publish build artifacts without explicit
+  confirmation each time — these are shared-state/irreversible actions on a public repo.
+
+## If something fails
+- If a command fails, paste the full error and summarize likely causes.
+- Don't "fix" by deleting tests or weakening assertions unless explicitly instructed.