ferromic 0.1.4__tar.gz

ferromic-0.1.4/.cargo/config.toml

@@ -0,0 +1,15 @@
+[env]
+SSL_CERT_FILE = { value = "/etc/ssl/certs/ca-certificates.crt", force = true }
+SSL_CERT_DIR = { value = "/etc/ssl/certs", force = true }
+OPENSSL_NO_PKG_CONFIG = { value = "1", force = true }
+
+[target.s390x-unknown-linux-gnu.env]
+OPENSSL_NO_ASM = "1"
+
+# Allow unresolved Python symbols on macOS to be resolved at runtime by the
+# interpreter when building the PyO3 extension module.
+[target.x86_64-apple-darwin]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
+
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]

ferromic-0.1.4/.gitattributes

@@ -0,0 +1 @@
+data/phy_outputs.zip filter=lfs diff=lfs merge=lfs -text

ferromic-0.1.4/.github/workflows/CI.yml

@@ -0,0 +1,229 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+      - master
+    tags:
+      - '*'
+    paths:
+      - '**.rs'
+      - 'Cargo.toml'
+  pull_request:
+    paths:
+      - '**.rs'
+      - 'Cargo.toml'
+  workflow_dispatch:
+    inputs:
+      ci_targets:
+        description: Select which platforms to build
+        type: choice
+        default: default
+        options:
+          - default
+          - all
+
+permissions:
+  contents: read
+
+jobs:
+  linux:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform: >-
+          ${{ fromJson(
+            (startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && github.event.inputs.ci_targets == 'all'))
+            && '[{"runner":"ubuntu-22.04","target":"x86_64","openblas_target":"SANDYBRIDGE"},{"runner":"ubuntu-22.04","target":"x86","openblas_target":"ATOM"},{"runner":"ubuntu-22.04","target":"aarch64","openblas_target":"ARMV8"},{"runner":"ubuntu-22.04","target":"armv7","openblas_target":"ARMV7"},{"runner":"ubuntu-22.04","target":"ppc64le","openblas_target":"POWER9"}]'
+            || '[{"runner":"ubuntu-22.04","target":"x86_64","openblas_target":"SANDYBRIDGE"}]'
+          ) }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@nightly
+      - name: Install build dependencies
+        if: ${{ matrix.platform.target == 'x86_64' }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential gfortran perl wget
+          perl -MIPC::Cmd -e1 >/dev/null 2>&1 || (wget -O - https://cpanmin.us | perl - --notest IPC::Cmd)
+      - name: Cargo tests
+        if: ${{ matrix.platform.target == 'x86_64' }}
+        env:
+          OPENBLAS_TARGET: ${{ matrix.platform.openblas_target }}
+        run: cargo test --release
+      - name: Work around missing s390x assembler support
+        if: ${{ matrix.platform.target == 's390x' }}
+        run: |
+          echo "OPENSSL_NO_ASM=1" >> $GITHUB_ENV
+          echo "OPENSSL_SRC_CONFIGURE_ARGS=no-asm" >> $GITHUB_ENV
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        env:
+          OPENSSL_STATIC: 1
+          OPENSSL_VENDORED: 1
+          OPENBLAS_TARGET: ${{ matrix.platform.openblas_target }}
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --find-interpreter
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+          manylinux: auto
+          rust-toolchain: nightly
+          before-script-linux: |
+            if command -v microdnf >/dev/null 2>&1; then
+              microdnf install -y gcc gcc-c++ gcc-gfortran make perl perl-IPC-Cmd wget
+            elif command -v dnf >/dev/null 2>&1; then
+              dnf install -y gcc gcc-c++ gcc-gfortran make perl perl-IPC-Cmd wget
+            elif command -v yum >/dev/null 2>&1; then
+              yum install -y gcc gcc-c++ gcc-gfortran make perl perl-IPC-Cmd wget
+            elif command -v apt-get >/dev/null 2>&1; then
+              apt-get update
+              apt-get install -y build-essential gfortran perl wget
+            elif command -v apk >/dev/null 2>&1; then
+              apk add --no-cache build-base gfortran perl perl-utils wget
+            fi
+            perl -MIPC::Cmd -e1 >/dev/null 2>&1 || (wget -O - https://cpanmin.us | perl - --notest IPC::Cmd)
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-linux-${{ matrix.platform.target }}
+          path: dist
+
+  musllinux:
+    if: ${{ startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && github.event.inputs.ci_targets == 'all') }}
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform: >-
+          ${{ fromJson('[{"runner":"ubuntu-22.04","target":"x86_64","openblas_target":"SANDYBRIDGE"},{"runner":"ubuntu-22.04","target":"x86","openblas_target":"ATOM"},{"runner":"ubuntu-22.04","target":"aarch64","openblas_target":"ARMV8"},{"runner":"ubuntu-22.04","target":"armv7","openblas_target":"ARMV7"}]') }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        env:
+          OPENSSL_STATIC: 1
+          OPENSSL_VENDORED: 1
+          OPENBLAS_TARGET: ${{ matrix.platform.openblas_target }}
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --find-interpreter
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+          manylinux: musllinux_1_2
+          rust-toolchain: nightly
+          before-script-linux: |
+            if command -v microdnf >/dev/null 2>&1; then
+              microdnf install -y gcc gcc-c++ gcc-gfortran make perl perl-IPC-Cmd wget
+            elif command -v dnf >/dev/null 2>&1; then
+              dnf install -y gcc gcc-c++ gcc-gfortran make perl perl-IPC-Cmd wget
+            elif command -v yum >/dev/null 2>&1; then
+              yum install -y gcc gcc-c++ gcc-gfortran make perl perl-IPC-Cmd wget
+            elif command -v apt-get >/dev/null 2>&1; then
+              apt-get update
+              apt-get install -y build-essential gfortran perl wget
+            elif command -v apk >/dev/null 2>&1; then
+              apk add --no-cache build-base gfortran perl perl-utils wget
+            fi
+            perl -MIPC::Cmd -e1 >/dev/null 2>&1 || (wget -O - https://cpanmin.us | perl - --notest IPC::Cmd)
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-musllinux-${{ matrix.platform.target }}
+          path: dist
+
+  windows:
+    if: ${{ startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && github.event.inputs.ci_targets == 'all') }}
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform: >-
+          ${{ fromJson('[{"runner":"windows-latest","target":"x64","vcpkg_triplet":"x64-windows"},{"runner":"windows-latest","target":"x86","vcpkg_triplet":"x86-windows"}]') }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+          architecture: ${{ matrix.platform.target }}
+      - name: Install OpenBLAS dependencies
+        shell: pwsh
+        run: |
+          $vcpkgRoot = Join-Path $Env:USERPROFILE "vcpkg"
+          if (-Not (Test-Path $vcpkgRoot)) {
+            git clone https://github.com/microsoft/vcpkg.git $vcpkgRoot
+            & (Join-Path $vcpkgRoot "bootstrap-vcpkg.bat") -disableMetrics
+          }
+          & (Join-Path $vcpkgRoot "vcpkg.exe") install openblas --triplet ${{ matrix.platform.vcpkg_triplet }}
+          "VCPKG_ROOT=$vcpkgRoot" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
+          "VCPKG_DEFAULT_TRIPLET=${{ matrix.platform.vcpkg_triplet }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --find-interpreter
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+          rust-toolchain: nightly
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-windows-${{ matrix.platform.target }}
+          path: dist
+
+  macos:
+    if: ${{ startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && github.event.inputs.ci_targets == 'all') }}
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform: >-
+          ${{ fromJson('[{"runner":"macos-13","target":"x86_64"},{"runner":"macos-14","target":"aarch64"}]') }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --find-interpreter
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+          rust-toolchain: nightly
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-macos-${{ matrix.platform.target }}
+          path: dist
+
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
+    needs: [linux, musllinux, windows, macos]
+    permissions:
+      # Use to sign the release artifacts
+      id-token: write
+      # Used to upload release artifacts
+      contents: write
+      # Used to generate artifact attestation
+      attestations: write
+    steps:
+      - uses: actions/download-artifact@v4
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-path: 'wheels-*/*'
+      - name: Publish to PyPI
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        uses: PyO3/maturin-action@v1
+        env:
+          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+        with:
+          command: upload
+          args: --non-interactive --skip-existing wheels-*/*
+          rust-toolchain: nightly

ferromic-0.1.4/.github/workflows/analysis_pipeline.yml

@@ -0,0 +1,387 @@
+name: Analysis Pipeline
+
+on:
+  workflow_dispatch:
+    inputs:
+      region_override:
+        description: 'Specific region (e.g., chr22:123-456). Leave empty for full run.'
+        required: false
+        type: string
+
+jobs:
+  prep-and-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      regions: ${{ steps.set-matrix.outputs.regions }}
+      gene_batches: ${{ steps.set-matrix.outputs.gene_batches }}
+      cache_key: ${{ steps.calc-key.outputs.key }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          lfs: false
+
+      - name: Create LFS file list
+        run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id
+
+      - name: Calculate Cache Key
+        id: calc-key
+        env:
+          THE_KEY: prep-results-v1-${{ runner.os }}-${{ hashFiles('cds/*.py', 'data/*.tsv', '.lfs-assets-id') }}
+        run: echo "key=$THE_KEY" >> $GITHUB_OUTPUT
+
+      - name: Check Output Cache
+        id: prep-cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            combined_*.phy
+            outgroup_*.phy
+          key: ${{ steps.calc-key.outputs.key }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Install dependencies
+        run: pip install numpy pandas scipy statsmodels ete3 lxml requests tqdm
+
+      - name: Cache LFS
+        if: steps.prep-cache.outputs.cache-hit != 'true'
+        uses: actions/cache@v4
+        with:
+          path: .git/lfs
+          key: ${{ runner.os }}-lfs-${{ hashFiles('.lfs-assets-id') }}
+          restore-keys: |
+            ${{ runner.os }}-lfs-
+
+      - name: Pull LFS or Download Artifact
+        if: steps.prep-cache.outputs.cache-hit != 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          if git lfs pull; then
+            echo "LFS pull successful"
+          else
+            echo "LFS pull failed. Falling back to artifact download."
+
+            set -euo pipefail
+            MANUAL_WORKFLOW_FILE="manual_run_vcf.yml"
+            api_root="https://api.github.com"
+            headers=(
+              -H "Authorization: Bearer $GH_TOKEN"
+              -H "Accept: application/vnd.github+json"
+              -H "X-GitHub-Api-Version: 2022-11-28"
+            )
+
+            echo "Finding latest successful run of $MANUAL_WORKFLOW_FILE..."
+            run_json="$(
+              curl -sS -L "${headers[@]}" \
+                "$api_root/repos/${{ github.repository }}/actions/workflows/${MANUAL_WORKFLOW_FILE}/runs?status=success&per_page=1"
+            )"
+            run_id="$(echo "$run_json" | jq '.workflow_runs[0].id')"
+
+            if [[ -z "$run_id" || "$run_id" == "null" ]]; then
+              echo "No successful Manual Run VCF Pipeline runs found." >&2
+              exit 1
+            fi
+            echo "Found Run ID: $run_id"
+
+            echo "Listing artifacts..."
+            artifact_json="$(
+              curl -sS -L "${headers[@]}" \
+                "$api_root/repos/${{ github.repository }}/actions/runs/$run_id/artifacts" \
+            )"
+
+            download_url=$(echo "$artifact_json" | jq -r '.artifacts[] | select(.name == "run-vcf-phy-outputs") | .archive_download_url')
+
+            if [[ -z "$download_url" || "$download_url" == "null" ]]; then
+               echo "Artifact 'run-vcf-phy-outputs' not found in run $run_id" >&2
+               exit 1
+            fi
+            echo "Found Artifact URL."
+
+            echo "Downloading artifact..."
+            curl -sS -L \
+              -H "Authorization: Bearer $GH_TOKEN" \
+              -H "Accept: application/vnd.github+json" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              "$download_url" > artifact.zip
+
+            echo "Extracting artifact..."
+            unzip -o artifact.zip
+
+            if [[ -f "phy_outputs.zip" ]]; then
+                mkdir -p data
+                mv phy_outputs.zip data/phy_outputs.zip
+                echo "Successfully restored data/phy_outputs.zip from artifact."
+            else
+                echo "Expected phy_outputs.zip inside the artifact, but not found."
+                ls -l
+                exit 1
+            fi
+
+            rm artifact.zip
+          fi
+
+      - name: Cache AXT file
+        if: steps.prep-cache.outputs.cache-hit != 'true'
+        uses: actions/cache@v4
+        with:
+          path: hg38.panTro6.net.axt.gz
+          key: axt-file-v1
+
+      - name: Unzip PHY files
+        if: steps.prep-cache.outputs.cache-hit != 'true'
+        run: |
+          unzip -o data/phy_outputs.zip
+          rm data/phy_outputs.zip
+          unzip -o '*.zip' || echo "No nested zip files found"
+          rm -f '*.zip'
+          ls -R
+      - name: Run AXT to PHY
+        if: steps.prep-cache.outputs.cache-hit != 'true'
+        run: |
+          python3 cds/axt_to_phy.py
+          ls outgroup_* || echo "No outgroup files found in root"
+        env:
+          AXT_WORKERS: 4
+
+      - name: Run Combine PHY
+        if: steps.prep-cache.outputs.cache-hit != 'true'
+        run: python3 cds/combine_phy.py
+
+      - name: Prepare Tools (Download & Zip)
+        run: |
+          # Create a temp python script to reuse pipeline_lib logic
+          cat <<EOF > prepare_tools.py
+          import os, sys
+          sys.path.append('cds')
+          import pipeline_lib as lib
+          lib.setup_external_tools(os.getcwd())
+          EOF
+          python3 prepare_tools.py
+
+          # Zip them up
+          zip -r tools.zip paml/ iqtree-3.0.1-Linux/
+
+      - name: Generate GHA Matrix
+        id: set-matrix
+        env:
+          REGION_INPUT: ${{ inputs.region_override }}
+        run: |
+          # Run the generator and capture JSON output
+          REGION_ARG=""
+          if [[ -n "$REGION_INPUT" ]]; then
+             REGION_ARG="--region-override $REGION_INPUT"
+          fi
+
+          JSON=$(python3 cds/generate_gha_matrix.py --batch-size 4 $REGION_ARG)
+
+          # Parse JSON to get regions and batches
+          REGIONS=$(echo "$JSON" | python3 -c "import sys, json; print(json.dumps(json.load(sys.stdin)['regions']))")
+          BATCHES=$(echo "$JSON" | python3 -c "import sys, json; print(json.dumps(json.load(sys.stdin)['gene_batches']))")
+
+          # Output to GHA environment
+          echo "regions=$REGIONS" >> $GITHUB_OUTPUT
+          echo "gene_batches=$BATCHES" >> $GITHUB_OUTPUT
+
+      - name: Upload Tools Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: tools-zip
+          path: tools.zip
+
+      - name: Upload Combined PHY Artifacts
+        if: steps.prep-cache.outputs.cache-hit != 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: combined-phy-files
+          path: |
+            combined_*.phy
+          retention-days: 3
+
+  iqtree-phase:
+    needs: prep-and-matrix
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 20
+      matrix:
+        region: ${{ fromJson(needs.prep-and-matrix.outputs.regions) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Install dependencies
+        run: pip install numpy pandas scipy statsmodels ete3 lxml requests tqdm
+
+      - name: Download Tools
+        uses: actions/download-artifact@v4
+        with:
+          name: tools-zip
+
+      - name: Unzip Tools
+        run: unzip tools.zip
+
+      - name: Restore PHY Cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            combined_*.phy
+            outgroup_*.phy
+          key: ${{ needs.prep-and-matrix.outputs.cache_key }}
+          fail-on-cache-miss: true
+
+      - name: Calculate Input Hash
+        id: calc-hash
+        run: |
+          python3 -c "
+          import sys, glob, hashlib, os
+          sys.path.append('cds')
+          import pipeline_lib as lib
+
+          target_region = '${{ matrix.region }}'
+          files = glob.glob('combined_inversion_*.phy')
+          found = None
+          for f in files:
+              try:
+                  if lib.parse_region_filename(f)['label'] == target_region:
+                      found = f
+                      break
+              except:
+                  continue
+
+          if found:
+              print(f'Found file: {found}')
+              with open(found, 'rb') as f:
+                  h = hashlib.sha256(f.read()).hexdigest()
+                  print(f'Hash: {h}')
+                  with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+                      fh.write(f'hash={h}\n')
+          else:
+              print(f'Error: Could not find file for region {target_region}')
+              sys.exit(1)
+          "
+
+      - name: Restore IQ-TREE Cache
+        uses: actions/cache@v4
+        id: iqtree-cache
+        with:
+          path: region_trees
+          key: iqtree-v1-${{ matrix.region }}-${{ steps.calc-hash.outputs.hash }}
+
+      - name: Run IQ-TREE
+        run: python3 cds/iqtree_runner.py
+        env:
+          TARGET_REGION: ${{ matrix.region }}
+
+      - name: Upload Region Tree
+        uses: actions/upload-artifact@v4
+        with:
+          name: region-tree-${{ matrix.region }}
+          path: |
+            region_trees/${{ matrix.region }}.treefile
+            region_trees/${{ matrix.region }}.FAILED
+          if-no-files-found: ignore
+          retention-days: 3
+
+  paml-phase:
+    needs: [prep-and-matrix, iqtree-phase]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 20
+      matrix:
+        batch: ${{ fromJson(needs.prep-and-matrix.outputs.gene_batches) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Install dependencies
+        run: pip install numpy pandas scipy statsmodels ete3 lxml requests tqdm
+
+      - name: Download Tools
+        uses: actions/download-artifact@v4
+        with:
+          name: tools-zip
+
+      - name: Unzip Tools
+        run: unzip tools.zip
+
+      - name: Restore PHY Cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            combined_*.phy
+            outgroup_*.phy
+          key: ${{ needs.prep-and-matrix.outputs.cache_key }}
+          fail-on-cache-miss: true
+
+      - name: Download All Region Trees
+        uses: actions/download-artifact@v4
+        with:
+          pattern: region-tree-*
+          merge-multiple: true
+          path: region_trees
+
+      - name: Run PAML (Batch)
+        run: python3 cds/codeml_runner.py
+        env:
+          GENE_BATCH: ${{ matrix.batch }}
+          # Optional: configure timeouts etc
+          PAML_TIMEOUT: 21100
+          REGION_OVERRIDE_FILTER: ${{ inputs.region_override }}
+          KEEP_PAML_OUT: 1
+
+      - name: Upload Partial Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: paml-results-${{ strategy.job-index }}
+          path: |
+            partial_results_*.tsv
+            paml_runs/
+          retention-days: 3
+
+  finalize-phase:
+    needs: paml-phase
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Install dependencies
+        run: pip install numpy pandas scipy statsmodels ete3 lxml requests tqdm
+
+      - name: Download Partial Results
+        uses: actions/download-artifact@v4
+        with:
+          pattern: paml-results-*
+          merge-multiple: true
+
+      - name: Run Final Stats
+        run: python3 cds/finalize_results.py
+
+      - name: Upload Final Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: full-paml-results
+          path: |
+            full_paml_results_*.tsv
+          retention-days: 3