qpx 1.0.1__tar.gz

qpx-1.0.1/.codacy.yml

@@ -0,0 +1,7 @@
+---
+engines:
+  flake8:
+    options:
+      per-file-ignores: "tests/*:S101"
+exclude_paths:
+  - "docs/spec/**"

qpx-1.0.1/.github/workflows/deploy-docs.yml

@@ -0,0 +1,47 @@
+# This workflow will build and deploy documentation to GitHub Pages
+# For more information see: https://www.mkdocs.org/user-guide/deploying-your-docs/
+
+name: Deploy Documentation
+
+on:
+  # TODO: re-enable after mkdocs config is fixed for next release
+  # push:
+  #   branches: ["main", "dev"]
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  deploy-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Cache pip dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+
+      - name: Build MkDocs documentation
+        run: |
+          mkdocs build --strict
+
+      - name: Deploy to GitHub Pages
+        run: |
+          mkdocs gh-deploy --force --message "docs: deploy documentation [skip ci]"

qpx-1.0.1/.github/workflows/integration-tests.yml

@@ -0,0 +1,114 @@
+name: Integration Tests
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      test_type:
+        description: "Type of integration test to run"
+        required: true
+        default: "all"
+        type: choice
+        options:
+          - all
+          - full-conversion-suite
+          - lfq-only
+          - tmt-only
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60 # 1 hour timeout for the entire job
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Sync workspace
+        run: uv sync --extra all --extra dev
+
+      - name: Run Full Conversion Suite Tests
+        if: ${{ github.event.inputs.test_type == 'all' || github.event.inputs.test_type == 'full-conversion-suite' || github.event_name == 'release' }}
+        run: uv run pytest -vv tests/core/quantms/test_full_conversion_suite.py --timeout=3000
+        env:
+          PYTHONIOENCODING: utf-8
+
+      - name: Run LFQ Tests Only
+        if: ${{ github.event.inputs.test_type == 'lfq-only' }}
+        run: uv run pytest -vv tests/core/quantms/test_full_conversion_suite.py -k "lfq" --timeout=3000
+        env:
+          PYTHONIOENCODING: utf-8
+
+      - name: Run TMT Tests Only
+        if: ${{ github.event.inputs.test_type == 'tmt-only' }}
+        run: uv run pytest -vv tests/core/quantms/test_full_conversion_suite.py -k "tmt" --timeout=3000
+        env:
+          PYTHONIOENCODING: utf-8
+
+      - name: Run All Integration Tests
+        if: ${{ github.event.inputs.test_type == 'all' || github.event_name == 'release' }}
+        run: uv run pytest -vv -m "integration" --timeout=3000
+        env:
+          PYTHONIOENCODING: utf-8
+
+      - name: Generate Example Outputs
+        if: ${{ github.event_name == 'release' }}
+        run: |
+          mkdir -p examples/output
+
+          echo "=== QuantMS LFQ Examples ==="
+          uv run qpxc convert quantms \
+            --mztab-path tests/examples/quantms/dda-lfq-full/PXD007683-LFQ.sdrf_openms_design_openms.mzTab.gz \
+            --sdrf-file tests/examples/quantms/dda-lfq-full/PXD007683-LFQ.sdrf.tsv \
+            --msstats-file tests/examples/quantms/dda-lfq-full/PXD007683-LFQ.sdrf_openms_design_msstats_in.csv.gz \
+            --output-folder examples/output \
+            --output-prefix quantms-lfq \
+            --structures psm,feature,pg \
+            --verbose
+
+          echo "=== DIA-NN Examples ==="
+          uv run qpxc convert diann \
+            --report-path tests/examples/diann/small/diann_report.tsv \
+            --qvalue-threshold 0.05 \
+            --mzml-info-folder tests/examples/diann/small/mzml \
+            --sdrf-file tests/examples/diann/small/PXD019909-DIA.sdrf.tsv \
+            --pg-matrix-path tests/examples/diann/small/diann_report.pg_matrix.tsv \
+            --output-folder examples/output \
+            --output-prefix diann \
+            --duckdb-max-memory 4GB \
+            --duckdb-threads 4 \
+            --verbose
+
+          echo "=== MaxQuant Examples ==="
+          uv run qpxc convert maxquant \
+            --msms-file tests/examples/maxquant/maxquant_simple/msms.txt \
+            --evidence-file tests/examples/maxquant/maxquant_full/evidence.txt.gz \
+            --sdrf-file tests/examples/maxquant/maxquant_full/PXD001819.sdrf.tsv \
+            --protein-groups-file tests/examples/maxquant/maxquant_full/proteinGroups.txt \
+            --output-folder examples/output \
+            --output-prefix maxquant \
+            --structures psm,feature,pg \
+            --verbose
+
+          echo "Generated example files:"
+          ls -la examples/output/
+        env:
+          PYTHONIOENCODING: utf-8
+
+      - name: Upload Example Outputs
+        if: ${{ github.event_name == 'release' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: qpx-example-outputs-${{ github.ref_name }}
+          path: examples/output/
+          retention-days: 90

qpx-1.0.1/.github/workflows/python-app.yml

@@ -0,0 +1,58 @@
+# This workflow installs dependencies with uv, runs pre-commit, and tests.
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: CI
+
+on:
+  push:
+    branches: ["main", "dev"]
+  pull_request:
+    branches: ["main", "dev"]
+  schedule:
+    # Run full test suite daily at 2 AM UTC
+    - cron: "0 2 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - name: Sync workspace
+        run: uv sync --extra dev
+      - name: Run pre-commit hooks
+        run: uv run pre-commit run --all-files
+
+  unit-tests:
+    runs-on: ubuntu-latest
+    needs: pre-commit
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - name: Sync workspace
+        run: uv sync --extra dev
+      - name: Run unit tests
+        run: uv run pytest -vv -m "not integration and not large_data"

qpx-1.0.1/.github/workflows/python-package.yml

@@ -0,0 +1,59 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python package
+
+on:
+  push:
+    branches: ["dev", "main"]
+  pull_request:
+    branches: ["dev", "main"]
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+      - name: Lint with ruff
+        run: |
+          ruff check .
+      - name: Test with pytest (Unit Tests Only)
+        run: |
+          pytest -vv -m "not integration and not large_data"
+
+  integration-tests:
+    runs-on: ubuntu-latest
+    needs: unit-tests # Only run if unit tests pass
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+      - name: Test with pytest (Integration Tests)
+        run: |
+          pytest -vv -m "integration" --timeout=3000
+        env:
+          PYTHONIOENCODING: utf-8

qpx-1.0.1/.github/workflows/python-publish.yml

@@ -0,0 +1,36 @@
+# Publish to PyPI via Trusted Publishing (OIDC) when a GitHub Release is created.
+# Prerequisites: configure a "Trusted Publisher" on PyPI for this repository.
+# See https://docs.pypi.org/trusted-publishers/creating-a-project-through-oidc/
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+  id-token: write  # Required for OIDC Trusted Publishing
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment: pypi
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+      - name: Build package
+        run: python -m build
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

qpx-1.0.1/.gitignore

@@ -0,0 +1,116 @@
+qpx.egg-info
+dist
+build
+.idea
+/data/*
+!/data/ontology/
+docs/.vscode
+__pycache__
+.vscode
+
+# DuckDB database files
+*duckdb*.db
+*.db
+*.db.wal
+*.db-shm
+
+# Temporary files
+*.tmp
+*.temp
+temp/
+tmp/
+.tmp/
+.temp/
+
+# Temporary mzTab files (created during processing)
+mztab_temp*.mzTab
+mztab_temp*.mzTab.gz
+mztab_temp*.mzTab.bz2
+
+# Test output and temporary data
+test_output*/
+output*/
+results*/
+*.parquet
+!examples/psm/*.parquet
+!qpx/core/ontology/data/*.parquet
+!data/ontology/*.parquet
+.hypothesis/
+
+# Log files
+*.log
+logs/
+
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Python cache and compiled files
+*.pyc
+*.pyo
+*.pyd
+.Python
+*.so
+__pycache__/
+*.egg-info/
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Jupyter Notebook checkpoints
+.ipynb_checkpoints/
+
+# Environment files
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Cursor
+.cursorrules
+.cursor/
+
+# MkDocs build output
+site/
+
+# Big examples data
+tissues/
+msnet_qpx/
+
+#Ignore cursor AI rules
+.cursor/rules/codacy.mdc
+
+
+# Claude Code
+.claude/
+
+# Kiro
+.kiro/
+workflow.md
+docs/metadata.avsc
+.claude/
+.codacy/
+
+
+#Ignore vscode AI rules
+.github/instructions/codacy.instructions.md
+
+.ruff*
+# Ruff cache lock
+.ruff_cache.lock
+
+uv.lock
+absexpr*
+scripts/convert_absexpr.py
+scripts/plasma_infiltration_model.py
+scripts/plot_absexpr_analysis.py
+scripts/plot_blood_biology.py
+docs/superpowers/*

qpx-1.0.1/.markdownlint.json

@@ -0,0 +1,8 @@
+{
+  "MD046": {
+    "style": "fenced"
+  },
+  "MD013": false,
+  "MD033": false,
+  "MD041": false
+}

qpx-1.0.1/.pre-commit-config.yaml

@@ -0,0 +1,15 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: end-of-file-fixer
+        exclude: '\.svg$'
+      - id: trailing-whitespace
+        exclude: '\.svg$'
+      - id: detect-private-key
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.8
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format

qpx-1.0.1/CHANGELOG.md

@@ -0,0 +1,55 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [1.0.0] - 2025-03-22
+
+### Added
+
+- **CLI command groups**: `convert`, `transform`, `query`, `info`, `validate`, `ontology`
+- **Converters**: DIA-NN, MaxQuant, QuantMS (mzTab LFQ & TMT), FragPipe, mzIdentML, SDRF
+- **Transform commands**:
+  - `gene-map` — map gene names from FASTA to QPX parquet data
+  - `quantify` — protein-level quantification via mokume (DirectLFQ, MaxLFQ, iBAQ, TopN, Sum)
+  - `normalize-accessions` — normalize protein accession formats (full ↔ bare UniProt)
+  - `update-metadata` — update sample/run metadata from a revised SDRF
+- **Query commands**: `sql`, `filter`, `head` for interactive dataset exploration
+- **Info commands**: dataset summary, Arrow schema display, Parquet metadata inspection
+- **Validate command**: schema validation with column presence, type matching, null checks, and PK uniqueness
+- **Ontology management**: `info`, `update`, `build`, `search` for PSI-MS and PRIDE CV terms
+- **Python API**: `qpx.open()`, `qpx.read_feature()`, `qpx.read_psm()`, `qpx.read_pg()`, etc.
+- **Dataset class**: unified access to all QPX structures with DuckDB-backed SQL queries
+- **DatasetCollection**: work with multiple datasets simultaneously
+- **Writers**: Parquet writers for all QPX structures (Feature, PSM, PG, Sample, Run, MzSpectra, PepMap, Ontology, Provenance)
+- **Views**: analytical views for protein, peptide, QC, and sample summaries
+- **Schema validation**: YAML-defined canonical schemas for all data structures
+- **Score normalization**: automatic PSI-MS ontology mapping for search engine scores
+- **PRIDE API integration**: `--enrich-pride` flag for automatic project metadata enrichment
+- **S3 support**: read QPX datasets from S3-compatible storage
+- **MkDocs documentation**: full CLI reference with auto-generated parameter tables and usage examples
+
+### Changed
+
+- Replaced per-converter `_safe_float_val` with shared `safe_float` utility
+- Optimized `_table_exists` to use parameterized `information_schema` query
+- Increased DIA-NN `file_num` default from 50 to 100 for larger cohort support
+- Defensive handling of None/NaN in gene mapping (`_resolve_gene_names`, `annotate_dataframe`)
+
+### Fixed
+
+- `.gitignore` duplicate entry removal
+- SQL injection safety in `_table_exists` (parameterized query)
+- Gene mapping crash on None/NaN protein accessions
+- Gene mapping `list(dict)` producing keys instead of preserving struct
+
+## [0.0.4] - 2025-03-01
+
+### Added
+
+- Initial PyPI release
+- Basic converter framework for QuantMS, DIA-NN, MaxQuant
+- QPX Parquet schema definition
+- CLI entry point (`qpxc`)

qpx-1.0.1/CONTRIBUTING.md

@@ -0,0 +1,92 @@
+# Contributing to QPX
+
+Thank you for your interest in contributing to QPX!
+
+## Code of Conduct
+
+We follow the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/2/0/code_of_conduct/).
+
+## Getting Started
+
+### Setup
+
+1. Fork and clone the repository:
+
+   ```bash
+   git clone https://github.com/your-username/qpx.git
+   cd qpx
+   git remote add upstream https://github.com/bigbio/qpx.git
+   ```
+
+2. Install dependencies (Python 3.10+):
+   ```bash
+   pip install -e ".[dev]"
+   ```
+
+### Development Workflow
+
+Create a feature branch:
+
+```bash
+git checkout -b feature/your-feature-name
+```
+
+## Code Standards
+
+Format code with ruff:
+
+```bash
+ruff check .
+ruff format .
+```
+
+## Testing
+
+Run tests (excludes integration tests by default):
+
+```bash
+pytest -vv -m "not integration"
+```
+
+Test markers:
+
+- `integration`: Integration tests requiring external resources
+- `large_data`: Tests requiring large data files
+
+## Submitting Changes
+
+1. Commit and push your changes:
+
+   ```bash
+   git add .
+   git commit -m "Brief description of changes"
+   git push origin feature/your-feature-name
+   ```
+
+2. Create a pull request on GitHub with:
+   - Clear description of changes
+   - Related issue numbers (e.g., "Fixes #123")
+   - Test results
+
+## Reporting Issues
+
+**Bugs**: Include title, description, steps to reproduce, environment (QPX version, Python version, OS), and error messages.
+
+**Features**: Describe the use case, proposed solution, and alternatives considered.
+
+## Documentation
+
+Build and preview documentation locally:
+
+```bash
+mkdocs serve  # Available at http://127.0.0.1:8000/
+```
+
+## Contact
+
+- **Maintainers**: Yasset Perez-Riverol (PRIDE Team, EMBL-EBI), Ping Zheng (Chongqing University)
+- **Issues**: [GitHub Issues](https://github.com/bigbio/qpx/issues)
+
+## License
+
+Contributions are licensed under [Apache License 2.0](LICENSE). Please add your name to the contributors list in README.md.