bioflowkit 0.2.0__tar.gz

bioflowkit-0.2.0/.github/workflows/candidate-smoke-test.yml

@@ -0,0 +1,111 @@
+name: Candidate smoke test
+
+# T5 (event-driven) cadence: when a PR adds/edits files under
+# update/candidates/, run `bioflow update auto` against just those
+# changes and post a comment summarising the per-candidate results.
+#
+# This catches malformed candidate YAMLs and image-pull failures
+# before a maintainer merges, so the monthly cron never has to deal
+# with broken candidates.
+
+on:
+  pull_request:
+    paths:
+      - 'update/candidates/**'
+
+permissions:
+  contents: read
+  pull-requests: write   # for the summary comment
+
+jobs:
+  smoke:
+    name: Validate + smoke-test changed candidates
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout PR head
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2   # we need base + head to compute the diff
+
+      - name: Identify changed candidate dirs
+        id: diff
+        run: |
+          # Discover which YYYY-MM dirs under update/candidates/ were touched.
+          # We pass each to `bioflow update auto --candidates-dir`.
+          changed=$(git diff --name-only HEAD^ HEAD -- 'update/candidates/**/*.yaml' \
+                    | xargs -I{} dirname {} \
+                    | sort -u \
+                    | tr '\n' ' ')
+          echo "dirs=$changed" >> "$GITHUB_OUTPUT"
+          if [ -z "$changed" ]; then
+            echo "No candidate YAMLs changed; nothing to do."
+          else
+            echo "Changed candidate dirs: $changed"
+          fi
+
+      - name: Set up Python
+        if: steps.diff.outputs.dirs != ''
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install bioflow
+        if: steps.diff.outputs.dirs != ''
+        run: pip install -e ".[dev]"
+
+      - name: Run smoke test on each changed candidate dir
+        if: steps.diff.outputs.dirs != ''
+        id: smoke
+        run: |
+          set -euo pipefail
+          mkdir -p _ci_reports
+          rc=0
+          for d in ${{ steps.diff.outputs.dirs }}; do
+            echo "==> Smoke-testing $d"
+            base=$(basename "$d")
+            bioflow update auto --candidates-dir "$d" \
+              --report "_ci_reports/$base.json" || rc=$?
+          done
+          echo "exitcode=$rc" >> "$GITHUB_OUTPUT"
+
+      - name: Upload reports
+        if: steps.diff.outputs.dirs != ''
+        uses: actions/upload-artifact@v4
+        with:
+          name: candidate-smoke-reports
+          path: _ci_reports/
+
+      - name: Comment summary on PR
+        if: steps.diff.outputs.dirs != ''
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+            const dir = '_ci_reports';
+            if (!fs.existsSync(dir)) return;
+            let body = '## bioflow candidate smoke test\n\n';
+            for (const f of fs.readdirSync(dir)) {
+              const data = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf-8'));
+              body += `### \`${f.replace(/\.json$/, '')}\`\n\n`;
+              body += `- scanned: **${data.candidates_scanned}**\n`;
+              body += `- auto_approve: ${data.auto_approve}\n`;
+              body += `- real docker: ${data.real_docker}\n\n`;
+              body += '| candidate | passed | error |\n|---|---|---|\n';
+              for (const r of (data.results || [])) {
+                const ok = r.passed ? '✅' : '❌';
+                body += `| \`${path.basename(r.candidate)}\` | ${ok} | ${(r.error || '').slice(0, 80)} |\n`;
+              }
+              body += '\n';
+            }
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body,
+            });
+
+      - name: Fail if any candidate failed
+        if: steps.diff.outputs.dirs != '' && steps.smoke.outputs.exitcode != '0'
+        run: exit ${{ steps.smoke.outputs.exitcode }}

bioflowkit-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,130 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  unit-tests:
+    name: Unit tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Run unit tests
+        run: python -m pytest tests/unit -q --tb=short
+
+  lint:
+    name: Lint (ruff)
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install ruff
+        run: pip install ruff
+
+      - name: Run ruff
+        run: ruff check .
+
+  typecheck:
+    # Advisory only — mypy issues do not block merging.  Provides
+    # visibility while we gradually add type annotations.  Will be
+    # upgraded to a blocking check when the error count reaches 0.
+    name: Type check (mypy, advisory)
+    runs-on: ubuntu-latest
+    continue-on-error: true
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install mypy
+        run: pip install -e ".[dev]" && pip install types-PyYAML types-requests
+
+      - name: Run mypy
+        run: mypy bioflow --ignore-missing-imports || true
+
+  registry-schema:
+    name: Validate registry YAMLs
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Validate all tool YAMLs against schema
+        run: |
+          python - <<'EOF'
+          import yaml, jsonschema, pathlib, sys
+
+          schema = yaml.safe_load(
+              pathlib.Path("registry/schema.yaml").read_text(encoding="utf-8")
+          )
+          errors = []
+          for p in pathlib.Path("registry/tools").rglob("*.yaml"):
+              doc = yaml.safe_load(p.read_text(encoding="utf-8"))
+              # strip update_meta before validating
+              doc.pop("update_meta", None)
+              try:
+                  jsonschema.validate(doc, schema)
+              except jsonschema.ValidationError as e:
+                  errors.append(f"{p}: {e.message}")
+          if errors:
+              for err in errors:
+                  print("FAIL:", err, file=sys.stderr)
+              sys.exit(1)
+          print(f"OK — {len(list(pathlib.Path('registry/tools').rglob('*.yaml')))} tool YAMLs validated")
+          EOF
+
+  digest-audit:
+    # Advisory until enough tools are pinned — prints the missing-digest
+    # count and lists the first ~30 unpinned tools.  Upgrade to a blocking
+    # check (drop `|| true`) once the bulk of the registry is pinned.
+    name: Container digest pin audit (advisory)
+    runs-on: ubuntu-latest
+    continue-on-error: true
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install pyyaml
+
+      - name: Count missing image_digest entries
+        run: python scripts/pin_digests.py --audit || true

bioflowkit-0.2.0/.github/workflows/docs.yml

@@ -0,0 +1,62 @@
+name: Docs
+
+# Build the MkDocs site and deploy it to GitHub Pages on every push to
+# main.  The tools/recipes reference pages are regenerated from the
+# registry first so the published docs always match the code.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docs/**'
+      - 'mkdocs.yml'
+      - 'registry/**'
+      - 'bioflow/recipes/**'
+      - 'scripts/gen_docs.py'
+      - '.github/workflows/docs.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow one concurrent deployment; cancel in-progress runs of this group
+concurrency:
+  group: pages
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install bioflow + docs deps
+        run: pip install -e ".[docs]"
+
+      - name: Regenerate reference pages from the registry
+        run: python scripts/gen_docs.py
+
+      - name: Build site (strict)
+        run: mkdocs build --strict
+
+      - name: Upload Pages artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: site
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

bioflowkit-0.2.0/.github/workflows/nightly-smoke.yml

@@ -0,0 +1,58 @@
+name: Nightly recipe smoke
+
+# Real-container smoke tests for the recipe matrix.  Runs once a day at
+# 03:00 UTC and on manual dispatch.  Failures are loud (workflow goes red
+# in the README badge) but never block PR merges — see ci.yml for the
+# unit-test gate.
+
+on:
+  schedule:
+    - cron: "0 3 * * *"
+  workflow_dispatch:
+
+jobs:
+  smoke-matrix:
+    name: Recipe smoke matrix
+    runs-on: ubuntu-latest
+    # Each recipe pulls 50–600 MB and runs for ~30s; ~10 entries fits well
+    # inside this limit even on a cold runner.
+    timeout-minutes: 45
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Confirm Docker is up on the runner
+        run: |
+          docker --version
+          docker info | head -20
+
+      - name: Install bioflow with dev extras
+        run: pip install -e ".[dev]"
+
+      - name: Run smoke matrix (real BioContainers)
+        env:
+          # The smoke matrix is Docker-gated; reduce log noise.
+          BIOFLOW_LOG_LEVEL: INFO
+        run: |
+          python -m pytest tests/integration/test_recipe_smoke_matrix.py \
+            -v -m docker \
+            --junitxml=reports/smoke.xml
+
+      - name: Upload junit report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: smoke-junit
+          path: reports/smoke.xml
+
+      - name: Fail the job on red results
+        if: failure()
+        run: |
+          echo "::error::Nightly recipe smoke matrix failed — inspect the junit"
+          echo "         artifact and the per-recipe stderr in the log above."
+          exit 1

bioflowkit-0.2.0/.github/workflows/release.yml

@@ -0,0 +1,179 @@
+name: Release to PyPI
+
+# Triggered by pushing a v* tag.  Pipeline:
+#   build      → sdist + wheel + twine check, artifacts uploaded
+#   testpypi   → publish to TestPyPI for one final smoke check
+#   pypi       → publish to PyPI
+#   github     → cut a GitHub Release with the artifacts attached
+#
+# Authentication uses PyPI Trusted Publishing (OIDC) — no long-lived
+# tokens.  Setup steps live in docs/MAINTAINER.md.
+#
+# Manual override: workflow_dispatch lets the maintainer re-run a stage
+# without recreating the tag (e.g. retry pypi after fixing a TestPyPI
+# issue).
+
+on:
+  push:
+    tags:
+      - "v*.*.*"          # v0.2.0, v0.2.1, v1.0.0, …
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build sdist + wheel
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0     # for setuptools_scm-style version lookups (future)
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build + twine
+        run: pip install --upgrade build twine
+
+      - name: Sanity-check version match
+        # Refuse to build if pyproject.toml and bioflow.__version__ have
+        # drifted.  Caught a real bug in 0.1.13.
+        run: |
+          python - <<'PY'
+          import tomllib, pathlib, re, sys
+          pp = tomllib.loads(pathlib.Path("pyproject.toml").read_text())
+          py_version = pp["project"]["version"]
+          init = pathlib.Path("bioflow/__init__.py").read_text(encoding="utf-8")
+          m = re.search(r'__version__\s*=\s*"([^"]+)"', init)
+          if not m or m.group(1) != py_version:
+              print(f"FAIL: pyproject={py_version} but __version__={m.group(1) if m else 'missing'}", file=sys.stderr)
+              sys.exit(1)
+          print(f"OK — both report {py_version}")
+          PY
+
+      - name: Build sdist + wheel
+        run: python -m build
+
+      - name: twine check
+        run: python -m twine check dist/*
+
+      - name: Verify wheel ships sdk/, cli/, _bundled_registry/
+        # Guard against pyproject.toml regressions that would drop the
+        # bundled registry or one of the freshly-split packages.  The
+        # glob matches whatever PyPI distribution name we ship under
+        # (`bioflowkit`, `bioflow`, etc.) — there is only ever one
+        # wheel per release in the artifact directory.
+        run: |
+          python - <<'PY'
+          import zipfile, glob, sys
+          wheels = glob.glob("dist/*.whl")
+          if not wheels:
+              print(f"FAIL: no wheel under dist/ (have: {sorted(glob.glob('dist/*'))})",
+                    file=sys.stderr)
+              sys.exit(1)
+          whl = wheels[0]
+          names = zipfile.ZipFile(whl).namelist()
+          required_prefixes = (
+              "bioflow/sdk/",
+              "bioflow/cli/",
+              "bioflow/_bundled_registry/tools/",
+          )
+          missing = [p for p in required_prefixes
+                     if not any(n.startswith(p) for n in names)]
+          if missing:
+              print(f"FAIL: {whl} missing prefixes {missing}", file=sys.stderr)
+              sys.exit(1)
+          print(f"OK — {whl} ships {len(names)} entries with all required prefixes")
+          PY
+
+      - name: Upload dist as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-dist
+          path: dist/
+          if-no-files-found: error
+
+  testpypi:
+    name: Publish to TestPyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/bioflowkit
+    permissions:
+      id-token: write          # OIDC token for trusted publishing
+    steps:
+      - name: Download dist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: python-dist
+          path: dist/
+
+      - name: Publish to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  pypi:
+    name: Publish to PyPI
+    needs: testpypi
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/bioflowkit
+    permissions:
+      id-token: write
+    steps:
+      - name: Download dist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: python-dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  github:
+    name: Cut GitHub Release
+    needs: pypi
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write          # required to create a release
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download dist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: python-dist
+          path: dist/
+
+      - name: Extract release notes for this tag
+        # Slice the CHANGELOG section whose header matches the tag (v0.2.0
+        # → look for "## [0.2.0]").  Falls back to "(see CHANGELOG.md)".
+        id: notes
+        run: |
+          TAG="${GITHUB_REF_NAME}"
+          VERSION="${TAG#v}"
+          python - <<PY > release_notes.md
+          import pathlib, re
+          src = pathlib.Path("CHANGELOG.md").read_text(encoding="utf-8")
+          pat = re.compile(rf"^## \[{re.escape('$VERSION')}\][^\n]*\n(.*?)(?=^## \[|\Z)",
+                           re.MULTILINE | re.DOTALL)
+          m = pat.search(src)
+          print(m.group(1).strip() if m else "(see CHANGELOG.md for details)")
+          PY
+          echo "RELEASE_NOTES_PATH=release_notes.md" >> "$GITHUB_OUTPUT"
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          name: ${{ github.ref_name }}
+          body_path: release_notes.md
+          files: dist/*
+          generate_release_notes: false      # we provided our own
+          draft: false
+          prerelease: ${{ contains(github.ref_name, '-') }}   # e.g. v0.2.0-rc1

bioflowkit-0.2.0/.gitignore

@@ -0,0 +1,100 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+
+# Bioflow runtime
+.bioflow_state.json
+data/test/**/out/
+data/references/dbs/**
+data/references/genomes/**
+!data/references/.gitkeep
+!data/references/dbs/.gitkeep
+!data/references/genomes/.gitkeep
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Comparative genomics analysis outputs (regeneratable)
+analysis/*/genomes/
+analysis/*/genomes_full/
+analysis/*/inputs/
+analysis/*/prokka/
+analysis/*/roary/
+analysis/*/fastani/
+analysis/*/fastani_full/ani_full.tsv
+analysis/*/scoary/results/
+analysis/*/run.log
+analysis/*/full_ani.log
+analysis/*/download_full.log
+analysis/*/fastani_full/genomes.txt
+# IQ-TREE intermediate files (keep only the *.treefile)
+analysis/*/phylogeny/*.bionj
+analysis/*/phylogeny/*.iqtree
+analysis/*/phylogeny/*.log
+analysis/*/phylogeny/*.mldist
+analysis/*/phylogeny/*.splits.nex
+analysis/*/phylogeny/*.uniqueseq.phy
+analysis/*/phylogeny/*.contree
+analysis/*/phylogeny/*.ckp.gz
+analysis/*/phylogeny/*.model.gz
+analysis/*/prokka_full/
+analysis/*/roary_full/
+analysis/*/full_pangenome.log
+analysis/*/scoary_full/results/
+analysis/*/eggnog/cog_db/
+analysis/*/eggnog/results/pangenome_vs_cog.tsv
+analysis/*/eggnog/db/
+analysis/*/eggnog/run.log
+analysis/*/eggnog/pangenome_reps.faa
+analysis/*/eggnog/pangenome_reps.tsv
+analysis/*/abricate_full/
+analysis/*/abricate_full.log
+analysis/*/phylogeny_full/_genes_unaligned/
+analysis/*/phylogeny_full/_genes_aligned/
+analysis/*/phylogeny_full/core_supermatrix.fna
+analysis/*/phylogeny_full/iqtree.log
+analysis/*/phylogeny_full/iqtree_full.iqtree
+analysis/*/phylogeny_full/iqtree_full.log
+analysis/*/phylogeny_full/iqtree_full.mldist
+analysis/*/phylogeny_full/iqtree_full.splits.nex
+analysis/*/phylogeny_full/iqtree_full.uniqueseq.phy
+analysis/*/phylogeny_full/iqtree_full.bionj
+analysis/*/phylogeny_full/iqtree_full.contree
+analysis/*/phylogeny_full/iqtree_full.ckp.gz
+analysis/*/cafe/results/Base_asr.tre
+analysis/*/cafe/results/Base_branch_probabilities.tab
+analysis/*/cafe/results/Base_change.tab
+analysis/*/cafe/results/Base_count.tab
+analysis/*/cafe/results/Base_report.cafe
+analysis/*/cafe/tree_ultrametric.nwk
+examples/_stage_demo_ws/
+examples/_cache_demo_ws/
+examples/_parallel_demo_ws/
+examples/_pipeline_demo_ws/
+analysis/*/pangenome_via_sdk/
+analysis/*/pangenome_sdk_13/
+analysis/*/sdk_13_run.log
+analysis/*/.cache/
+examples/_pectobacterium_ws/
+examples/_pectobacterium_run.log
+
+# Update scheduler outputs (per-machine, not for version control)
+update/notifications/
+update/last_run.json
+update/release_watch_state.json
+update/notifications/cron.log
+update/notifications/release_watch.log
+
+# Build artifacts
+dist/
+build/
+*.egg-info/
+
+# MkDocs build output
+site/