cotlab 0.8.0__tar.gz

cotlab-0.8.0/.env.example

@@ -0,0 +1 @@
+HF_TOKEN=your_huggingface_token_here

cotlab-0.8.0/.github/workflows/docs.yml

@@ -0,0 +1,42 @@
+name: Deploy Documentation
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+      - 'mkdocs.yml'
+      - '.github/workflows/docs.yml'
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup micromamba
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          micromamba-version: "2.0.5-0"
+          environment-name: cotlab
+          create-args: >-
+            python=3.11
+            pip
+          init-shell: bash
+
+      - name: Install dependencies
+        run: |
+          pip install mkdocs mkdocs-material "mkdocstrings[python]"
+        shell: micromamba-shell {0}
+
+      - name: Install CoTLab package
+        run: pip install -e .
+        shell: micromamba-shell {0}
+
+      - name: Deploy to GitHub Pages
+        run: mkdocs gh-deploy --force
+        shell: micromamba-shell {0}

cotlab-0.8.0/.github/workflows/lint.yml

@@ -0,0 +1,35 @@
+name: Lint
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "0 6 * * 1"  # Every Monday at 06:00 UTC
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Install dependencies
+        run: uv pip install --system -e ".[dev]"
+
+      - name: Check formatting
+        run: ruff format --check src/ tests/
+
+      - name: Lint
+        run: ruff check src/ tests/

cotlab-0.8.0/.github/workflows/tests.yml

@@ -0,0 +1,364 @@
+name: Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "0 6 * * 1"  # Every Monday at 06:00 UTC
+
+jobs:
+  unit-tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: ["3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Free disk space (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Install dependencies
+        run: uv pip install --system -e ".[dev]"
+
+      - name: Run unit tests
+        run: python -m pytest tests/ -v --tb=short
+
+  vllm-metal:
+    runs-on: macos-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Print Apple Silicon info
+        run: |
+          sysctl -n machdep.cpu.brand_string
+          uname -m
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Install CoTLab
+        run: uv pip install --system -e ".[dev]"
+
+      - name: Install vLLM 0.13.0 from source
+        run: |
+          cd /tmp
+          curl -OL https://github.com/vllm-project/vllm/releases/download/v0.13.0/vllm-0.13.0.tar.gz
+          tar xf vllm-0.13.0.tar.gz && cd vllm-0.13.0
+          pip install -r requirements/cpu.txt
+          pip install .
+
+      - name: Pin transformers for vLLM
+        run: pip install "transformers>=4.56.0,<5"
+
+      - name: Install vllm-metal plugin
+        run: pip install vllm-metal
+
+      - name: Re-pin transformers (final)
+        run: pip install "transformers>=4.56.0,<5"
+
+      - name: Show versions
+        run: |
+          python - <<'PY'
+          import transformers, vllm
+          print("transformers:", transformers.__version__)
+          print("vllm:", vllm.__version__)
+          PY
+
+      - name: Test vllm-metal installation
+        run: |
+          python -c "
+          from vllm import LLM
+          import vllm_metal
+          print(f'vLLM-Metal ready!')
+          print(f'vllm-metal version: {vllm_metal.__version__}')
+          "
+
+      - name: Test VLLMBackend platform detection
+        run: |
+          python -c "
+          from cotlab.backends.vllm_backend import VLLMBackend, _detect_platform, _is_apple_silicon
+          assert _is_apple_silicon(), 'Should detect Apple Silicon'
+          assert _detect_platform() == 'metal', 'Should detect metal platform'
+          backend = VLLMBackend()
+          assert backend.platform == 'metal', 'Backend should use metal platform'
+          print('VLLMBackend metal platform detection works!')
+          "
+
+  integration:
+    runs-on: ${{ matrix.os }}
+    env:
+      HF_TOKEN: ${{ secrets.HF_TOKEN }}
+      HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Free disk space (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Install dependencies
+        run: uv pip install --system -e ".[dev]"
+
+      - name: Check Hugging Face token
+        run: |
+          if [ -z "${HF_TOKEN}" ]; then
+            echo "HF_TOKEN secret is required for private dataset access."
+            exit 1
+          fi
+
+      - name: Authenticate Hugging Face
+        run: |
+          python - <<'PY'
+          import os
+          from huggingface_hub import login
+
+          login(token=os.environ["HF_TOKEN"], add_to_git_credential=False)
+          print("Authenticated to Hugging Face Hub.")
+          PY
+
+      - name: Test framework imports
+        run: |
+          python -c "
+          from cotlab import __version__
+          from cotlab.backends import TransformersBackend
+          from cotlab.experiments import (
+              CoTFaithfulnessExperiment,
+              ClassificationExperiment,
+              CoTAblationExperiment,
+              ActivationPatchingExperiment,
+              ActivationCompareExperiment,
+              SycophancyHeadsExperiment,
+              MultiHeadPatchingExperiment,
+              FullLayerPatchingExperiment,
+              SteeringVectorsExperiment,
+              CoTHeadsExperiment,
+              LogitLensExperiment,
+              MultiHeadCoTExperiment,
+              ResidualNormOODExperiment,
+              CompositeShiftDetectorExperiment,
+          )
+          from cotlab.prompts import ChainOfThoughtStrategy, RadiologyPromptStrategy, CardiologyPromptStrategy, NeurologyPromptStrategy, OncologyPromptStrategy
+          from cotlab.datasets import RadiologyDataset, SyntheticMedicalDataset, CardiologyDataset, NeurologyDataset, OncologyDataset
+          from cotlab.analysis import CoTParser, FaithfulnessMetrics
+          from cotlab.patching import ActivationPatcher
+
+          print(f'CoTLab v{__version__} - All 15 experiments available!')
+          "
+
+      - name: Test prompt strategies
+        run: |
+          python -c "
+          from cotlab.prompts import create_prompt_strategy
+
+          strategies = [
+              'simple', 'chain_of_thought', 'direct_answer', 'arrogance', 'no_instruction',
+              'adversarial', 'uncertainty', 'socratic', 'contrarian', 'expert_persona',
+              'sycophantic', 'few_shot'
+          ]
+          for name in strategies:
+              s = create_prompt_strategy(name)
+              prompt = s.build_prompt({'question': 'Test question?'})
+              print(f'{name}: {len(prompt)} chars')
+
+          print(f'All {len(strategies)} prompt strategies work!')
+          "
+
+      - name: Test dataset loading
+        run: |
+          python -c "
+          from cotlab.datasets import SyntheticMedicalDataset, RadiologyDataset, CardiologyDataset, NeurologyDataset, OncologyDataset, PubHealthBenchDataset, MARCDataset, MedBulletsDataset, PLABDataset
+
+          for Dataset in [SyntheticMedicalDataset, RadiologyDataset, CardiologyDataset, NeurologyDataset, OncologyDataset, PubHealthBenchDataset, MARCDataset, MedBulletsDataset, PLABDataset]:
+              dataset = Dataset()
+              print(f'{Dataset.__name__}: {len(dataset)} samples')
+
+          print('All 9 datasets load correctly!')
+          "
+
+      - name: Test CoT parser
+        run: |
+          python -c "
+          from cotlab.analysis import CoTParser
+
+          parser = CoTParser()
+          cot = '''
+          1. Patient has fever
+          2. Symptoms suggest infection
+          Therefore, likely viral URI.
+          '''
+
+          result = parser.analyze(cot)
+          print(f'Steps: {result[\"num_steps\"]}')
+          print(f'Conclusion: {result[\"conclusion\"]}')
+          print('CoT parser works!')
+          "
+
+      - name: Test new prompt features (answer_first, few_shot, contrarian)
+        run: |
+          python -c "
+          from cotlab.prompts import RadiologyPromptStrategy, CardiologyPromptStrategy
+
+          answer_first = RadiologyPromptStrategy(answer_first=True)
+          af_prompt = answer_first.build_prompt({'report': 'Test'})
+          assert 'Initial' in af_prompt, 'answer_first prompt should contain Initial Assessment'
+          print('answer_first mode works')
+
+          with_examples = RadiologyPromptStrategy(few_shot=True)
+          without_examples = RadiologyPromptStrategy(few_shot=False)
+          text_with = with_examples.build_prompt({'report': 'Test'})
+          text_without = without_examples.build_prompt({'report': 'Test'})
+          assert 'Example 1:' in text_with, 'Should have examples when few_shot=true'
+          assert 'Example 1:' not in text_without, 'Should not have examples when few_shot=false'
+          assert len(text_without) < len(text_with), 'Prompt without examples should be shorter'
+          print('few_shot parameter works')
+
+          standard = RadiologyPromptStrategy(contrarian=False)
+          contrarian = RadiologyPromptStrategy(contrarian=True)
+          std_prompt = standard.build_prompt({'report': 'Test'})
+          con_prompt = contrarian.build_prompt({'report': 'Test'})
+          assert std_prompt != con_prompt, 'Contrarian prompt should differ from standard'
+          print('contrarian mode works')
+
+          for Strategy in [RadiologyPromptStrategy, CardiologyPromptStrategy]:
+              s = Strategy(answer_first=True, few_shot=False)
+              prompt = s.build_prompt({'report': 'Test'})
+              assert 'Initial' in prompt and 'Example 1:' not in prompt
+          print('Parameter combinations work across specialties')
+          "
+
+      - name: Test histopathology dataset and prompt
+        run: |
+          python -c "
+          from cotlab.datasets import HistopathologyDataset
+          from cotlab.prompts import HistopathologyPromptStrategy
+
+          print('Testing HistopathologyPromptStrategy...')
+
+          p = HistopathologyPromptStrategy()
+          prompt = p.build_prompt({'text': 'Test histopathology report'})
+          assert 'quality' in prompt.lower(), 'Prompt should mention quality'
+          assert 'Example 1' in prompt, 'Should have few-shot examples by default'
+          print('Standard prompt works')
+
+          p_no_fs = HistopathologyPromptStrategy(few_shot=False)
+          prompt_no_fs = p_no_fs.build_prompt({'text': 'Test'})
+          assert 'Example 1' not in prompt_no_fs, 'Should not have examples when few_shot=false'
+          print('few_shot=False works')
+
+          p_con = HistopathologyPromptStrategy(contrarian=True)
+          prompt_con = p_con.build_prompt({'text': 'Test'})
+          assert 'skeptical' in prompt_con.lower() or 'critical' in prompt_con.lower(), 'Contrarian should be skeptical'
+          assert 'skeptical' in p_con.get_system_message().lower(), 'System message should be skeptical'
+          print('contrarian mode works')
+
+          p_af = HistopathologyPromptStrategy(answer_first=True)
+          prompt_af = p_af.build_prompt({'text': 'Test'})
+          assert 'First' in prompt_af, 'Answer-first should ask for rating first'
+          print('answer_first mode works')
+
+          result = p.parse_response('{\"quality_score\": 2, \"reasoning\": \"Good report\"}')
+          assert result['quality_score'] == 2, 'Should parse quality_score'
+          assert result['parse_success'], 'Should mark as success'
+          print('parse_response works')
+
+          print('All histopathology tests passed!')
+          "
+
+  notebook-test:
+    runs-on: ${{ matrix.os }}
+    env:
+      HF_TOKEN: ${{ secrets.HF_TOKEN }}
+      HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Free disk space (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Install dependencies
+        run: |
+          uv pip install --system -e ".[dev]"
+          pip install papermill ipykernel
+          python -m ipykernel install --user --name cotlab --display-name "CoTLab"
+
+      - name: Check Hugging Face token
+        run: |
+          if [ -z "${HF_TOKEN}" ]; then
+            echo "HF_TOKEN secret is required for private dataset access."
+            exit 1
+          fi
+
+      - name: Authenticate Hugging Face
+        run: |
+          python - <<'PY'
+          import os
+          from huggingface_hub import login
+
+          login(token=os.environ["HF_TOKEN"], add_to_git_credential=False)
+          print("Authenticated to Hugging Face Hub.")
+          PY
+
+      - name: Execute tutorial notebook
+        run: |
+          echo "Running notebook: cotlab_tutorial.ipynb (CI mode: 5 samples, 64 max_tokens)"
+          cd notebooks
+          papermill cotlab_tutorial.ipynb cotlab_tutorial_executed.ipynb -k cotlab --progress-bar -p NUM_SAMPLES 5 -p MAX_TOKENS 64
+          echo "Notebook executed successfully on ${{ matrix.os }}!"

cotlab-0.8.0/.gitignore

@@ -0,0 +1,64 @@
+# Virtual environments
+/cot/
+/cotlab/
+/.venv/
+/venv/
+/env/
+
+# Experiment outputs
+outputs/
+multirun/
+
+# Model configs (local only — never commit)
+conf/model/
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# Environment
+.env
+.envrc
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+.nox/
+
+# Jupyter
+.ipynb_checkpoints/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log
+uv.lock

cotlab-0.8.0/.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+repos:
+  # Ruff - linting and formatting
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.14.10
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  # Basic file checks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+        args: ['--maxkb=600']

cotlab-0.8.0/CLAUDE.md

@@ -0,0 +1,75 @@
+# CoTLab — Claude Instructions
+
+## Commit style
+- Always prefix with a conventional type: `feat`, `fix`, `refactor`, `chore`, `docs`, `test`
+- One line only — no body, no co-author trailer, no bullet points
+- Example: `feat: add composite shift detector experiment`
+
+## Model names
+Always use HF repo ID format directly — never safe-name format:
+- `model=google/gemma-3-270m-it` (local smoke test)
+- `model=google/medgemma-4b-it` (local full test)
+- `model=google/medgemma-27b-text-it` (H100)
+
+## Git — never commit model config files
+Never stage or commit anything under `conf/model/`. These are local only.
+Always exclude: `conf/model/google/`, `conf/model/*.yaml`
+
+## Run commands
+Always use `\` line continuation with **one arg per line** — this is the working pattern.
+Always include `backend.device=` explicitly.
+
+Local (Mac, MPS):
+```bash
+python -m cotlab.main \
+    experiment=residual_norm_ood \
+    dataset=afrimedqa \
+    backend=transformers \
+    backend.device=mps \
+    model=google/gemma-3-270m-it \
+    experiment.num_samples=10
+
+python -m cotlab.main \
+    experiment=composite_shift_detector \
+    dataset=afrimedqa \
+    backend=transformers \
+    backend.device=mps \
+    model=google/gemma-3-270m-it \
+    experiment.num_samples=10
+
+python -m cotlab.main \
+    experiment=activation_patching \
+    dataset=afrimedqa \
+    backend=transformers \
+    backend.device=mps \
+    model=google/gemma-3-270m-it \
+    experiment.patching_mode=cot_contrast \
+    experiment.num_samples=10
+```
+
+H100 (use `backend.device=cuda` and `model=google/medgemma-27b-text-it`, drop `experiment.num_samples`):
+```bash
+python -m cotlab.main \
+    experiment=residual_norm_ood \
+    dataset=afrimedqa \
+    backend=transformers \
+    backend.device=cuda \
+    model=google/medgemma-27b-text-it
+```
+
+## Multirun (H100 batch across datasets)
+Define DS variable with `\` line continuation, then quote `"dataset=${DS}"`:
+```bash
+DS="afrimedqa,medbullets,medmcqa,medqa,medxpertqa,mmlu_medical,cardiology, \
+    histopathology,neurology,oncology,pediatrics,plab"
+
+python -m cotlab.main --multirun \
+    experiment=residual_norm_ood \
+    backend=transformers \
+    backend.device=cuda \
+    model=google/medgemma-27b-text-it \
+    "dataset=${DS}"
+```
+Set DS once, reuse for all experiments. The `\` inside the double-quoted string joins the lines so Hydra sees one clean comma-separated value.
+
+Always test locally with 270m + 10 samples before running on H100.

cotlab-0.8.0/Dockerfile.rocm

@@ -0,0 +1,18 @@
+FROM rocm/vllm-dev:rocm7.1.1_navi_ubuntu24.04_py3.12_pytorch_2.8_vllm_0.10.2rc1
+
+# Ensure cache directory exists (base image already has user with UID 1000)
+RUN mkdir -p /home/user/.cache/huggingface && \
+    chown -R 1000:1000 /home/user 2>/dev/null || true
+
+WORKDIR /app
+
+# Copy project files
+COPY pyproject.toml README.md ./
+COPY src/ ./src/
+COPY conf/ ./conf/
+COPY data/ ./data/
+# Install CoTLab (without vllm since it's already in the base image)
+RUN pip install -e .
+
+# Default entrypoint
+ENTRYPOINT ["python", "-m", "cotlab.main"]

cotlab-0.8.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025-2026 Hüseyin Çavuş
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.