cotlab 0.8.0__tar.gz → 0.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cotlab-0.8.2/.github/dependabot.yml +10 -0
- cotlab-0.8.2/.github/workflows/container.yml +53 -0
- cotlab-0.8.2/.github/workflows/docs.yml +38 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/.github/workflows/lint.yml +9 -6
- cotlab-0.8.2/.github/workflows/release.yml +34 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/.github/workflows/tests.yml +47 -36
- {cotlab-0.8.0 → cotlab-0.8.2}/.gitignore +0 -1
- cotlab-0.8.2/Containerfile +27 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/PKG-INFO +19 -2
- {cotlab-0.8.0 → cotlab-0.8.2}/README.md +12 -0
- cotlab-0.8.2/conf/experiment/confabulation_analysis.yaml +14 -0
- cotlab-0.8.2/conf/experiment/entropy_neuron_overlap.yaml +8 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/h_neuron_analysis.yaml +0 -1
- cotlab-0.8.2/conf/experiment/sae_feature_neuron_overlap.yaml +12 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/pyproject.toml +14 -1
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/__init__.py +1 -1
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/backends/transformers_backend.py +10 -1
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/core/base.py +1 -1
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/__init__.py +6 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/activation_patching.py +9 -9
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/composite_shift_detector.py +8 -5
- cotlab-0.8.2/src/cotlab/experiments/confabulation_analysis.py +360 -0
- cotlab-0.8.2/src/cotlab/experiments/entropy_neuron_overlap.py +231 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/h_neuron_analysis.py +11 -9
- cotlab-0.8.2/src/cotlab/experiments/sae_feature_neuron_overlap.py +230 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/tests/test_experiments.py +162 -0
- cotlab-0.8.2/uv.lock +6054 -0
- cotlab-0.8.0/.github/workflows/docs.yml +0 -42
- cotlab-0.8.0/CLAUDE.md +0 -75
- {cotlab-0.8.0 → cotlab-0.8.2}/.env.example +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/.pre-commit-config.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/Dockerfile.rocm +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/LICENSE +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/backend/transformers.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/backend/vllm.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/config.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/afrimedqa.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/cardiology.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/histopathology.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/m_arc.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/medbullets.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/medmcqa.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/medqa.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/medxpertqa.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/mmlu_medical.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/movie_ood.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/neurology.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/oncology.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/patching_pairs.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/pediatrics.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/plab.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/probing_diagnosis.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/pubhealthbench.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/pubmedqa.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/radiology.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/synthetic.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/dataset/tcga.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/activation_compare.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/activation_patching.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/attention_analysis.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/classification.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/composite_shift_detector.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/cot_ablation.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/cot_faithfulness.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/cot_heads.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/full_layer_cot.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/full_layer_patching.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/logit_lens.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/multi_head_cot.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/multi_head_patching.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/probing_classifier.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/residual_norm_ood.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/sae_feature_analysis.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/steering_vectors.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/sycophancy_heads.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/experiment/token_group_contrast.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/adversarial.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/arrogance.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/cardiology.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/chain_of_thought.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/contrarian.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/contrarian_matched.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/cot_matched.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/direct_answer.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/direct_matched.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/expert_persona.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/few_shot.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/histopathology.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/mcq.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/neurology.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/no_instruction.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/oncology.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/plab.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/pubhealthbench.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/pubmedqa.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/radiology.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/simple.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/socratic.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/sycophantic.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/tcga.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/conf/prompt/uncertainty.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/data/datasets.yaml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/data/movie_ood.jsonl +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/docker-compose.rocm.yml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/docs/api/core.md +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/docs/api/experiments.md +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/docs/api/prompts.md +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/docs/getting-started/installation.md +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/docs/getting-started/quickstart.md +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/docs/guide/configuration.md +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/docs/guide/experiments.md +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/docs/guide/models.md +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/docs/guide/prompts.md +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/docs/index.md +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/docs/rocm-setup.md +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/mkdocs.yml +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/notebooks/cotlab_tutorial.ipynb +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/scripts/analyze_json_compliance.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/scripts/build_movie_ood_dataset.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/scripts/compare_json_vs_freetext.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/scripts/cotlab-rocm.sh +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/analyse_experiments.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/analysis/__init__.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/analysis/cot_parser.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/analysis/faithfulness_metrics.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/backends/__init__.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/backends/base.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/backends/vllm_backend.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/cli.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/core/__init__.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/core/config.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/core/registry.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/datasets/__init__.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/datasets/loaders.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiment/__init__.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/activation_compare.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/attention_analysis.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/classification.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/cot_ablation.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/cot_faithfulness.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/cot_heads.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/full_layer_cot.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/full_layer_patching.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/logit_lens.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/multi_head_cot.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/multi_head_patching.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/probing_classifier.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/residual_norm_ood.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/sae_feature_analysis.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/steering_vectors.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/experiments/sycophancy_heads.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/logging/__init__.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/logging/json_logger.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/main.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/patching/__init__.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/patching/cache.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/patching/hooks.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/patching/interventions.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/patching/patcher.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/patching/sae.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/prompts/__init__.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/prompts/cardiology.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/prompts/histopathology.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/prompts/length_matched_strategies.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/prompts/mcq.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/prompts/neurology.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/prompts/oncology.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/prompts/plab.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/prompts/pubhealthbench.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/prompts/pubmedqa.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/prompts/radiology.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/prompts/strategies.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/prompts/tcga.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/src/cotlab/runner.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/tests/conftest.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/tests/test_analysis.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/tests/test_backends.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/tests/test_core.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/tests/test_datasets.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/tests/test_experiment_docs.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/tests/test_patching.py +0 -0
- {cotlab-0.8.0 → cotlab-0.8.2}/tests/test_prompts.py +0 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
name: Container
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
build-and-push:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read
|
|
13
|
+
packages: write
|
|
14
|
+
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
|
17
|
+
|
|
18
|
+
- name: Set up Docker Buildx
|
|
19
|
+
uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2
|
|
20
|
+
|
|
21
|
+
- name: Log in to GHCR
|
|
22
|
+
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
|
|
23
|
+
with:
|
|
24
|
+
registry: ghcr.io
|
|
25
|
+
username: ${{ github.actor }}
|
|
26
|
+
password: ${{ secrets.GITHUB_TOKEN }}
|
|
27
|
+
|
|
28
|
+
- name: Log in to Docker Hub
|
|
29
|
+
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
|
|
30
|
+
with:
|
|
31
|
+
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
32
|
+
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
33
|
+
continue-on-error: true
|
|
34
|
+
|
|
35
|
+
- name: Build and push to GHCR
|
|
36
|
+
uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75
|
|
37
|
+
with:
|
|
38
|
+
context: .
|
|
39
|
+
file: ./Containerfile
|
|
40
|
+
push: true
|
|
41
|
+
tags: ghcr.io/huseyincavusbi/cotlab:latest
|
|
42
|
+
cache-from: type=gha
|
|
43
|
+
cache-to: type=gha,mode=max
|
|
44
|
+
|
|
45
|
+
- name: Push to Docker Hub
|
|
46
|
+
uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75
|
|
47
|
+
with:
|
|
48
|
+
context: .
|
|
49
|
+
file: ./Containerfile
|
|
50
|
+
push: true
|
|
51
|
+
tags: docker.io/huseyincavus/cotlab:latest
|
|
52
|
+
cache-from: type=gha
|
|
53
|
+
continue-on-error: true
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
name: Deploy Documentation
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
paths:
|
|
8
|
+
- 'docs/**'
|
|
9
|
+
- 'mkdocs.yml'
|
|
10
|
+
- '.github/workflows/docs.yml'
|
|
11
|
+
|
|
12
|
+
permissions:
|
|
13
|
+
contents: read
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
deploy:
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
permissions:
|
|
19
|
+
contents: write
|
|
20
|
+
steps:
|
|
21
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
22
|
+
|
|
23
|
+
- name: Set up Python
|
|
24
|
+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
|
25
|
+
with:
|
|
26
|
+
python-version: "3.11"
|
|
27
|
+
|
|
28
|
+
- name: Install uv
|
|
29
|
+
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
30
|
+
with:
|
|
31
|
+
enable-cache: true
|
|
32
|
+
cache-dependency-glob: "pyproject.toml"
|
|
33
|
+
|
|
34
|
+
- name: Install dependencies
|
|
35
|
+
run: uv sync --frozen --extra docs
|
|
36
|
+
|
|
37
|
+
- name: Deploy to GitHub Pages
|
|
38
|
+
run: uv run mkdocs gh-deploy --force
|
|
@@ -8,28 +8,31 @@ on:
|
|
|
8
8
|
schedule:
|
|
9
9
|
- cron: "0 6 * * 1" # Every Monday at 06:00 UTC
|
|
10
10
|
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read
|
|
13
|
+
|
|
11
14
|
jobs:
|
|
12
15
|
lint:
|
|
13
16
|
runs-on: ubuntu-latest
|
|
14
17
|
steps:
|
|
15
|
-
- uses: actions/checkout@v6
|
|
18
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
16
19
|
|
|
17
20
|
- name: Set up Python
|
|
18
|
-
uses: actions/setup-python@v6
|
|
21
|
+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
|
19
22
|
with:
|
|
20
23
|
python-version: "3.11"
|
|
21
24
|
|
|
22
25
|
- name: Install uv
|
|
23
|
-
uses: astral-sh/setup-uv@
|
|
26
|
+
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
24
27
|
with:
|
|
25
28
|
enable-cache: true
|
|
26
29
|
cache-dependency-glob: "pyproject.toml"
|
|
27
30
|
|
|
28
31
|
- name: Install dependencies
|
|
29
|
-
run: uv
|
|
32
|
+
run: uv sync --frozen --all-extras
|
|
30
33
|
|
|
31
34
|
- name: Check formatting
|
|
32
|
-
run: ruff format --check src/ tests/
|
|
35
|
+
run: uv run ruff format --check src/ tests/
|
|
33
36
|
|
|
34
37
|
- name: Lint
|
|
35
|
-
run: ruff check src/ tests/
|
|
38
|
+
run: uv run ruff check src/ tests/
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
release:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
environment: release
|
|
15
|
+
permissions:
|
|
16
|
+
contents: write
|
|
17
|
+
id-token: write
|
|
18
|
+
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
21
|
+
|
|
22
|
+
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v5
|
|
23
|
+
|
|
24
|
+
- name: Build
|
|
25
|
+
run: uv build
|
|
26
|
+
|
|
27
|
+
- name: Publish to PyPI
|
|
28
|
+
run: uv publish
|
|
29
|
+
|
|
30
|
+
- name: Create GitHub Release
|
|
31
|
+
uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v2
|
|
32
|
+
with:
|
|
33
|
+
generate_release_notes: true
|
|
34
|
+
files: dist/*
|
|
@@ -8,6 +8,9 @@ on:
|
|
|
8
8
|
schedule:
|
|
9
9
|
- cron: "0 6 * * 1" # Every Monday at 06:00 UTC
|
|
10
10
|
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read
|
|
13
|
+
|
|
11
14
|
jobs:
|
|
12
15
|
unit-tests:
|
|
13
16
|
runs-on: ${{ matrix.os }}
|
|
@@ -17,7 +20,7 @@ jobs:
|
|
|
17
20
|
python-version: ["3.11", "3.12", "3.13"]
|
|
18
21
|
|
|
19
22
|
steps:
|
|
20
|
-
- uses: actions/checkout@v6
|
|
23
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
21
24
|
|
|
22
25
|
- name: Free disk space (Linux)
|
|
23
26
|
if: runner.os == 'Linux'
|
|
@@ -25,48 +28,50 @@ jobs:
|
|
|
25
28
|
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
|
26
29
|
|
|
27
30
|
- name: Set up Python ${{ matrix.python-version }}
|
|
28
|
-
uses: actions/setup-python@v6
|
|
31
|
+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
|
29
32
|
with:
|
|
30
33
|
python-version: ${{ matrix.python-version }}
|
|
31
34
|
|
|
32
35
|
- name: Install uv
|
|
33
|
-
uses: astral-sh/setup-uv@
|
|
36
|
+
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
34
37
|
with:
|
|
35
38
|
enable-cache: true
|
|
36
39
|
cache-dependency-glob: "pyproject.toml"
|
|
37
40
|
|
|
38
41
|
- name: Install dependencies
|
|
39
|
-
run: uv
|
|
42
|
+
run: uv sync --frozen --all-extras
|
|
40
43
|
|
|
41
44
|
- name: Run unit tests
|
|
42
|
-
run: python -m pytest tests/ -v --tb=short
|
|
45
|
+
run: uv run python -m pytest tests/ -v --tb=short
|
|
43
46
|
|
|
44
47
|
vllm-metal:
|
|
45
48
|
runs-on: macos-latest
|
|
46
49
|
continue-on-error: true
|
|
47
50
|
steps:
|
|
48
|
-
- uses: actions/checkout@v6
|
|
51
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
49
52
|
|
|
50
53
|
- name: Print Apple Silicon info
|
|
51
54
|
run: |
|
|
52
55
|
sysctl -n machdep.cpu.brand_string
|
|
53
56
|
uname -m
|
|
54
57
|
|
|
55
|
-
- name:
|
|
56
|
-
uses:
|
|
58
|
+
- name: Setup micromamba
|
|
59
|
+
uses: mamba-org/setup-micromamba@d7c9bd84e824b79d2af72a2d4196c7f4300d3476 # v3.0.0
|
|
57
60
|
with:
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
cache-dependency-glob: "pyproject.toml"
|
|
61
|
+
micromamba-version: "2.0.5-0"
|
|
62
|
+
environment-name: cotlab
|
|
63
|
+
create-args: >-
|
|
64
|
+
python=3.12
|
|
65
|
+
pip
|
|
66
|
+
init-shell: bash
|
|
65
67
|
|
|
66
68
|
- name: Install CoTLab
|
|
67
|
-
|
|
69
|
+
shell: bash -el {0}
|
|
70
|
+
run: |
|
|
71
|
+
pip install -e ".[dev]"
|
|
68
72
|
|
|
69
73
|
- name: Install vLLM 0.13.0 from source
|
|
74
|
+
shell: bash -el {0}
|
|
70
75
|
run: |
|
|
71
76
|
cd /tmp
|
|
72
77
|
curl -OL https://github.com/vllm-project/vllm/releases/download/v0.13.0/vllm-0.13.0.tar.gz
|
|
@@ -75,15 +80,19 @@ jobs:
|
|
|
75
80
|
pip install .
|
|
76
81
|
|
|
77
82
|
- name: Pin transformers for vLLM
|
|
83
|
+
shell: bash -el {0}
|
|
78
84
|
run: pip install "transformers>=4.56.0,<5"
|
|
79
85
|
|
|
80
86
|
- name: Install vllm-metal plugin
|
|
87
|
+
shell: bash -el {0}
|
|
81
88
|
run: pip install vllm-metal
|
|
82
89
|
|
|
83
90
|
- name: Re-pin transformers (final)
|
|
91
|
+
shell: bash -el {0}
|
|
84
92
|
run: pip install "transformers>=4.56.0,<5"
|
|
85
93
|
|
|
86
94
|
- name: Show versions
|
|
95
|
+
shell: bash -el {0}
|
|
87
96
|
run: |
|
|
88
97
|
python - <<'PY'
|
|
89
98
|
import transformers, vllm
|
|
@@ -92,6 +101,7 @@ jobs:
|
|
|
92
101
|
PY
|
|
93
102
|
|
|
94
103
|
- name: Test vllm-metal installation
|
|
104
|
+
shell: bash -el {0}
|
|
95
105
|
run: |
|
|
96
106
|
python -c "
|
|
97
107
|
from vllm import LLM
|
|
@@ -101,6 +111,7 @@ jobs:
|
|
|
101
111
|
"
|
|
102
112
|
|
|
103
113
|
- name: Test VLLMBackend platform detection
|
|
114
|
+
shell: bash -el {0}
|
|
104
115
|
run: |
|
|
105
116
|
python -c "
|
|
106
117
|
from cotlab.backends.vllm_backend import VLLMBackend, _detect_platform, _is_apple_silicon
|
|
@@ -120,7 +131,7 @@ jobs:
|
|
|
120
131
|
matrix:
|
|
121
132
|
os: [ubuntu-latest, macos-latest]
|
|
122
133
|
steps:
|
|
123
|
-
- uses: actions/checkout@v6
|
|
134
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
124
135
|
|
|
125
136
|
- name: Free disk space (Linux)
|
|
126
137
|
if: runner.os == 'Linux'
|
|
@@ -128,18 +139,18 @@ jobs:
|
|
|
128
139
|
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
|
129
140
|
|
|
130
141
|
- name: Set up Python
|
|
131
|
-
uses: actions/setup-python@v6
|
|
142
|
+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
|
132
143
|
with:
|
|
133
144
|
python-version: "3.11"
|
|
134
145
|
|
|
135
146
|
- name: Install uv
|
|
136
|
-
uses: astral-sh/setup-uv@
|
|
147
|
+
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
137
148
|
with:
|
|
138
149
|
enable-cache: true
|
|
139
150
|
cache-dependency-glob: "pyproject.toml"
|
|
140
151
|
|
|
141
152
|
- name: Install dependencies
|
|
142
|
-
run: uv
|
|
153
|
+
run: uv sync --frozen --all-extras
|
|
143
154
|
|
|
144
155
|
- name: Check Hugging Face token
|
|
145
156
|
run: |
|
|
@@ -150,7 +161,7 @@ jobs:
|
|
|
150
161
|
|
|
151
162
|
- name: Authenticate Hugging Face
|
|
152
163
|
run: |
|
|
153
|
-
python - <<'PY'
|
|
164
|
+
uv run python - <<'PY'
|
|
154
165
|
import os
|
|
155
166
|
from huggingface_hub import login
|
|
156
167
|
|
|
@@ -160,7 +171,7 @@ jobs:
|
|
|
160
171
|
|
|
161
172
|
- name: Test framework imports
|
|
162
173
|
run: |
|
|
163
|
-
python -c "
|
|
174
|
+
uv run python -c "
|
|
164
175
|
from cotlab import __version__
|
|
165
176
|
from cotlab.backends import TransformersBackend
|
|
166
177
|
from cotlab.experiments import (
|
|
@@ -189,7 +200,7 @@ jobs:
|
|
|
189
200
|
|
|
190
201
|
- name: Test prompt strategies
|
|
191
202
|
run: |
|
|
192
|
-
python -c "
|
|
203
|
+
uv run python -c "
|
|
193
204
|
from cotlab.prompts import create_prompt_strategy
|
|
194
205
|
|
|
195
206
|
strategies = [
|
|
@@ -207,7 +218,7 @@ jobs:
|
|
|
207
218
|
|
|
208
219
|
- name: Test dataset loading
|
|
209
220
|
run: |
|
|
210
|
-
python -c "
|
|
221
|
+
uv run python -c "
|
|
211
222
|
from cotlab.datasets import SyntheticMedicalDataset, RadiologyDataset, CardiologyDataset, NeurologyDataset, OncologyDataset, PubHealthBenchDataset, MARCDataset, MedBulletsDataset, PLABDataset
|
|
212
223
|
|
|
213
224
|
for Dataset in [SyntheticMedicalDataset, RadiologyDataset, CardiologyDataset, NeurologyDataset, OncologyDataset, PubHealthBenchDataset, MARCDataset, MedBulletsDataset, PLABDataset]:
|
|
@@ -219,7 +230,7 @@ jobs:
|
|
|
219
230
|
|
|
220
231
|
- name: Test CoT parser
|
|
221
232
|
run: |
|
|
222
|
-
python -c "
|
|
233
|
+
uv run python -c "
|
|
223
234
|
from cotlab.analysis import CoTParser
|
|
224
235
|
|
|
225
236
|
parser = CoTParser()
|
|
@@ -237,7 +248,7 @@ jobs:
|
|
|
237
248
|
|
|
238
249
|
- name: Test new prompt features (answer_first, few_shot, contrarian)
|
|
239
250
|
run: |
|
|
240
|
-
python -c "
|
|
251
|
+
uv run python -c "
|
|
241
252
|
from cotlab.prompts import RadiologyPromptStrategy, CardiologyPromptStrategy
|
|
242
253
|
|
|
243
254
|
answer_first = RadiologyPromptStrategy(answer_first=True)
|
|
@@ -270,7 +281,7 @@ jobs:
|
|
|
270
281
|
|
|
271
282
|
- name: Test histopathology dataset and prompt
|
|
272
283
|
run: |
|
|
273
|
-
python -c "
|
|
284
|
+
uv run python -c "
|
|
274
285
|
from cotlab.datasets import HistopathologyDataset
|
|
275
286
|
from cotlab.prompts import HistopathologyPromptStrategy
|
|
276
287
|
|
|
@@ -315,7 +326,7 @@ jobs:
|
|
|
315
326
|
matrix:
|
|
316
327
|
os: [ubuntu-latest, macos-latest]
|
|
317
328
|
steps:
|
|
318
|
-
- uses: actions/checkout@v6
|
|
329
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
319
330
|
|
|
320
331
|
- name: Free disk space (Linux)
|
|
321
332
|
if: runner.os == 'Linux'
|
|
@@ -323,21 +334,21 @@ jobs:
|
|
|
323
334
|
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
|
324
335
|
|
|
325
336
|
- name: Set up Python
|
|
326
|
-
uses: actions/setup-python@v6
|
|
337
|
+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
|
327
338
|
with:
|
|
328
339
|
python-version: "3.11"
|
|
329
340
|
|
|
330
341
|
- name: Install uv
|
|
331
|
-
uses: astral-sh/setup-uv@
|
|
342
|
+
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
332
343
|
with:
|
|
333
344
|
enable-cache: true
|
|
334
345
|
cache-dependency-glob: "pyproject.toml"
|
|
335
346
|
|
|
336
347
|
- name: Install dependencies
|
|
337
348
|
run: |
|
|
338
|
-
uv
|
|
339
|
-
pip install papermill ipykernel
|
|
340
|
-
python -m ipykernel install --user --name cotlab --display-name "CoTLab"
|
|
349
|
+
uv sync --frozen --all-extras
|
|
350
|
+
uv pip install --system papermill ipykernel
|
|
351
|
+
uv run python -m ipykernel install --user --name cotlab --display-name "CoTLab"
|
|
341
352
|
|
|
342
353
|
- name: Check Hugging Face token
|
|
343
354
|
run: |
|
|
@@ -348,17 +359,17 @@ jobs:
|
|
|
348
359
|
|
|
349
360
|
- name: Authenticate Hugging Face
|
|
350
361
|
run: |
|
|
351
|
-
python - <<'PY'
|
|
362
|
+
uv run python - <<'PY'
|
|
352
363
|
import os
|
|
353
364
|
from huggingface_hub import login
|
|
354
365
|
|
|
355
366
|
login(token=os.environ["HF_TOKEN"], add_to_git_credential=False)
|
|
356
|
-
print("Authenticated to Hugging
|
|
367
|
+
print("Authenticated to Hugging Hub.")
|
|
357
368
|
PY
|
|
358
369
|
|
|
359
370
|
- name: Execute tutorial notebook
|
|
360
371
|
run: |
|
|
361
372
|
echo "Running notebook: cotlab_tutorial.ipynb (CI mode: 5 samples, 64 max_tokens)"
|
|
362
373
|
cd notebooks
|
|
363
|
-
papermill cotlab_tutorial.ipynb cotlab_tutorial_executed.ipynb -k cotlab --progress-bar -p NUM_SAMPLES 5 -p MAX_TOKENS 64
|
|
374
|
+
uv run papermill cotlab_tutorial.ipynb cotlab_tutorial_executed.ipynb -k cotlab --progress-bar -p NUM_SAMPLES 5 -p MAX_TOKENS 64
|
|
364
375
|
echo "Notebook executed successfully on ${{ matrix.os }}!"
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
FROM nvidia/cuda:12.8.0-runtime-ubuntu24.04
|
|
2
|
+
|
|
3
|
+
ENV DEBIAN_FRONTEND=noninteractive \
|
|
4
|
+
PYTHONUNBUFFERED=1 \
|
|
5
|
+
PATH="/opt/venv/bin:$PATH"
|
|
6
|
+
|
|
7
|
+
COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
|
|
8
|
+
|
|
9
|
+
RUN apt-get update && \
|
|
10
|
+
apt-get install -y --no-install-recommends python3 python3-venv && \
|
|
11
|
+
rm -rf /var/lib/apt/lists/*
|
|
12
|
+
|
|
13
|
+
RUN uv venv /opt/venv
|
|
14
|
+
|
|
15
|
+
# Install torch with CUDA 12 support explicitly to avoid version detection bugs
|
|
16
|
+
RUN uv pip install torch==2.6.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/cu126
|
|
17
|
+
|
|
18
|
+
WORKDIR /app
|
|
19
|
+
COPY . .
|
|
20
|
+
|
|
21
|
+
# Install local package
|
|
22
|
+
RUN uv pip install .
|
|
23
|
+
|
|
24
|
+
RUN mkdir /data /results
|
|
25
|
+
WORKDIR /workspace
|
|
26
|
+
|
|
27
|
+
ENTRYPOINT ["cotlab"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cotlab
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.2
|
|
4
4
|
Summary: CoTLab - Chain of Thought Research Toolkit for LLMs
|
|
5
5
|
Project-URL: Repository, https://github.com/huseyincavusbi/CoTLab
|
|
6
6
|
Project-URL: Documentation, https://huseyincavusbi.github.io/CoTLab
|
|
@@ -21,6 +21,7 @@ Requires-Python: >=3.10
|
|
|
21
21
|
Requires-Dist: accelerate>=0.27.0
|
|
22
22
|
Requires-Dist: bitsandbytes>=0.46.1
|
|
23
23
|
Requires-Dist: click>=8.1.0
|
|
24
|
+
Requires-Dist: defusedxml>=0.7.1
|
|
24
25
|
Requires-Dist: huggingface-hub>=0.20.0
|
|
25
26
|
Requires-Dist: hydra-core>=1.3.0
|
|
26
27
|
Requires-Dist: numpy>=1.24.0
|
|
@@ -33,7 +34,7 @@ Requires-Dist: safetensors>=0.4.0
|
|
|
33
34
|
Requires-Dist: scikit-learn>=1.3.0
|
|
34
35
|
Requires-Dist: torch>=2.1.0
|
|
35
36
|
Requires-Dist: tqdm>=4.65.0
|
|
36
|
-
Requires-Dist: transformers<
|
|
37
|
+
Requires-Dist: transformers<6,>=4.56.0
|
|
37
38
|
Requires-Dist: xmltodict>=0.13.0
|
|
38
39
|
Provides-Extra: dev
|
|
39
40
|
Requires-Dist: ipython>=8.0.0; extra == 'dev'
|
|
@@ -43,6 +44,10 @@ Requires-Dist: pre-commit>=3.0.0; extra == 'dev'
|
|
|
43
44
|
Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
|
|
44
45
|
Requires-Dist: pytest>=7.0.0; extra == 'dev'
|
|
45
46
|
Requires-Dist: ruff>=0.4.4; extra == 'dev'
|
|
47
|
+
Provides-Extra: docs
|
|
48
|
+
Requires-Dist: mkdocs-material>=9.4.0; extra == 'docs'
|
|
49
|
+
Requires-Dist: mkdocs>=1.5.0; extra == 'docs'
|
|
50
|
+
Requires-Dist: mkdocstrings[python]>=0.23.0; extra == 'docs'
|
|
46
51
|
Provides-Extra: vllm
|
|
47
52
|
Requires-Dist: vllm>=0.6.0; extra == 'vllm'
|
|
48
53
|
Description-Content-Type: text/markdown
|
|
@@ -161,6 +166,18 @@ python -m cotlab.main \
|
|
|
161
166
|
experiment.top_k=10
|
|
162
167
|
```
|
|
163
168
|
|
|
169
|
+
## Acknowledgements
|
|
170
|
+
|
|
171
|
+
This research is conducted in collaboration with the
|
|
172
|
+
[Great Ormond Street Hospital DRIVE Unit](https://www.gosh.nhs.uk/our-research/drive-unit-for-digital-innovation/).
|
|
173
|
+
|
|
174
|
+
## Contributors
|
|
175
|
+
|
|
176
|
+
- **Huseyin Cavus** — Core Contributor
|
|
177
|
+
- **Dr. Pavithra Rajendran** — Machine Learning Lead, GOSH DRIVE
|
|
178
|
+
- **Sebin Sabu** — Senior AI Scientist, GOSH DRIVE
|
|
179
|
+
- **Jaskaran Singh Kawatra** — ML Engineer, GOSH DRIVE
|
|
180
|
+
|
|
164
181
|
## License
|
|
165
182
|
|
|
166
183
|
MIT
|
|
@@ -112,6 +112,18 @@ python -m cotlab.main \
|
|
|
112
112
|
experiment.top_k=10
|
|
113
113
|
```
|
|
114
114
|
|
|
115
|
+
## Acknowledgements
|
|
116
|
+
|
|
117
|
+
This research is conducted in collaboration with the
|
|
118
|
+
[Great Ormond Street Hospital DRIVE Unit](https://www.gosh.nhs.uk/our-research/drive-unit-for-digital-innovation/).
|
|
119
|
+
|
|
120
|
+
## Contributors
|
|
121
|
+
|
|
122
|
+
- **Huseyin Cavus** — Core Contributor
|
|
123
|
+
- **Dr. Pavithra Rajendran** — Machine Learning Lead, GOSH DRIVE
|
|
124
|
+
- **Sebin Sabu** — Senior AI Scientist, GOSH DRIVE
|
|
125
|
+
- **Jaskaran Singh Kawatra** — ML Engineer, GOSH DRIVE
|
|
126
|
+
|
|
115
127
|
## License
|
|
116
128
|
|
|
117
129
|
MIT
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Confabulation Analysis — Test if H-Neurons encode confident errors vs uncertainty
|
|
2
|
+
_target_: cotlab.experiments.ConfabulationAnalysisExperiment
|
|
3
|
+
|
|
4
|
+
name: confabulation_analysis
|
|
5
|
+
description: "Test if H-Neurons encode confabulation vs uncertainty"
|
|
6
|
+
|
|
7
|
+
probe_path: sweep_C_1.json
|
|
8
|
+
ood_dataset_path: CoTLab/data/movie_ood.jsonl
|
|
9
|
+
num_samples: 50
|
|
10
|
+
conf_high: 13.0
|
|
11
|
+
conf_low: 10.0
|
|
12
|
+
seed: 42
|
|
13
|
+
max_input_tokens: 1024
|
|
14
|
+
answer_cue: "\n\nAnswer:"
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Entropy Neuron Overlap — Test overlap between H-Neurons and entropy neurons
|
|
2
|
+
_target_: cotlab.experiments.EntropyNeuronOverlapExperiment
|
|
3
|
+
|
|
4
|
+
name: entropy_neuron_overlap
|
|
5
|
+
description: "Test overlap between H-Neurons and entropy neurons"
|
|
6
|
+
|
|
7
|
+
probe_path: sweep_C_1.json
|
|
8
|
+
percentile: 99.0
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# SAE Feature to H-Neuron Overlap — Test if SAE features overlap with H-Neurons
|
|
2
|
+
_target_: cotlab.experiments.SAEFeatureNeuronOverlapExperiment
|
|
3
|
+
|
|
4
|
+
name: sae_feature_neuron_overlap
|
|
5
|
+
description: "Test if SAE features overlap with H-Neurons"
|
|
6
|
+
|
|
7
|
+
sae_repo_id: "google/gemma-scope-2-1b-it"
|
|
8
|
+
sae_layer: 9
|
|
9
|
+
sae_feature_id: 1000
|
|
10
|
+
sae_width: "16k"
|
|
11
|
+
probe_path: sweep_C_1.json
|
|
12
|
+
top_k_neurons: 50
|
|
@@ -26,7 +26,7 @@ classifiers = [
|
|
|
26
26
|
dependencies = [
|
|
27
27
|
"click>=8.1.0",
|
|
28
28
|
"torch>=2.1.0",
|
|
29
|
-
"transformers>=4.56.0,<
|
|
29
|
+
"transformers>=4.56.0,<6",
|
|
30
30
|
"hydra-core>=1.3.0",
|
|
31
31
|
"omegaconf>=2.3.0",
|
|
32
32
|
"accelerate>=0.27.0",
|
|
@@ -37,6 +37,7 @@ dependencies = [
|
|
|
37
37
|
"python-dotenv>=1.0.0",
|
|
38
38
|
"scikit-learn>=1.3.0",
|
|
39
39
|
"xmltodict>=0.13.0",
|
|
40
|
+
"defusedxml>=0.7.1",
|
|
40
41
|
"bitsandbytes>=0.46.1",
|
|
41
42
|
"pyarrow>=14.0.0",
|
|
42
43
|
"pandas>=2.0.0",
|
|
@@ -59,6 +60,11 @@ dev = [
|
|
|
59
60
|
"jupyter>=1.0.0",
|
|
60
61
|
"pre-commit>=3.0.0",
|
|
61
62
|
]
|
|
63
|
+
docs = [
|
|
64
|
+
"mkdocs>=1.5.0",
|
|
65
|
+
"mkdocs-material>=9.4.0",
|
|
66
|
+
"mkdocstrings[python]>=0.23.0",
|
|
67
|
+
]
|
|
62
68
|
|
|
63
69
|
[project.urls]
|
|
64
70
|
Repository = "https://github.com/huseyincavusbi/CoTLab"
|
|
@@ -94,3 +100,10 @@ check_untyped_defs = false
|
|
|
94
100
|
warn_return_any = false
|
|
95
101
|
warn_unused_ignores = false
|
|
96
102
|
no_implicit_optional = false
|
|
103
|
+
|
|
104
|
+
[tool.uv]
|
|
105
|
+
environments = [
|
|
106
|
+
"sys_platform == 'darwin' and platform_machine == 'arm64'",
|
|
107
|
+
"sys_platform == 'linux' and platform_machine == 'x86_64'",
|
|
108
|
+
"sys_platform == 'linux' and platform_machine == 'aarch64'"
|
|
109
|
+
]
|
|
@@ -38,7 +38,7 @@ class TransformersBackend(InferenceBackend):
|
|
|
38
38
|
device: str = "cuda",
|
|
39
39
|
dtype: str = "bfloat16",
|
|
40
40
|
enable_hooks: bool = True,
|
|
41
|
-
trust_remote_code: bool =
|
|
41
|
+
trust_remote_code: bool = False,
|
|
42
42
|
**kwargs,
|
|
43
43
|
):
|
|
44
44
|
self._device_map = device # Used for model loading (supports "auto")
|
|
@@ -47,6 +47,15 @@ class TransformersBackend(InferenceBackend):
|
|
|
47
47
|
self.enable_hooks = enable_hooks
|
|
48
48
|
self.trust_remote_code = trust_remote_code
|
|
49
49
|
|
|
50
|
+
if self.trust_remote_code:
|
|
51
|
+
import warnings
|
|
52
|
+
|
|
53
|
+
warnings.warn(
|
|
54
|
+
"trust_remote_code is set to True. This will execute code downloaded from the "
|
|
55
|
+
"Hugging Face Hub. Ensure you trust the repository before proceeding!",
|
|
56
|
+
UserWarning,
|
|
57
|
+
)
|
|
58
|
+
|
|
50
59
|
self._model = None
|
|
51
60
|
self._tokenizer = None
|
|
52
61
|
self._model_name: Optional[str] = None
|
|
@@ -586,7 +586,7 @@ class StructuredOutputMixin:
|
|
|
586
586
|
except Exception:
|
|
587
587
|
# Fallback to ElementTree with recursive parsing
|
|
588
588
|
try:
|
|
589
|
-
import
|
|
589
|
+
import defusedxml.ElementTree as ET
|
|
590
590
|
|
|
591
591
|
def elem_to_dict(elem):
|
|
592
592
|
text = elem.text.strip() if elem.text else None
|