fdq 0.0.55.dev0__tar.gz → 0.0.56.dev7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/.github/workflows/build-package.yml +7 -7
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/.github/workflows/github-release.yml +8 -8
- fdq-0.0.56.dev7/.github/workflows/on_branches.yml +30 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/.github/workflows/on_merge.yml +2 -2
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/.github/workflows/on_pr.yml +11 -10
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/.github/workflows/publish-to-pypi.yml +5 -6
- fdq-0.0.56.dev7/.github/workflows/ruff.yml +64 -0
- fdq-0.0.55.dev0/.github/workflows/test.yml → fdq-0.0.56.dev7/.github/workflows/unittest.yml +11 -6
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/PKG-INFO +1 -3
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/experiment_templates/mnist/mnist_preparator.py +4 -12
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/experiment_templates/mnist/mnist_test.py +4 -12
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/experiment_templates/mnist/train_mnist.py +3 -11
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/experiment_templates/segment_pets/oxfordpet_preparator.py +2 -8
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/experiment_templates/segment_pets/train_oxpets.py +2 -7
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/fdq_submit.py +16 -48
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/pyproject.toml +12 -8
- fdq-0.0.56.dev7/src/fdq/__about__.py +3 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/src/fdq/dump.py +6 -18
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/src/fdq/experiment.py +40 -121
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/src/fdq/inference.py +8 -24
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/src/fdq/misc.py +12 -36
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/src/fdq/run_experiment.py +9 -29
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/src/fdq/tensorrt_engine.py +11 -31
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/src/fdq/testing.py +19 -58
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/src/fdq/transformers.py +17 -51
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/src/fdq/ui_functions.py +8 -24
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/src/networks/simpleNet.py +1 -4
- fdq-0.0.55.dev0/.github/workflows/on_branches.yml +0 -35
- fdq-0.0.55.dev0/src/fdq/__about__.py +0 -3
- fdq-0.0.55.dev0/temp/not_used/test-install.yml +0 -72
- fdq-0.0.55.dev0/temp/old_orig/publish.yml +0 -112
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/.bumpversion.cfg +0 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/.gitattributes +0 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/.github/workflows/on_push.yml +0 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/.gitignore +0 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/LICENSE +0 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/README.md +0 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/experiment_templates/mnist/dummy_crossentropy_loss.py +0 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/experiment_templates/mnist/mnist_class_dense.json +0 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/experiment_templates/mnist/mnist_parent.json +0 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/experiment_templates/segment_pets/oxpets_test.py +0 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/experiment_templates/segment_pets/segment_pets.json +0 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/experiment_templates/segment_pets/segment_pets_distributed.json +0 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/experiment_templates/segment_pets/segment_pets_distributed_v4.json +0 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/src/fdq/__init__.py +0 -0
- {fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/submit.sh +0 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
# Build the
|
|
2
|
-
#
|
|
1
|
+
# Build the Python package for distribution
|
|
2
|
+
# based on https://github.com/CAIIVS/chuchichaestli
|
|
3
3
|
# https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
|
|
4
4
|
|
|
5
|
-
name: Build distribution
|
|
5
|
+
name: Build distribution
|
|
6
6
|
|
|
7
7
|
on:
|
|
8
8
|
workflow_call:
|
|
@@ -21,20 +21,20 @@ jobs:
|
|
|
21
21
|
steps:
|
|
22
22
|
- uses: actions/checkout@v4
|
|
23
23
|
|
|
24
|
-
- name: Set up
|
|
24
|
+
- name: Set up 3.x
|
|
25
25
|
uses: actions/setup-python@v5
|
|
26
26
|
with:
|
|
27
27
|
python-version: "3.x"
|
|
28
28
|
|
|
29
|
-
- name: Install
|
|
29
|
+
- name: Install
|
|
30
30
|
run: |
|
|
31
31
|
python -m pip install build --user
|
|
32
32
|
|
|
33
|
-
- name: Build
|
|
33
|
+
- name: Build distribution
|
|
34
34
|
run: |
|
|
35
35
|
python -m build
|
|
36
36
|
|
|
37
|
-
- name:
|
|
37
|
+
- name: Build artifacts
|
|
38
38
|
uses: actions/upload-artifact@v4
|
|
39
39
|
with:
|
|
40
40
|
name: ${{ inputs.artifact-name }}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
#
|
|
2
|
-
#
|
|
1
|
+
# Uploads a GitHub release from stored artifact dists signed with Sigstore
|
|
2
|
+
# based on https://github.com/CAIIVS/chuchichaestli
|
|
3
3
|
# https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
|
|
4
4
|
|
|
5
|
-
name: Upload GitHub release
|
|
5
|
+
name: Upload GitHub release
|
|
6
6
|
|
|
7
7
|
on:
|
|
8
8
|
workflow_call:
|
|
@@ -27,20 +27,20 @@ jobs:
|
|
|
27
27
|
id-token: write # IMPORTANT: mandatory for sigstore
|
|
28
28
|
|
|
29
29
|
steps:
|
|
30
|
-
- name: Download
|
|
30
|
+
- name: Download
|
|
31
31
|
uses: actions/download-artifact@v4
|
|
32
32
|
with:
|
|
33
33
|
name: ${{ inputs.artifact-name }}
|
|
34
34
|
path: dist/
|
|
35
35
|
|
|
36
|
-
- name: Sign
|
|
36
|
+
- name: Sign with Sigstore
|
|
37
37
|
uses: sigstore/gh-action-sigstore-python@v3.0.0
|
|
38
38
|
with:
|
|
39
39
|
inputs: >-
|
|
40
40
|
./dist/*.tar.gz
|
|
41
41
|
./dist/*.whl
|
|
42
42
|
|
|
43
|
-
- name: Fetch release tag
|
|
43
|
+
- name: Fetch release tag
|
|
44
44
|
env:
|
|
45
45
|
GH_TOKEN: ${{ secrets.READ_WRITE_PAT }}
|
|
46
46
|
run: >-
|
|
@@ -48,7 +48,7 @@ jobs:
|
|
|
48
48
|
&& echo "RELEASE_TAG=${{ inputs.tag }}" >> $GITHUB_ENV
|
|
49
49
|
|| echo "RELEASE_TAG=$(gh api repos/${{ github.repository }}/tags --jq '.[0].name')" >> $GITHUB_ENV
|
|
50
50
|
|
|
51
|
-
- name: Create GitHub Release
|
|
51
|
+
- name: Create GitHub Release
|
|
52
52
|
env:
|
|
53
53
|
GH_TOKEN: ${{ secrets.READ_WRITE_PAT }}
|
|
54
54
|
run: >-
|
|
@@ -56,7 +56,7 @@ jobs:
|
|
|
56
56
|
--repo '${{ github.repository }}'
|
|
57
57
|
--notes ""
|
|
58
58
|
|
|
59
|
-
- name:
|
|
59
|
+
- name: GitHub Release
|
|
60
60
|
env:
|
|
61
61
|
GH_TOKEN: ${{ secrets.READ_WRITE_PAT }}
|
|
62
62
|
run: >-
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Triggers when commits are pushed to a non-default branch, but aren't in a PR
|
|
2
|
+
|
|
3
|
+
name: Branch workflow
|
|
4
|
+
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
branches-ignore: [main]
|
|
8
|
+
paths-ignore: ["**/__about__.py"]
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
in-pr-check:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: insurgent-lab/is-in-pr-action@v0.2.0
|
|
15
|
+
id: isInPR
|
|
16
|
+
outputs:
|
|
17
|
+
result: ${{ steps.isInPR.outputs.result }}
|
|
18
|
+
|
|
19
|
+
run-unittests:
|
|
20
|
+
name: Run Unittests
|
|
21
|
+
needs: in-pr-check
|
|
22
|
+
if: ${{ needs.in-pr-check.outputs.result == 'false' }}
|
|
23
|
+
uses: ./.github/workflows/unittest.yml
|
|
24
|
+
|
|
25
|
+
build-package:
|
|
26
|
+
name: Build distribution 📦
|
|
27
|
+
needs: run-unittests
|
|
28
|
+
uses: ./.github/workflows/build-package.yml
|
|
29
|
+
with:
|
|
30
|
+
artifact-name: "package-dist"
|
|
@@ -9,7 +9,7 @@ on:
|
|
|
9
9
|
|
|
10
10
|
jobs:
|
|
11
11
|
version-bump-on-merge:
|
|
12
|
-
name: Version bump
|
|
12
|
+
name: Version bump (micro)
|
|
13
13
|
if: ${{ github.event.pull_request.merged == true }}
|
|
14
14
|
permissions:
|
|
15
15
|
contents: write
|
|
@@ -20,7 +20,7 @@ jobs:
|
|
|
20
20
|
ref: ${{ github.base_ref }}
|
|
21
21
|
fetch-depth: 0
|
|
22
22
|
token: ${{ secrets.READ_WRITE_PAT }}
|
|
23
|
-
- name: Hatch version bump
|
|
23
|
+
- name: Hatch version bump
|
|
24
24
|
uses: phdenzel/hatch-bump@v2
|
|
25
25
|
with:
|
|
26
26
|
type: "micro"
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Triggers when commits are pushed to an unmerged PR targeting main
|
|
2
2
|
|
|
3
3
|
name: PR workflow
|
|
4
4
|
|
|
@@ -9,18 +9,19 @@ on:
|
|
|
9
9
|
paths-ignore: ["**/__about__.py"]
|
|
10
10
|
|
|
11
11
|
jobs:
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
12
|
+
|
|
13
|
+
run-ruff:
|
|
14
|
+
name: Run Ruff
|
|
15
|
+
uses: ./.github/workflows/ruff.yml
|
|
16
|
+
|
|
17
|
+
run-unittests:
|
|
18
|
+
name: Run Unittests
|
|
19
|
+
needs: run-ruff
|
|
20
|
+
uses: ./.github/workflows/unittest.yml
|
|
20
21
|
|
|
21
22
|
build-package:
|
|
22
23
|
name: Build distribution 📦
|
|
23
|
-
|
|
24
|
+
needs: run-unittests
|
|
24
25
|
uses: ./.github/workflows/build-package.yml
|
|
25
26
|
with:
|
|
26
27
|
artifact-name: "package-dist"
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
#
|
|
2
|
-
#
|
|
1
|
+
# Publishes the Python package distribution to PyPI or TestPyPI
|
|
2
|
+
# based on https://github.com/CAIIVS/chuchichaestli
|
|
3
3
|
# https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
|
|
4
|
-
# Note:
|
|
5
4
|
# This workflow requires a token from PyPI/TestPyPI. It should be stored as 'pypi_token' secret.
|
|
6
5
|
|
|
7
|
-
name: Publish
|
|
6
|
+
name: Publish to PyPI
|
|
8
7
|
|
|
9
8
|
on:
|
|
10
9
|
workflow_call:
|
|
@@ -38,13 +37,13 @@ jobs:
|
|
|
38
37
|
# id-token: write
|
|
39
38
|
|
|
40
39
|
steps:
|
|
41
|
-
- name: Download
|
|
40
|
+
- name: Download distribution artifact
|
|
42
41
|
uses: actions/download-artifact@v4
|
|
43
42
|
with:
|
|
44
43
|
name: ${{ inputs.artifact-name }}
|
|
45
44
|
path: dist/
|
|
46
45
|
|
|
47
|
-
- name: Publish
|
|
46
|
+
- name: Publish to ${{ inputs.test && 'TestPyPI' || 'PyPI' }}
|
|
48
47
|
uses: pypa/gh-action-pypi-publish@release/v1
|
|
49
48
|
with:
|
|
50
49
|
repository-url: ${{ inputs.test && 'https://test.pypi.org/legacy/' || 'https://upload.pypi.org/legacy/' }}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
name: Run Ruff
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_call:
|
|
5
|
+
inputs:
|
|
6
|
+
versions:
|
|
7
|
+
description: "Test the installation on these Python versions"
|
|
8
|
+
type: string
|
|
9
|
+
# default: '["3.10", "3.11", "3.12"]'
|
|
10
|
+
default: '["3.12"]'
|
|
11
|
+
required: false
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
run-ruff:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
|
|
17
|
+
strategy:
|
|
18
|
+
matrix:
|
|
19
|
+
python-version: ${{ fromJSON(inputs.versions) }}
|
|
20
|
+
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@v4
|
|
23
|
+
|
|
24
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
25
|
+
uses: actions/setup-python@v5
|
|
26
|
+
with:
|
|
27
|
+
python-version: ${{ matrix.python-version }}
|
|
28
|
+
|
|
29
|
+
- name: Cache pip dependencies
|
|
30
|
+
uses: actions/cache@v4
|
|
31
|
+
with:
|
|
32
|
+
path: ~/.cache/pip
|
|
33
|
+
key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }}
|
|
34
|
+
restore-keys: |
|
|
35
|
+
${{ runner.os }}-pip-
|
|
36
|
+
|
|
37
|
+
# - name: Install system dependencies
|
|
38
|
+
# run: |
|
|
39
|
+
# sudo apt-get update
|
|
40
|
+
# sudo apt-get install -y gnuplot
|
|
41
|
+
|
|
42
|
+
# - name: Install Python dependencies
|
|
43
|
+
# run: |
|
|
44
|
+
# python -m pip install --upgrade pip
|
|
45
|
+
# python -m pip install .[dev]
|
|
46
|
+
|
|
47
|
+
- name: Install Ruff
|
|
48
|
+
run: |
|
|
49
|
+
python -m pip install --upgrade pip
|
|
50
|
+
pip install ruff==0.11.8
|
|
51
|
+
|
|
52
|
+
- name: Check formatting with ruff
|
|
53
|
+
run: |
|
|
54
|
+
ruff format --check --diff .
|
|
55
|
+
# manually format: $ ruff format .
|
|
56
|
+
|
|
57
|
+
- name: Lint with ruff
|
|
58
|
+
run: |
|
|
59
|
+
ruff check . --output-format=github --exit-zero
|
|
60
|
+
# auto fix: $ ruff check . --fix
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
|
|
@@ -1,17 +1,22 @@
|
|
|
1
|
-
name: Run
|
|
1
|
+
name: Run Unittests
|
|
2
2
|
|
|
3
3
|
on:
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
workflow_call:
|
|
5
|
+
inputs:
|
|
6
|
+
versions:
|
|
7
|
+
description: "Test the installation on these Python versions"
|
|
8
|
+
type: string
|
|
9
|
+
# default: '["3.10", "3.11", "3.12"]'
|
|
10
|
+
default: '["3.12"]'
|
|
11
|
+
required: false
|
|
7
12
|
|
|
8
13
|
jobs:
|
|
9
|
-
|
|
14
|
+
run-unittests:
|
|
10
15
|
runs-on: ubuntu-latest
|
|
11
16
|
|
|
12
17
|
strategy:
|
|
13
18
|
matrix:
|
|
14
|
-
python-version:
|
|
19
|
+
python-version: ${{ fromJSON(inputs.versions) }}
|
|
15
20
|
|
|
16
21
|
steps:
|
|
17
22
|
- uses: actions/checkout@v4
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fdq
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.56.dev7
|
|
4
4
|
Summary: ML runtime (https://pypi.org/project/fdq/)
|
|
5
5
|
Project-URL: Homepage, https://github.com/mstadelmann/fonduecaquelon
|
|
6
6
|
Project-URL: Repository, https://github.com/mstadelmann/fonduecaquelon.git
|
|
@@ -706,8 +706,6 @@ Requires-Dist: torchvision==0.22.0
|
|
|
706
706
|
Requires-Dist: tpl==0.10
|
|
707
707
|
Requires-Dist: wandb==0.19.10
|
|
708
708
|
Provides-Extra: dev
|
|
709
|
-
Requires-Dist: pre-commit==3.7.0; extra == 'dev'
|
|
710
|
-
Requires-Dist: pytest==8.3.5; extra == 'dev'
|
|
711
709
|
Requires-Dist: ruff==0.11.8; extra == 'dev'
|
|
712
710
|
Description-Content-Type: text/markdown
|
|
713
711
|
|
|
@@ -26,9 +26,7 @@ def create_datasets(experiment, args):
|
|
|
26
26
|
|
|
27
27
|
transform = experiment.transformers["resize_norm_inp"]
|
|
28
28
|
|
|
29
|
-
train_all_set = datasets.MNIST(
|
|
30
|
-
args.base_path, train=True, download=True, transform=transform
|
|
31
|
-
)
|
|
29
|
+
train_all_set = datasets.MNIST(args.base_path, train=True, download=True, transform=transform)
|
|
32
30
|
test_set = datasets.MNIST(args.base_path, train=False, transform=transform)
|
|
33
31
|
|
|
34
32
|
n_train_all = len(train_all_set)
|
|
@@ -38,17 +36,13 @@ def create_datasets(experiment, args):
|
|
|
38
36
|
subset_ratio = args.get("subset_train", 1)
|
|
39
37
|
if subset_ratio < 1:
|
|
40
38
|
n_subset_samples = int(n_train_all * subset_ratio)
|
|
41
|
-
train_all_set, _ = random_split(
|
|
42
|
-
train_all_set, [n_subset_samples, n_train_all - n_subset_samples]
|
|
43
|
-
)
|
|
39
|
+
train_all_set, _ = random_split(train_all_set, [n_subset_samples, n_train_all - n_subset_samples])
|
|
44
40
|
n_train_all = len(train_all_set)
|
|
45
41
|
|
|
46
42
|
subset_ratio = args.get("subset_test", 1)
|
|
47
43
|
if subset_ratio < 1:
|
|
48
44
|
n_subset_samples = int(n_test_samples * subset_ratio)
|
|
49
|
-
test_set, _ = random_split(
|
|
50
|
-
test_set, [n_subset_samples, n_test_samples - n_subset_samples]
|
|
51
|
-
)
|
|
45
|
+
test_set, _ = random_split(test_set, [n_subset_samples, n_test_samples - n_subset_samples])
|
|
52
46
|
n_test_samples = len(test_set)
|
|
53
47
|
|
|
54
48
|
# val set = subset from train
|
|
@@ -56,9 +50,7 @@ def create_datasets(experiment, args):
|
|
|
56
50
|
if val_ratio is not None and val_ratio > 0:
|
|
57
51
|
n_val_samples = int(n_train_all * val_ratio)
|
|
58
52
|
n_train_samples = n_train_all - n_val_samples
|
|
59
|
-
train_set, val_set = random_split(
|
|
60
|
-
train_all_set, [n_train_samples, n_val_samples]
|
|
61
|
-
)
|
|
53
|
+
train_set, val_set = random_split(train_all_set, [n_train_samples, n_val_samples])
|
|
62
54
|
else:
|
|
63
55
|
n_val_samples = 0
|
|
64
56
|
n_train_samples = n_train_all
|
|
@@ -33,9 +33,7 @@ def interactive_test(experiment, test_loader):
|
|
|
33
33
|
Returns:
|
|
34
34
|
accuracy (float or None): The computed accuracy of the model, or None if no samples were tested.
|
|
35
35
|
"""
|
|
36
|
-
max_samples_to_print = getIntInput(
|
|
37
|
-
"How many test batches do you want to check?\n", drange=[1, 1000]
|
|
38
|
-
)
|
|
36
|
+
max_samples_to_print = getIntInput("How many test batches do you want to check?\n", drange=[1, 1000])
|
|
39
37
|
labels_gt = []
|
|
40
38
|
labels_pred = []
|
|
41
39
|
total_accuracy = []
|
|
@@ -132,9 +130,7 @@ def fdq_test(experiment):
|
|
|
132
130
|
accuracy = None
|
|
133
131
|
|
|
134
132
|
if experiment.exp_def.data.MNIST.args.test_batch_size != 1:
|
|
135
|
-
raise ValueError(
|
|
136
|
-
"Error: Test batch size must be 1 for this experiment. Please change the experiment file."
|
|
137
|
-
)
|
|
133
|
+
raise ValueError("Error: Test batch size must be 1 for this experiment. Please change the experiment file.")
|
|
138
134
|
|
|
139
135
|
if experiment.mode.op_mode.unittest or experiment.inargs.test_model_auto:
|
|
140
136
|
# no interactive for test experiments
|
|
@@ -155,9 +151,7 @@ def fdq_test(experiment):
|
|
|
155
151
|
|
|
156
152
|
if tmode == 3:
|
|
157
153
|
in_scalar = getIntInput("Int input value?", drange=[0, 5000])
|
|
158
|
-
in_tensor = in_scalar * torch.ones(
|
|
159
|
-
(1, 3, experiment.net_input_size[0], experiment.net_input_size[1])
|
|
160
|
-
)
|
|
154
|
+
in_tensor = in_scalar * torch.ones((1, 3, experiment.net_input_size[0], experiment.net_input_size[1]))
|
|
161
155
|
|
|
162
156
|
print(in_tensor.shape)
|
|
163
157
|
|
|
@@ -165,8 +159,6 @@ def fdq_test(experiment):
|
|
|
165
159
|
pred_sm = F.softmax(pred, dim=1)
|
|
166
160
|
pred_am = pred_sm.argmax()
|
|
167
161
|
|
|
168
|
-
print(
|
|
169
|
-
f"Prediction: {pred.tolist()} \nSoftmax: {pred_sm.tolist()} \nArgmax: {pred_am.tolist()}"
|
|
170
|
-
)
|
|
162
|
+
print(f"Prediction: {pred.tolist()} \nSoftmax: {pred_sm.tolist()} \nArgmax: {pred_am.tolist()}")
|
|
171
163
|
|
|
172
164
|
return accuracy
|
|
@@ -36,18 +36,13 @@ def fdq_train(experiment: fdqExperiment) -> None:
|
|
|
36
36
|
|
|
37
37
|
with torch.autocast(device_type=device_type, enabled=experiment.useAMP):
|
|
38
38
|
output = model(inputs)
|
|
39
|
-
loss_tensor = (
|
|
40
|
-
experiment.losses["cross_ent"](output, targets)
|
|
41
|
-
/ experiment.gradacc_iter
|
|
42
|
-
)
|
|
39
|
+
loss_tensor = experiment.losses["cross_ent"](output, targets) / experiment.gradacc_iter
|
|
43
40
|
if experiment.useAMP and experiment.scaler is not None:
|
|
44
41
|
experiment.scaler.scale(loss_tensor).backward()
|
|
45
42
|
else:
|
|
46
43
|
loss_tensor.backward()
|
|
47
44
|
|
|
48
|
-
experiment.update_gradients(
|
|
49
|
-
b_idx=nb_batch, loader_name="MNIST", model_name="simpleNet"
|
|
50
|
-
)
|
|
45
|
+
experiment.update_gradients(b_idx=nb_batch, loader_name="MNIST", model_name="simpleNet")
|
|
51
46
|
|
|
52
47
|
train_loss_sum += loss_tensor.detach().item()
|
|
53
48
|
|
|
@@ -91,10 +86,7 @@ def fdq_train(experiment: fdqExperiment) -> None:
|
|
|
91
86
|
"dataformats": "CHW",
|
|
92
87
|
}
|
|
93
88
|
|
|
94
|
-
captions = [
|
|
95
|
-
f"Predicted: {preds[idx].item()}, True: {targets[idx].item()}"
|
|
96
|
-
for idx in range(len(preds))
|
|
97
|
-
]
|
|
89
|
+
captions = [f"Predicted: {preds[idx].item()}, True: {targets[idx].item()}" for idx in range(len(preds))]
|
|
98
90
|
imgs_wandb = {
|
|
99
91
|
"name": "inputs",
|
|
100
92
|
"data": inputs[:max_log_size],
|
{fdq-0.0.55.dev0 → fdq-0.0.56.dev7}/experiment_templates/segment_pets/oxfordpet_preparator.py
RENAMED
|
@@ -65,18 +65,12 @@ class OxfordPetDataset(torch.utils.data.Dataset):
|
|
|
65
65
|
|
|
66
66
|
if self.binary:
|
|
67
67
|
mask = torch.where(mask == 2.0, torch.tensor(0.0, dtype=mask.dtype), mask)
|
|
68
|
-
mask = torch.where(
|
|
69
|
-
(mask == 1.0) | (mask == 3.0), torch.tensor(1.0, dtype=mask.dtype), mask
|
|
70
|
-
)
|
|
68
|
+
mask = torch.where((mask == 1.0) | (mask == 3.0), torch.tensor(1.0, dtype=mask.dtype), mask)
|
|
71
69
|
# add channel dimension
|
|
72
70
|
mask = mask.unsqueeze(0)
|
|
73
71
|
else:
|
|
74
72
|
# one hot encoding
|
|
75
|
-
mask = (
|
|
76
|
-
torch.nn.functional.one_hot((mask - 1).long(), num_classes=3)
|
|
77
|
-
.permute(2, 0, 1)
|
|
78
|
-
.float()
|
|
79
|
-
)
|
|
73
|
+
mask = torch.nn.functional.one_hot((mask - 1).long(), num_classes=3).permute(2, 0, 1).float()
|
|
80
74
|
|
|
81
75
|
if self.transform_img is not None:
|
|
82
76
|
image = self.transform_img(image)
|
|
@@ -39,18 +39,13 @@ def fdq_train(experiment: fdqExperiment) -> None:
|
|
|
39
39
|
|
|
40
40
|
with torch.autocast(device_type=device_type, enabled=experiment.useAMP):
|
|
41
41
|
output = model(inputs)
|
|
42
|
-
train_loss_tensor = (
|
|
43
|
-
experiment.losses["cross_ent"](output, targets)
|
|
44
|
-
/ experiment.gradacc_iter
|
|
45
|
-
)
|
|
42
|
+
train_loss_tensor = experiment.losses["cross_ent"](output, targets) / experiment.gradacc_iter
|
|
46
43
|
if experiment.useAMP and experiment.scaler is not None:
|
|
47
44
|
experiment.scaler.scale(train_loss_tensor).backward()
|
|
48
45
|
else:
|
|
49
46
|
train_loss_tensor.backward()
|
|
50
47
|
|
|
51
|
-
experiment.update_gradients(
|
|
52
|
-
b_idx=nb_tbatch, loader_name="OXPET", model_name="ccUNET"
|
|
53
|
-
)
|
|
48
|
+
experiment.update_gradients(b_idx=nb_tbatch, loader_name="OXPET", model_name="ccUNET")
|
|
54
49
|
|
|
55
50
|
train_loss_sum += train_loss_tensor.detach().item()
|
|
56
51
|
|
|
@@ -383,14 +383,10 @@ def load_json(path: str) -> dict:
|
|
|
383
383
|
raise FDQSubmitError(f"Cannot read file {path}: {exc}") from exc
|
|
384
384
|
|
|
385
385
|
if not isinstance(data, dict):
|
|
386
|
-
raise FDQSubmitError(
|
|
387
|
-
f"JSON file {path} must contain a dictionary at root level"
|
|
388
|
-
)
|
|
386
|
+
raise FDQSubmitError(f"JSON file {path} must contain a dictionary at root level")
|
|
389
387
|
|
|
390
388
|
if data.get("globals") is None:
|
|
391
|
-
raise FDQSubmitError(
|
|
392
|
-
f"Experiment config {path} missing required 'globals' section"
|
|
393
|
-
)
|
|
389
|
+
raise FDQSubmitError(f"Experiment config {path} missing required 'globals' section")
|
|
394
390
|
|
|
395
391
|
return data
|
|
396
392
|
|
|
@@ -451,9 +447,7 @@ def load_conf_file(path: str) -> "DictToObj":
|
|
|
451
447
|
# Check for circular references
|
|
452
448
|
abs_parent_path = os.path.abspath(parent_path)
|
|
453
449
|
if abs_parent_path in processed_files:
|
|
454
|
-
raise FDQSubmitError(
|
|
455
|
-
f"Circular reference detected in parent hierarchy: {parent_path}"
|
|
456
|
-
)
|
|
450
|
+
raise FDQSubmitError(f"Circular reference detected in parent hierarchy: {parent_path}")
|
|
457
451
|
processed_files.add(abs_parent_path)
|
|
458
452
|
|
|
459
453
|
log_info(f"Loading parent configuration: {parent_path}")
|
|
@@ -463,18 +457,14 @@ def load_conf_file(path: str) -> "DictToObj":
|
|
|
463
457
|
parent = parent_conf.get("globals", {}).get("parent", {})
|
|
464
458
|
|
|
465
459
|
if depth >= max_depth:
|
|
466
|
-
raise FDQSubmitError(
|
|
467
|
-
f"Maximum parent hierarchy depth ({max_depth}) exceeded"
|
|
468
|
-
)
|
|
460
|
+
raise FDQSubmitError(f"Maximum parent hierarchy depth ({max_depth}) exceeded")
|
|
469
461
|
|
|
470
462
|
return DictToObj(conf)
|
|
471
463
|
|
|
472
464
|
except FDQSubmitError:
|
|
473
465
|
raise
|
|
474
466
|
except Exception as exc:
|
|
475
|
-
raise FDQSubmitError(
|
|
476
|
-
f"Failed to load configuration from {path}: {exc}"
|
|
477
|
-
) from exc
|
|
467
|
+
raise FDQSubmitError(f"Failed to load configuration from {path}: {exc}") from exc
|
|
478
468
|
|
|
479
469
|
|
|
480
470
|
class DictToObj:
|
|
@@ -614,8 +604,7 @@ def check_config(job_config: dict[str, Any]) -> dict[str, Any]:
|
|
|
614
604
|
|
|
615
605
|
if missing_fields:
|
|
616
606
|
raise FDQSubmitError(
|
|
617
|
-
f"Missing mandatory configuration fields: {', '.join(missing_fields)}. "
|
|
618
|
-
"Please update your config file!"
|
|
607
|
+
f"Missing mandatory configuration fields: {', '.join(missing_fields)}. Please update your config file!"
|
|
619
608
|
)
|
|
620
609
|
|
|
621
610
|
# Validate and normalize values
|
|
@@ -636,22 +625,16 @@ def check_config(job_config: dict[str, Any]) -> dict[str, Any]:
|
|
|
636
625
|
os.makedirs(job_config["results_path"], exist_ok=True)
|
|
637
626
|
log_info(f"Created results directory: {job_config['results_path']}")
|
|
638
627
|
except OSError as exc:
|
|
639
|
-
raise FDQSubmitError(
|
|
640
|
-
f"Cannot create results directory {job_config['results_path']}: {exc}"
|
|
641
|
-
) from exc
|
|
628
|
+
raise FDQSubmitError(f"Cannot create results directory {job_config['results_path']}: {exc}") from exc
|
|
642
629
|
|
|
643
630
|
# Validate resource specifications
|
|
644
631
|
if job_config.get("mem") and not re.match(r"^\d+[GMK]?$", str(job_config["mem"])):
|
|
645
|
-
log_warning(
|
|
646
|
-
f"Memory specification '{job_config['mem']}' may be invalid. Expected format: number + G/M/K"
|
|
647
|
-
)
|
|
632
|
+
log_warning(f"Memory specification '{job_config['mem']}' may be invalid. Expected format: number + G/M/K")
|
|
648
633
|
|
|
649
634
|
return job_config
|
|
650
635
|
|
|
651
636
|
|
|
652
|
-
def create_submit_file(
|
|
653
|
-
job_config: dict[str, Any], slurm_conf: Any, submit_path: str
|
|
654
|
-
) -> None:
|
|
637
|
+
def create_submit_file(job_config: dict[str, Any], slurm_conf: Any, submit_path: str) -> None:
|
|
655
638
|
"""Create a SLURM submit file from the job configuration.
|
|
656
639
|
|
|
657
640
|
Args:
|
|
@@ -686,20 +669,11 @@ def create_submit_file(
|
|
|
686
669
|
template_content = template_content.replace("#additional_pip_packages#", "")
|
|
687
670
|
elif isinstance(slurm_conf.additional_pip_packages, list):
|
|
688
671
|
if slurm_conf.additional_pip_packages: # Only if list is not empty
|
|
689
|
-
packages_cmd = "\n".join(
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
)
|
|
693
|
-
log_info(
|
|
694
|
-
f"Adding {len(slurm_conf.additional_pip_packages)} additional pip packages"
|
|
695
|
-
)
|
|
696
|
-
template_content = template_content.replace(
|
|
697
|
-
"#additional_pip_packages#", packages_cmd
|
|
698
|
-
)
|
|
672
|
+
packages_cmd = "\n".join(f"uv pip install '{pkg}'" for pkg in slurm_conf.additional_pip_packages)
|
|
673
|
+
log_info(f"Adding {len(slurm_conf.additional_pip_packages)} additional pip packages")
|
|
674
|
+
template_content = template_content.replace("#additional_pip_packages#", packages_cmd)
|
|
699
675
|
else:
|
|
700
|
-
template_content = template_content.replace(
|
|
701
|
-
"#additional_pip_packages#", ""
|
|
702
|
-
)
|
|
676
|
+
template_content = template_content.replace("#additional_pip_packages#", "")
|
|
703
677
|
else:
|
|
704
678
|
raise FDQSubmitError(
|
|
705
679
|
f"additional_pip_packages must be a list of strings, got {type(slurm_conf.additional_pip_packages)}"
|
|
@@ -772,9 +746,7 @@ def submit_slurm_job(submit_path: str) -> str:
|
|
|
772
746
|
)
|
|
773
747
|
|
|
774
748
|
if result.returncode != 0:
|
|
775
|
-
raise FDQSubmitError(
|
|
776
|
-
f"SLURM job submission failed (exit code {result.returncode}): {result.stderr}"
|
|
777
|
-
)
|
|
749
|
+
raise FDQSubmitError(f"SLURM job submission failed (exit code {result.returncode}): {result.stderr}")
|
|
778
750
|
|
|
779
751
|
# Extract job ID from output
|
|
780
752
|
match = re.search(r"Submitted batch job (\d+)", result.stdout)
|
|
@@ -790,9 +762,7 @@ def submit_slurm_job(submit_path: str) -> str:
|
|
|
790
762
|
log_info(f"Successfully submitted batch job {job_id}")
|
|
791
763
|
return job_id
|
|
792
764
|
else:
|
|
793
|
-
raise FDQSubmitError(
|
|
794
|
-
f"Could not extract job ID from SLURM output: {result.stdout}"
|
|
795
|
-
)
|
|
765
|
+
raise FDQSubmitError(f"Could not extract job ID from SLURM output: {result.stdout}")
|
|
796
766
|
|
|
797
767
|
except subprocess.TimeoutExpired as exc:
|
|
798
768
|
raise FDQSubmitError("SLURM submission timed out after 30 seconds") from exc
|
|
@@ -827,9 +797,7 @@ def main() -> None:
|
|
|
827
797
|
job_config["user"] = getpass.getuser()
|
|
828
798
|
|
|
829
799
|
# Validate required paths
|
|
830
|
-
if not hasattr(exp_config, "store") or not hasattr(
|
|
831
|
-
exp_config.store, "results_path"
|
|
832
|
-
):
|
|
800
|
+
if not hasattr(exp_config, "store") or not hasattr(exp_config.store, "results_path"):
|
|
833
801
|
raise FDQSubmitError("Configuration missing 'store.results_path' setting")
|
|
834
802
|
|
|
835
803
|
job_config["results_path"] = exp_config.store.results_path
|
|
@@ -35,18 +35,15 @@ dependencies = [
|
|
|
35
35
|
"opencv-python==4.11.0.86",
|
|
36
36
|
"ruff==0.11.8",
|
|
37
37
|
]
|
|
38
|
+
|
|
38
39
|
[project.optional-dependencies]
|
|
39
40
|
dev = [
|
|
40
41
|
"ruff==0.11.8",
|
|
41
|
-
"pre-commit==3.7.0",
|
|
42
|
-
"pytest==8.3.5",
|
|
43
42
|
]
|
|
44
43
|
|
|
45
44
|
[dependency-groups]
|
|
46
45
|
dev = [
|
|
47
46
|
"ruff==0.11.8",
|
|
48
|
-
"pre-commit==3.7.0",
|
|
49
|
-
"pytest==8.3.5",
|
|
50
47
|
]
|
|
51
48
|
|
|
52
49
|
[project.urls]
|
|
@@ -74,11 +71,18 @@ dependencies = ["pytest", "pytest-cov"]
|
|
|
74
71
|
[tool.pytest.ini_options]
|
|
75
72
|
addopts = ["--import-mode=importlib"]
|
|
76
73
|
|
|
74
|
+
[tool.ruff]
|
|
75
|
+
line-length = 120
|
|
76
|
+
|
|
77
77
|
[tool.ruff.lint]
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
78
|
+
select = ["E9", "F63", "F7", "F82", "UP", "D"]
|
|
79
|
+
# E9 = pycodestyle syntax errors
|
|
80
|
+
# F63 = pyflakes duplicate arguments
|
|
81
|
+
# F7 = pyflakes syntax errors in type annotations
|
|
82
|
+
# F82 = pyflakes undefined name errors
|
|
83
|
+
# UP = pyupgrade
|
|
84
|
+
# D = pydocstyle
|
|
85
|
+
|
|
82
86
|
ignore = ["D100","D102"]
|
|
83
87
|
|
|
84
88
|
[tool.ruff.lint.pydocstyle]
|