lightly-train 0.2.1__tar.gz → 0.2.2__tar.gz

lightly_train-0.2.2/.github/workflows/build_docker_image.yml

@@ -0,0 +1,80 @@
+name: Docker image build and push to gcloud registry
+
+# Pushes a release candidate build to lightly-main/train after a merge into main
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+env:
+  BUILD_NUMBER: ${{ github.run_number }}
+  NAMESPACE: lightly-main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-docker-image:
+    name: build docker image
+    runs-on: cpu
+    timeout-minutes: 60
+    outputs:
+      job_status: ${{ job.status }}
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v4
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+      - name: set up cloud sdk
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          version: "421.0.0" # fix version so that our runners don't download trash
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+      - name: build docker image and push to gcloud registry
+        run: |
+          cd ./docker && make main-deploy-amd64-cuda
+
+  slack-notification:
+    name: send slack notification
+    needs:
+      - build-docker-image
+    if: always()
+    runs-on: cpu
+    timeout-minutes: 5
+    steps:
+      - name: Slack Notification - Success
+        if: ${{ needs.build-docker-image.result == 'success' }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_RELEASES }}
+          SLACK_ICON_EMOJI: ":train:"
+          SLACK_USERNAME: "Build of LightlyTrain Docker Image"
+          SLACK_COLOR: "good"
+          SLACK_MESSAGE: |
+            :tada: Build status: :github-check-mark:
+            LightlyTrain docker image built successfully from main branch and pushed to private GCR.
+
+            Pull it from the registry with:
+            ```
+            docker pull eu.gcr.io/boris-250909/${{ env.NAMESPACE }}/train:${{ github.sha }}
+            ```
+
+            See it in the registry at:
+            https://console.cloud.google.com/artifacts/docker/boris-250909/europe/eu.gcr.io/${{ env.NAMESPACE }}%2Ftrain
+      - name: Slack Notification - Failure
+        if: ${{ needs.build-docker-image.result != 'success' }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_RELEASES }}
+          SLACK_ICON_EMOJI: ":x:"
+          SLACK_USERNAME: "Build of LightlyTrain Docker Image"
+          SLACK_COLOR: "danger"
+          SLACK_MESSAGE: |
+            :x: LightlyTrain docker image build failed.
+            Please check the logs and take necessary action.

lightly_train-0.2.2/.github/workflows/release_dockerhub.yml

@@ -0,0 +1,61 @@
+name: Docker Hub Release
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  dockerhub-push:
+    name: Push the latest main image to DockerHub
+    runs-on: cpu
+    defaults:
+      run:
+        working-directory: ./docker
+    steps:
+    - name: checkout code
+      uses: actions/checkout@v4
+      with:
+          ref: main
+    - name: push to DockerHub
+      id: push-to-dockerhub
+      run: |
+        echo "push up to docker hub"
+        docker login -u borislightly -p ${{ secrets.DOCKERHUB_PASSWORD_BORISLIGHTLY }}
+        make release
+  slack-notification:
+    name: send slack notification
+    needs:
+      - dockerhub-push
+    if: always()
+    runs-on: cpu
+    timeout-minutes: 5
+    steps:
+    - name: Slack Notification - Success
+      if: ${{ needs.dockerhub-push.result == 'success' }}
+      uses: rtCamp/action-slack-notify@v2
+      env:
+        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_RELEASES }}
+        SLACK_ICON_EMOJI: ":train:"
+        SLACK_USERNAME: "Release of LightlyTrain Docker Image to DockerHub"
+        SLACK_COLOR: "good"
+        SLACK_MESSAGE: |
+          :tada: Release status: :github-check-mark:
+          LightlyTrain Docker Image pushed to DockerHub successfully.
+
+          Pull it from the registry with:
+          ```
+          docker pull lighly/train:latest
+          ```
+
+          See it in DockerHub at https://hub.docker.com/repository/docker/lightly/train/general
+    - name: Slack Notification - Failure
+      if: ${{ needs.dockerhub-push.result != 'success' }}
+      uses: rtCamp/action-slack-notify@v2
+      env:
+        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_RELEASES }}
+        SLACK_ICON_EMOJI: ":x:"
+        SLACK_USERNAME: "Release of LightlyTrain Docker Image to DockerHub"
+        SLACK_COLOR: "danger"
+        SLACK_MESSAGE: |
+          :x: Push of LightlyTrain Docker Image to DockerHub failed.
+          Please check the logs and take necessary action.

{lightly_train-0.2.1 → lightly_train-0.2.2}/.github/workflows/release_pypi.yml

@@ -1,7 +1,6 @@
 name: Build and release LightlyTrain package to PyPI
 
 on:
-  push:
   workflow_dispatch:
 # TODO(Philipp, 03/23): Enable me after proper testing.
 #  release:
@@ -32,7 +31,7 @@ jobs:
           echo "Version mismatch: Tag is $LATEST_TAG, but __version__ in __init__.py is $VERSION_FROM_INIT"
           exit 1
         fi
-        echo "Version matches tag: $LATEST_TAG"
+        echo "__version__ from __init__.py $VERSION_FROM_INIT matches latest tag $LATEST_TAG"
     - name: Set up Python 3.10
       uses: actions/setup-python@v5
       with:
@@ -41,14 +40,13 @@ jobs:
       run: |
         make install-uv reset-venv
         source .venv/bin/activate
+        make install-dist
     - name: Build and release
       id: build_and_release
       run: |
-        uv pip install wheel
-        uv pip install twine
-        uv pip install build
+        source .venv/bin/activate
         # NOTE: This uses the files checked out from the latest tag.
-        make release-pypi
+        make dist
         twine upload -u ${{ secrets.PYPI_USER_NAME }} -p ${{ secrets.PYPI_PASSWORD }} dist/*
     - name: Convert success/failure strings to emojis
       id: emoji_status
@@ -64,7 +62,7 @@ jobs:
       if: always()
       uses: rtCamp/action-slack-notify@v2
       env:
-        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_TEST_CHANNEL }}
+        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_RELEASES }}
         SLACK_ICON_EMOJI: ":python:"
         SLACK_USERNAME: Release of LightlyTrain Package ${{steps.checkout_latest_release_tag.outputs.tag_name}}
         SLACK_COLOR: ${{ steps.build_and_release.outcome }}

lightly_train-0.2.2/.github/workflows/test.yml

@@ -0,0 +1,58 @@
+name: Unit Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  detect-code-changes:
+    name: Detect Code Changes
+    runs-on: cpu
+    outputs:
+      run-tests: ${{ steps.filter.outputs.run-tests }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          list-files: shell
+          filters: |
+            run-tests:
+              - '!docs/**'
+              - '!docker/**'
+              - '!.github/**'
+              - '.github/workflows/test.yml'
+  test:
+    name: Test
+    needs: detect-code-changes
+    if: needs.detect-code-changes.outputs.run-tests == 'true'
+    runs-on: cpu
+    strategy:
+      matrix:
+        python: ["3.8", "3.11"]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set Up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Set Up Environment
+        run: |
+          make install-uv reset-venv
+          source .venv/bin/activate
+          make install-pinned-${{ matrix.python }}
+          make install-ffmpeg-ubuntu
+      - name: Run Pytest
+        env:
+          CUDA_VISIBLE_DEVICES: ""
+        run: |
+          source .venv/bin/activate
+          make test-ci

lightly_train-0.2.2/.github/workflows/test_code_format.yml

@@ -0,0 +1,65 @@
+name: Code Format Check
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  detect-code-changes:
+    name: Detect Code Changes
+    runs-on: cpu
+    outputs:
+      run-tests: ${{ steps.filter.outputs.run-tests }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          list-files: shell
+          filters: |
+            run-tests:
+              - '!docs/**'
+              - '!docker/**'
+              - '!.github/**'
+              - '.github/workflows/test_code_format.yml'
+  test:
+    name: Check
+    runs-on: cpu
+    strategy:
+      matrix:
+        python: ["3.8"]
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v4
+    - name: Set Up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Set Up Environment
+      run: |
+        make install-uv reset-venv
+        source .venv/bin/activate
+        make install-pinned-${{ matrix.python }}
+        make install-ffmpeg-ubuntu
+    - name: Run Format Check
+      run: |
+        source .venv/bin/activate
+        make format-check
+    - name: Run Type Check
+      run: |
+        source .venv/bin/activate
+        make type-check
+    - name: Run Header check
+      run: |
+        source .venv/bin/activate
+        make add-header
+        git diff --name-only .
+        git diff --quiet --name-only . || exit -1
+  

lightly_train-0.2.2/.github/workflows/test_docker.yml

@@ -0,0 +1,56 @@
+name: Docker Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  detect-code-changes:
+    name: Detect Code Changes
+    runs-on: cpu
+    outputs:
+      run-tests: ${{ steps.filter.outputs.run-tests }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          list-files: shell
+          filters: |
+            run-tests:
+              - '!docs/**'
+              - '!.github/**'
+              - '.github/workflows/test_docker.yml'
+  test:
+    name: Test
+    needs: detect-code-changes
+    if: needs.detect-code-changes.outputs.run-tests == 'true'
+    runs-on: cpu
+    strategy:
+      matrix:
+        python: ["3.11"]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set Up Environment
+        run: |
+          make install-uv reset-venv
+          source .venv/bin/activate
+          # Store new PATH in GITHUB_ENV to keep environment enabled for subsequent steps.
+          echo "PATH=$PATH" >> $GITHUB_ENV
+          make install-pinned-${{ matrix.python }}
+      - name: Build Docker Image
+        working-directory: docker
+        run: |
+          make build-docker-amd64-cuda
+      - name: Run Docker Tests
+        working-directory: docker
+        run: |
+          make test

lightly_train-0.2.2/.github/workflows/test_minimal_deps.yml

@@ -0,0 +1,61 @@
+name: Minimal Dependency Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  detect-code-changes:
+    name: Detect Code Changes
+    runs-on: cpu
+    outputs:
+      run-tests: ${{ steps.filter.outputs.run-tests }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          list-files: shell
+          filters: |
+            run-tests:
+              - '!docs/**'
+              - '!docker/**'
+              - '!.github/**'
+              - '.github/workflows/test_minimal_deps.yml'
+  test:
+    name: Test
+    needs: detect-code-changes
+    if: needs.detect-code-changes.outputs.run-tests == 'true'
+    runs-on: cpu
+    strategy:
+      matrix:
+        python: ["3.8"]
+        dependencies: ["minimal", "minimal-extras"]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set Up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Set Up Environment
+        run: |
+          make install-uv reset-venv
+          source .venv/bin/activate
+          make install-${{ matrix.dependencies }}
+          if [[ ${{ matrix.dependencies }} == "minimal-extras" ]]; then
+            make install-ffmpeg-ubuntu
+          fi
+      - name: Run Pytest
+        env:
+          CUDA_VISIBLE_DEVICES: ""
+        run: |
+          source .venv/bin/activate
+          make test-ci

lightly_train-0.2.2/.github/workflows/weekly_dependency_tests.yml

@@ -0,0 +1,56 @@
+name: Weekly Dependency Test
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 22 * * THU' # each Thursday night, at 22:00 UTC
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  detect-code-changes:
+    name: Detect Code Changes
+    runs-on: cpu
+    outputs:
+      run-tests: ${{ steps.filter.outputs.run-tests }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          list-files: shell
+          filters: |
+            run-tests:
+              - '!docs/**'
+              - '!docker/**'
+              - '!.github/**'
+              - '.github/workflows/weekly_dependency_tests.yml'
+  test:
+    name: Test
+    needs: detect-code-changes
+    if: needs.detect-code-changes.outputs.run-tests == 'true'
+    runs-on: cpu
+    strategy:
+      matrix:
+        python: ["3.8", "3.11"]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Setup Environment
+        run: |
+          make install-uv reset-venv
+          source .venv/bin/activate
+          make install-latest
+          make install-ffmpeg-ubuntu
+      - name: Run Pytest
+        env:
+          CUDA_VISIBLE_DEVICES: ""
+        run: |
+          source .venv/bin/activate
+          make test-ci

lightly_train-0.2.1/README.md → lightly_train-0.2.2/DOCS.md

@@ -1,11 +1,6 @@
-# LightlyTrain
-
-*Train models with self-supervised learning in a single command*
-
 ## Installation
 
-```
-git clone https://github.com/lightly-ai/lightly-train.git
+```bash
 pip install lightly-train[tensorboard,timm]
 ```
 

{lightly_train-0.2.1 → lightly_train-0.2.2}/Makefile

@@ -1,20 +1,24 @@
-## make clean
+### Cleaning
+
 .PHONY: clean
 clean: clean-build clean-pyc clean-out
 
-## remove build artifacts
+# remove build artifacts
+.PHONY: clean-build
 clean-build:
 	rm -fr build/
 	rm -fr dist/
 	rm -fr .eggs/
 	find . -name '*.egg-info' -exec rm -fr {} +
-	find . -name '*.egg' -exec rm -f {} +
+	find . -name '*.egg' -exec rm -fr {} +
 
-## remove python file artifacts
+# remove python file artifacts
+.PHONY: clean-pyc
 clean-pyc:
 	find . -name '__pycache__' -exec rm -fr {} +
 
-## remove hydra outputs
+# remove hydra outputs
+.PHONY: clean-out
 clean-out:
 	rm -fr outputs/
 	rm -fr lightly_outputs/
@@ -22,6 +26,7 @@ clean-out:
 	rm -fr lightly_epoch_*.ckpt
 	rm -fr last.ckpt
 
+### Formatting and type-checking
 
 # run format and type checks and tests
 .PHONY: all-checks
@@ -50,6 +55,13 @@ format-check:
 type-check:
 	mypy src tests
 
+# adding the license header to all files
+.PHONY: add-header
+add-header:
+	licenseheaders -t dev_tools/licenseheader.tmpl -d src
+	licenseheaders -t dev_tools/licenseheader.tmpl -d tests
+
+
 
 ### Testing
 
@@ -98,7 +110,7 @@ DOCKER_EXTRAS = --extra tensorboard --extra timm
 
 # Date until which dependencies installed with --exclude-newer must have been released.
 # Dependencies released after this date are ignored.
-EXCLUDE_NEWER_DATE="2024-08-07"
+EXCLUDE_NEWER_DATE="2024-08-28"
 
 # Install ffmpeg on Ubuntu.
 . PHONY: install-ffmpeg-ubuntu
@@ -174,9 +186,13 @@ install-docker-dependencies:
 install-docker:
 	uv pip install -v --no-deps .
 
+# Install dependencies for building and publishing the package.
+.PHONY: install-dist
+install-dist:
+	uv pip install --exclude-newer ${EXCLUDE_NEWER_DATE} wheel twine build 
 
-## build source and wheel package
-release-pypi: clean
+### Building source and wheel package for publishing to pypi
+.PHONY: dist
+dist: clean
 	python -m build
 	ls -l dist
-

lightly_train-0.2.2/PKG-INFO

@@ -0,0 +1,75 @@
+Metadata-Version: 2.1
+Name: lightly_train
+Version: 0.2.2
+Summary: Train models with self-supervised learning in a single command
+Author: Lightly Team
+License: AGPL-3.0
+Keywords: machine-learning,computer-vision,deep-learning,self-supervised-learning,contrastive-learning,pytorch,python,pretrained-models,embeddings
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: lightly>=1.5.12
+Requires-Dist: omegaconf>=2.3
+Requires-Dist: psutil>=5.0
+Requires-Dist: pytorch_lightning>=2.1
+Requires-Dist: torch>=2.0.0
+Requires-Dist: torchvision>=0.15.1
+Requires-Dist: tqdm>=4.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: pytest-mock>=3.14.0; extra == "dev"
+Requires-Dist: ruff>=0.4.0; extra == "dev"
+Requires-Dist: mypy>=1.10.0; extra == "dev"
+Requires-Dist: types-tqdm>=4.0.0; extra == "dev"
+Requires-Dist: licenseheaders>=0.8.8; extra == "dev"
+Provides-Extra: super-gradients
+Requires-Dist: super-gradients>=3.7; extra == "super-gradients"
+Provides-Extra: tensorboard
+Requires-Dist: tensorboard>=2.10.0; extra == "tensorboard"
+Provides-Extra: timm
+Requires-Dist: timm>=1.0.3; extra == "timm"
+
+# LightlyTrain
+
+*Train models with self-supervised learning in a single command*
+
+
+## Why LightlyTrain
+
+LightlyTrain uses self-supervised learning (SSL) to train models on large datasets
+without the need for labels. It provides simple Python, Command Line, and Docker
+interfaces to train models with popular SSL methods such as SimCLR or DINO. The trained
+models are ideal starting points for fine-tuning on downstream tasks such as image 
+classification, object detection, and segmentation or for generating image embeddings.
+Models trained with LightlyTrain result in improved performance, faster convergence, and
+better generalization compared to models trained without SSL. Image embeddings created
+with LightlyTrain capture more relevant information than their supervised counterparts
+and seamlessly extend to new classes due to the unsupervised nature of SSL.
+
+Lightly is the expert in SSL for computer vision and developed LightlyTrain to simplify
+model training for any task and dataset.
+
+
+## Features
+
+* Train models on any image data without labels
+* Train models from popular libraries such as [torchvision](https://github.com/pytorch/vision), [TIMM](https://github.com/huggingface/pytorch-image-models), and [SuperGradients](https://github.com/Deci-AI/super-gradients)
+* Train custom models
+* No SSL expertise required
+* Automatic SSL method selection (soon!)
+* Python, Command Line, and Docker support
+* Multi-GPU and multi-node (soon!) support
+* Export models for fine-tuning or inference
+* Generate and export image embeddings
+* Monitor training progress with TensorBoard, Weights & Biases, Neptune, etc. (soon!)
+
+## License
+
+LightlyTrain is available under an AGPL-3.0 and a commercial license. Please contact us
+at info@lightly.ai for more information.
+
+## Contact
+
+* [**Email**](info@lightly.ai)  
+* [**Website**](https://www.lightly.ai/lightlytrain)  
+* [**Discord**](https://discord.gg/xvNJW94)

lightly_train-0.2.2/README.md

@@ -0,0 +1,44 @@
+# LightlyTrain
+
+*Train models with self-supervised learning in a single command*
+
+
+## Why LightlyTrain
+
+LightlyTrain uses self-supervised learning (SSL) to train models on large datasets
+without the need for labels. It provides simple Python, Command Line, and Docker
+interfaces to train models with popular SSL methods such as SimCLR or DINO. The trained
+models are ideal starting points for fine-tuning on downstream tasks such as image 
+classification, object detection, and segmentation or for generating image embeddings.
+Models trained with LightlyTrain result in improved performance, faster convergence, and
+better generalization compared to models trained without SSL. Image embeddings created
+with LightlyTrain capture more relevant information than their supervised counterparts
+and seamlessly extend to new classes due to the unsupervised nature of SSL.
+
+Lightly is the expert in SSL for computer vision and developed LightlyTrain to simplify
+model training for any task and dataset.
+
+
+## Features
+
+* Train models on any image data without labels
+* Train models from popular libraries such as [torchvision](https://github.com/pytorch/vision), [TIMM](https://github.com/huggingface/pytorch-image-models), and [SuperGradients](https://github.com/Deci-AI/super-gradients)
+* Train custom models
+* No SSL expertise required
+* Automatic SSL method selection (soon!)
+* Python, Command Line, and Docker support
+* Multi-GPU and multi-node (soon!) support
+* Export models for fine-tuning or inference
+* Generate and export image embeddings
+* Monitor training progress with TensorBoard, Weights & Biases, Neptune, etc. (soon!)
+
+## License
+
+LightlyTrain is available under an AGPL-3.0 and a commercial license. Please contact us
+at info@lightly.ai for more information.
+
+## Contact
+
+* [**Email**](info@lightly.ai)  
+* [**Website**](https://www.lightly.ai/lightlytrain)  
+* [**Discord**](https://discord.gg/xvNJW94)