gammasimtools 0.12.0__tar.gz → 0.14.0__tar.gz

gammasimtools-0.14.0/.env_template

@@ -0,0 +1,18 @@
+# Template file for environment variables settings
+# Copy this file to '.env' and add the required
+# credentials for simulation model database access
+SIMTOOLS_DB_API_PORT=27017 #Port on the MongoDB server
+SIMTOOLS_DB_SERVER='cta-simpipe-protodb.zeuthen.desy.de' # MongoDB server
+SIMTOOLS_DB_API_USER=YOUR_USERNAME # username for MongoDB: ask the responsible person
+SIMTOOLS_DB_API_PW=YOUR_PASSWORD # Password for MongoDB: ask the responsible person
+SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin'
+SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-v0-5-0'
+SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray'
+SIMTOOLS_USER_NAME='simtools-user'
+SIMTOOLS_USER_ORGANIZATION='CTAO'
+SIMTOOLS_USER_EMAIL="default@email.org"
+SIMTOOLS_USER_ORCID='0000-0000-0000-0000'
+
+# The dashboards to monitor the MongoDB instance are in (accessible only from within DESY)
+# https://statspub.zeuthen.desy.de/d/4vXnWwMGz/mongodb?orgId=1&refresh=30s
+# https://statspub.zeuthen.desy.de/d/tBkrQGNmz/mongodb-wiredtiger?orgId=1&refresh=1m

gammasimtools-0.14.0/.github/workflows/CI-docs.yml

@@ -0,0 +1,89 @@
+---
+name: CI-docs
+# Build and deploy docs to gh-pages
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [main]
+    types: [opened, closed, synchronize]
+
+jobs:
+
+  docs:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Python dependencies
+        run: |
+          pip install '.[doc]'
+
+      - name: Check for non-ASCII characters
+        run: |
+          output=$(perl -ne 'print if /[^[:ascii:]]/' docs/source/*)
+          if [ -n "$output" ]; then
+            echo "Non-ASCII characters found in documentation."
+            exit 1
+          fi
+
+      - name: Check that all applications are documented
+        run: |-
+          scripts=$(grep 'scripts.simtools-' pyproject.toml | sed -e 's/^scripts\.//' -e 's/ =.*$//')
+          FULLY_DOCUMENTED="TRUE"
+          for S in $scripts; do
+              if [ ! -e "docs/source/user-guide/applications/$S.rst" ]; then
+                echo "Undocumented script: $S"
+                FULLY_DOCUMENTED="FALSE"
+              fi
+          done
+          if [[ "$FULLY_DOCUMENTED" = "FALSE" ]]; then
+              exit 1
+          fi
+        shell: /usr/bin/bash -e {0}
+
+      - name: Check for complete API documentation
+        run: |-
+          MODULES=$(find src/simtools -type f -name "*.py" \
+          ! -path "src/simtools/applications/*" \
+          ! -name "__init__.py" \
+          ! -path "src/simtools/_*" -prune)
+
+          for M in $MODULES; do
+              module=$(basename "$M" .py)
+              if ! grep -q "$module" docs/source/api-reference/*.md; then
+                  echo "Undocumented module: $module"
+                  FULLY_DOCUMENTED="FALSE"
+              fi
+          done
+          if [[ "$FULLY_DOCUMENTED" = "FALSE" ]]; then
+              exit 1
+          fi
+
+      - name: Build docs
+        shell: bash -l {0}
+        run: |
+          make -C docs/ html
+          exit_code=$?
+          if [ $exit_code -ne 0 ]; then
+            echo "Make command failed with exit code $exit_code"
+            exit $exit_code
+          fi
+          touch docs/build/html/.nojekyll
+
+      - name: Deploy to github pages
+        # only run when PR is merged
+        if: github.event.pull_request.merged
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BRANCH: gh-pages
+          FOLDER: docs/build/html
+          CLEAN: true

gammasimtools-0.14.0/.github/workflows/CI-integrationtests.yml

@@ -0,0 +1,118 @@
+---
+name: CI-integrationtests
+# Integration tests for applications
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize]
+  schedule:
+    - cron: "0 0 * * *"
+  release:
+    types: [published]
+
+jobs:
+
+  test_building:
+    # Build the package and check if it is installable
+    # (tests among others that all components are there)
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
+    steps:
+
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Build the package
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade build
+          python -m build
+
+  integration_tests:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/gammasim/simtools-dev:latest
+      options: --user 0
+
+    services:
+      mongodb:
+        image: mongo:latest
+        env:
+          MONGO_INITDB_ROOT_USERNAME: api
+          MONGO_INITDB_ROOT_PASSWORD: password
+        options: >-
+          --health-cmd "mongosh --host localhost --port 27017 -u api -p password --authenticationDatabase admin --eval 'db.runCommand({ ping: 1 })'"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    strategy:
+      fail-fast: false
+      matrix:
+        type: ['6.0.0', '5.0.0']
+
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set PATH
+        run: |
+          echo "PATH=\$PATH:/usr/bin:/usr/local/bin:$SIMTOOLS_SIMTEL_PATH" >> "$GITHUB_ENV"
+
+      - name: Create environment file (remote DB; schedule only)
+        if: ${{ github.event_name == 'schedule' }}
+        run: |
+          {
+            echo "SIMTOOLS_DB_SERVER=${{ secrets.DB_SERVER }}"
+            echo "SIMTOOLS_DB_API_USER=${{ secrets.DB_API_USER }}"
+            echo "SIMTOOLS_DB_API_PW=${{ secrets.DB_API_PW }}"
+            echo "SIMTOOLS_DB_API_PORT=${{ secrets.DB_API_PORT }}"
+            echo "SIMTOOLS_SIMTEL_PATH=/workdir/sim_telarray/"
+          } > .env
+
+      - name: Extract SIMTOOLS_DB_SIMULATION_MODEL
+        run: |
+          SIMTOOLS_DB_SIMULATION_MODEL=$(grep 'SIMTOOLS_DB_SIMULATION_MODEL=' .env_template | cut -d '=' -f2- | tr -d '"')
+          SIMTOOLS_DB_SIMULATION_MODEL="${SIMTOOLS_DB_SIMULATION_MODEL//\'/}"
+          echo "SIMTOOLS_DB_SIMULATION_MODEL=$SIMTOOLS_DB_SIMULATION_MODEL" >> "$GITHUB_ENV"
+
+      - name: Create environment file (local DB)
+        if: github.event_name != 'schedule'
+        run: |
+          {
+            echo "SIMTOOLS_DB_SERVER=mongodb"
+            echo "SIMTOOLS_DB_API_USER=api"
+            echo "SIMTOOLS_DB_API_PW=password"
+            echo "SIMTOOLS_DB_API_PORT=27017"
+            echo "SIMTOOLS_DB_SIMULATION_MODEL=${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}"
+            echo "SIMTOOLS_SIMTEL_PATH=/workdir/sim_telarray/"
+          } > .env
+
+      - name: Upload data to MongoDB
+        if: github.event_name != 'schedule'
+        run: |
+          source /workdir/env/bin/activate
+          pip install --no-cache-dir -e '.[tests,dev,doc]'
+          cd database_scripts/
+          ./upload_from_model_repository_to_db.sh ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}
+
+      - name: Integration tests
+        shell: bash -l {0}
+        env:
+          SIMTOOLS_DB_SIMULATION_MODEL: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}
+          SIMTOOLS_DB_SERVER: ${{ github.event_name == 'schedule' && secrets.DB_SERVER || 'mongodb' }}
+        run: |
+          source /workdir/env/bin/activate
+          pip install --no-cache-dir -e '.[tests,dev,doc]'
+          pytest --model_version=${{ matrix.type }} --color=yes --durations=20 -n 4 --dist loadscope --no-cov tests/integration_tests/

gammasimtools-0.14.0/.github/workflows/CI-linter.yml

@@ -0,0 +1,100 @@
+---
+name: CI-linting
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize]
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+
+  linting:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
+    permissions:
+      contents: read
+      packages: read
+      statuses: write
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          # super-linter needs the full git history to get the
+          # list of files that changed across commits
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+
+      - name: Check for non-ASCII characters
+        run: |
+          output=$(find . -type f \
+          \( -name "*.py" -o -name "*.rst" -o -name "*.yml" -o -name "*.toml" \) \
+          -exec perl -ne 'print if /[^[:ascii:]]/ && !/latin/i' {} \;)
+          if [ -n "$output" ]; then
+            echo "Non-ASCII characters found in documentation."
+            exit 1
+          fi
+
+      - name: install packages not included in super-linter
+        run: |
+          pip install validate-pyproject
+
+      - name: pyproject.toml
+        run: |
+          validate-pyproject pyproject.toml
+
+      - name: Check whether the citation metadata from CITATION.cff is valid
+        uses: citation-file-format/cffconvert-github-action@2.0.0
+        with:
+          args: "--validate"
+
+      # Dependencies required to avoid errors
+      # reported by linters
+      - name: Install mamba dependencies
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: environment.yml
+          init-shell: bash
+
+      - name: Python dependencies
+        run: |
+          pip install -e '.[tests,dev,doc]'
+          pip install pylint
+
+      - name: pylint
+        run: |
+          git ls-files 'src/simtools/*.py' | xargs pylint -v
+
+      - name: Pre-commit
+        run: |
+          pre-commit run --all-files
+
+      - name: Lint Code Base
+        uses: super-linter/super-linter@v7
+        env:
+          VALIDATE_ALL_CODEBASE: false
+          # docker
+          VALIDATE_DOCKERFILE_HADOLINT: true
+          # .env file
+          VALIDATE_ENV: true
+          # language
+          VALIDATE_NATURAL_LANGUAGE: true
+          # bash
+          VALIDATE_BASH: true
+          # path for linter rules
+          LINTER_RULES_PATH: ./
+          # create a log file
+          CREATE_LOG_FILE: true
+          LOG_FILE: superlinter.log
+          DEFAULT_BRANCH: main
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

gammasimtools-0.14.0/.github/workflows/CI-schema-validation.yml

@@ -0,0 +1,66 @@
+---
+name: CI-schema-validation
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'src/simtools/schemas/model_parameters/*'
+      - 'tests/integration_tests/config/*'
+
+jobs:
+
+  schema-validation:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
+    permissions:
+      contents: read
+      packages: read
+      statuses: write
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Installation
+        run: |
+          pip install -e .
+
+      - name: Check for files without the suffix *.schema.yml
+        run: |
+          for file in src/simtools/schemas/model_parameters/*.yml; do
+            if [[ ! $file =~ .*schema.yml$ ]]; then
+              echo "File $file does not have the suffix *.schema.yml"
+              exit 1
+            fi
+          done
+
+      - name: Model parameter schema validation
+        run: |
+          for file in src/simtools/schemas/model_parameters/*.yml; do
+            echo "Testing schema $file"
+            python src/simtools/applications/validate_file_using_schema.py \
+            --file_name "$file" \
+            --schema model_parameter_and_data_schema.metaschema.yml \
+            --data_type "schema" \
+            --log_level DEBUG
+          done
+
+      - name: Integration test config file validation
+        run: |
+          for file in tests/integration_tests/config/*.yml; do
+            echo "Testing config file $file"
+            python src/simtools/applications/validate_file_using_schema.py \
+            --file_name "$file" \
+            --schema application_workflow.metaschema.yml \
+            --data_type "schema" \
+            --log_level DEBUG
+          done

gammasimtools-0.14.0/.github/workflows/CI-software-update.yml

@@ -0,0 +1,36 @@
+---
+name: Software updates
+# Run regularly to update software dependencies
+
+on:
+  # Run on the first day of each month at midnight
+  schedule:
+    - cron: "0 0 1 * *"
+  workflow_dispatch:
+
+jobs:
+  pre-commit-auto-update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install packages (as pre-commit is running on all files after autoupdate)
+        run: |
+          pip install -e '.[tests,dev,doc]'
+          pip install pylint
+
+      - uses: browniebroke/pre-commit-autoupdate-action@main
+
+      - uses: peter-evans/create-pull-request@v7
+        if: always()
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          branch: update/pre-commit-hooks
+          title: Update pre-commit hooks
+          commit-message: "chore: update pre-commit hooks"
+          body: Update versions of pre-commit hooks to latest version.

gammasimtools-0.14.0/.github/workflows/CI-unittests.yml

@@ -0,0 +1,151 @@
+---
+name: CI-unittests
+# Execute unit tests
+# (includes CTAO-DPPS-SonarQube)
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize]
+  schedule:
+    - cron: "0 0 * * *"
+  release:
+    types: [published]
+
+jobs:
+  unit_tests:
+    env:
+      # For scheduled runs, use production DB
+      SIMTOOLS_DB_SERVER: ${{ github.event_name != 'schedule' && 'localhost' || secrets.DB_SERVER }}
+      SIMTOOLS_DB_API_USER: ${{ github.event_name != 'schedule' && 'api' || secrets.DB_API_USER }}
+      SIMTOOLS_DB_API_PW: ${{ github.event_name != 'schedule' && 'password' || secrets.DB_API_PW }}
+      SIMTOOLS_DB_API_PORT: ${{ github.event_name != 'schedule' && '27017' || secrets.DB_API_PORT }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+            python-version: "3.11"
+            install-method: mamba
+
+          - os: ubuntu-latest
+            python-version: "3.12"
+            install-method: mamba
+            extra-args: ["codecov", "random-order"]
+
+          - os: ubuntu-latest
+            python-version: "3.12"
+            install-method: pip
+
+          - os: ubuntu-latest
+            python-version: "3.13"
+            install-method: mamba
+
+    services:
+      mongodb:
+        image: mongo:latest
+        env:
+          MONGO_INITDB_ROOT_USERNAME: api
+          MONGO_INITDB_ROOT_PASSWORD: password
+        ports:
+          - 27017:27017
+        options: >-
+          --health-cmd "mongosh --host localhost --port 27017 -u api -p password --authenticationDatabase admin --eval 'db.runCommand({ ping: 1 })'"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Prepare mamba installation
+        if: matrix.install-method == 'mamba'
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+        run: |
+          sed -i -e "s/- python=.*/- python=$PYTHON_VERSION/g" environment.yml
+
+      - name: Install mamba dependencies
+        if: matrix.install-method == 'mamba'
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: environment.yml
+          create-args: python=${{ matrix.python-version }}
+          init-shell: bash
+
+      - name: Python setup
+        if: matrix.install-method == 'pip'
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          check-latest: true
+
+      - name: Python dependencies (pip)
+        if: matrix.install-method == 'pip'
+        run: |
+          pip install '.[tests,dev,doc]'
+
+      - name: Python dependencies (mamba)
+        if: matrix.install-method == 'mamba'
+        run: |
+          pip install -e '.[tests,dev,doc]'
+
+      - name: Extract SIMTOOLS_DB_SIMULATION_MODEL
+        run: |
+          SIMTOOLS_DB_SIMULATION_MODEL=$(grep 'SIMTOOLS_DB_SIMULATION_MODEL=' .env_template | cut -d '=' -f2- | tr -d '"')
+          SIMTOOLS_DB_SIMULATION_MODEL="${SIMTOOLS_DB_SIMULATION_MODEL//\'/}"
+          echo "SIMTOOLS_DB_SIMULATION_MODEL=$SIMTOOLS_DB_SIMULATION_MODEL" >> "$GITHUB_ENV"
+
+      - name: Print DB connection variables
+        run: |
+          echo "Event name: ${{ github.event_name }}"
+          echo "DB Server: $SIMTOOLS_DB_SERVER"
+          echo "DB API User: $SIMTOOLS_DB_API_USER"
+          echo "DB API Port: $SIMTOOLS_DB_API_PORT"
+          echo "DB Simulation Model: $SIMTOOLS_DB_SIMULATION_MODEL"
+
+      - name: Upload data to MongoDB
+        if: github.event_name != 'schedule'
+        run: |
+          cd database_scripts/
+          ./upload_from_model_repository_to_db.sh ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}
+
+      - name: Unit tests
+        shell: bash -l {0}
+        env:
+          SIMTOOLS_DB_SIMULATION_MODEL: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}
+        run: |
+          pytest --durations=10 --color=yes -n 4 --dist loadscope \
+            --cov=simtools --cov-report=xml
+
+      # CTAO-DPPS-SonarQube
+      - uses: SonarSource/sonarqube-scan-action@v5.0.0
+        if: contains(matrix.extra-args, 'codecov')
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+
+      - name: Random order
+        if: github.event_name == 'schedule' && contains(matrix.extra-args, 'random-order')
+        shell: bash -l {0}
+        env:
+          SIMTOOLS_DB_SIMULATION_MODEL: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}
+        run: |
+          pytest --color=yes -n 4 --dist loadscope --count 5 --random-order
+
+      - name: Upload coverage report to codecov
+        uses: codecov/codecov-action@v5
+        if: contains(matrix.extra-args, 'codecov')
+        with:
+          token: ${{ secrets.CODECOV_SIMTOOLS_TOKEN }}

gammasimtools-0.14.0/.github/workflows/build-simtools-dev-image.yml

@@ -0,0 +1,93 @@
+---
+name: docker-simtools-dev
+# Build simtools images for developers based on CORSIKA / sim_telarray images (no optimizations)
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*'
+  pull_request:
+    branches: ["main"]
+    types: [review_requested, ready_for_review]
+  release:
+    types: [published]
+  schedule:
+    - cron: '0 0 * * 0'  # Every Sunday at 00:00 UTC (build only)
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+  TEST_TAG: user/app:test
+
+jobs:
+
+  build-simtools-container:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      matrix:
+        type: ['dev']
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set build arguments (correct branch)
+        run: |
+          if [[ "${{ github.event_name }}" == 'pull_request' ]]; then
+            echo "BUILD_BRANCH=${GITHUB_HEAD_REF}" >> "$GITHUB_ENV"
+          else
+            echo "BUILD_BRANCH=${GITHUB_REF#refs/heads/}" >> "$GITHUB_ENV"
+          fi
+          echo "BUILD BRANCH ${{ env.BUILD_BRANCH }}"
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          tags: |
+            type=ref,event=pr
+            type=semver,pattern={{major}}.{{minor}}.{{patch}}
+            type=schedule,pattern={{date 'YYYYMMDD'}}
+            type=raw,value={{date 'YYYYMMDD-HHmmss'}}
+          images: ${{ env.REGISTRY }}/gammasim/simtools-${{ matrix.type }}
+          flavor: latest=true
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          build-args: BUILD_BRANCH=${{ env.BUILD_BRANCH }}
+          load: true
+          file: ./docker/Dockerfile-${{ matrix.type }}
+          tags: ${{ env.TEST_TAG }}
+
+      - name: Build and push image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64/v8
+          build-args: BUILD_BRANCH=${{ env.BUILD_BRANCH }}
+          push:
+            ${{ github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
+          file: ./docker/Dockerfile-${{ matrix.type }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}-${{ matrix.type }}