gammasimtools 0.8.1__tar.gz → 0.9.0__tar.gz

gammasimtools-0.9.0/.env_template

@@ -0,0 +1,14 @@
+# Template file for environment variables settings
+# Copy this file to '.env' and add the required
+# credentials for simulation model database access
+SIMTOOLS_DB_API_PORT=27017 #Port on the MongoDB server
+SIMTOOLS_DB_SERVER='cta-simpipe-protodb.zeuthen.desy.de' # MongoDB server
+SIMTOOLS_DB_API_USER=YOUR_USERNAME # username for MongoDB: ask the responsible person
+SIMTOOLS_DB_API_PW=YOUR_PASSWORD # Password for MongoDB: ask the responsible person
+SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin'
+SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-LATEST'
+SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray'
+
+# The dashboards to monitor the MongoDB instance are in (accessible only from within DESY)
+# https://statspub.zeuthen.desy.de/d/4vXnWwMGz/mongodb?orgId=1&refresh=30s
+# https://statspub.zeuthen.desy.de/d/tBkrQGNmz/mongodb-wiredtiger?orgId=1&refresh=1m

gammasimtools-0.9.0/.github/workflows/CI-docs.yml

@@ -0,0 +1,89 @@
+---
+name: CI-docs
+# Build and deploy docs to gh-pages
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [main]
+    types: [opened, closed, synchronize]
+
+jobs:
+
+  docs:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Python dependencies
+        run: |
+          pip install '.[doc]'
+
+      - name: Check for non-ASCII characters
+        run: |
+          output=$(perl -ne 'print if /[^[:ascii:]]/' docs/source/*)
+          if [ -n "$output" ]; then
+            echo "Non-ASCII characters found in documentation."
+            exit 1
+          fi
+
+      - name: Check that all applications are documented
+        run: |-
+          scripts=$(grep 'scripts.simtools-' pyproject.toml | sed -e 's/^scripts\.//' -e 's/ =.*$//')
+          FULLY_DOCUMENTED="TRUE"
+          for S in $scripts; do
+              if [ ! -e "docs/source/user-guide/applications/$S.rst" ]; then
+                echo "Undocumented script: $S"
+                FULLY_DOCUMENTED="FALSE"
+              fi
+          done
+          if [[ "$FULLY_DOCUMENTED" = "FALSE" ]]; then
+              exit 1
+          fi
+        shell: /usr/bin/bash -e {0}
+
+      - name: Check for complete API documentation
+        run: |-
+          MODULES=$(find src/simtools -type f -name "*.py" \
+            ! -path "src/simtools/applications/*" \
+            ! -name "__init__.py" ! -name "version.py" \
+            ! -path "src/simtools/_*" -prune)
+          FULLY_DOCUMENTED="TRUE"
+          for M in "${MODULES[@]}"; do
+              module=$(basename "$M" .py)
+              if ! grep -q "$module" docs/source/api-reference/*.md; then
+                echo "Undocumented module: $module"
+                FULLY_DOCUMENTED="FALSE"
+              fi
+          done
+          if [[ "$FULLY_DOCUMENTED" = "FALSE" ]]; then
+              exit 1
+          fi
+
+      - name: Build docs
+        shell: bash -l {0}
+        run: |
+          make -C docs/ html
+          exit_code=$?
+          if [ $exit_code -ne 0 ]; then
+            echo "Make command failed with exit code $exit_code"
+            exit $exit_code
+          fi
+          touch docs/build/html/.nojekyll
+
+      - name: Deploy to github pages
+        # only run when PR is merged
+        if: github.event.pull_request.merged
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BRANCH: gh-pages
+          FOLDER: docs/build/html
+          CLEAN: true

gammasimtools-0.9.0/.github/workflows/CI-integrationtests.yml

@@ -0,0 +1,71 @@
+---
+name: CI-integrationtests
+# Integration tests for applications
+
+env:
+  SIMTOOLS_DB_SERVER: ${{ secrets.DB_SERVER }}
+  SIMTOOLS_DB_API_USER: ${{ secrets.DB_API_USER }}
+  SIMTOOLS_DB_API_PW: ${{ secrets.DB_API_PW }}
+  SIMTOOLS_DB_API_PORT: ${{ secrets.DB_API_PORT }}
+  SIMTOOLS_DB_SIMULATION_MODEL: "CTAO-Simulation-Model-LATEST"
+  SIMTOOLS_SIMTEL_PATH: "/workdir/sim_telarray/"
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize]
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+
+  testbuilding:
+    # Build the package and check if it is installable
+    # (tests among others that all components are there)
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
+    steps:
+
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Build the package
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade build
+          python -m build
+
+  integrationtests:
+    # Run integration tests
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/gammasim/simtools-dev:latest
+      options: --user 0
+    strategy:
+      fail-fast: false
+      matrix:
+        type: ['6.0.0', '5.0.0']
+
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
+    steps:
+
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Set PATH
+        run: |
+          echo "PATH=\$PATH:/usr/bin:/usr/local/bin:$SIMTOOLS_SIMTEL_PATH" >> "$GITHUB_ENV"
+
+      - name: Run integration tests
+        shell: bash -l {0}
+        run: |
+          source /workdir/env/bin/activate
+          pip install -e '.[tests,dev,doc]'
+          pytest --model_version=${{ matrix.type }} --color=yes --durations=20 -n auto --dist loadscope --no-cov tests/integration_tests/

gammasimtools-0.9.0/.github/workflows/CI-linter.yml

@@ -0,0 +1,100 @@
+---
+name: CI-linting
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize]
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+
+  linting:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
+    permissions:
+      contents: read
+      packages: read
+      statuses: write
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          # super-linter needs the full git history to get the
+          # list of files that changed across commits
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+
+      - name: Check for non-ASCII characters
+        run: |
+          output=$(find . -type f \
+          \( -name "*.py" -o -name "*.rst" -o -name "*.yml" -o -name "*.toml" \) \
+          -exec perl -ne 'print if /[^[:ascii:]]/ && !/latin/i' {} \;)
+          if [ -n "$output" ]; then
+            echo "Non-ASCII characters found in documentation."
+            exit 1
+          fi
+
+      - name: install packages not included in super-linter
+        run: |
+          pip install validate-pyproject
+
+      - name: pyproject.toml
+        run: |
+          validate-pyproject pyproject.toml
+
+      - name: Check whether the citation metadata from CITATION.cff is valid
+        uses: citation-file-format/cffconvert-github-action@2.0.0
+        with:
+          args: "--validate"
+
+      # Dependencies required to avoid errors
+      # reported by linters
+      - name: Install mamba dependencies
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: environment.yml
+          init-shell: bash
+
+      - name: Python dependencies
+        run: |
+          pip install -e '.[tests,dev,doc]'
+          pip install pylint
+
+      - name: pylint
+        run: |
+          git ls-files 'src/simtools/*.py' | xargs pylint -v
+
+      - name: Pre-commit
+        run: |
+          pre-commit run --all-files
+
+      - name: Lint Code Base
+        uses: super-linter/super-linter@v7
+        env:
+          VALIDATE_ALL_CODEBASE: false
+          # docker
+          VALIDATE_DOCKERFILE_HADOLINT: true
+          # .env file
+          VALIDATE_ENV: true
+          # language
+          VALIDATE_NATURAL_LANGUAGE: true
+          # bash
+          VALIDATE_BASH: true
+          # path for linter rules
+          LINTER_RULES_PATH: ./
+          # create a log file
+          CREATE_LOG_FILE: true
+          LOG_FILE: superlinter.log
+          DEFAULT_BRANCH: main
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

gammasimtools-0.9.0/.github/workflows/CI-schema-validation.yml

@@ -0,0 +1,65 @@
+name: CI-schema-validation
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'src/simtools/schemas/model_parameters/*'
+      - 'tests/integration_tests/config/*'
+
+jobs:
+
+  schema-validation:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
+    permissions:
+      contents: read
+      packages: read
+      statuses: write
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Installation
+        run: |
+          pip install -e .
+
+      - name: Check for files without the suffix *.schema.yml
+        run: |
+          for file in src/simtools/schemas/model_parameters/*.yml; do
+            if [[ ! $file =~ .*schema.yml$ ]]; then
+              echo "File $file does not have the suffix *.schema.yml"
+              exit 1
+            fi
+          done
+
+      - name: Model parameter schema validation
+        run: |
+          for file in src/simtools/schemas/model_parameters/*.yml; do
+            echo "Testing schema $file"
+            python src/simtools/applications/validate_file_using_schema.py \
+            --file_name "$file" \
+            --schema model_parameter_and_data_schema.metaschema.yml \
+            --data_type "schema" \
+            --log_level DEBUG
+          done
+
+      - name: Integration test config file validation
+        run: |
+          for file in tests/integration_tests/config/*.yml; do
+            echo "Testing config file $file"
+            python src/simtools/applications/validate_file_using_schema.py \
+            --file_name "$file" \
+            --schema integration_tests_config.metaschema.yml \
+            --data_type "schema" \
+            --log_level DEBUG
+          done

gammasimtools-0.9.0/.github/workflows/CI-unittests.yml

@@ -0,0 +1,120 @@
+---
+name: CI-unittests
+# Execute unit tests
+# (includes CTAO-DPPS-SonarQube)
+env:
+  SIMTOOLS_DB_SERVER: ${{ secrets.DB_SERVER }}
+  SIMTOOLS_DB_API_USER: ${{ secrets.DB_API_USER }}
+  SIMTOOLS_DB_API_PW: ${{ secrets.DB_API_PW }}
+  SIMTOOLS_DB_API_PORT: ${{ secrets.DB_API_PORT }}
+  SIMTOOLS_DB_SIMULATION_MODEL: "CTAO-Simulation-Model-LATEST"
+  SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize]
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+
+  unittests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: macos-latest
+            python-version: "3.11"
+            install-method: mamba
+
+          - os: ubuntu-latest
+            python-version: "3.11"
+            install-method: mamba
+            extra-args: ["codecov", "random-order"]
+
+          - os: ubuntu-latest
+            python-version: "3.11"
+            install-method: pip
+
+          - os: ubuntu-latest
+            python-version: "3.12"
+            install-method: mamba
+
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Prepare mamba installation
+        if: matrix.install-method == 'mamba'
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+        run: |
+          # setup correct python version
+          sed -i -e "s/- python=.*/- python=$PYTHON_VERSION/g" environment.yml
+
+      - name: Install mamba dependencies
+        if: matrix.install-method == 'mamba'
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: environment.yml
+          create-args: python=${{ matrix.python-version }}
+          init-shell: bash
+
+      - name: Python setup
+        if: matrix.install-method == 'pip'
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          check-latest: true
+
+      - if: ${{ matrix.install-method == 'pip' && runner.os == 'macOS' }}
+        name: Fix Python PATH on macOS
+        # from https://github.com/cta-observatory/ctapipe/blob/cfacbe1eeb2d9a4634980e5f1ef39b944bca6a7a/.github/workflows/ci.yml#L100
+        run: |
+          tee -a "$HOME/.bash_profile" <<< "export PATH=$pythonLocation/bin:$PATH"
+
+      - name: Python dependencies (pip)
+        if: matrix.install-method == 'pip'
+        run: |
+          pip install '.[tests,dev,doc]'
+
+      - name: Python dependencies (mamba)
+        if: matrix.install-method == 'mamba'
+        run: |
+          pip install -e '.[tests,dev,doc]'
+
+      - name: Unit tests
+        shell: bash -l {0}
+        run: |
+          pytest --durations=10 --color=yes -n auto --dist loadscope \
+            --cov=simtools --cov-report=xml
+
+      # CTAO-DPPS-SonarQube
+      - uses: SonarSource/sonarqube-scan-action@v4.2.1
+        if: contains(matrix.extra-args, 'codecov')
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+
+      - name: Random order
+        if: github.event_name == 'schedule' && contains(matrix.extra-args, 'random-order')
+        shell: bash -l {0}
+        run: |
+          pytest --color=yes -n auto --dist loadscope --count 5 --random-order
+
+      - name: Upload coverage report to codecov
+        uses: codecov/codecov-action@v5
+        if: contains(matrix.extra-args, 'codecov')
+        with:
+          token: ${{ secrets.CODECOV_SIMTOOLS_TOKEN }}

gammasimtools-0.9.0/.github/workflows/build-docker-images.yml

@@ -0,0 +1,129 @@
+# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
+name: docker-simtools
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*'
+  pull_request:
+    branches: ["main"]
+    types: [review_requested, ready_for_review]
+  release:
+    types: [published]
+  schedule:
+    - cron: '0 0 * * 0'  # Every Sunday at 00:00 UTC
+
+env:
+  SIMTOOLS_DB_SERVER: ${{ secrets.DB_SERVER }}
+  SIMTOOLS_DB_API_USER: ${{ secrets.DB_READ_USER }}
+  SIMTOOLS_DB_API_PW: ${{ secrets.DB_READ_PW }}
+  SIMTOOLS_DB_API_PORT: ${{ secrets.DB_API_PORT }}
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+  TEST_TAG: user/app:test
+
+jobs:
+
+  build-simtools-container:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      matrix:
+        type: ['dev', 'prod']
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set build arguments (correct branch)
+        run: |
+          if [[ "${{ github.event_name }}" == 'pull_request' ]]; then
+            echo "BUILD_BRANCH=${GITHUB_HEAD_REF}" >> "$GITHUB_ENV"
+          else
+            echo "BUILD_BRANCH=${GITHUB_REF#refs/heads/}" >> "$GITHUB_ENV"
+          fi
+          echo "BUILD BRANCH ${{ env.BUILD_BRANCH }}"
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          tags: |
+            type=ref,event=pr
+            type=semver,pattern={{major}}.{{minor}}.{{patch}}
+            type=schedule,pattern={{date 'YYYYMMDD'}}
+            type=raw,value={{date 'YYYYMMDD-HHmmss'}}
+          images: ${{ env.REGISTRY }}/gammasim/simtools-${{ matrix.type }}
+          flavor: latest=true
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v6
+        if: matrix.type == 'prod'
+        with:
+          context: .
+          build-args: BUILD_BRANCH=${{ env.BUILD_BRANCH }}
+          load: true
+          file: ./docker/Dockerfile-${{ matrix.type }}
+          tags: ${{ env.TEST_TAG }}
+
+      # test the prod image with one application using the database
+      # and the sim_telarray installation
+      - name: Test Docker image
+        if: matrix.type == 'prod'
+        run: >
+          # Clone simulation model repo for faster file access
+          git clone https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/model_parameters.git
+          # run docker
+          docker run --rm
+          -e "SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray'"
+          -e "SIMTOOLS_DB_API_PORT=${{ env.SIMTOOLS_DB_API_PORT }}"
+          -e "SIMTOOLS_DB_API_USER=${{ env.SIMTOOLS_DB_API_USER }}"
+          -e "SIMTOOLS_DB_API_PW=${{ env.SIMTOOLS_DB_API_PW }}"
+          -e "SIMTOOLS_DB_SERVER=${{ env.SIMTOOLS_DB_SERVER }}"
+          -e "SIMTOOLS_DB_SIMULATION_MODEL_URL='./model_parameters'"
+          -v "$(pwd):/workdir/external" ${{ env.TEST_TAG }}
+          bash -c "cd /workdir/external && simtools-simulate-prod --config /workdir/external/tests/integration_tests/config/simulate_prod_gamma_20_deg_pack_for_grid.yml"
+        shell: /usr/bin/bash -e {0}
+
+      - name: Test Command Line Tools
+        if: matrix.type == 'prod'
+        run: |
+          set -e
+          echo "Reading scripts defined in pyproject.toml"
+          ls -l pyproject.toml
+          scripts=$(grep 'scripts.simtools-' pyproject.toml | sed -e 's/^scripts\.//' -e 's/ =.*$//')
+          for script in $scripts; do
+            echo "Testing command: $script --help"
+            docker run --rm ${{ env.TEST_TAG }} $script --help
+          done
+        shell: /usr/bin/bash -e {0}
+
+      - name: Push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          build-args: BUILD_BRANCH=${{ env.BUILD_BRANCH }}
+          platforms: linux/amd64,linux/arm64/v8
+          push:
+            ${{ github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
+          file: ./docker/Dockerfile-${{ matrix.type }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}-${{ matrix.type }}

gammasimtools-0.9.0/.gitignore

@@ -0,0 +1,37 @@
+# Compiled files
+*.py[co]
+*.a
+*.o
+*.so
+__pycache__
+
+# Sphinx
+docs/build
+
+# environmental variables
+.env
+
+# Mac users
+.DS_Store
+
+telarray_rand.conf*
+
+# Distribution / packaging
+build/
+*.egg-info/
+.egg
+
+# Generic output path of
+# simtools data products
+simtools-output
+
+# Dynamic version file
+# generated by setuptools_scm
+src/simtools/_version.py
+
+# Coverage files
+coverage*
+.coverage*
+
+# Ignore temporary output of database scripts
+database_scripts/mongo-data

gammasimtools-0.9.0/.pre-commit-config.yaml

@@ -0,0 +1,74 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 24.10.0
+    hooks:
+      - id: black
+        args: ["--line-length=100"]
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff
+    rev: v0.9.2
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+  # pylint
+  - repo: local
+    hooks:
+      - id: pylint
+        name: pylint
+        entry: pylint
+        language: system
+        types: [python]
+        exclude: ^tests/|^simtools/applications/db_development_tools/
+        args:
+          [
+            "-rn",  # Only display messages
+            "-sn",  # Don't display the score
+          ]
+  # https://github.com/pre-commit/pre-commit-hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-added-large-files
+        args: ['--maxkb=500']
+  # https://github.com/HunterMcGushion/docstr_coverage
+  - repo: https://github.com/HunterMcGushion/docstr_coverage
+    rev: v2.3.2  # most recent docstr-coverage release or commit sha
+    hooks:
+      - id: docstr-coverage
+        args: ["--verbose", "2", "--fail-under", "70.", "src/simtools"]
+  # Github action
+  - repo: https://github.com/rhysd/actionlint
+    rev: v1.7.7
+    hooks:
+      - id: actionlint
+  # https://pyproject-fmt.readthedocs.io/en/latest/
+  - repo: https://github.com/tox-dev/pyproject-fmt
+    rev: "v2.5.0"
+    hooks:
+      - id: pyproject-fmt
+  # codespell
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.3.0
+    hooks:
+      - id: codespell
+  # markdownlint
+  - repo: https://github.com/igorshubovych/markdownlint-cli
+    rev: v0.43.0
+    hooks:
+      - id: markdownlint
+        args: ["--disable", "MD041"]
+  # yamllint
+  - repo: https://github.com/adrienverge/yamllint.git
+    rev: v1.35.1
+    hooks:
+      - id: yamllint
+        args: ["--strict", "-d", '{rules: {line-length: {max: 250}}}']
+  # towncrier
+  - repo: https://github.com/twisted/towncrier
+    rev: 24.8.0
+    hooks:
+      - id: towncrier-update
+        files: $docs/changes/
+        args: ['--keep']