gammasimtools 0.5.1__tar.gz → 0.6.1__tar.gz

gammasimtools-0.6.1/.env_template

@@ -0,0 +1,13 @@
+# Template file for environment variables settings
+# Copy this file to '.env' and add the required
+# credentials for simulation model database access
+SIMTOOLS_DB_API_PORT=27017 #Port on the MongoDB server
+SIMTOOLS_DB_SERVER='cta-simpipe-protodb.zeuthen.desy.de' # MongoDB server
+SIMTOOLS_DB_API_USER=YOUR_USERNAME # username for MongoDB: ask the responsible person
+SIMTOOLS_DB_API_PW=YOUR_PASSWORD # Password for MongoDB: ask the responsible person
+SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin'
+SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray'
+
+# The dashboards to monitor the MongoDB instance are in (accessible only from within DESY)
+# https://statspub.zeuthen.desy.de/d/4vXnWwMGz/mongodb?orgId=1&refresh=30s
+# https://statspub.zeuthen.desy.de/d/tBkrQGNmz/mongodb-wiredtiger?orgId=1&refresh=1m

{gammasimtools-0.5.1 → gammasimtools-0.6.1}/.github/workflows/CI-docs.yml

@@ -14,31 +14,42 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Python 3.10.4
-        uses: actions/setup-python@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.10.4
+          python-version: "3.11"
 
       - name: Python dependencies
         run: |
-          python --version
           pip install '.[doc]'
 
+      - name: Check for non-ASCII characters
+        run: |
+          output=$(perl -ne 'print if /[^[:ascii:]]/' docs/source/*)
+          if [ -n "$output" ]; then
+            echo "Non-ASCII characters found in documentation."
+            exit 1
+          fi
+
       - name: Build docs
         shell: bash -l {0}
         run: |
           make -C docs/ html
+          exit_code=$?
+          if [ $exit_code -ne 0 ]; then
+            echo "Make command failed with exit code $exit_code"
+            exit $exit_code
+          fi
           touch docs/build/html/.nojekyll
 
-      - name: Deploy to gihub pages
+      - name: Deploy to github pages
         # only run when PR is merged
         if: github.event.pull_request.merged
         uses: JamesIves/github-pages-deploy-action@v4
         with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TOKEN: ${{ secrets.GITHUB_TOKEN }}
           BRANCH: gh-pages
           FOLDER: docs/build/html
           CLEAN: true
-          SINGLE_COMMIT: true

gammasimtools-0.6.1/.github/workflows/CI-integrationtests.yml

@@ -0,0 +1,45 @@
+---
+name: CI-integrationtests
+# Integration tests for applications
+
+env:
+  SIMTOOLS_DB_SERVER: ${{ secrets.DB_SERVER }}
+  SIMTOOLS_DB_API_USER: ${{ secrets.DB_API_USER }}
+  SIMTOOLS_DB_API_PW: ${{ secrets.DB_API_PW }}
+  SIMTOOLS_DB_API_PORT: ${{ secrets.DB_API_PORT }}
+  SIMTOOLS_SIMTEL_PATH: "/workdir/sim_telarray/"
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+
+  integrationtests:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/gammasim/simtools-dev:latest
+      options: --user 0
+
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
+    steps:
+
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Set PATH
+        run: |
+          echo "PATH=\$PATH:/usr/bin:/usr/local/bin:$SIMTOOLS_SIMTEL_PATH" >> $GITHUB_ENV
+
+      - name: Run integration tests
+        shell: bash -l {0}
+        run: |
+          source /workdir/env/bin/activate
+          pip install '.[tests,dev,doc]'
+          export PYTHONPATH=$PYTHONPATH:.
+          pytest --color=yes --durations=10 --no-cov tests/integration_tests/

gammasimtools-0.6.1/.github/workflows/CI-linter.yml

@@ -0,0 +1,136 @@
+---
+name: CI-linting
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+
+  linting:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
+    permissions:
+      contents: read
+      packages: read
+      statuses: write
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Check for non-ASCII characters
+        run: |
+          output=$(find . -type f \
+          \( -name "*.py" -o -name "*.rst" -o -name "*.yml" -o -name "*.toml" \) \
+          -exec perl -ne 'print if /[^[:ascii:]]/' {} \;)
+          if [ -n "$output" ]; then
+            echo "Non-ASCII characters found in documentation."
+            exit 1
+          fi
+
+      - name: install packages not included in super-linter
+        run: |
+          pip install validate-pyproject restructuredtext-lint
+
+      - name: pyproject.toml
+        run: |
+          validate-pyproject pyproject.toml
+
+      # RST linter
+      # Note: unclear how to suppress error messages
+      # (use grep -v in this case)
+    #  - name: restructuredtext-lint
+    #    run: |
+    #      rst-lint README.rst docs/source |
+    #       grep -v "Unknown directive type" |
+    #       grep -v "Unknown interpreted text role" |
+    #       grep -v "Cannot analyze code. Pygments package not found."
+
+      - name: Check whether the citation metadata from CITATION.cff is valid
+        uses: citation-file-format/cffconvert-github-action@2.0.0
+        with:
+          args: "--validate"
+
+      - name: yaml_config used by super-linter
+        run: |
+          # TODO - very large line length
+          echo 'rules:' > yaml_config.yaml
+          echo '  line-length:' >> yaml_config.yaml
+          echo '    max: 250' >> yaml_config.yaml
+
+      # Dependencies required to avoid errors
+      # reported by linters
+      - name: Install mamba dependencies
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: environment.yml
+          init-shell: bash
+
+      - name: Python dependencies
+        run: |
+          pip install -e '.[tests,dev,doc]'
+          pip install pylint
+
+      - name: pylint
+        run: |
+          pylint $(git ls-files 'simtools/*.py')
+
+
+      - name: Lint Code Base
+        uses: super-linter/super-linter@v5
+        env:
+          VALIDATE_ALL_CODEBASE: false
+          # github actions
+          VALIDATE_GITHUB_ACTIONS: true
+          # yaml
+          VALIDATE_YAML: true
+          YAML_CONFIG_FILE: yaml_config.yaml
+          YAML_ERROR_ON_WARNING: false
+          # isort
+          VALIDATE_PYTHON_ISORT: true
+          PYTHON_ISORT_CONFIG_FILE: pyproject.toml
+          # flake8
+          VALIDATE_PYTHON_FLAKE8: true
+          # black
+          VALIDATE_PYTHON_BLACK: true
+          PYTHON_BLACK_CONFIG_FILE: pyproject.toml
+          # markdown
+          VALIDATE_MARKDOWN: true
+          # docker
+          VALIDATE_DOCKERFILE_HADOLINT: true
+          # copy and paste
+          VALIDATE_JSCPD_ALL_CODEBASE: true
+          # .env file
+          VALIDATE_ENV: true
+          # language
+          VALIDATE_NATURAL_LANGUAGE: true
+          # bash
+          VALIDATE_BASH: true
+          # path for linter rules
+          LINTER_RULES_PATH: ./
+          # create a log file
+          CREATE_LOG_FILE: true
+          LOG_FILE: superlinter.log
+          DEFAULT_BRANCH: main
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      # Upload super-linter log file
+      # and keep it for 5 days
+      - name: Archive production artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: super-linter reports
+          path: |
+             superlinter.log
+             retention-days: 5

{gammasimtools-0.5.1 → gammasimtools-0.6.1}/.github/workflows/CI-unittests.yml

@@ -1,20 +1,18 @@
 ---
 name: CI-unittests
-# Execute unit tests - pyflakes and pytest
+# Execute unit tests - flake8 and pytest
 #
 env:
-  DB_SERVER: ${{ secrets.DB_SERVER }}
-  DB_API_USER: ${{ secrets.DB_API_USER }}
-  DB_API_PW: ${{ secrets.DB_API_PW }}
-  DB_API_PORT: ${{ secrets.DB_API_PORT }}
+  SIMTOOLS_DB_SERVER: ${{ secrets.DB_SERVER }}
+  SIMTOOLS_DB_API_USER: ${{ secrets.DB_API_USER }}
+  SIMTOOLS_DB_API_PW: ${{ secrets.DB_API_PW }}
+  SIMTOOLS_DB_API_PORT: ${{ secrets.DB_API_PORT }}
 
 on:
   pull_request:
-    branches: [main]
     types: [opened, synchronize]
-  push:
-    branches:
-      - main
+  schedule:
+    - cron: "0 0 * * *"
 
 jobs:
 
@@ -27,29 +25,14 @@ jobs:
             python-version: "3.9"
             install-method: mamba
 
-          - os: ubuntu-latest
-            python-version: "3.9"
-            install-method: pip
-
-          - os: macos-latest
-            python-version: "3.9"
-            install-method: mamba
-
           - os: macos-latest
             python-version: "3.9"
-            install-method: pip
-
-          - os: ubuntu-latest
-            python-version: "3.10"
             install-method: mamba
 
-          - os: ubuntu-latest
-            python-version: "3.10"
-            install-method: pip
-
           - os: ubuntu-latest
             python-version: "3.11"
             install-method: mamba
+            extra-args: ["codecov"]
 
           - os: ubuntu-latest
             python-version: "3.11"
@@ -61,10 +44,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -86,7 +69,7 @@ jobs:
 
       - name: Python setup
         if: matrix.install-method == 'pip'
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           check-latest: true
@@ -97,17 +80,27 @@ jobs:
         run: |
           tee -a ~/.bash_profile <<<'export PATH="$pythonLocation/bin:$PATH"'
 
-      - name: Python dependencies
+      - name: Python dependencies (pip)
+        if: matrix.install-method == 'pip'
         run: |
-          python --version
           pip install '.[tests,dev,doc]'
 
-      - name: pyflakes liniting
+      - name: Python dependencies (mamba)
+        if: matrix.install-method == 'mamba'
+        run: |
+          pip install -e '.[tests,dev,doc]'
+
+      - name: flake8
         run: |
-          pyflakes .
+          flake8 --max-line-length 100 --per-file-ignores="_version.py:F401 E501" .
 
       - name: Unit tests
         shell: bash -l {0}
-        run: pytest --durations=10 --color=yes --cov --cov-report=xml
+        run: |
+          pytest --durations=10 --color=yes
 
-      - uses: codecov/codecov-action@v3.1.4
+      - name: Upload coverage report to codecov
+        uses: codecov/codecov-action@v3
+        if: contains(matrix.extra-args, 'codecov')
+        with:
+          token: ${{ secrets.CODECOV_SIMTOOLS_TOKEN }}

gammasimtools-0.5.1/.github/workflows/build-docker-images.yml → gammasimtools-0.6.1/.github/workflows/build-docker-corsika-simtelarray-image.yml

@@ -1,15 +1,16 @@
 # https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
-name: docker-simtools
+name: docker-corsika-simtelarray
 
 on:
   workflow_dispatch:
+  pull_request:
+    paths:
+      - './docker/Dockerfile-simtelarray'
   push:
     branches:
       - main
     tags:
       - 'v*'
-  pull_request:
-    branches: ["main"]
   release:
     types: [published]
   schedule:
@@ -21,27 +22,32 @@ env:
 
 jobs:
 
-  build-simtools-container:
+  corsika-simtelarray-container:
     runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
     strategy:
       matrix:
-        type: ['dev', 'prod']
+        type: ['corsika_sim_telarray']
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: download corsikasimtelpackage
+        run: |
+          wget --no-verbose https://syncandshare.desy.de/index.php/s/${{ secrets.CLOUD_SIMTEL }}/download
+          mv download corsika7.7_simtelarray.tar.gz
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -49,7 +55,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           tags: |
             type=ref,event=pr
@@ -60,12 +66,12 @@ jobs:
           flavor: latest=true
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/amd64,linux/arm64/v8
           push:
-            ${{ github.event_name == 'pull_request' || github.event_name == 'release' || github.event_name == 'workflow_dispatch' }}
-          file: ./docker/Dockerfile-${{ matrix.type }}
+            ${{ github.event_name == 'release' || github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' }}
+          file: ./docker/Dockerfile-simtelarray
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}-${{ matrix.type }}

gammasimtools-0.6.1/.github/workflows/build-docker-images.yml

@@ -0,0 +1,108 @@
+# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
+name: docker-simtools
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*'
+  pull_request:
+    branches: ["main"]
+    types: [review_requested, ready_for_review]
+  release:
+    types: [published]
+  schedule:
+    - cron: '0 0 * * 0'  # Every Sunday at 00:00 UTC
+
+env:
+  SIMTOOLS_DB_SERVER: ${{ secrets.DB_SERVER }}
+  SIMTOOLS_DB_API_USER: ${{ secrets.DB_READ_USER }}
+  SIMTOOLS_DB_API_PW: ${{ secrets.DB_READ_PW }}
+  SIMTOOLS_DB_API_PORT: ${{ secrets.DB_API_PORT }}
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+  TEST_TAG: user/app:test
+
+jobs:
+
+  build-simtools-container:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      matrix:
+        type: ['dev', 'prod']
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set build arguments
+        run: |
+          if [[ "${{ github.event_name }}" == 'pull_request' ]]; then
+            echo "BUILD_BRANCH=${GITHUB_HEAD_REF}" >> "$GITHUB_ENV"
+          else
+            echo "BUILD_BRANCH=${GITHUB_REF#refs/heads/}" >> "$GITHUB_ENV"
+          fi
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          tags: |
+            type=ref,event=pr
+            type=semver,pattern={{major}}.{{minor}}.{{patch}}
+            type=schedule,pattern={{date 'YYYYMMDD'}}
+            type=raw,value={{date 'YYYYMMDD-HHmmss'}}
+          images: ${{ env.REGISTRY }}/gammasim/simtools-${{ matrix.type }}
+          flavor: latest=true
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v5
+        if: matrix.type == 'prod'
+        with:
+          context: .
+          build-args: BUILD_BRANCH=${{ env.BUILD_BRANCH }}
+          load: true
+          file: ./docker/Dockerfile-${{ matrix.type }}
+          tags: ${{ env.TEST_TAG }}
+
+      # test the prod image with one application using the database
+      # and the sim_telarray installation
+      - name: Test Docker image
+        if: matrix.type == 'prod'
+        run: >
+          docker run --rm
+          -e "SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray'"
+          -e "SIMTOOLS_DB_API_PORT=${{ env.SIMTOOLS_DB_API_PORT }}"
+          -e "SIMTOOLS_DB_API_USER=${{ env.SIMTOOLS_DB_API_USER }}"
+          -e "SIMTOOLS_DB_API_PW=${{ env.SIMTOOLS_DB_API_PW }}"
+          -e "SIMTOOLS_DB_SERVER=${{ env.SIMTOOLS_DB_SERVER }}" -v "$(pwd):/workdir/external" ${{ env.TEST_TAG }}
+          bash -c "cd /workdir/external && simtools-simulate-prod --config /workdir/external/tests/integration_tests/config/simulate_prod_gamma_20_deg_pack_for_grid.yml"
+        shell: /usr/bin/bash -e {0}
+
+      - name: Push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64/v8
+          push:
+            ${{ github.event_name == 'release' || github.ref == 'refs/heads/main' }}
+          file: ./docker/Dockerfile-${{ matrix.type }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}-${{ matrix.type }}

{gammasimtools-0.5.1 → gammasimtools-0.6.1}/.github/workflows/pypi.yml

@@ -10,15 +10,15 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       # make sure we have version info
       - run: git fetch --tags
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.9"
+          python-version: "3.11"
 
       - name: Install dependencies
         run: |

{gammasimtools-0.5.1 → gammasimtools-0.6.1}/.gitignore

@@ -8,11 +8,8 @@ __pycache__
 # Sphinx
 docs/build
 
-# DB config gfile
-dbDetails.yml
-
-# set environment
-set_env_db.sh
+# environmental variables
+.env
 
 # Mac users
 .DS_Store
@@ -31,3 +28,7 @@ simtools-output
 # Dynamic version file
 # generated by setuptools_scm
 simtools/_version.py
+
+# Coverage files
+coverage*
+.coverage*

{gammasimtools-0.5.1 → gammasimtools-0.6.1}/.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
       - id: isort
         args: ["--profile", "black", "--filter-files"]
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 23.11.0
     hooks:
       - id: black
         args: ["--line-length=100"]
@@ -15,16 +15,39 @@ repos:
     rev: 6.1.0
     hooks:
       - id: flake8
-        args: ["--max-line-length=100", "--extend-ignore=E203,E712"]
+        args: ["--max-line-length=100"]
   # https://github.com/pre-commit/pre-commit-hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
+  - repo: local
+    hooks:
+      - id: pylint
+        name: pylint
+        entry: pylint
+        language: system
+        types: [python]
+        exclude: ^tests/|^simtools/applications/db_development_tools/
+        args:
+          [
+            "-rn",  # Only display messages
+            "-sn",  # Don't display the score
+          ]
   # https://github.com/HunterMcGushion/docstr_coverage
   - repo: https://github.com/HunterMcGushion/docstr_coverage
-    rev: v2.3.0 # most recent docstr-coverage release or commit sha
+    rev: v2.3.0  # most recent docstr-coverage release or commit sha
     hooks:
       - id: docstr-coverage
         args: ["--verbose", "2", "--fail-under", "70.", "simtools"]
+  # gitup action
+  - repo: https://github.com/rhysd/actionlint
+    rev: v1.6.26
+    hooks:
+      - id: actionlint
+  # https://pyproject-fmt.readthedocs.io/en/latest/
+  - repo: https://github.com/tox-dev/pyproject-fmt
+    rev: "1.0.0"
+    hooks:
+      - id: pyproject-fmt

{gammasimtools-0.5.1 → gammasimtools-0.6.1}/CITATION.cff

@@ -18,6 +18,11 @@ authors:
     family-names: Gueta
     orcid: 'https://orcid.org/0000-0002-9440-2398'
     affiliation: DESY
+  - given-names: Tobias
+    name-particle: K.
+    family-names: Kleiner
+    orcid: 'https://orcid.org/0000-0002-4260-9186'
+    affiliation: DESY
   - given-names: Victor
     family-names: Barbosa Martins
     orcid: 'https://orcid.org/0000-0002-5085-8828'