ctao-datapipe 0.1.0__tar.gz

ctao_datapipe-0.1.0/.gitignore

@@ -0,0 +1,133 @@
+# setuptools_scm generated version file
+_version.py
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+docs/api
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

ctao_datapipe-0.1.0/.gitlab-ci.yml

@@ -0,0 +1,83 @@
+include:
+  - project: 'cta-computing/dpps/aiv/dpps-aiv-toolkit'
+    ref: dd87c1aaf379ece916a8e11c069e79d07b521b56
+    file: "ci-functions.yml"
+  - "aiv-config.yml"
+
+stages:
+  - prepare
+  - lint
+  - build
+  - sign
+  - tests
+  - sonarqube
+  - publish
+  - report
+
+
+#  full test for report, including summary and coverage xml artifacts
+test:
+  stage: tests
+  image: harbor.cta-observatory.org/proxy_cache/python:3.12
+
+  before_script:
+    - python --version
+    - pip install -e .[test]
+    - apt update --yes && apt install --yes nodejs
+
+  script:
+    - >-
+      pytest -v --color=yes
+      --doctest-modules --doctest-glob='docs/**/*.rst'
+      --ignore-glob='src/datapipe/_dev_version/*'
+      --cov --cov-report=xml:test/coverage.xml
+      --junitxml=test/report.xml
+      --log-cli-level=INFO
+
+  after_script: []
+
+  artifacts:
+    paths:
+      - test/coverage.xml
+      - test/report.xml
+
+
+collect-test-artifacts:
+  needs:
+    - job: collect-manual-tests
+      artifacts: true
+    - job: test
+      artifacts: true
+
+# unit tests using a matrix of python versions
+unittests:
+  stage: tests
+  parallel:
+    matrix:
+      - PYTHON_VERSION:
+        - "3.11"
+        - "3.12"
+        - "3.13"
+
+  image: "harbor.cta-observatory.org/proxy_cache/python:$PYTHON_VERSION"
+
+  before_script:
+    - apt update --yes
+    - apt install --yes nodejs # needed for cwl with JS
+    - python --version
+    - pip install -e .[test]
+
+  script:
+    - >-
+      pytest -v --color=yes
+      --doctest-modules
+      --doctest-glob='docs/**/*.rst' --ignore-glob='src/datapipe/_dev_version/*'
+
+
+k8s-integration-tests:
+  rules:
+    - when: never
+
+publish-chart:
+  rules:
+    - when: never

ctao_datapipe-0.1.0/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "dpps-aiv-toolkit"]
+	path = dpps-aiv-toolkit
+	url = ../../aiv/dpps-aiv-toolkit.git

ctao_datapipe-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,42 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0  # Use the ref you want to point at
+    hooks:
+    - id: trailing-whitespace
+      exclude: ".*\\.fits(\\.fz)?"
+    - id: check-added-large-files
+    - id: check-case-conflict
+    - id: check-merge-conflict
+      exclude: ".*\\.fits(\\.fz)?"
+    - id: end-of-file-fixer
+      exclude: ".*\\.fits(\\.fz)?"
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.1
+    hooks:
+    - id: codespell
+      additional_dependencies:
+        - tomli
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.5
+    hooks:
+      - id: ruff
+        args: [ --fix, --show-fixes ]
+      - id: ruff-format
+
+  - repo: https://github.com/scientific-python/cookie
+    rev: "2025.01.22"
+    hooks:
+      - id: sp-repo-review
+
+  - repo: local
+    hooks:
+    - id: CI yaml check
+      args: [ dpps-aiv-toolkit ]
+      name: CI yaml check
+      description: CI yaml check
+      language: script
+      entry: dpps-aiv-toolkit/cicheck.sh
+      always_run: true
+      pass_filenames: false

ctao_datapipe-0.1.0/AUTHORS

@@ -0,0 +1,34 @@
+Karl Kosack <karl.kosack@cta-observatory.org>
+Maximilian Linhoff <maximillian.linhoff@cta-observatory.org>
+Thierry Stolarczyk <thierry.stolarczyk@cta-consortium.org>
+Abelardo Moralejo Olaizola <abelardo.moralejo@cta-consortium.org>
+Armelle Jardin-Blicq <armelle.jardin-blicq@cta-consortium.org>
+Clara Escanuela <clara.escanuelanieves@cta-consortium.org>
+Daniel Morcuende-Parrilla <daniel.morcuende@cta-observatory.org>
+Daniel Parsons <daniel.parsons@cta-consortium.org>
+David Berge <david.berge@cta-consortium.org>
+David Sanchez <david.sanchez@cta-consortium.org>
+Denis Dumora <denis.dumora@cta-consortium.org>
+Dominik Elsässer <dominik.elsasser@cta-consortium.org>
+Franca Cassol <franca.cassol@cta-consortium.org>
+Gabriel Emery <gabriel.emery@cta-consortium.org>
+Georg Schwefer <georg.schwefer@cta-consortium.org>
+Gilles Maurin <gilles.maurin@cta-consortium.org>
+Heide Costantini <heide.costantini@cta-consortium.org>
+Ivan Agudo <ivan.agudo@cta-consortium.org>
+Jim Hinton <jim.hinton@cta-observatory.org>
+Jonas Hackfeld <jonas.hackfeld@cta-consortium.org>
+Jose Contreras <jose.contreras@cta-consortium.org>
+Lukas Beiske <lukas.beiske@cta-consortium.org>
+Lukas Nickel <lukas.nickel@cta-consortium.org>
+Marianne Lemoine-Goumard <marianne.lemoine-goumard@cta-consortium.org>
+Mathieu de Bony <mathieu.debony@cta-consortium.org>
+Rubén López Coto <ruben.lopez@cta-consortium.org>
+Rune Dominik <rune.dominik@cta-consortium.org>
+Ruo Yu Shang <ruoyu.shang@cta-consortium.org>
+Stefan Fröse <stefan.frose@cta-consortium.org>
+Thomas Vuillaume <thomas.vuillaume@cta-consortium.org>
+Tjark Miener <tjark.miener@cta-consortium.org>
+Tomas Bylund <tomas.bylund@cta-consortium.org>
+Vincent Pollet <vincent.pollet@cta-consortium.org>
+Weidong Jin <weidong.jin@cta-consortium.org>

ctao_datapipe-0.1.0/CHANGES.rst

@@ -0,0 +1,9 @@
+datapipe v0.1.0-rc1 (2025-04-17)
+--------------------------------
+
+This is the first (pre)release of the datapipe package.
+
+New Features
+~~~~~~~~~~~~
+
+- CWL workflows covering UC-DPPS-130-1.2, 1.2.1, and 1.2.2

ctao_datapipe-0.1.0/Dockerfile

@@ -0,0 +1,33 @@
+# See https://testdriven.io/blog/docker-best-practices/ for recommendations
+# on writing dockerfiles for python projects.
+
+# We use a build stage to build a wheel which we then copy and install
+# in the second stage to minimize image size. This is mostly needed
+# because setuptools_scm needs the full version info from git and git
+# itself but including that in the final image would bloat its size.
+
+# we are using the official python image with just a version tag here
+# it comes with many tools needed to build and compile python packages
+# which increases it's size but is helpful for building
+FROM python:3.12 AS builder
+
+# add necessary sources, including .git for version info
+COPY . /repo/
+
+# build the wheel
+RUN python -m pip install --no-cache-dir build \
+    && cd repo \
+    && python -m build --wheel
+
+
+# second stage, copy and install wheel
+# We are using the official python 3.12 image
+# as base image in the slim variant to reduce image size.
+FROM python:3.12-slim
+COPY --from=builder /repo/dist /tmp/dist
+
+RUN python -m pip install --no-cache-dir /tmp/dist/* \
+    && rm -r /tmp/dist
+
+RUN useradd --create-home --system --user-group ctao
+USER ctao

ctao_datapipe-0.1.0/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2022, Cherenkov Telescope Array Observatory Consortium
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ctao_datapipe-0.1.0/MANIFEST.in

@@ -0,0 +1,2 @@
+prune src/datapipe/_dev_version
+prune .github

ctao_datapipe-0.1.0/PKG-INFO

@@ -0,0 +1,51 @@
+Metadata-Version: 2.4
+Name: ctao-datapipe
+Version: 0.1.0
+Summary: CTAO Data Processing Pipeline Software
+Maintainer-email: Karl Kosack <karl.kosack@cta-observatory.org>, Maximilian Linhoff <maximilian.linhoff@cta-observatory.org>
+License-Expression: BSD-3-Clause
+Project-URL: repository, https://gitlab.cta-observatory.org/cta-computing/dpps/datapipe/datapipe
+Project-URL: documentation, http://cta-computing.gitlab-pages.cta-observatory.org/dpps/datapipe/datapipe
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: AUTHORS
+Requires-Dist: ctapipe~=0.25.0
+Requires-Dist: ctapipe-io-zfits~=0.2.0
+Requires-Dist: pyirf~=0.12
+Requires-Dist: eventio~=1.15.0
+Requires-Dist: iminuit>=2
+Requires-Dist: matplotlib~=3.0
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-requirements; extra == "test"
+Requires-Dist: cwltool; extra == "test"
+Provides-Extra: doc
+Requires-Dist: sphinx; extra == "doc"
+Requires-Dist: numpydoc; extra == "doc"
+Requires-Dist: ctao-sphinx-theme~=0.1.2; extra == "doc"
+Requires-Dist: sphinx-changelog; extra == "doc"
+Provides-Extra: dev
+Requires-Dist: setuptools_scm; extra == "dev"
+Requires-Dist: towncrier; extra == "dev"
+Provides-Extra: all
+Requires-Dist: ctao-datapipe[dev,doc,test]; extra == "all"
+Dynamic: license-file
+
+# CTAO Data Processing Pipeline Subsystem
+
+This package provides the software and workflow definitions for the [CTAO][1]
+_Data Processing and Preservation System_ (DPPS) **Data Processing Pipeline**
+subsystem (DataPipe). It is responsible for transforming raw (DL0) observed and
+simulated data products produced by the observatory into "science-ready" data
+products (DL3), including reconstructed event lists and instrumental response
+functions.
+
+- [Documentation][2]
+- [Code Repository][3]
+- [PyPI: ctao-datapipe](https://pypi.org/project/ctao-datapipe/)
+
+[1]: https://www.ctao.org "CTAO website"
+[2]: http://cta-computing.gitlab-pages.cta-observatory.org/dpps/datapipe/datapipe/latest/ "DataPipe documentation"
+[3]: https://gitlab.cta-observatory.org/cta-computing/dpps/datapipe/datapipe "DataPipe code"

ctao_datapipe-0.1.0/README.md

@@ -0,0 +1,16 @@
+# CTAO Data Processing Pipeline Subsystem
+
+This package provides the software and workflow definitions for the [CTAO][1]
+_Data Processing and Preservation System_ (DPPS) **Data Processing Pipeline**
+subsystem (DataPipe). It is responsible for transforming raw (DL0) observed and
+simulated data products produced by the observatory into "science-ready" data
+products (DL3), including reconstructed event lists and instrumental response
+functions.
+
+- [Documentation][2]
+- [Code Repository][3]
+- [PyPI: ctao-datapipe](https://pypi.org/project/ctao-datapipe/)
+
+[1]: https://www.ctao.org "CTAO website"
+[2]: http://cta-computing.gitlab-pages.cta-observatory.org/dpps/datapipe/datapipe/latest/ "DataPipe documentation"
+[3]: https://gitlab.cta-observatory.org/cta-computing/dpps/datapipe/datapipe "DataPipe code"

ctao_datapipe-0.1.0/aiv-config.yml

@@ -0,0 +1,13 @@
+variables:
+  # used to select requirements
+  DPPS_UC_GROUPS: DataPipe
+  DPPS_RELEASE: v0.1
+
+  RELEASE_PLAN_FN: dpps-aiv-toolkit/dpps-release-plan/release_development_document/dpps-release-plan.tex
+  TRACEABILITY_CSV_FN: dpps-aiv-toolkit/requirements-traceability/DPPS_Requirements_Traceability_Matrix.csv
+
+  APPLICATION_NAME: DataPipe
+  APPLICATION_AUTHOR: DataPipe Team
+  APPLICATION_AUTHOR_ORGANIZATION: CTAO
+  DOCKER_IMAGE_CONTEXT: $CI_PROJECT_DIR
+  #EXTRA_CONFIG_FILES: aiv-config-dependencies.yml

ctao_datapipe-0.1.0/docs/Makefile

@@ -0,0 +1,23 @@
+# Minimal makefile for Sphinx documentation
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?= -W --keep-going -n --color
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile clean
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+clean:
+	rm -rf docs/api
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

ctao_datapipe-0.1.0/docs/_static/custom.css

@@ -0,0 +1 @@
+@import url('https://fonts.googleapis.com/css2?family=Inter:ital,opsz,wght@0,14..32,100..900;1,14..32,100..900&family=Space+Grotesk:wght@300..700&display=swap');

ctao_datapipe-0.1.0/docs/changelog.rst

@@ -0,0 +1,7 @@
+Change Log
+==========
+
+.. changelog::
+   :towncrier: ../
+   :towncrier-skip-if-empty:
+   :changelog_file: ../CHANGES.rst

ctao_datapipe-0.1.0/docs/changes/22.maintenance.rst

@@ -0,0 +1,2 @@
+- Update docker URL in the installation page of the documentation to point to the CTAO Harbor, where the image is deployed
+- Simplify the README so that it is appropriate for PyPI.

ctao_datapipe-0.1.0/docs/changes/README.md

@@ -0,0 +1,22 @@
+# How to use towncrier
+
+An overview can be found [here](https://towncrier.readthedocs.io/en/stable/quickstart.html#).
+
+1. Create a new file for your changes `<MERGE REQUEST NUMBER>.<TYPE>.rst` in the corresponding folder.
+   The following types are available:
+    - feature: `New feature`
+    - bugfix: `Bugfix`
+    - api: `API Changes`
+    - optimization: `Refactoring and Optimization`
+    - maintenance: `Maintenance`
+
+2. Write a suitable message for the change:
+    ```
+    Fixed ``crazy_function`` to be consistent with ``not_so_crazy_function``
+    ```
+
+3. (For maintainers) How to generate a change log:
+    Execute the following command in the base directory of the project
+    ```
+    towncrier build --version=<VERSION NUMBER>
+    ```

ctao_datapipe-0.1.0/docs/changes/template.rst

@@ -0,0 +1,43 @@
+{% if render_title %}
+{% if versiondata.name %}
+{{ versiondata.name | lower }} {{ versiondata.version }} ({{ versiondata.date }})
+{{ top_underline * ((versiondata.name + versiondata.version + versiondata.date)|length + 4)}}
+{% else %}
+{{ versiondata.version }} ({{ versiondata.date }})
+{{ top_underline * ((versiondata.version + versiondata.date)|length + 3)}}
+{% endif %}
+{% endif %}
+
+{% for category, val in definitions.items() %}
+
+{% set underline = underlines[0] %}
+{{ definitions[category]['name'] }}
+{{ underline * definitions[category]['name']|length }}
+{% set underline = underlines[1] %}
+
+{% for section, _ in sections.items() %}
+{% if section and category in sections[section] %}
+{{section}}
+{{ underline * section|length }}
+
+{% endif %}
+{% if sections[section] and category in sections[section] %}
+{% if definitions[category]['showcontent'] %}
+{% for text, values in sections[section][category].items() %}
+- {{ text }} [{{ values|join(', ') }}]
+
+{% endfor %}
+{% else %}
+- {{ sections[section][category]['']|join(', ') }}
+
+{% endif %}
+{% if sections[section][category]|length == 0 %}
+No significant changes.
+
+{% else %}
+{% endif %}
+{% else %}
+{# No significant changes. #}
+{% endif %}
+{% endfor %}
+{% endfor %}

ctao_datapipe-0.1.0/docs/conf.py

@@ -0,0 +1,78 @@
+"""Configuration file for the Sphinx documentation builder."""
+
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+import datapipe
+
+project = "datapipe"
+copyright = "2023, Cherenkov Telescope Array Observatory gGmbH"
+author = "Karl Kosack"
+version = datapipe.__version__
+# The full version, including alpha/beta/rc tags.
+release = version
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx.ext.githubpages",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "numpydoc",
+    "sphinx_changelog",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = []
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["changes"]
+
+# have all links automatically associated with the right domain.
+default_role = "py:obj"
+
+
+# intersphinx allows referencing other packages sphinx docs
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3/", None),
+}
+
+# -- Options for HTML output -------------------------------------------------
+
+html_theme = "ctao"
+html_theme_options = dict(
+    navigation_with_keys=False,
+    # setup for displaying multiple versions, also see setup in .gitlab-ci.yml
+    switcher=dict(
+        json_url="http://cta-computing.gitlab-pages.cta-observatory.org/dpps/datapipe/datapipe/versions.json",  # noqa: E501
+        version_match="latest" if ".dev" in version else f"v{version}",
+    ),
+    navbar_center=["version-switcher", "navbar-nav"],
+    gitlab_url="https://gitlab.cta-observatory.org/cta-computing/dpps/datapipe/datapipe",
+    logo=dict(
+        image_light="_static/cta.png",
+        image_dark="_static/cta_dark.png",
+        alt_text="ctao-logo",
+        text=" | data<b>pipe</b>",
+    ),
+)
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = [
+    "_static",
+]
+html_css_files = [
+    "custom.css",
+]