triggerflow 0.1.12__py3-none-any.whl → 0.2.1__py3-none-any.whl

triggerflow/starter/{{ cookiecutter.repo_name }}/.gitlab-ci.yml

@@ -0,0 +1,56 @@
+image: continuumio/miniconda3
+
+stages:
+  - load
+  - preprocess
+  - train
+  - validate
+  - compile
+
+variables:
+  KEDRO_ENV: "base" # TODO: add production env
+
+before_script:
+  - eval "$(conda shell.bash hook)"
+  - conda env create --file=environment.yml
+  - conda activate triggerflow
+
+load_data:
+  stage: load
+  script:
+    - kedro run --pipeline=load_data
+  artifacts:
+    paths:
+      - data/02_loaded/
+
+preprocess_data:
+  stage: preprocess
+  script:
+    - kedro run --pipeline=data_processing
+  artifacts:
+    paths:
+      - data/03_preprocessed/
+
+train_model:
+  stage: train
+  script:
+    - kedro run --pipeline=model_training
+  artifacts:
+    paths:
+      - data/04_models/
+
+validate_model:
+  stage: validate
+  script:
+    - kedro run --pipeline=model_validation
+  artifacts:
+    paths:
+      - data/05_validation/
+
+compile:
+  stage: compile
+  script:
+    - kedro run --pipeline=compile
+  artifacts:
+    paths:
+      - data/06_compile/

triggerflow/starter/{{ cookiecutter.repo_name }}/README.md

@@ -0,0 +1,29 @@
+# {{ cookiecutter.project_name }}
+
+- conda env create --file=environment.yml
+- conda activate {{ cookiecutter.project_name }}
+
+## Data versioning
+When a dataset changes one can do (TODO: add this to pipeline to compare the hash):
+- dvc add data/01_raw/companies.csv
+- git add data/01_raw/companies.csv.dvc
+- git commit -m "Track dataset changes with DVC"
+
+## Run CI local
+- brew install gitlab-ci-local
+- gitlab-ci-local --list
+- gitlab-ci-local
+
+## ToDos:
+- move functionality of uhh_mlatl1 to pipeline
+- if case in base dataloader for classification or not
+- add model evaluation steps
+- automation of dvc in CI pipeline
+- move {{ cookiecutter.project_name }} meta data json to dvc
+- add linting and type checking
+- write tests
+- write out reporting / logging / plots etc.
+- track plots with dvc?
+- cross check pipeline afterwards with {{ cookiecutter.project_name }} team
+- make starter pipeline as template
+- add {{ cookiecutter.project_name }} model

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/README.md

@@ -0,0 +1,26 @@
+# What is this for?
+
+This folder should be used to store configuration files used by Kedro or by separate tools.
+
+This file can be used to provide users with instructions for how to reproduce local configuration with their own credentials. You can edit the file however you like, but you may wish to retain the information below and add your own section in the [Instructions](#Instructions) section.
+
+## Local configuration
+
+The `local` folder should be used for configuration that is either user-specific (e.g. IDE configuration) or protected (e.g. security keys).
+
+> *Note:* Please do not check in any local configuration to version control.
+
+## Base configuration
+
+The `base` folder is for shared configuration, such as non-sensitive and project-related configuration that may be shared across team members.
+
+WARNING: Please do not put access credentials in the base configuration folder.
+
+## Instructions
+
+
+
+
+## Need help?
+
+[Find out more about configuration from the Kedro documentation](https://docs.kedro.org/en/stable/kedro_project_setup/configuration.html).

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/catalog.yml

@@ -0,0 +1,84 @@
+{{ cookiecutter.python_package }}_meta_data:
+  filepath: data/01_raw/samples_dummy.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
+
+{{ cookiecutter.python_package }}_data:
+  sample_info: data/01_raw/samples_dummy.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.{{ cookiecutter.python_package }}_dataset.{{ cookiecutter.project_name }}Dataset
+
+{{ cookiecutter.python_package }}_meta_data_loaded:
+  filepath: data/02_loaded/{{ cookiecutter.python_package }}_meta_data.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
+
+{{ cookiecutter.python_package }}_data_loaded:
+  filepath: data/02_loaded/{{ cookiecutter.python_package }}_data.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_X_train:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_X_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_X_test:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_X_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_y_train:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_y_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_y_test:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_y_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+event_ids_train:
+  filepath: data/03_preprocessed/event_ids_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+event_ids_test:
+  filepath: data/03_preprocessed/event_ids_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+scaler:
+  filepath: data/03_preprocessed/scaler.pkl
+  type: pickle.PickleDataset
+
+train_model:
+  filepath: data/04_models/trained_model.pkl
+  type: pickle.PickleDataset
+
+training_history:
+  type: matplotlib.MatplotlibDataset
+  filepath: data/07_reporting/training_history.png
+  save_args:
+    format: png
+
+model_pred:
+  filepath: data/05_validation/model_pred.pkl
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_compile.yml

@@ -0,0 +1,14 @@
+# This is a boilerplate parameters config generated for pipeline 'compile'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html
+
+
+compile:
+  name: "munet"
+  ml_backend: "Keras"
+  compiler: "hls4ml"
+  mlflow_url: "https://mlflow-deploy-mflow.app.cern.ch"
+  compiler_config:
+    test: 123

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_data_processing.yml

@@ -0,0 +1,8 @@
+# This is a boilerplate parameters config generated for pipeline 'data_processing'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html
+
+random_state: 42
+test_size: 0.5

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_load_data.yml

@@ -0,0 +1,5 @@
+# This is a boilerplate parameters config generated for pipeline 'load_data'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_model_training.yml

@@ -0,0 +1,9 @@
+# This is a boilerplate parameters config generated for pipeline 'model_training'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html
+
+{{ cookiecutter.python_package }}_model:
+  hps:
+    name: "{{ cookiecutter.python_package }}"

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_model_validation.yml

@@ -0,0 +1,5 @@
+# This is a boilerplate parameters config generated for pipeline 'model_validation'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/catalog.yml

@@ -0,0 +1,84 @@
+{{ cookiecutter.python_package }}_meta_data:
+  filepath: data/01_raw/samples_dummy.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
+
+{{ cookiecutter.python_package }}_data:
+  sample_info: data/01_raw/samples_dummy.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.{{ cookiecutter.python_package }}_dataset.{{ cookiecutter.project_name }}Dataset
+
+{{ cookiecutter.python_package }}_meta_data_loaded:
+  filepath: data/02_loaded/{{ cookiecutter.python_package }}_meta_data.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
+
+{{ cookiecutter.python_package }}_data_loaded:
+  filepath: data/02_loaded/{{ cookiecutter.python_package }}_data.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_X_train:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_X_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_X_test:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_X_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_y_train:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_y_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_y_test:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_y_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+event_ids_train:
+  filepath: data/03_preprocessed/event_ids_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+event_ids_test:
+  filepath: data/03_preprocessed/event_ids_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+scaler:
+  filepath: data/03_preprocessed/scaler.pkl
+  type: pickle.PickleDataset
+
+train_model:
+  filepath: data/04_models/trained_model.pkl
+  type: pickle.PickleDataset
+
+training_history:
+  type: matplotlib.MatplotlibDataset
+  filepath: data/07_reporting/training_history.png
+  save_args:
+    format: png
+
+model_pred:
+  filepath: data/05_validation/model_pred.pkl
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_compile.yml

@@ -0,0 +1,14 @@
+# This is a boilerplate parameters config generated for pipeline 'compile'
+# using Kedro 0.19.14.
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/0.19.14/configuration/parameters.html
+
+
+compile:
+  name: "munet"
+  ml_backend: "Keras"
+  compiler: "hls4ml"
+  mlflow_url: "https://mlflow-deploy-mflow.app.cern.ch"
+  compiler_config:
+    test: 123

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_data_processing.yml

@@ -0,0 +1,8 @@
+# This is a boilerplate parameters config generated for pipeline 'data_processing'
+# using Kedro 0.19.14.
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/0.19.14/configuration/parameters.html
+
+random_state: 42
+test_size: 0.5

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_load_data.yml

@@ -0,0 +1,5 @@
+# This is a boilerplate parameters config generated for pipeline 'load_data'
+# using Kedro 0.19.14.
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/0.19.14/configuration/parameters.html

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_model_training.yml

@@ -0,0 +1,9 @@
+# This is a boilerplate parameters config generated for pipeline 'model_training'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html
+
+{{ cookiecutter.python_package }}_model:
+  hps:
+    name: "{{ cookiecutter.python_package }}"

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_model_validation.yml

@@ -0,0 +1,5 @@
+# This is a boilerplate parameters config generated for pipeline 'model_validation'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/0.19.14/configuration/parameters.html

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/logging.yml

@@ -0,0 +1,43 @@
+# To enable this custom logging configuration, set KEDRO_LOGGING_CONFIG to the path of this file.
+# More information available at https://docs.kedro.org/en/stable/logging/logging.html
+version: 1
+
+disable_existing_loggers: False
+
+formatters:
+  simple:
+    format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: INFO
+    formatter: simple
+    stream: ext://sys.stdout
+
+  info_file_handler:
+    class: logging.handlers.RotatingFileHandler
+    level: INFO
+    formatter: simple
+    filename: info.log
+    maxBytes: 10485760 # 10MB
+    backupCount: 20
+    encoding: utf8
+    delay: True
+
+  rich:
+    class: kedro.logging.RichHandler
+    rich_tracebacks: True
+    # Advance options for customisation.
+    # See https://docs.kedro.org/en/stable/logging/logging.html#project-side-logging-configuration
+    # tracebacks_show_locals: False
+
+loggers:
+  kedro:
+    level: INFO
+
+  {{ cookiecutter.python_package }}:
+    level: INFO
+
+root:
+  handlers: [rich, info_file_handler]

triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/samples.json

@@ -0,0 +1,15 @@
+{
+    "samples" : {
+        "testSample" : {
+            "all_file_path":"data/01_raw/test.root",
+            "path":"data/01_raw/test.root",
+            "file_pattern":["test.root"],
+            "DAS" : "Blabla",
+            "type" : "123",
+            "data" : false,
+            "era" : "phase1",
+            "run" : "run3",
+            "is_signal": true
+        }
+    }
+}

triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/samples_dummy.json

@@ -0,0 +1,26 @@
+{
+    "samples" : {
+        "muon24I" : {
+            "path": "data/01_raw/samples_dummy.json",
+            "folder":"data/01_raw/",
+            "file_pattern":["samples_dummy.json"],
+            "DAS" : "-",
+            "type" : "-",
+            "data" : true,
+            "era" : "phase1",
+            "run" : "run3",
+            "is_signal": true
+        },
+        "zb24I" : {
+            "path": "data/01_raw/samples_dummy.json",
+            "folder":"data/01_raw/",
+            "file_pattern":["samples_dummy.json"],
+            "DAS" : "-",
+            "type" : "-",
+            "data" : true,
+            "era" : "phase1",
+            "run" : "run3",
+            "is_signal": false
+        }
+    }
+}

triggerflow/starter/{{ cookiecutter.repo_name }}/dvc.yaml

@@ -0,0 +1,7 @@
+stages:
+  process_samples:
+    cmd: 
+    deps:
+      - 
+    outs:
+      - 

triggerflow/starter/{{ cookiecutter.repo_name }}/environment.yml

@@ -0,0 +1,21 @@
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.11
+  - pip
+  - pip:
+    - kedro
+    - kedro-viz
+    - kedro-datasets
+    - matplotlib
+    - mplhep
+    - shap
+    - scikit-learn
+    - pandas
+    - dvc
+    - shap
+    - ruff
+    - uproot
+    - awkward
+    - triggerflow

triggerflow/starter/{{ cookiecutter.repo_name }}/pyproject.toml

@@ -0,0 +1,50 @@
+[build-system]
+requires = [ "setuptools",]
+build-backend = "setuptools.build_meta"
+
+[project]
+requires-python = ">=3.10"
+name = "{{ cookiecutter.python_package }}"
+readme = "README.md"
+dynamic = [ "version",]
+dependencies = [ "ipython>=8.10", "jupyterlab>=3.0", "notebook", "kedro~=1.0.0",]
+
+[project.scripts]
+{{ cookiecutter.project_name }} = "{{ cookiecutter.python_package }}.__main__:main"
+
+[project.optional-dependencies]
+dev = [ "pytest-cov~=3.0", "pytest-mock>=1.7.1, <2.0", "pytest~=7.2", "ruff~=0.1.8",]
+
+[tool.kedro]
+package_name = "{{ cookiecutter.python_package }}"
+project_name = "{{ cookiecutter.project_name }}"
+kedro_init_version = "1.0.0"
+tools = "['Linting', 'Testing', 'Custom Logging', 'Data Structure']"
+example_pipeline = "False"
+source_dir = "src"
+
+[tool.ruff]
+line-length = 88
+show-fixes = true
+select = [ "F", "W", "E", "I", "UP", "PL", "T201",]
+ignore = [ "E501",]
+
+[project.entry-points."kedro.hooks"]
+
+[tool.pytest.ini_options]
+addopts = "--cov-report term-missing --cov src/{{ cookiecutter.python_package }} -ra"
+
+[tool.coverage.report]
+fail_under = 0
+show_missing = true
+exclude_lines = [ "pragma: no cover", "raise NotImplementedError",]
+
+[tool.ruff.format]
+docstring-code-format = true
+
+[tool.setuptools.dynamic.version]
+attr = "{{ cookiecutter.python_package }}.__version__"
+
+[tool.setuptools.packages.find]
+where = [ "src",]
+namespaces = false

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/__init__.py

@@ -0,0 +1,3 @@
+"""{{ cookiecutter.project_name }}"""
+
+__version__ = "0.1"

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/__main__.py

@@ -0,0 +1,25 @@
+"""{{ cookiecutter.project_name }} file for ensuring the package is executable
+as `{{ cookiecutter.project_name }}` and `python -m {{ cookiecutter.python_package }}`
+"""
+
+import sys
+from pathlib import Path
+from typing import Any
+
+from kedro.framework.cli.utils import find_run_command
+from kedro.framework.project import configure_project
+
+
+def main(*args, **kwargs) -> Any:
+    package_name = Path(__file__).parent.name
+    configure_project(package_name)
+
+    interactive = hasattr(sys, "ps1")
+    kwargs["standalone_mode"] = not interactive
+
+    run = find_run_command(package_name)
+    return run(*args, **kwargs)
+
+
+if __name__ == "__main__":
+    main()

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/any_object.py

@@ -0,0 +1,20 @@
+from kedro.io import AbstractDataset
+from typing import Any
+
+
+class AnyObject(AbstractDataset):
+    """
+    Abstract class which can be used for passing "Any" object
+    """
+
+    def __init__(self):
+        pass
+
+    def _load(self) -> None:
+        pass
+
+    def _save(self, data: Any) -> Any:
+        return data
+
+    def _describe(self) -> dict:
+        return {}