triggerflow 0.3.4__py3-none-any.whl

triggerflow/starter/{{ cookiecutter.repo_name }}/.gitignore

@@ -0,0 +1,143 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+.vscode/
+info.log
+
+# IntelliJ
+.idea/
+*.iml
+out/
+.idea_modules/

triggerflow/starter/{{ cookiecutter.repo_name }}/.gitlab-ci.yml

@@ -0,0 +1,56 @@
+image: continuumio/miniconda3
+
+stages:
+  - load
+  - preprocess
+  - train
+  - validate
+  - compile
+
+variables:
+  KEDRO_ENV: "base" # TODO: add production env
+
+before_script:
+  - eval "$(conda shell.bash hook)"
+  - conda env create --file=environment.yml
+  - conda activate triggerflow
+
+load_data:
+  stage: load
+  script:
+    - kedro run --pipeline=load_data
+  artifacts:
+    paths:
+      - data/02_loaded/
+
+preprocess_data:
+  stage: preprocess
+  script:
+    - kedro run --pipeline=data_processing
+  artifacts:
+    paths:
+      - data/03_preprocessed/
+
+train_model:
+  stage: train
+  script:
+    - kedro run --pipeline=model_training
+  artifacts:
+    paths:
+      - data/04_models/
+
+validate_model:
+  stage: validate
+  script:
+    - kedro run --pipeline=model_validation
+  artifacts:
+    paths:
+      - data/05_validation/
+
+compile:
+  stage: compile
+  script:
+    - kedro run --pipeline=compile
+  artifacts:
+    paths:
+      - data/06_compile/

triggerflow/starter/{{ cookiecutter.repo_name }}/README.md

@@ -0,0 +1,29 @@
+# {{ cookiecutter.project_name }}
+
+- conda env create --file=environment.yml
+- conda activate {{ cookiecutter.project_name }}
+
+## Data versioning
+When a dataset changes one can do (TODO: add this to pipeline to compare the hash):
+- dvc add data/01_raw/companies.csv
+- git add data/01_raw/companies.csv.dvc
+- git commit -m "Track dataset changes with DVC"
+
+## Run CI local
+- brew install gitlab-ci-local
+- gitlab-ci-local --list
+- gitlab-ci-local
+
+## ToDos:
+- move functionality of uhh_mlatl1 to pipeline
+- if case in base dataloader for classification or not
+- add model evaluation steps
+- automation of dvc in CI pipeline
+- move {{ cookiecutter.project_name }} meta data json to dvc
+- add linting and type checking
+- write tests
+- write out reporting / logging / plots etc.
+- track plots with dvc?
+- cross check pipeline afterwards with {{ cookiecutter.project_name }} team
+- make starter pipeline as template
+- add {{ cookiecutter.project_name }} model

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/README.md

@@ -0,0 +1,26 @@
+# What is this for?
+
+This folder should be used to store configuration files used by Kedro or by separate tools.
+
+This file can be used to provide users with instructions for how to reproduce local configuration with their own credentials. You can edit the file however you like, but you may wish to retain the information below and add your own section in the [Instructions](#Instructions) section.
+
+## Local configuration
+
+The `local` folder should be used for configuration that is either user-specific (e.g. IDE configuration) or protected (e.g. security keys).
+
+> *Note:* Please do not check in any local configuration to version control.
+
+## Base configuration
+
+The `base` folder is for shared configuration, such as non-sensitive and project-related configuration that may be shared across team members.
+
+WARNING: Please do not put access credentials in the base configuration folder.
+
+## Instructions
+
+
+
+
+## Need help?
+
+[Find out more about configuration from the Kedro documentation](https://docs.kedro.org/en/stable/kedro_project_setup/configuration.html).

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/catalog.yml

@@ -0,0 +1,84 @@
+{{ cookiecutter.python_package }}_meta_data:
+  filepath: data/01_raw/samples.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
+
+{{ cookiecutter.python_package }}_meta_data_loaded:
+  filepath: data/02_loaded/{{ cookiecutter.python_package }}_meta_data.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
+
+{{ cookiecutter.python_package }}_data:
+  sample_info: data/01_raw/samples.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.{{ cookiecutter.python_package }}_dataset.{{ cookiecutter.project_name }}Dataset
+
+{{ cookiecutter.python_package }}_data_loaded:
+  filepath: data/02_loaded/{{ cookiecutter.python_package }}_data.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_X_train:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_X_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_X_test:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_X_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_y_train:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_y_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_y_test:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_y_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+event_ids_train:
+  filepath: data/03_preprocessed/event_ids_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+event_ids_test:
+  filepath: data/03_preprocessed/event_ids_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+scaler:
+  filepath: data/03_preprocessed/scaler.pkl
+  type: pickle.PickleDataset
+
+train_model:
+  filepath: data/04_models/trained_model.pkl
+  type: pickle.PickleDataset
+
+training_history:
+  type: matplotlib.MatplotlibDataset
+  filepath: data/07_reporting/training_history.png
+  save_args:
+    format: png
+
+model_pred:
+  filepath: data/05_validation/model_pred.pkl
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_compile.yml

@@ -0,0 +1,14 @@
+# This is a boilerplate parameters config generated for pipeline 'compile'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html
+
+
+compile:
+  name: "munet"
+  ml_backend: "Keras"
+  compiler: "hls4ml"
+  mlflow_url: "https://mlflow-deploy-mflow.app.cern.ch"
+  compiler_config:
+    test: 123

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_data_processing.yml

@@ -0,0 +1,8 @@
+# This is a boilerplate parameters config generated for pipeline 'data_processing'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html
+
+random_state: 42
+test_size: 0.5

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_load_data.yml

@@ -0,0 +1,5 @@
+# This is a boilerplate parameters config generated for pipeline 'load_data'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_model_training.yml

@@ -0,0 +1,9 @@
+# This is a boilerplate parameters config generated for pipeline 'model_training'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html
+
+{{ cookiecutter.python_package }}_model:
+  hps:
+    name: "{{ cookiecutter.python_package }}"

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_model_validation.yml

@@ -0,0 +1,5 @@
+# This is a boilerplate parameters config generated for pipeline 'model_validation'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/catalog.yml

@@ -0,0 +1,90 @@
+{{ cookiecutter.python_package }}_meta_data:
+  filepath: data/01_raw/samples.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
+
+{{ cookiecutter.python_package }}_data:
+  sample_info: data/01_raw/samples.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.{{ cookiecutter.python_package }}_dataset.{{ cookiecutter.project_name }}Dataset
+
+{{ cookiecutter.python_package }}_meta_data_loaded:
+  filepath: data/02_loaded/{{ cookiecutter.python_package }}_meta_data.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
+
+{{ cookiecutter.python_package }}_loader:
+  sample_json: data/01_raw/samples.json
+  settings: data/01_raw/settings.json
+  config: data/01_raw/condor_config.json
+  type: {{ cookiecutter.python_package }}.datasets.{{ cookiecutter.python_package }}_loader.{{ cookiecutter.project_name }}Loader
+
+{{ cookiecutter.python_package }}_data_loaded:
+  filepath: data/02_loaded/{{ cookiecutter.python_package }}_data.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_X_train:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_X_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_X_test:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_X_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_y_train:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_y_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_y_test:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_y_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+event_ids_train:
+  filepath: data/03_preprocessed/event_ids_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+event_ids_test:
+  filepath: data/03_preprocessed/event_ids_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+scaler:
+  filepath: data/03_preprocessed/scaler.pkl
+  type: pickle.PickleDataset
+
+train_model:
+  filepath: data/04_models/trained_model.pkl
+  type: pickle.PickleDataset
+
+training_history:
+  type: matplotlib.MatplotlibDataset
+  filepath: data/07_reporting/training_history.png
+  save_args:
+    format: png
+
+model_pred:
+  filepath: data/05_validation/model_pred.pkl
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_compile.yml

@@ -0,0 +1,14 @@
+# This is a boilerplate parameters config generated for pipeline 'compile'
+# using Kedro 0.19.14.
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/0.19.14/configuration/parameters.html
+
+
+compile:
+  name: "munet"
+  ml_backend: "Keras"
+  compiler: "hls4ml"
+  mlflow_url: "https://mlflow-deploy-mflow.app.cern.ch"
+  compiler_config:
+    test: 123

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_data_processing.yml

@@ -0,0 +1,8 @@
+# This is a boilerplate parameters config generated for pipeline 'data_processing'
+# using Kedro 0.19.14.
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/0.19.14/configuration/parameters.html
+
+random_state: 42
+test_size: 0.5

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_load_data.yml

@@ -0,0 +1,5 @@
+# This is a boilerplate parameters config generated for pipeline 'load_data'
+# using Kedro 0.19.14.
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/0.19.14/configuration/parameters.html

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_model_training.yml

@@ -0,0 +1,9 @@
+# This is a boilerplate parameters config generated for pipeline 'model_training'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html
+
+{{ cookiecutter.python_package }}_model:
+  hps:
+    name: "{{ cookiecutter.python_package }}"

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_model_validation.yml

@@ -0,0 +1,5 @@
+# This is a boilerplate parameters config generated for pipeline 'model_validation'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/0.19.14/configuration/parameters.html

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/logging.yml

@@ -0,0 +1,43 @@
+# To enable this custom logging configuration, set KEDRO_LOGGING_CONFIG to the path of this file.
+# More information available at https://docs.kedro.org/en/stable/logging/logging.html
+version: 1
+
+disable_existing_loggers: False
+
+formatters:
+  simple:
+    format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: INFO
+    formatter: simple
+    stream: ext://sys.stdout
+
+  info_file_handler:
+    class: logging.handlers.RotatingFileHandler
+    level: INFO
+    formatter: simple
+    filename: info.log
+    maxBytes: 10485760 # 10MB
+    backupCount: 20
+    encoding: utf8
+    delay: True
+
+  rich:
+    class: kedro.logging.RichHandler
+    rich_tracebacks: True
+    # Advance options for customisation.
+    # See https://docs.kedro.org/en/stable/logging/logging.html#project-side-logging-configuration
+    # tracebacks_show_locals: False
+
+loggers:
+  kedro:
+    level: INFO
+
+  {{ cookiecutter.python_package }}:
+    level: INFO
+
+root:
+  handlers: [rich, info_file_handler]

triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/condor_config.json

@@ -0,0 +1,11 @@
+{
+    "log_directory": "./condor_logs",
+    "cores": 2,
+    "memory": "2GB",
+    "disk": "2GB",
+    "job_extra_directives": {
+        "+JobFlavour": "longlunch",
+        "getenv": true
+    },
+    "death_timeout": "60"
+}

triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/cuda_config.json

@@ -0,0 +1,4 @@
+{
+    "n_workers": 2,
+    "device_memory_limit": "16GB"
+}

triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/samples.json

@@ -0,0 +1,24 @@
+{
+    "samples" : {
+        "signal" : {
+            "files":"data/01_raw/test.root",
+            "file_pattern":["test.root"],
+            "DAS" : "Blabla",
+            "type" : "123",
+            "data" : false,
+            "era" : "phase1",
+            "run" : "run3",
+            "is_signal": true
+        },
+        "background" : {
+            "files":"data/01_raw/test.root",
+            "file_pattern":["test.root"],
+            "DAS" : "Blabla",
+            "type" : "123",
+            "data" : false,
+            "era" : "phase1",
+            "run" : "run3",
+            "is_signal": false
+        }
+    }
+}

triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/settings.json

@@ -0,0 +1,8 @@
+{
+    "output_dir": "./data/02_loaded/",
+    "run_local": true,
+    "num_workers": 4,
+    "chunksize": 50000,
+    "cluster_type": "local",
+    "jobs": 4
+}

triggerflow/starter/{{ cookiecutter.repo_name }}/dvc.yaml

@@ -0,0 +1,7 @@
+stages:
+  process_samples:
+    cmd: 
+    deps:
+      - 
+    outs:
+      -