triggerflow 0.1.12__py3-none-any.whl → 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trigger_dataset/__init__.py +0 -0
- trigger_dataset/core.py +88 -0
- trigger_loader/__init__.py +0 -0
- trigger_loader/cluster_manager.py +107 -0
- trigger_loader/loader.py +95 -0
- trigger_loader/processor.py +211 -0
- triggerflow/cli.py +122 -0
- triggerflow/core.py +29 -9
- triggerflow/mlflow_wrapper.py +54 -49
- triggerflow/starter/.gitignore +143 -0
- triggerflow/starter/README.md +0 -0
- triggerflow/starter/cookiecutter.json +5 -0
- triggerflow/starter/prompts.yml +9 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/.dvcignore +3 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/.gitignore +143 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/.gitlab-ci.yml +56 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/README.md +29 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/README.md +26 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/catalog.yml +84 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters.yml +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_compile.yml +14 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_data_processing.yml +8 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_load_data.yml +5 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_model_training.yml +9 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_model_validation.yml +5 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/catalog.yml +84 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters.yml +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_compile.yml +14 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_data_processing.yml +8 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_load_data.yml +5 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_model_training.yml +9 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_model_validation.yml +5 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/conf/logging.yml +43 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/.gitkeep +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/samples.json +15 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/samples_dummy.json +26 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/data/02_loaded/.gitkeep +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/data/03_preprocessed/.gitkeep +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/data/04_models/.gitkeep +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/data/05_validation/.gitkeep +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/data/06_compile/.gitkeep +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/data/07_reporting/.gitkeep +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/dvc.yaml +7 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/environment.yml +21 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/pyproject.toml +50 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/__init__.py +3 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/__main__.py +25 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/any_object.py +20 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/base_dataset.py +137 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/meta_dataset.py +88 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/{{ cookiecutter.python_package }}_dataset.py +35 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/models/__init__.py +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/models/base_model.py +155 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/models/{{ cookiecutter.python_package }}_model.py +16 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipeline_registry.py +17 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/compile/__init__.py +10 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/compile/nodes.py +50 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/compile/pipeline.py +10 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/__init__.py +10 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/nodes.py +40 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py +28 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/load_data/__init__.py +10 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/load_data/nodes.py +12 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/load_data/pipeline.py +20 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/model_training/__init__.py +10 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/model_training/nodes.py +31 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/model_training/pipeline.py +24 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/model_validation/__init__.py +10 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/model_validation/nodes.py +29 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/model_validation/pipeline.py +24 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py +46 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/utils/__init__.py +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/utils/metric.py +4 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/utils/plotting.py +598 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/tests/__init__.py +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/tests/pipelines/__init__.py +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/tests/pipelines/compile/__init__.py +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/tests/pipelines/compile/test_pipeline.py +9 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/tests/pipelines/data_processing/__init__.py +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/tests/pipelines/data_processing/test_pipeline.py +9 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/tests/pipelines/load_data/__init__.py +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/tests/pipelines/load_data/test_pipeline.py +9 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/tests/pipelines/model_training/__init__.py +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/tests/pipelines/model_training/test_pipeline.py +9 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/tests/pipelines/model_validation/__init__.py +0 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/tests/pipelines/model_validation/test_pipeline.py +9 -0
- triggerflow/starter/{{ cookiecutter.repo_name }}/tests/test_run.py +27 -0
- triggerflow/templates/makefile +3 -3
- triggerflow/templates/makefile_version +2 -2
- triggerflow/templates/model_template.cpp +19 -18
- triggerflow/templates/scales.h +1 -1
- triggerflow-0.2.1.dist-info/METADATA +97 -0
- triggerflow-0.2.1.dist-info/RECORD +97 -0
- triggerflow-0.2.1.dist-info/entry_points.txt +2 -0
- triggerflow-0.2.1.dist-info/top_level.txt +3 -0
- triggerflow-0.1.12.dist-info/METADATA +0 -61
- triggerflow-0.1.12.dist-info/RECORD +0 -11
- triggerflow-0.1.12.dist-info/top_level.txt +0 -1
- {triggerflow-0.1.12.dist-info → triggerflow-0.2.1.dist-info}/WHEEL +0 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This is a boilerplate pipeline 'model_training'
|
|
3
|
+
generated using Kedro 1.0.0
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import pandas as pd
|
|
7
|
+
from {{ cookiecutter.python_package }}.utils.plotting import plotTrainingHistory, get_dummy
|
|
8
|
+
from {{ cookiecutter.python_package }}.models.{{ cookiecutter.python_package }}_model import {{ cookiecutter.python_package }}
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def train_model(
|
|
12
|
+
X_train: pd.DataFrame, y_train: pd.DataFrame, params: dict
|
|
13
|
+
) -> pd.DataFrame:
|
|
14
|
+
"""Trains a dummy model on some data.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
X_train:
|
|
18
|
+
y_train:
|
|
19
|
+
params:
|
|
20
|
+
Returns:
|
|
21
|
+
Trained model.
|
|
22
|
+
"""
|
|
23
|
+
params["hps"]["nInputs"] = X_train.shape[-1]
|
|
24
|
+
model = {{ cookiecutter.python_package }}(name=params["hps"]["name"], hps=params["hps"])
|
|
25
|
+
model.train(X_train, y_train)
|
|
26
|
+
|
|
27
|
+
f, _ = get_dummy()
|
|
28
|
+
# NOTE: one can also plot the history
|
|
29
|
+
# f, _ = plotTrainingHistory(model.history)
|
|
30
|
+
|
|
31
|
+
return model, f
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This is a boilerplate pipeline 'model_training'
|
|
3
|
+
generated using Kedro 1.0.0
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from kedro.pipeline import node, Pipeline, pipeline # noqa
|
|
7
|
+
from .nodes import train_model
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def create_pipeline(**kwargs) -> Pipeline:
|
|
11
|
+
return pipeline(
|
|
12
|
+
[
|
|
13
|
+
node(
|
|
14
|
+
func=train_model,
|
|
15
|
+
inputs=[
|
|
16
|
+
"processed_{{ cookiecutter.python_package }}_X_train",
|
|
17
|
+
"processed_{{ cookiecutter.python_package }}_y_train",
|
|
18
|
+
"params:{{ cookiecutter.python_package }}_model",
|
|
19
|
+
],
|
|
20
|
+
outputs=["train_model", "training_history"],
|
|
21
|
+
name="train_model_node",
|
|
22
|
+
)
|
|
23
|
+
]
|
|
24
|
+
)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This is a boilerplate pipeline 'model_validation'
|
|
3
|
+
generated using Kedro 1.0.0
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import logging
|
|
7
|
+
import pandas as pd
|
|
8
|
+
from sklearn.metrics import roc_auc_score
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def validated_model(model, X_test: pd.DataFrame, y_test: pd.DataFrame) -> pd.DataFrame:
|
|
12
|
+
"""Trains a dummy model on some data.
|
|
13
|
+
|
|
14
|
+
Args:
|
|
15
|
+
model:
|
|
16
|
+
X_test:
|
|
17
|
+
y_test:
|
|
18
|
+
Returns:
|
|
19
|
+
Model prediction.
|
|
20
|
+
"""
|
|
21
|
+
# get logger for reporting
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
pred = model.predict(X_test)
|
|
25
|
+
auc = roc_auc_score(y_test, pred)
|
|
26
|
+
|
|
27
|
+
logger.info(f"Area under ROC curve: {auc:.4f}")
|
|
28
|
+
|
|
29
|
+
return pred
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This is a boilerplate pipeline 'model_validation'
|
|
3
|
+
generated using Kedro 1.0.0
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from kedro.pipeline import node, Pipeline, pipeline # noqa
|
|
7
|
+
from .nodes import validated_model
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def create_pipeline(**kwargs) -> Pipeline:
|
|
11
|
+
return pipeline(
|
|
12
|
+
[
|
|
13
|
+
node(
|
|
14
|
+
func=validated_model,
|
|
15
|
+
inputs=[
|
|
16
|
+
"train_model",
|
|
17
|
+
"processed_{{ cookiecutter.python_package }}_X_test",
|
|
18
|
+
"processed_{{ cookiecutter.python_package }}_y_test",
|
|
19
|
+
],
|
|
20
|
+
outputs="model_pred",
|
|
21
|
+
name="validated_model_node",
|
|
22
|
+
)
|
|
23
|
+
]
|
|
24
|
+
)
|
triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Project settings. There is no need to edit this file unless you want to change values
|
|
2
|
+
from the Kedro defaults. For further information, including these default values, see
|
|
3
|
+
https://docs.kedro.org/en/stable/kedro_project_setup/settings.html."""
|
|
4
|
+
|
|
5
|
+
# Instantiated project hooks.
|
|
6
|
+
# For example, after creating a hooks.py and defining a ProjectHooks class there, do
|
|
7
|
+
# from {{ cookiecutter.python_package }}.hooks import ProjectHooks
|
|
8
|
+
# Hooks are executed in a Last-In-First-Out (LIFO) order.
|
|
9
|
+
# HOOKS = (ProjectHooks(),)
|
|
10
|
+
|
|
11
|
+
# Installed plugins for which to disable hook auto-registration.
|
|
12
|
+
# DISABLE_HOOKS_FOR_PLUGINS = ("kedro-viz",)
|
|
13
|
+
|
|
14
|
+
# Class that manages storing KedroSession data.
|
|
15
|
+
# from kedro.framework.session.store import BaseSessionStore
|
|
16
|
+
# SESSION_STORE_CLASS = BaseSessionStore
|
|
17
|
+
# Keyword arguments to pass to the `SESSION_STORE_CLASS` constructor.
|
|
18
|
+
# SESSION_STORE_ARGS = {
|
|
19
|
+
# "path": "./sessions"
|
|
20
|
+
# }
|
|
21
|
+
|
|
22
|
+
# Directory that holds configuration.
|
|
23
|
+
# CONF_SOURCE = "conf"
|
|
24
|
+
|
|
25
|
+
# Class that manages how configuration is loaded.
|
|
26
|
+
# from kedro.config import OmegaConfigDataset
|
|
27
|
+
|
|
28
|
+
# CONFIG_DATASET_CLASS = OmegaConfigDataset
|
|
29
|
+
|
|
30
|
+
# Keyword arguments to pass to the `CONFIG_DATASET_CLASS` constructor.
|
|
31
|
+
CONFIG_DATASET_ARGS = {
|
|
32
|
+
"base_env": "base",
|
|
33
|
+
"default_run_env": "local",
|
|
34
|
+
# "config_patterns": {
|
|
35
|
+
# "spark" : ["spark*/"],
|
|
36
|
+
# "parameters": ["parameters*", "parameters*/**", "**/parameters*"],
|
|
37
|
+
# }
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
# Class that manages Kedro's library components.
|
|
41
|
+
# from kedro.framework.context import KedroContext
|
|
42
|
+
# CONTEXT_CLASS = KedroContext
|
|
43
|
+
|
|
44
|
+
# Class that manages the Data Catalog.
|
|
45
|
+
# from kedro.io import DataCatalog
|
|
46
|
+
# DATA_CATALOG_CLASS = DataCatalog
|
|
File without changes
|