champions 0.9.0__py3-none-any.whl

champions/__init__.py

@@ -0,0 +1,24 @@
+import logging.config
+import yaml
+
+try:
+    with open("logging.yaml", "r") as f:
+        config = yaml.safe_load(f)
+except FileNotFoundError:
+    config = {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "simple": {"format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"}
+        },
+        "handlers": {
+            "console": {
+                "class": "logging.StreamHandler",
+                "formatter": "simple",
+                "level": "INFO",
+            }
+        },
+        "root": {"handlers": ["console"], "level": "INFO"},
+    }
+
+logging.config.dictConfig(config)

champions/cli.py

@@ -0,0 +1,39 @@
+import logging
+from typing import Annotated
+import typer
+import yaml
+
+from champions.model.datacard import DataCard
+from champions.model.settings import EvalSettings, TrainSettings
+from champions.service.eval import Eval
+from champions.service.train import Train
+
+
+logger = logging.getLogger(__name__)
+
+app = typer.Typer()
+
+
+# def main(config: Annotated[typer.FileText, typer.Option()]):
+@app.command()
+def train(
+    datacard: Annotated[typer.FileText, typer.Option()],
+    trainsettings: Annotated[typer.FileText, typer.Option()],
+):
+    train = Train(
+        dc=DataCard(**yaml.safe_load(datacard)),
+        settings=TrainSettings(**yaml.safe_load(trainsettings)),
+    )
+    train.run()
+
+
+@app.command()
+def eval(
+    datacard: Annotated[typer.FileText, typer.Option()],
+    evalsettings: Annotated[typer.FileText, typer.Option()],
+):
+    train = Eval(
+        dc=DataCard(**yaml.safe_load(datacard)),
+        settings=EvalSettings(**yaml.safe_load(evalsettings)),
+    )
+    train.run()

champions/model/champions.py

@@ -0,0 +1,35 @@
+import logging
+from pydantic import BaseModel
+
+logger = logging.getLogger(__name__)
+
+
+class Spore(BaseModel):
+    cut: list[str]
+    score: float
+    depth: str
+
+
+class Champion(BaseModel):
+    spore: list[Spore]
+    target: str | int
+
+    def get_sql(self, res_name: str) -> str:
+        sql_ref = "CASE \n"
+        for spore in self.spore:
+            sql_ref += (
+                f"  WHEN {' AND '.join(spore.cut)} THEN CAST({spore.score} AS DOUBLE)\n"
+            )
+        sql_ref += f"END AS {res_name}"
+        return sql_ref
+
+
+class Champions(BaseModel):
+    champions: dict[str, Champion]
+    target: str
+
+    def get_sql(self) -> dict[str, str]:
+        return {
+            f"res_{name}": champion.get_sql(f"res_{name}")
+            for name, champion in self.champions.items()
+        }

champions/model/datacard.py

@@ -0,0 +1,48 @@
+import logging
+from typing import Literal
+from pydantic import BaseModel
+
+logger = logging.getLogger(__name__)
+
+
+class Feature(BaseModel):
+    name: str
+    statistical: Literal["category", "continus"]
+    type: str
+
+
+class Target(BaseModel):
+    feature_name: str
+    values: list[int | str]
+
+
+class DataCard(BaseModel):
+    features: list[Feature]
+    infos: dict
+    target: Target
+    test_files: list[str]
+    train_files: list[str]
+
+    @property
+    def feature_names(self) -> list[str]:
+        return [f'"{feat.name}"' for feat in self.features]
+
+    @property
+    def train_feature(self) -> list[Feature]:
+        return [feat for feat in self.features if feat.name != self.target.feature_name]
+
+    @property
+    def train_cat_feature(self) -> list[Feature]:
+        return [
+            feat
+            for feat in self.features
+            if feat.name != self.target.feature_name and feat.statistical == "category"
+        ]
+
+    @property
+    def train_con_feature(self) -> list[Feature]:
+        return [
+            feat
+            for feat in self.features
+            if feat.name != self.target.feature_name and feat.statistical == "continus"
+        ]

champions/model/dataframes.py

@@ -0,0 +1,225 @@
+import itertools
+import math
+import time
+import polars as pl
+from pydantic import BaseModel
+from dataclasses import dataclass
+
+from champions.model.datacard import Feature
+from champions.model.filter import CombineFilter, Filter, SingleFilter
+import sys
+
+
+class ContCats(BaseModel):
+    cuts: list[float]
+    labels: list[str]
+    feat_name: str
+
+    def cut(self, series: pl.Series) -> pl.Series:
+        # return series.cut(self.cuts)
+        return series.cut(self.cuts, labels=self.labels)
+
+    def get_single_filter(self, i: int) -> SingleFilter:
+        if i == 2 and len(self.cuts) == 1:
+            sys.exit()
+
+        if i == 0:
+            return SingleFilter(
+                feat_name=self.feat_name,
+                operator="<=",
+                value=self.cuts[i],
+            )
+
+        if i == len(self.labels) - 1:
+            return SingleFilter(
+                feat_name=self.feat_name,
+                operator=">",
+                value=self.cuts[i - 1],
+            )
+        return SingleFilter(
+            feat_name=self.feat_name,
+            operator="between",
+            value=[self.cuts[i - 1], self.cuts[i]],
+        )
+
+
+class CategorizedFeatureMixin:
+    diff_df: pl.DataFrame
+    cut_list: list[ContCats]
+    feature_list: list[str]
+
+    def calc_diff(
+        self,
+    ) -> float:
+        # self.diff_sr.entropy()
+        return self.diff_df["diff"].abs().sum()
+
+    def set_diff_df(
+        self,
+        df_count_target: pl.DataFrame,
+        df_count_non_target: pl.DataFrame,
+        join_on: list[str] | str,
+    ):
+        self.diff_df = (
+            df_count_non_target.join(df_count_target, on=join_on, how="outer")
+            .fill_null(0)
+            .with_columns(
+                (pl.col("proportion_right") - pl.col("proportion")).alias("diff"),
+                pl.max_horizontal(
+                    pl.col("proportion"), pl.col("proportion_right")
+                ).alias("max_proportion"),
+            )
+        )
+
+    def get_left_right_filter(self) -> tuple[Filter, Filter]:
+        # for fea_name, cut in zip(self.feature_list, self.cut_list):
+
+        target_res_df = self.diff_df.filter(pl.col("diff") > 0)[
+            [f"{cut.feat_name}_right" for cut in self.cut_list]
+        ]
+        target_lists = [
+            target_res_df[f"{cut.feat_name}_right"].to_list() for cut in self.cut_list
+        ]
+        target_list_tupels = [akt_tuple for akt_tuple in zip(*target_lists)]
+
+        all_labels = [cut.labels for cut in self.cut_list]
+        target_filter = []
+        non_targer_filter = []
+
+        for possible_combs in itertools.product(*all_labels):
+            single_filter_list = [
+                cut.get_single_filter(int(index))
+                for index, cut in zip(possible_combs, self.cut_list)
+            ]
+            comb_filter = CombineFilter(combine=single_filter_list)
+            if possible_combs in target_list_tupels:
+                target_filter.append(comb_filter)
+            else:
+                non_targer_filter.append(comb_filter)
+
+        return Filter(combine=non_targer_filter, invert=False), Filter(
+            combine=target_filter, invert=False
+        )
+
+
+@dataclass
+class CategorizedFeature(CategorizedFeatureMixin):
+    feature: Feature
+    cuts: ContCats
+    target_sr: pl.Series
+    non_target_sr: pl.Series
+
+    def __post_init__(self):
+        df_count_target = self.target_sr.value_counts(normalize=True)
+        df_count_non_target = self.non_target_sr.value_counts(normalize=True)
+        self.set_diff_df(
+            df_count_target=df_count_target,
+            df_count_non_target=df_count_non_target,
+            join_on=self.feature.name,
+        )
+
+
+    @property
+    def cut_list(self) -> list[ContCats]:
+        return [self.cuts]
+
+    def is_diff_to_low(self, threshold: float = 0.90) -> bool:
+        max_wert = self.diff_df["max_proportion"].max()
+        min_prop_of_max = self.diff_df.filter(pl.col("max_proportion") == max_wert)[
+            "proportion", "proportion_right"
+        ].min_horizontal()[0]
+        return min_prop_of_max > threshold
+
+
+class CombinedCategorizedFeature(CategorizedFeatureMixin):
+    def __init__(
+        self,
+        train_features: tuple[CategorizedFeature],
+        non_target_size: int,
+        target_size: int,
+    ):
+        groub_by = [train.feature.name for train in train_features]
+        non_target_df = pl.DataFrame([train.non_target_sr for train in train_features])
+
+        df_count_non_target = (
+            non_target_df.group_by(groub_by)
+            .len(name="proportion")
+            .with_columns(pl.col("proportion") / non_target_size)
+        )
+        target_df = pl.DataFrame([train.target_sr for train in train_features])
+        df_count_target = (
+            target_df.group_by(groub_by)
+            .len(name="proportion")
+            .with_columns(pl.col("proportion") / target_size)
+        )
+        self.set_diff_df(
+            df_count_target=df_count_target,
+            df_count_non_target=df_count_non_target,
+            join_on=groub_by,
+        )
+        self.cut_list = [train.cuts for train in train_features]
+
+
+class TrainDataframes:
+    def __init__(
+        self,
+        target_df: pl.DataFrame,
+        non_target_df: pl.DataFrame,
+        frac_eval_cat: float,
+        min_size: int,
+    ):
+        self.target_df_size = target_df.height
+        self.non_target_df_size = non_target_df.height
+
+        self.min_size = min_size
+
+        self.n_count_target, n_group_target = self._calc_split(target_df.height, frac_eval_cat)
+        self.target_df_count = target_df.head(self.n_count_target)
+        target_df_group = target_df.tail(-self.n_count_target)
+        #target_df_group['weight'] = 0.5 /  len(target_df_group)
+ 
+        self.n_count_non_target, n_group_non_target = self._calc_split(non_target_df.height, frac_eval_cat)
+        self.non_target_df_count = non_target_df.head(self.n_count_non_target)
+        non_target_df_group = non_target_df.tail(-self.n_count_non_target)
+
+        if n_group_non_target > 0 and n_group_target > 0:       
+
+            self.df_group = pl.concat([target_df_group.with_columns(pl.lit(0.5 /n_group_target).alias('weight')),
+                              non_target_df_group.with_columns(pl.lit(0.5 /n_group_non_target).alias('weight'))
+                              ])
+        self.train_features: list[CategorizedFeature] = []
+
+    def create_categorized_features(
+        self, feat: Feature, cuts: ContCats
+    ) -> CategorizedFeature:
+        target_sr = cuts.cut(series=self.target_df_count[feat.name])
+        non_target_sr = cuts.cut(series=self.non_target_df_count[feat.name])
+        return CategorizedFeature(
+            feature=feat, cuts=cuts, target_sr=target_sr, non_target_sr=non_target_sr
+        )
+
+    def _calc_split(self, n: int, frac: float):
+        split = round(n * frac)
+        if split < 1:
+            return 1 ,0
+        if split >= n:
+            return n - 1 , 1
+        return split, n- split
+
+    def score(self) -> float:
+        if self.non_target_df_size + self.target_df_size == 0:
+            return 0.0
+        return (self.target_df_size - self.non_target_df_size) / (
+            self.non_target_df_size + self.target_df_size
+        )
+
+    def is_final_size(self) -> bool:
+        return (
+            self.target_df_size < self.min_size
+            or self.non_target_df_size < self.min_size
+        )
+
+
+class EvalDataframe:
+    def __init__(self, df: pl.DataFrame, target: str | int) -> None:
+        self.df = df

champions/model/filter.py

@@ -0,0 +1,37 @@
+from typing import Literal
+from pydantic import BaseModel
+
+
+class SingleFilter(BaseModel):
+    feat_name: str
+    operator: Literal["=", "<", "<=", ">", ">=", "in", "between"]
+    value: str | int | float | list[str] | list[int] | list[float]
+
+    def sql(self) -> str:
+        if self.operator == "between":
+            return f'( "{self.feat_name}" > {self.value[0]} AND "{self.feat_name}" <= {self.value[1]} )'
+        if isinstance(self.value, list):
+            return f"{self.feat_name} {self.operator} ( {', '.join(self.value)})"
+        return f'"{self.feat_name}"{self.operator}{self.value}'
+
+
+class CombineFilter(BaseModel):
+    combine: list[SingleFilter]
+    invert: bool = False
+
+    def sql(self, do_invert: bool = False) -> str:
+        prefix = "not" if self.invert ^ do_invert else ""  # python xor
+        if len(self.combine) == 1:
+            return f" {prefix} {self.combine[0].sql()} "
+        return f" {prefix} ( {' AND '.join([sf.sql() for sf in self.combine])} )"
+
+
+class Filter(BaseModel):
+    combine: list[CombineFilter]
+    invert: bool
+
+    def sql(self, do_invert: bool = False) -> str:
+        prefix = "not" if self.invert ^ do_invert else ""  # python xor
+        if len(self.combine) == 1:
+            return f" {prefix} {self.combine[0].sql()} "
+        return f" {prefix} ( {' OR '.join([sf.sql() for sf in self.combine])} )"

champions/model/settings.py

@@ -0,0 +1,55 @@
+import logging
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+import yaml
+
+from champions.model.champions import Champion, Champions
+
+logger = logging.getLogger(__name__)
+
+
+class EvalSettings(BaseModel):
+    in_folder: Path = Field(description="Name where the champion is stored")
+    out_folder: Path = Field(description="Name where the evaluation is stored")
+    roc_points: int = Field(description="Number of points for roc curve", default=500)
+
+
+class TrainSettings(BaseModel):
+    n: int = Field(description="Number of Champions per target value", default=1)
+    out_folder: Path = Field(description="Name where the champion is stored")
+    n_cat: int = Field(description="Number of categories in one featerue", default=3)
+    max_depth: int = Field(description="Max depth of the champion", default=10)
+
+    max_eval_fit: int = Field(
+        description="Wieviel sollen max für training gleichzeitig benutzt werden",
+        default=1000,
+    )
+    min_eval_fit: int = Field(
+        description="Wieviel müssen fürs training mindestens benutzt werden", default=10
+    )
+    n_dims: int = Field(
+        description="Wieviele dimensionen sollen auf einmal benutzt werden", default=3
+    )
+    calcs_per_dim: int | None = Field(
+        description="Wieviele berechnungen sollen pro dimension gemacht werden",
+        default=5000,
+    )
+    frac_eval_cat: float = Field(
+        description="Wie groß ist der anteil der für eval benutzt werden soll",
+        default=0.5,
+    )
+
+    def champion_exists(self, target: str | int, n: int) -> bool:
+        if self.out_folder.exists():
+            if (self.out_folder / f"{target}" / f"{n}.yaml").exists():
+                return True
+        return False
+
+    def save_champion(self, champion: Champion, n: int):
+        target = champion.target
+        out_folder = self.out_folder / f"{target}"
+        logger.info(f"save champion {n} for target {target} to {out_folder}")
+        out_folder.mkdir(parents=True, exist_ok=True)
+        with open(out_folder / f"{n}.yaml", "w") as f:
+            f.write(yaml.safe_dump(champion.model_dump()))

champions/service/darkwing.py

@@ -0,0 +1,96 @@
+import logging
+import sys
+from typing import Any
+import duckdb
+from pydantic import BaseModel
+import polars as pl
+
+from champions.model.champions import Champions
+from champions.model.datacard import DataCard
+from champions.model.dataframes import EvalDataframe, TrainDataframes
+from champions.model.filter import Filter, SingleFilter
+from champions.model.settings import TrainSettings
+
+
+logger = logging.getLogger(__name__)
+
+
+class Darkwing(BaseModel):
+    dc: DataCard
+    df_train_cach: Any | None = None
+
+    def read_akt_train(
+        self,
+        targer_filter: Filter,
+        train_settings: TrainSettings,
+        akt_filters: list[Filter] = [],
+    ) -> TrainDataframes:
+        """
+        Reads a list of CSV files into a DuckDB relation.
+
+        Args:
+            filepaths: A list of filepaths to CSV files.
+
+        Returns:
+            A DuckDB relation containing the data from all CSV files.
+        """
+
+        full_filters_target = [targer_filter.sql()] + [f.sql() for f in akt_filters]
+        target_df = self._get_pl_train_df(
+            full_filters=full_filters_target,
+            max_eval_fit=train_settings.max_eval_fit,
+        )
+        full_filters_non_target = [targer_filter.sql(do_invert=True)] + [
+            f.sql() for f in akt_filters
+        ]
+        non_target_df = self._get_pl_train_df(
+            full_filters=full_filters_non_target,
+            max_eval_fit=train_settings.max_eval_fit,
+        )
+
+        return TrainDataframes(
+            target_df=target_df,
+            non_target_df=non_target_df,
+            frac_eval_cat=train_settings.frac_eval_cat,
+            min_size=train_settings.min_eval_fit,
+        )
+
+    def _get_pl_train_df(self, full_filters: list[str], max_eval_fit) -> pl.DataFrame:
+        df = self.get_cached_train_df()
+        sql = f"""
+                SELECT {", ".join(self.dc.feature_names)}
+                FROM df
+                WHERE {" AND ".join(full_filters)}
+                LIMIT {max_eval_fit}; 
+              """
+        return duckdb.sql(sql).pl().sample(fraction=1, shuffle=True)
+
+    def get_cached_train_df(self) -> pl.DataFrame:
+        if self.df_train_cach is None:
+            self.df_train_cach = pl.read_csv(",".join(self.dc.train_files))
+        return self.df_train_cach
+
+    def get_eval_sr(self, champions: Champions) -> EvalDataframe:
+        df_sum = None
+        norm = 0.0
+
+        for name, champion in champions.champions.items():
+            norm += 1.0
+            col_name = f"res_{name}"
+            case_sql = champion.get_sql(col_name)
+            df = self._get_pl_eval_df(col_sql=case_sql)
+            df_sum = df_sum.hstack(df) if df_sum is not None else df
+
+        return (df_sum.sum_horizontal() / norm).alias(champions.target)
+
+        # for feat_name, case_sql in champions.get_sql().items():
+        #    df = self._get_pl_eval_df(case_sql=case_sql)
+        #    logger.info(f"Evaluate {df}")
+        #    logger.info(f"{case_sql}")
+
+    def _get_pl_eval_df(self, col_sql: str) -> pl.DataFrame:
+        sql = f"""
+                SELECT {col_sql}\n
+                FROM read_csv({self.dc.test_files}); 
+              """
+        return duckdb.sql(sql).pl()

champions/service/eval.py

@@ -0,0 +1,130 @@
+import os
+import sys
+from typing import Optional
+import duckdb
+from pydantic import BaseModel
+import yaml
+from sklearn.metrics import roc_curve
+import altair as alt
+
+from champions.model.champions import Champion, Champions
+from champions.model.datacard import DataCard
+from champions.model.settings import EvalSettings
+from champions.service.darkwing import Darkwing
+import polars as pl
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class Eval(BaseModel):
+    dc: DataCard
+    settings: EvalSettings
+    darkwing: Optional[Darkwing] = None
+
+    def model_post_init(self, __context) -> None:
+        self.darkwing = Darkwing(dc=self.dc)
+        os.makedirs(self.settings.out_folder, exist_ok=True)
+        super().model_post_init(__context)
+
+    def run(self):
+        logger.info("Start Eval")
+
+        df_res = self.darkwing._get_pl_eval_df(col_sql=self.dc.target.feature_name)
+        # df_res = pl.read_parquet("test.parquet")
+        plots = None
+
+        for target in self.dc.target.values:
+            logger.info(f"Evaluate target {target}")
+            target_champions = self.load_champions(target=f"{target}")
+            sr = self.darkwing.get_eval_sr(champions=target_champions)
+            df_res = df_res.with_columns(sr)
+            new_plot = self.plot_roc(df=df_res, target_label=target)
+            new_plot.save(self.settings.out_folder / f"{target}_roc.html")
+            if plots is None:
+                plots = new_plot
+            else:
+                plots = plots + new_plot
+
+        plots.save(self.settings.out_folder / "all_roc.html")
+
+        if len(self.dc.target.values) > 1:
+            df_multi_res = self.add_multi_class_result(
+                df=df_res, target_values=self.dc.target.values
+            )
+            df_res = pl.concat(
+                [df_res, df_multi_res],
+                how="horizontal",
+            )
+
+            multi_res_plot = (
+                df_res.group_by("label")
+                .agg(pl.col("correct").sum() / pl.count())
+                .sort("label")
+                .plot.bar(x="label", y="correct")
+            )
+            prec = df_res["correct"].sum() / df_res["correct"].count()
+            all_pred_labels = (
+                alt.Chart(pl.DataFrame({"correct": [prec]}))
+                .mark_rule()
+                .encode(y="correct")
+            )
+            (multi_res_plot + all_pred_labels).save(
+                self.settings.out_folder / "multi_class_result.html"
+            )
+        logger.info(f"{df_res}")
+
+    def add_multi_class_result(self, df: pl.DataFrame, target_values: list[str | int]):
+        greates = ", ".join([f'"{value}"' for value in target_values])
+        cases = [
+            f'WHEN "{value}" = GREATEST({greates}) THEN {value}'
+            for value in target_values
+        ]
+        return (
+            duckdb.sql(f"""
+            SELECT
+                label,
+                CASE
+                    {" \n ".join(cases)}
+                END AS predicted_label
+            FROM df
+        """)
+            .pl()
+            .with_columns(
+                (pl.col("label") == pl.col("predicted_label"))
+                .cast(pl.Int64)
+                .alias("correct")
+            )
+            .select("predicted_label", "correct")
+        )
+
+    def load_champions(self, target: str) -> Champions:
+        target_folder = self.settings.in_folder / target
+        data = {}
+        for file in target_folder.glob("*.yaml"):
+            with open(file, "r") as f:
+                logger.debug(f"Load Champion from {file}")
+                data[file.name.strip(".yaml")] = Champion(**yaml.safe_load(f))
+        return Champions(champions=data, target=target)
+
+    def plot_roc(self, df: pl.DataFrame, target_label: str | int):
+        score_feat = f"{target_label}"
+        df_plot = (
+            duckdb.sql(f"""
+            SELECT fp,tp
+            FROM (
+                SELECT
+                    ROUND(SUM(CASE WHEN label == '{target_label}' THEN 1 ELSE 0 END) OVER (ORDER BY "{score_feat}" DESC) / SUM(CASE WHEN label == '{target_label}' THEN 1 ELSE 0 END) OVER (), 3)  AS tp,
+                    ROUND(SUM(CASE WHEN label == '{target_label}' THEN 0 ELSE 1 END) OVER (ORDER BY "{score_feat}" DESC) / SUM(CASE WHEN label == '{target_label}' THEN 0 ELSE 1 END) OVER (), 3)  AS fp
+                FROM df
+                )
+            GROUP BY fp,tp
+            ORDER BY fp,tp
+        """)
+            .pl(
+                # ).vstack(pl.DataFrame({'fp': 0.0, 'tp': 0.0})
+            )
+            .with_columns(pl.lit(score_feat).alias("target"))
+        )
+        plot = df_plot.plot.line(x="fp", y="tp", color="target").interactive()
+        return plot