champions 0.9.0__py3-none-any.whl

champions/service/train.py

@@ -0,0 +1,213 @@
+import itertools
+import math
+import sys
+import time
+from typing import Optional
+from pydantic import BaseModel
+from champions.model.datacard import DataCard, Feature
+from champions.model.dataframes import (
+    CombinedCategorizedFeature,
+    ContCats,
+    TrainDataframes,
+    CategorizedFeature,
+)
+from champions.model.filter import CombineFilter, Filter, SingleFilter
+from champions.model.settings import TrainSettings
+from champions.model.champions import Champion, Spore
+import polars as pl
+
+
+import logging
+
+from champions.service.darkwing import Darkwing
+
+logger = logging.getLogger(__name__)
+
+
+class Train(BaseModel):
+    dc: DataCard
+    settings: TrainSettings
+    darkwing: Optional[Darkwing] = None
+
+    def model_post_init(self, __context) -> None:
+        self.darkwing = Darkwing(dc=self.dc)
+        super().model_post_init(__context)
+
+    def run(self):
+        logger.info("start training")
+        for n in range(self.settings.n):
+            for cat in self.dc.target.values:
+                if self.settings.champion_exists(target=cat, n=n):
+                    logger.info(f"tree {n} for label {cat} already exists")
+                    continue
+
+                logger.info(f"train tree {n} for label {cat}")
+
+                train_res = Champion(
+                    spore=self.train_champion(
+                        target_filter=self.gen_target_filter(cat=cat),
+                        path_filter=[],
+                    ),
+                    target=cat,
+                )
+                self.settings.save_champion(champion=train_res, n=n)
+
+    def gen_target_filter(self, cat) -> Filter:
+        return Filter(
+            combine=[
+                CombineFilter(
+                    combine=[
+                        SingleFilter(
+                            feat_name=self.dc.target.feature_name,
+                            operator="=",
+                            value=cat,
+                        )
+                    ]
+                )
+            ],
+            invert=False,
+        )
+
+    def train_champion(
+        self, target_filter: Filter, path_filter: list[Filter], depth: str = ""
+    ) -> list[Spore]:
+        train_df = self.darkwing.read_akt_train(
+            targer_filter=target_filter,
+            train_settings=self.settings,
+            akt_filters=path_filter,
+        )
+
+        if train_df.is_final_size() or len(depth) >= self.settings.max_depth:
+            score = train_df.score()
+            logging.info(
+                f"final spore depth: {depth}, {score} non target {train_df.non_target_df_size} target: {train_df.target_df_size}"
+            )
+            return [
+                Spore(
+                    cut=[fil.sql() for fil in path_filter],
+                    score=score,
+                    depth=depth,
+                )
+            ]
+
+        self.cater(train_df=train_df)
+        left_filter, right_filter = self.counter(train_df=train_df)
+        left_spores = self.train_champion(
+            target_filter=target_filter,
+            path_filter=path_filter + [left_filter],
+            depth=depth + "l",
+        )
+        right_spores = self.train_champion(
+            target_filter=target_filter,
+            path_filter=path_filter + [right_filter],
+            depth=depth + "r",
+        )
+        return left_spores + right_spores
+
+    def counter(self, train_df: TrainDataframes) -> tuple[Filter, Filter]:
+        sorted_train_feats = sorted(
+            train_df.train_features,
+            key=lambda x: x.calc_diff(),
+            reverse=True,
+        )
+        best_feat = sorted_train_feats[0]
+        best_feat_diff = best_feat.calc_diff()
+
+        for dim in range(2, self.settings.n_dims + 1):
+            counter = 0
+            for comb in itertools.combinations(sorted_train_feats, r=dim):
+                counter += 1
+                akt_feature = CombinedCategorizedFeature(
+                    comb,
+                    non_target_size=train_df.n_count_non_target,
+                    target_size=train_df.n_count_target,
+                )
+                if akt_feature.calc_diff() > best_feat_diff:
+                    best_feat = akt_feature
+                    best_feat_diff = akt_feature.calc_diff()
+
+                if (
+                    self.settings.calcs_per_dim
+                    and counter > self.settings.calcs_per_dim
+                ):
+                    break
+
+        # with pl.Config(tbl_rows=100, tbl_cols=100):
+        #    print(best_feat.diff_df)
+
+        # print(best_feat_diff, train_df.n_count_non_target)
+        # print(best_feat_diff, train_df.n_count_target)
+
+        return best_feat.get_left_right_filter()
+
+    def cater(self, train_df: TrainDataframes):
+        for feat in self.dc.train_cat_feature:
+            self.cat_cater(feat=feat, train_df=train_df)
+        for feat in self.dc.train_con_feature:
+            self.cont_cater(feat=feat, train_df=train_df)
+
+    def cat_cater(self, feat: Feature, train_df: TrainDataframes):
+        raise NotImplementedError("gibt noch net")
+        df = train_df.non_target_df_cat
+        print(df.select(pl.col(feat.name).value_counts()))
+
+    def get_n_quantiles(self, series: pl.Series, n: int) -> ContCats:
+        res = []
+        label = ["0"]
+        idx = 0
+        for i in range(1, n):
+            quantile = series.quantile(quantile=i / n, interpolation="lower")
+            if quantile in res:
+                continue
+            idx += 1
+            label.append(f"{idx}")
+            res.append(quantile)
+        return ContCats(cuts=res, labels=label, feat_name=series.name)
+
+    def cont_cater(self, feat: Feature, train_df: TrainDataframes):
+        categorized_feature = self.cont_cater_impl(feat=feat, train_df=train_df, n=2)
+
+        for n in range(3,self.settings.n_cat+1):
+            cat_feature = self.cont_cater_impl(feat=feat, train_df=train_df, n=n)
+            if cat_feature.calc_diff() > categorized_feature.calc_diff():
+                categorized_feature = cat_feature
+
+        if not categorized_feature.is_diff_to_low():
+            train_df.train_features.append(categorized_feature)
+
+
+    def cont_cater_impl(
+        self, feat: Feature, train_df: TrainDataframes, n: int
+    ) -> CategorizedFeature:
+        
+
+        df_sorted = train_df.df_group[[feat.name,'weight']].sort(feat.name)
+        cumulative_weights = df_sorted['weight'].cum_sum()
+
+        res = []
+        label = ["0"]
+        idx = 0
+        for i in range(1, n):
+            quantile = i /n 
+            index = (cumulative_weights >= quantile).arg_true()[0]
+
+            
+            if index > 0:
+                weight_below = cumulative_weights[index - 1]
+                value_below = df_sorted[feat.name][index - 1]
+                value_at_index = df_sorted[feat.name][index]
+                fraction = (quantile - weight_below) / (cumulative_weights[index] - weight_below)
+                value =  value_below + fraction * (value_at_index - value_below)
+            else:
+                value = df_sorted[feat.name][0]
+            if value in res:
+                continue
+            idx += 1
+            label.append(f"{idx}")
+            res.append(value)
+
+
+        return train_df.create_categorized_features(
+            feat=feat,
+            cuts=ContCats(cuts=res, labels=label, feat_name=feat.name),
+        )

champions-0.9.0.dist-info/METADATA

@@ -0,0 +1,33 @@
+Metadata-Version: 2.4
+Name: champions
+Version: 0.9.0
+Summary: Add your description here
+Project-URL: Homepage, https://gitlab.com/gwhe/champions
+Project-URL: Issues, https://gitlab.com/gwhe/champions/-/issues
+Author-email: swayand <stefan.wayand@gmail.com>
+License-Expression: GPL-3.0-or-later
+License-File: LICENSE
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.13
+Requires-Dist: altair>=5.5.0
+Requires-Dist: duckdb>=1.3.2
+Requires-Dist: polars>=1.32.0
+Requires-Dist: pyarrow>=19.0.1
+Requires-Dist: pydantic>=2.10.6
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: scikit-learn>=1.7.1
+Requires-Dist: typer>=0.15.2
+Requires-Dist: vl-convert-python>=1.8.0
+Description-Content-Type: text/markdown
+
+# Champions 
+
+
+## Status
+
+Disclaimer Doku muss noch gemahct werden.
+
+## Initial Thoughts
+
+The idea of this project is to make binary classification. It departs from some well-established Machine Learning methods in order to test some new ones. Some are purely technical, while others are more fundamental. This is a complete rewrite of my first private package, which I called Pilze (mushrooms). In German, "Champions" and "Champignons" sound similar, which is why I liked the name. But "Mushrooms" explains the project's purpose very well. Mushrooms are neither plants nor animals, but somewhere in between. In ML, it's all about correlations, and in a very abstract way, neural networks and tree-based algorithms solve this problem in opposite directions. Neural networks look at a single event and try to learn all correlations at one stage, while tree-based ML algorithms look at all (or many) events and try to find the correlations stage by stage. SVMs are a step further than trees, but in my personal opinion, they focus on the wrong thing. They focus on separation and not on the correlation of the individual classes. The initial idea of this project was to build a tree in which nodes not only make one cut but a set of cuts in more than one dimension. When it comes to implementations, the obvious choice is to use sklearn. It is one of the best software projects ever built, in my opinion. But some things are really annoying, and overall, it is hard to fit new ideas into this very strict framework.

champions-0.9.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+champions/__init__.py,sha256=Wbt9W5dYpbtuqqoh7K24jtGSMZSrzY5t1fnql0k6_No,641
+champions/cli.py,sha256=iXfoAxS9-oDyK1gsDonT7O0lj-SgK5rBRnNKiF44rWQ,991
+champions/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+champions/model/champions.py,sha256=2avlOipAR78d1N-xqExWT0a4WdDKEoF_wwcrPGPNJn0,799
+champions/model/datacard.py,sha256=I2yhhsw5JN9Au6sKIJvvfAYmHzflsKh4eiOyTb9XlsY,1175
+champions/model/dataframes.py,sha256=nGdec3ubQI5z-Xsf_aljtN1K3PtGElKy1Ms7akMjEZE,7516
+champions/model/filter.py,sha256=T-M-W9rYMY_Ool7N-CbDGe3ecPMU2gFrSLS5NDVPrjM,1364
+champions/model/settings.py,sha256=qjLw9fUx2KScYzsh3-53gEcE6tdc0HI-vH0di8Uf6Jg,2118
+champions/service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+champions/service/darkwing.py,sha256=hXEzXhsOTUB0qEmIaWySXMiGkNfWjq5Hui1prAQEfD8,3189
+champions/service/eval.py,sha256=lxy5jkExMTpJdtNHeEQj3jYn4DezWnhv5urMwYwCnB8,4689
+champions/service/train.py,sha256=lJP01OikyjnLy4MgmBpool7IuNnzd9TK7zT53W4LOWo,7329
+champions-0.9.0.dist-info/METADATA,sha256=9YbdlkbofJ3DsOI8Otf1fsXmAJegZwGNIp-eGhHpfng,2274
+champions-0.9.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+champions-0.9.0.dist-info/entry_points.txt,sha256=Jrat4iyGkU0OttDrUreuSo9fbaA5iJSsV2TPdiy_Sok,48
+champions-0.9.0.dist-info/licenses/LICENSE,sha256=1neeRA1TYb6baPpggguJlo8nWoiIlnBvhF_Sb1DTWhs,35139
+champions-0.9.0.dist-info/RECORD,,

champions-0.9.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

champions-0.9.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+champions = champions.cli:app