spells-mtg 0.10.11__py3-none-any.whl → 0.11.16__py3-none-any.whl

spells/.ruff_cache/.gitignore

@@ -0,0 +1,2 @@
+# Automatically created by ruff.
+*

spells/.ruff_cache/CACHEDIR.TAG

@@ -0,0 +1 @@
+Signature: 8a477f597d28d172789f06886806bc55

spells/cache.py

@@ -7,6 +7,7 @@ and groupbys.
 Caches are cleared per-set when new files are downloaded.
 """
 
+import datetime as dt
 from enum import StrEnum
 import os
 from pathlib import Path
@@ -31,6 +32,8 @@ class EventType(StrEnum):
 class DataDir(StrEnum):
     CACHE = "cache"
     EXTERNAL = "external"
+    RATINGS = "ratings"
+    DECK_COLOR = "deck_color"
 
 
 def spells_print(mode, content):
@@ -141,6 +144,8 @@ def data_dir_path(cache_dir: DataDir) -> str:
     ext = {
         DataDir.CACHE: "Cache" if is_win else "cache",
         DataDir.EXTERNAL: "External" if is_win else "external",
+        DataDir.RATINGS: "Ratings" if is_win else "ratings",
+        DataDir.DECK_COLOR: "DeckColor" if is_win else "deck_color",
     }[cache_dir]
 
     data_dir = os.path.join(data_home(), ext)
@@ -163,6 +168,38 @@ def data_file_path(set_code, dataset_type: str, event_type=EventType.PREMIER):
     )
 
 
+def card_ratings_file_path(
+    set_code: str,
+    format: str,
+    user_group: str,
+    deck_color: str,
+    start_date: dt.date,
+    end_date: dt.date,
+) -> tuple[str, str]:
+    return os.path.join(
+        data_dir_path(DataDir.RATINGS),
+        set_code,
+    ), (
+        f"{format}_{user_group}_{deck_color}_{start_date.strftime('%Y-%m-%d')}"
+        f"_{end_date.strftime('%Y-%m-%d')}.json"
+    )
+
+def deck_color_file_path(
+    set_code: str,
+    format: str,
+    user_group: str,
+    start_date: dt.date,
+    end_date: dt.date,
+) -> tuple[str, str]:
+    return os.path.join(
+        data_dir_path(DataDir.DECK_COLOR),
+        set_code,
+    ), (
+        f"{format}_{user_group}_{start_date.strftime('%Y-%m-%d')}"
+        f"_{end_date.strftime('%Y-%m-%d')}.json"
+    )
+
+
 def cache_dir_for_set(set_code: str) -> str:
     return os.path.join(data_dir_path(DataDir.CACHE), set_code)
 

spells/card_data_files.py

@@ -0,0 +1,222 @@
+import datetime as dt
+import os
+import wget
+from time import sleep
+
+import polars as pl
+
+from spells import cache
+from spells.enums import ColName
+
+RATINGS_TEMPLATE = (
+    "https://www.17lands.com/card_ratings/data?expansion={set_code}&format={format}"
+    "{user_group_param}{deck_color_param}&start_date={start_date_str}&end_date={end_date_str}"
+)
+
+DECK_COLOR_DATA_TEMPLATE = (
+    "https://www.17lands.com/color_ratings/data?expansion={set_code}&event_type={format}"
+    "{user_group_param}&start_date={start_date_str}&end_date={end_date_str}&combine_splash=true"
+)
+
+START_DATE_MAP = {
+    "PIO": dt.date(2024, 12, 10),
+    "DFT": dt.date(2025, 2, 11),
+    "TDM": dt.date(2025, 4, 8),
+    "FIN": dt.date(2025, 6, 10),
+    "EOE": dt.date(2025, 7, 29),
+    "OM1": dt.date(2025, 9, 23),
+    "Cube+-+Powered": dt.date(2025, 10, 28),
+}
+
+ratings_col_defs = {
+    ColName.NAME: pl.col("name").cast(pl.String),
+    ColName.COLOR: pl.col("color").cast(pl.String),
+    ColName.RARITY: pl.col("rarity").cast(pl.String),
+    ColName.IMAGE_URL: pl.col("url").cast(pl.String),
+    ColName.NUM_SEEN: pl.col("seen_count").cast(pl.Int64),
+    ColName.LAST_SEEN: pl.col("seen_count") * pl.col("avg_seen").cast(pl.Float64),
+    ColName.NUM_TAKEN: pl.col("pick_count").cast(pl.Int64),
+    ColName.TAKEN_AT: pl.col("pick_count") * pl.col("avg_pick").cast(pl.Float64),
+    ColName.DECK: pl.col("game_count").cast(pl.Int64),
+    ColName.WON_DECK: pl.col("win_rate") * pl.col("game_count").cast(pl.Float64),
+    ColName.SIDEBOARD: (pl.col("pool_count") - pl.col("game_count")).cast(pl.Int64),
+    ColName.OPENING_HAND: pl.col("opening_hand_game_count").cast(pl.Int64),
+    ColName.WON_OPENING_HAND: pl.col("opening_hand_game_count")
+    * pl.col("opening_hand_win_rate").cast(pl.Float64),
+    ColName.DRAWN: pl.col("drawn_game_count").cast(pl.Int64),
+    ColName.WON_DRAWN: pl.col("drawn_win_rate")
+    * pl.col("drawn_game_count").cast(pl.Float64),
+    ColName.NUM_GIH: pl.col("ever_drawn_game_count").cast(pl.Int64),
+    ColName.NUM_GIH_WON: pl.col("ever_drawn_game_count")
+    * pl.col("ever_drawn_win_rate").cast(pl.Float64),
+    ColName.NUM_GNS: pl.col("never_drawn_game_count").cast(pl.Int64),
+    ColName.WON_NUM_GNS: pl.col("never_drawn_game_count")
+    * pl.col("never_drawn_win_rate").cast(pl.Float64),
+}
+
+deck_color_col_defs = {
+    ColName.MAIN_COLORS: pl.col("short_name").cast(pl.String),
+    ColName.NUM_GAMES: pl.col("games").cast(pl.Int64),
+    ColName.NUM_WON: pl.col("wins").cast(pl.Int64),
+}
+
+
+def deck_color_df(
+    set_code: str,
+    format: str = "PremierDraft",
+    player_cohort: str = "all",
+    start_date: dt.date | None = None,
+    end_date: dt.date | None = None,
+):
+    if start_date is None:
+        start_date = START_DATE_MAP[set_code]
+    if end_date is None:
+        end_date = dt.date.today() - dt.timedelta(days=1)
+
+    target_dir, filename = cache.deck_color_file_path(
+        set_code,
+        format,
+        player_cohort,
+        start_date,
+        end_date,
+    )
+
+    if not os.path.isdir(target_dir):
+        os.makedirs(target_dir)
+
+    deck_color_file_path = os.path.join(target_dir, filename)
+
+    if not os.path.isfile(deck_color_file_path):
+        user_group_param = (
+            "" if player_cohort == "all" else f"&user_group={player_cohort}"
+        )
+
+        url = DECK_COLOR_DATA_TEMPLATE.format(
+            set_code=set_code,
+            format=format,
+            user_group_param=user_group_param,
+            start_date_str=start_date.strftime("%Y-%m-%d"),
+            end_date_str=end_date.strftime("%Y-%m-%d"),
+        )
+
+        wget.download(
+            url,
+            out=deck_color_file_path,
+        )
+
+    df = (
+        pl.read_json(deck_color_file_path)
+        .filter(~pl.col("is_summary"))
+        .select(
+            [
+                pl.lit(set_code).alias(ColName.EXPANSION),
+                pl.lit(format).alias(ColName.EVENT_TYPE),
+                (pl.lit("Top") if player_cohort == "top" else pl.lit(None))
+                .alias(ColName.PLAYER_COHORT)
+                .cast(pl.String),
+                *[val.alias(key) for key, val in deck_color_col_defs.items()],
+            ]
+        )
+    )
+
+    return df
+
+
+def base_ratings_df(
+    set_code: str,
+    format: str = "PremierDraft",
+    player_cohort: str = "all",
+    deck_colors: str | list[str] = "any",
+    start_date: dt.date | None = None,
+    end_date: dt.date | None = None,
+) -> pl.DataFrame:
+    if start_date is None:
+        start_date = START_DATE_MAP[set_code]
+    if end_date is None:
+        end_date = dt.date.today() - dt.timedelta(days=1)
+
+    if isinstance(deck_colors, str):
+        deck_colors = [deck_colors]
+
+    concat_list = []
+    for i, deck_color in enumerate(deck_colors):
+        ratings_dir, filename = cache.card_ratings_file_path(
+            set_code,
+            format,
+            player_cohort,
+            deck_color,
+            start_date,
+            end_date,
+        )
+
+        if not os.path.isdir(ratings_dir):
+            os.makedirs(ratings_dir)
+
+        ratings_file_path = os.path.join(ratings_dir, filename)
+
+        if not os.path.isfile(ratings_file_path):
+            if i > 0:
+                sleep(5)
+            user_group_param = (
+                "" if player_cohort == "all" else f"&user_group={player_cohort}"
+            )
+            deck_color_param = "" if deck_color == "any" else f"&colors={deck_color}"
+
+            url = RATINGS_TEMPLATE.format(
+                set_code=set_code,
+                format=format,
+                user_group_param=user_group_param,
+                deck_color_param=deck_color_param,
+                start_date_str=start_date.strftime("%Y-%m-%d"),
+                end_date_str=end_date.strftime("%Y-%m-%d"),
+            )
+
+            wget.download(
+                url,
+                out=ratings_file_path,
+            )
+
+        concat_list.append(
+            pl.read_json(ratings_file_path, infer_schema_length=500)
+            .with_columns(
+                (pl.lit(deck_color) if deck_color != "any" else pl.lit(None))
+                .alias(ColName.MAIN_COLORS)
+                .cast(pl.String)
+            )
+            .select(
+                [
+                    pl.lit(set_code).alias(ColName.EXPANSION),
+                    pl.lit(format).alias(ColName.EVENT_TYPE),
+                    (pl.lit("Top") if player_cohort == "top" else pl.lit(None))
+                    .alias(ColName.PLAYER_COHORT)
+                    .cast(pl.String),
+                    ColName.MAIN_COLORS,
+                    *[val.alias(key) for key, val in ratings_col_defs.items()],
+                ]
+            )
+        )
+
+    raw_df = pl.concat(concat_list)
+
+    group_cols = [
+        ColName.NAME,
+        ColName.EXPANSION,
+        ColName.MAIN_COLORS,
+    ]
+
+    attr_cols = [
+        ColName.EVENT_TYPE,
+        ColName.PLAYER_COHORT,
+        ColName.COLOR,
+        ColName.RARITY,
+        ColName.IMAGE_URL,
+    ]
+
+    sum_cols = list(set(ratings_col_defs) - set(group_cols + attr_cols))
+
+    attr_df = raw_df.select(group_cols + attr_cols).group_by(group_cols).first()
+    sum_df = raw_df.select(group_cols + sum_cols).group_by(group_cols).sum()
+
+    df = attr_df.join(sum_df, on=group_cols, join_nulls=True)
+
+    return df

spells/columns.py

@@ -40,6 +40,11 @@ default_columns = [
     ColName.GIH_WR,
 ]
 
+
+def agg_col(expr: pl.Expr) -> ColSpec:
+    return ColSpec(col_type=ColType.AGG, expr=expr)
+
+
 _specs: dict[str, ColSpec] = {
     ColName.NAME: ColSpec(
         col_type=ColType.GROUP_BY,
@@ -168,7 +173,8 @@ _specs: dict[str, ColSpec] = {
     ),
     ColName.PICK_INDEX: ColSpec(
         col_type=ColType.GROUP_BY,
-        expr=lambda set_context: pl.col(ColName.PICK_NUMBER) + pl.col(ColName.PACK_NUMBER) * set_context['picks_per_pack']
+        expr=lambda set_context: pl.col(ColName.PICK_NUMBER)
+        + pl.col(ColName.PACK_NUMBER) * set_context["picks_per_pack"],
     ),
     ColName.TAKEN_AT: ColSpec(
         col_type=ColType.PICK_SUM,
@@ -441,149 +447,61 @@ _specs: dict[str, ColSpec] = {
     ColName.IMAGE_URL: ColSpec(
         col_type=ColType.CARD_ATTR,
     ),
-    ColName.PICKED_MATCH_WR: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.EVENT_MATCH_WINS_SUM) / pl.col(ColName.EVENT_MATCHES_SUM),
-    ),
-    ColName.TROPHY_RATE: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.IS_TROPHY_SUM) / pl.col(ColName.NUM_TAKEN),
-    ),
-    ColName.GAME_WR: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.NUM_WON) / pl.col(ColName.NUM_GAMES),
-    ),
-    ColName.ALSA: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.LAST_SEEN) / pl.col(ColName.NUM_SEEN),
-    ),
-    ColName.ATA: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.TAKEN_AT) / pl.col(ColName.NUM_TAKEN),
-    ),
-    ColName.NUM_GP: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.DECK),
-    ),
-    ColName.PCT_GP: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.DECK) / (pl.col(ColName.DECK) + pl.col(ColName.SIDEBOARD)),
-    ),
-    ColName.GP_WR: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.WON_DECK) / pl.col(ColName.DECK),
-    ),
-    ColName.NUM_OH: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.OPENING_HAND),
-    ),
-    ColName.OH_WR: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.WON_OPENING_HAND) / pl.col(ColName.OPENING_HAND),
-    ),
-    ColName.NUM_GIH: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.OPENING_HAND) + pl.col(ColName.DRAWN),
-    ),
-    ColName.NUM_GIH_WON: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.WON_OPENING_HAND) + pl.col(ColName.WON_DRAWN),
-    ),
-    ColName.GIH_WR: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.NUM_GIH_WON) / pl.col(ColName.NUM_GIH),
-    ),
-    ColName.GNS_WR: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.WON_NUM_GNS) / pl.col(ColName.NUM_GNS),
-    ),
-    ColName.IWD: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.GIH_WR) - pl.col(ColName.GNS_WR),
-    ),
-    ColName.NUM_IN_POOL: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.DECK) + pl.col(ColName.SIDEBOARD),
-    ),
-    ColName.NUM_IN_POOL_TOTAL: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.NUM_IN_POOL).sum(),
-    ),
-    ColName.IN_POOL_WR: ColSpec(
-        col_type=ColType.AGG,
-        expr=(pl.col(ColName.WON_DECK) + pl.col(ColName.WON_SIDEBOARD))
-        / pl.col(ColName.NUM_IN_POOL),
-    ),
-    ColName.DECK_TOTAL: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.DECK).sum(),
-    ),
-    ColName.WON_DECK_TOTAL: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.WON_DECK).sum(),
-    ),
-    ColName.GP_WR_MEAN: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.WON_DECK_TOTAL) / pl.col(ColName.DECK_TOTAL),
-    ),
-    ColName.GP_WR_EXCESS: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.GP_WR) - pl.col(ColName.GP_WR_MEAN),
-    ),
-    ColName.GP_WR_VAR: ColSpec(
-        col_type=ColType.AGG,
-        expr=(pl.col(ColName.GP_WR_EXCESS).pow(2) * pl.col(ColName.NUM_GP)).sum()
-        / pl.col(ColName.DECK_TOTAL),
-    ),
-    ColName.GP_WR_STDEV: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.GP_WR_VAR).sqrt(),
-    ),
-    ColName.GP_WR_Z: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.GP_WR_EXCESS) / pl.col(ColName.GP_WR_STDEV),
-    ),
-    ColName.GIH_TOTAL: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.NUM_GIH).sum(),
-    ),
-    ColName.WON_GIH_TOTAL: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.NUM_GIH_WON).sum(),
-    ),
-    ColName.GIH_WR_MEAN: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.WON_GIH_TOTAL) / pl.col(ColName.GIH_TOTAL),
-    ),
-    ColName.GIH_WR_EXCESS: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.GIH_WR) - pl.col(ColName.GIH_WR_MEAN),
-    ),
-    ColName.GIH_WR_VAR: ColSpec(
-        col_type=ColType.AGG,
-        expr=(pl.col(ColName.GIH_WR_EXCESS).pow(2) * pl.col(ColName.NUM_GIH)).sum()
-        / pl.col(ColName.GIH_TOTAL),
-    ),
-    ColName.GIH_WR_STDEV: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.GIH_WR_VAR).sqrt(),
-    ),
-    ColName.GIH_WR_Z: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.GIH_WR_EXCESS) / pl.col(ColName.GIH_WR_STDEV),
-    ),
-    ColName.DECK_MANA_VALUE_AVG: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.DECK_MANA_VALUE) / pl.col(ColName.DECK_SPELLS),
-    ),
-    ColName.DECK_LANDS_AVG: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.DECK_LANDS) / pl.col(ColName.NUM_GAMES),
-    ),
-    ColName.DECK_SPELLS_AVG: ColSpec(
-        col_type=ColType.AGG,
-        expr=pl.col(ColName.DECK_SPELLS) / pl.col(ColName.NUM_GAMES),
-    ),
+    ColName.PICKED_MATCH_WR: agg_col(
+        pl.col(ColName.EVENT_MATCH_WINS_SUM) / pl.col(ColName.EVENT_MATCHES_SUM)
+    ),
+    ColName.TROPHY_RATE: agg_col(
+        pl.col(ColName.IS_TROPHY_SUM) / pl.col(ColName.NUM_TAKEN),
+    ),
+    ColName.GAME_WR: agg_col(
+        pl.col(ColName.NUM_WON) / pl.col(ColName.NUM_GAMES),
+    ),
+    ColName.ALSA: agg_col(pl.col(ColName.LAST_SEEN) / pl.col(ColName.NUM_SEEN)),
+    ColName.ATA: agg_col(pl.col(ColName.TAKEN_AT) / pl.col(ColName.NUM_TAKEN)),
+    ColName.NUM_GP: agg_col(pl.col(ColName.DECK)),
+    ColName.PCT_GP: agg_col(
+        pl.col(ColName.DECK) / (pl.col(ColName.DECK) + pl.col(ColName.SIDEBOARD))
+    ),
+    ColName.GP_WR: agg_col(pl.col(ColName.WON_DECK) / pl.col(ColName.DECK)),
+    ColName.NUM_OH: agg_col(pl.col(ColName.OPENING_HAND)),
+    ColName.OH_WR: agg_col(
+        pl.col(ColName.WON_OPENING_HAND) / pl.col(ColName.OPENING_HAND)
+    ),
+    ColName.NUM_GIH: agg_col(pl.col(ColName.OPENING_HAND) + pl.col(ColName.DRAWN)),
+    ColName.NUM_GIH_WON: agg_col(
+        pl.col(ColName.WON_OPENING_HAND) + pl.col(ColName.WON_DRAWN)
+    ),
+    ColName.GIH_WR: agg_col(pl.col(ColName.NUM_GIH_WON) / pl.col(ColName.NUM_GIH)),
+    ColName.GNS_WR: agg_col(pl.col(ColName.WON_NUM_GNS) / pl.col(ColName.NUM_GNS)),
+    ColName.IWD: agg_col(pl.col(ColName.GIH_WR) - pl.col(ColName.GNS_WR)),
+    ColName.NUM_IN_POOL: agg_col(pl.col(ColName.DECK) + pl.col(ColName.SIDEBOARD)),
+    ColName.NUM_IN_POOL_TOTAL: agg_col(pl.col(ColName.NUM_IN_POOL).sum()),
+    ColName.IN_POOL_WR: agg_col(
+        (pl.col(ColName.WON_DECK) + pl.col(ColName.WON_SIDEBOARD))
+        / pl.col(ColName.NUM_IN_POOL)
+    ),
+    ColName.DECK_TOTAL: agg_col(pl.col(ColName.DECK).sum()),
+    ColName.WON_DECK_TOTAL: agg_col(pl.col(ColName.WON_DECK).sum()),
+    ColName.GP_WR_MEAN: agg_col(pl.col(ColName.WON_DECK_TOTAL) / pl.col(ColName.DECK_TOTAL)),
+    ColName.GP_WR_EXCESS: agg_col(pl.col(ColName.GP_WR) - pl.col(ColName.GP_WR_MEAN)),
+    ColName.GP_WR_VAR: agg_col((pl.col(ColName.GP_WR_EXCESS).pow(2) * pl.col(ColName.NUM_GP)).sum()
+        / pl.col(ColName.DECK_TOTAL)
+    ),
+    ColName.GP_WR_STDEV: agg_col(pl.col(ColName.GP_WR_VAR).sqrt()),
+    ColName.GP_WR_Z: agg_col(pl.col(ColName.GP_WR_EXCESS) / pl.col(ColName.GP_WR_STDEV)),
+    ColName.GIH_TOTAL: agg_col(pl.col(ColName.NUM_GIH).sum()),
+    ColName.WON_GIH_TOTAL: agg_col(pl.col(ColName.NUM_GIH_WON).sum()),
+    ColName.GIH_WR_MEAN: agg_col(pl.col(ColName.WON_GIH_TOTAL) / pl.col(ColName.GIH_TOTAL)),
+    ColName.GIH_WR_EXCESS: agg_col(pl.col(ColName.GIH_WR) - pl.col(ColName.GIH_WR_MEAN)),
+    ColName.GIH_WR_VAR: agg_col(
+        (pl.col(ColName.GIH_WR_EXCESS).pow(2) * pl.col(ColName.NUM_GIH)).sum()
+        / pl.col(ColName.GIH_TOTAL)
+    ),
+    ColName.GIH_WR_STDEV: agg_col(pl.col(ColName.GIH_WR_VAR).sqrt()),
+    ColName.GIH_WR_Z: agg_col(pl.col(ColName.GIH_WR_EXCESS) / pl.col(ColName.GIH_WR_STDEV)),
+    ColName.DECK_MANA_VALUE_AVG: agg_col(pl.col(ColName.DECK_MANA_VALUE) / pl.col(ColName.DECK_SPELLS)),
+    ColName.DECK_LANDS_AVG: agg_col(pl.col(ColName.DECK_LANDS) / pl.col(ColName.NUM_GAMES)),
+    ColName.DECK_SPELLS_AVG: agg_col(pl.col(ColName.DECK_SPELLS) / pl.col(ColName.NUM_GAMES)),
 }
 
 for item in ColName:

spells/config.py

@@ -1,4 +1,6 @@
 all_sets = [
+    "EOE",
+    "FIN",
     "TDM",
     "DFT",
     "PIO",

spells/draft_data.py

@@ -6,6 +6,8 @@ Aggregate dataframes containing raw counts are cached in the local file system
 for performance.
 """
 
+from dataclasses import dataclass
+import datetime
 import functools
 import hashlib
 import re
@@ -23,9 +25,19 @@ from spells import manifest
 from spells.columns import ColDef, ColSpec, get_specs
 from spells.enums import View, ColName, ColType
 from spells.log import make_verbose
+from spells.card_data_files import base_ratings_df
 
 DF = TypeVar("DF", pl.LazyFrame, pl.DataFrame)
 
+@dataclass
+class CardDataFileSpec():
+    set_code: str
+    format: str = "PremierDraft"
+    player_cohort: str = "all"
+    deck_colors: str | list[str] = "any"
+    start_date: datetime.date | None = None
+    end_date: datetime.date | None = None
+
 
 def _cache_key(args) -> str:
     """
@@ -37,20 +49,25 @@ def _cache_key(args) -> str:
 @functools.lru_cache(maxsize=None)
 def get_names(set_code: str) -> list[str]:
     card_fp = cache.data_file_path(set_code, View.CARD)
-    card_view = pl.read_parquet(card_fp)
-    card_names_set = frozenset(card_view.get_column("name").to_list())
+    try:
+        card_view = pl.read_parquet(card_fp)
+        card_names_set = frozenset(card_view.get_column("name").to_list())
 
-    draft_fp = cache.data_file_path(set_code, View.DRAFT)
-    draft_view = pl.scan_parquet(draft_fp)
-    cols = draft_view.collect_schema().names()
+        draft_fp = cache.data_file_path(set_code, View.DRAFT)
+        draft_view = pl.scan_parquet(draft_fp)
+        cols = draft_view.collect_schema().names()
 
-    prefix = "pack_card_"
-    names = [col[len(prefix) :] for col in cols if col.startswith(prefix)]
-    draft_names_set = frozenset(names)
+        prefix = "pack_card_"
+        names = [col[len(prefix) :] for col in cols if col.startswith(prefix)]
+        draft_names_set = frozenset(names)
+
+        assert (
+            draft_names_set == card_names_set
+        ), "names mismatch between card and draft file"
+    except FileNotFoundError:
+        ratings_data = base_ratings_df(set_code)
+        names = list(ratings_data['name'])
 
-    assert (
-        draft_names_set == card_names_set
-    ), "names mismatch between card and draft file"
     return names
 
 
@@ -330,7 +347,10 @@ def _view_select(
     cdefs = [col_def_map[c] for c in sorted(view_cols)]
     select = []
     for cdef in cdefs:
-        if is_agg_view:
+        if isinstance(df, pl.DataFrame) and cdef.name in df.columns:
+            base_cols = base_cols.union(frozenset({cdef.name}))
+            select.append(cdef.name)
+        elif is_agg_view:
             if cdef.col_type == ColType.AGG:
                 base_cols = base_cols.union(cdef.dependencies)
                 select.append(cdef.expr)
@@ -454,7 +474,7 @@ def _base_agg_df(
 
     if group_by:
         joined_df = functools.reduce(
-            lambda prev, curr: prev.join(curr, on=group_by, how="outer", coalesce=True),
+            lambda prev, curr: prev.join(curr, on=group_by, how="full", coalesce=True),
             join_dfs,
         )
     else:
@@ -476,6 +496,7 @@ def summon(
     write_cache: bool = True,
     card_context: pl.DataFrame | dict[str, Any] | None = None,
     set_context: pl.DataFrame | dict[str, Any] | None = None,
+    cdfs: CardDataFileSpec | None = None,
 ) -> pl.DataFrame:
     specs = get_specs()
 
@@ -515,33 +536,51 @@ def summon(
         else:
             this_set_context = None
 
-        col_def_map = _hydrate_col_defs(code, specs, set_card_context, this_set_context)
+        col_def_map = _hydrate_col_defs(
+            code, 
+            specs, 
+            set_card_context, 
+            this_set_context,
+            card_only=cdfs is not None,
+        )
         m = manifest.create(col_def_map, columns, group_by, filter_spec)
 
-        calc_fn = functools.partial(_base_agg_df, code, m, use_streaming=use_streaming)
-        agg_df = _fetch_or_cache(
-            calc_fn,
-            code,
-            (
+        if cdfs is None:
+            calc_fn = functools.partial(_base_agg_df, code, m, use_streaming=use_streaming)
+            agg_df = _fetch_or_cache(
+                calc_fn,
                 code,
-                sorted(m.view_cols.get(View.DRAFT, set())),
-                sorted(m.view_cols.get(View.GAME, set())),
-                sorted(c.signature or "" for c in m.col_def_map.values()),
-                sorted(m.base_view_group_by),
-                filter_spec,
-            ),
-            read_cache=read_cache,
-            write_cache=write_cache,
-        )
-
-        if View.CARD in m.view_cols:
-            card_cols = m.view_cols[View.CARD].union({ColName.NAME})
-            fp = cache.data_file_path(code, View.CARD)
-            card_df = pl.read_parquet(fp)
-            select_df = _view_select(
-                card_df, card_cols, m.col_def_map, is_agg_view=False
+                (
+                    code,
+                    sorted(m.view_cols.get(View.DRAFT, set())),
+                    sorted(m.view_cols.get(View.GAME, set())),
+                    sorted(c.signature or "" for c in m.col_def_map.values()),
+                    sorted(m.base_view_group_by),
+                    filter_spec,
+                ),
+                read_cache=read_cache,
+                write_cache=write_cache,
+            )
+            if View.CARD in m.view_cols:
+                card_cols = m.view_cols[View.CARD].union({ColName.NAME})
+                fp = cache.data_file_path(code, View.CARD)
+                card_df = pl.read_parquet(fp)
+                select_df = _view_select(
+                    card_df, card_cols, m.col_def_map, is_agg_view=False
+                )
+                agg_df = agg_df.join(select_df, on="name", how="full", coalesce=True)
+        else:
+            assert len(codes) == 1, "Only one set supported for loading from card data file"
+            assert codes[0] == cdfs.set_code, "Wrong set file specified"
+            agg_df = base_ratings_df(
+                set_code=cdfs.set_code,
+                format=cdfs.format,
+                player_cohort=cdfs.player_cohort,
+                deck_colors=cdfs.deck_colors,
+                start_date=cdfs.start_date,
+                end_date=cdfs.end_date,
             )
-            agg_df = agg_df.join(select_df, on="name", how="outer", coalesce=True)
+
         concat_dfs.append(agg_df)
 
     full_agg_df = pl.concat(concat_dfs, how="vertical")

spells/extension.py

@@ -101,6 +101,10 @@ def context_cols(attr, silent: bool = True) -> dict[str, ColSpec]:
                 [pl.col(f"seen_{attr}_{name}") for name in names]
             ),
         ),
+        f"not_picked_{attr}_sum": ColSpec(
+            col_type=ColType.PICK_SUM,
+            expr=pl.col(f"seen_{attr}_pack_sum") - pl.col(f"pick_{attr}_sum")
+        ),
         f"least_{attr}_seen": ColSpec(
             col_type=ColType.PICK_SUM,
             expr=lambda names: pl.min_horizontal(

spells/external.py

@@ -30,7 +30,6 @@ RESOURCE_TEMPLATE = (
     "https://17lands-public.s3.amazonaws.com/analysis_data/{dataset_type}_data/"
 )
 
-
 class FileFormat(StrEnum):
     CSV = "csv"
     PARQUET = "parquet"
@@ -121,7 +120,8 @@ def _context() -> int:
     for code in all_sets:
         write_card_file(code, force_download=True)
         get_set_context(code, force_download=True)
-    return 0 
+    return 0
+
 
 def _refresh(set_code: str):
     return _add(set_code, force_download=True)
@@ -359,4 +359,5 @@ def get_set_context(set_code: str, force_download=False) -> int:
     context_df.write_parquet(context_fp)
 
     cache.spells_print(mode, f"Wrote file {context_fp}")
+    
     return 0

{spells_mtg-0.10.11.dist-info → spells_mtg-0.11.16.dist-info}/METADATA

@@ -1,11 +1,11 @@
 Metadata-Version: 2.1
 Name: spells-mtg
-Version: 0.10.11
+Version: 0.11.16
 Summary: analaysis of 17Lands.com public datasets
 Author-Email: Joel Barnes <oelarnes@gmail.com>
 License: MIT
 Requires-Python: >=3.11
-Requires-Dist: polars>=1.14.0
+Requires-Dist: polars==1.22.0
 Requires-Dist: wget>=3.2
 Description-Content-Type: text/markdown
 

spells_mtg-0.11.16.dist-info/RECORD

@@ -0,0 +1,23 @@
+spells/.ruff_cache/.gitignore,sha256=njpg8ebsSuYCFcEdVLFxOSdF7CXp3e1DPVvZITY68xY,35
+spells/.ruff_cache/0.8.6/17785301476771359756,sha256=gPYLG8psuOSo37IefLzTu5wgRrbKLWEfhWP_yLC4VIY,159
+spells/.ruff_cache/CACHEDIR.TAG,sha256=WVMVbX4MVkpCclExbq8m-IcOZIOuIZf5FrYw5Pk-Ma4,43
+spells/__init__.py,sha256=0pnh2NLn9FrNKncE9-tBsighp3e8YAsN--_ovUpgWGs,333
+spells/cache.py,sha256=6cl0q62erR3LCANPSfxG5-J7JQfLwNdWjzlBpfiL4IE,7174
+spells/card_data_files.py,sha256=x_oiFXw-mH-xZrSLrnE6dV8f1C2mQr5iFp9skR_YGhY,7246
+spells/cards.py,sha256=6stFPhJOzHqvQnkSv9cDeylRa_7L9Y8sOaowiZhzz6I,4174
+spells/columns.py,sha256=HsmOSunZHs0RMyXoSo-gAVWMNZqMYs3IrO5v5-99BqM,16678
+spells/config.py,sha256=Nym660bbYt4ijzC5scVJ6PVOh-4vBi4hfFusBIPxcZk,257
+spells/draft_data.py,sha256=iYsfmVRPy67GS3Epv8xakYVZQZIr0JbrZMFG0_wpLm4,21380
+spells/enums.py,sha256=gbwfon6tQCoKDb-m4hSaHWi9slj82yqaH3qhYMVrsck,4991
+spells/extension.py,sha256=iTR_2a_elM-8q8qxeeZFV8145i97fW2c3PfE3rN-HFg,8625
+spells/external.py,sha256=I-f_vMx-h2kvunUlZthsauaeDn42vcRk0wvTURfImzs,11848
+spells/filter.py,sha256=J-YTOOAzOQpvIX29tviYL04RVoOUlfsbjBXoQBDCEdQ,3380
+spells/log.py,sha256=3avmg65hru8K9npKLvPp1wWWxq-hoEYDUCbxqhPkKUw,2175
+spells/manifest.py,sha256=ExWVk17BRw615UmvrV817xwz457yfTNdNMNE_M00aEg,8338
+spells/schema.py,sha256=DbMvV8PIThJTp0Xzp_XIorlW6JhE1ud1kWRGf5SQ4_c,6406
+spells/utils.py,sha256=IO3brrXVvZla0LRTEB5v6NgGqZb_rYA46XtKBURGMNk,1944
+spells_mtg-0.11.16.dist-info/METADATA,sha256=G-rx2_1SCdYXRT15YWkgTNA7kKnUdqVDFMf6qvFQSy0,47370
+spells_mtg-0.11.16.dist-info/WHEEL,sha256=9P2ygRxDrTJz3gsagc0Z96ukrxjr-LFBGOgv3AuKlCA,90
+spells_mtg-0.11.16.dist-info/entry_points.txt,sha256=a9Y1omdl9MdnKuIj3aOodgrp-zZII6OCdvqwgP6BFvI,63
+spells_mtg-0.11.16.dist-info/licenses/LICENSE,sha256=tS54XYbJSgmq5zuHhbsQGbNQLJPVgXqhF5nu2CSRMig,1068
+spells_mtg-0.11.16.dist-info/RECORD,,

{spells_mtg-0.10.11.dist-info → spells_mtg-0.11.16.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: pdm-backend (2.4.4)
+Generator: pdm-backend (2.4.5)
 Root-Is-Purelib: true
 Tag: py3-none-any

spells_mtg-0.10.11.dist-info/RECORD

@@ -1,19 +0,0 @@
-spells/__init__.py,sha256=0pnh2NLn9FrNKncE9-tBsighp3e8YAsN--_ovUpgWGs,333
-spells/cache.py,sha256=CjMsIOdUz43heJ4IZ5kTWfS_rAeOMFQ1qVAuq3mTWao,6192
-spells/cards.py,sha256=6stFPhJOzHqvQnkSv9cDeylRa_7L9Y8sOaowiZhzz6I,4174
-spells/columns.py,sha256=s_PYyg2QaRL6kLWFNKCBEfbMF0x7O7-Yd9SeG1ttYL4,18206
-spells/config.py,sha256=zpRUZ-6JKALE149L0yfeD1AgqKPaVBGHfD_TdHi-jqE,235
-spells/draft_data.py,sha256=1HKYjDu0O8fD2AhUL9Zgw0jVu4lKweJ4KBFMDkH5JLU,19937
-spells/enums.py,sha256=gbwfon6tQCoKDb-m4hSaHWi9slj82yqaH3qhYMVrsck,4991
-spells/extension.py,sha256=LBqGbJbe7iSRQkxJK7npkADCfzhdnIwwVvlmTn8xvjQ,8454
-spells/external.py,sha256=USqOtOVY7Mn39rdUPhmIeGm08s6TDnEV8-N3D14qJzE,11844
-spells/filter.py,sha256=J-YTOOAzOQpvIX29tviYL04RVoOUlfsbjBXoQBDCEdQ,3380
-spells/log.py,sha256=3avmg65hru8K9npKLvPp1wWWxq-hoEYDUCbxqhPkKUw,2175
-spells/manifest.py,sha256=ExWVk17BRw615UmvrV817xwz457yfTNdNMNE_M00aEg,8338
-spells/schema.py,sha256=DbMvV8PIThJTp0Xzp_XIorlW6JhE1ud1kWRGf5SQ4_c,6406
-spells/utils.py,sha256=IO3brrXVvZla0LRTEB5v6NgGqZb_rYA46XtKBURGMNk,1944
-spells_mtg-0.10.11.dist-info/METADATA,sha256=Z2vzKqgfKDsX85L8A26nhbRYiA83APhCgnN6DRZrQcc,47370
-spells_mtg-0.10.11.dist-info/WHEEL,sha256=tSfRZzRHthuv7vxpI4aehrdN9scLjk-dCJkPLzkHxGg,90
-spells_mtg-0.10.11.dist-info/entry_points.txt,sha256=a9Y1omdl9MdnKuIj3aOodgrp-zZII6OCdvqwgP6BFvI,63
-spells_mtg-0.10.11.dist-info/licenses/LICENSE,sha256=tS54XYbJSgmq5zuHhbsQGbNQLJPVgXqhF5nu2CSRMig,1068
-spells_mtg-0.10.11.dist-info/RECORD,,