spells-mtg 0.8.3__tar.gz → 0.9.0__tar.gz

{spells_mtg-0.8.3 → spells_mtg-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spells-mtg
-Version: 0.8.3
+Version: 0.9.0
 Summary: analaysis of 17Lands.com public datasets
 Author-Email: Joel Barnes <oelarnes@gmail.com>
 License: MIT

{spells_mtg-0.8.3 → spells_mtg-0.9.0}/pyproject.toml

@@ -11,7 +11,7 @@ dependencies = [
 ]
 requires-python = ">=3.11"
 readme = "README.md"
-version = "0.8.3"
+version = "0.9.0"
 
 [project.license]
 text = "MIT"
@@ -40,4 +40,8 @@ dev = [
     "pytest>=8.3.3",
     "ruff>=0.7.4",
     "jupyter>=1.1.1",
+    "matplotlib>=3.10.0",
+    "numpy>=2.2.0",
+    "jupyter-book>=1.0.3",
+    "ghp-import>=2.1.0",
 ]

{spells_mtg-0.8.3 → spells_mtg-0.9.0}/spells/cache.py

@@ -14,6 +14,11 @@ import sys
 import polars as pl
 
 
+class EventType(StrEnum):
+    PREMIER = "PremierDraft"
+    TRADITIONAL = "TradDraft"
+
+
 class DataDir(StrEnum):
     CACHE = "cache"
     EXTERNAL = "external"
@@ -52,6 +57,22 @@ def data_dir_path(cache_dir: DataDir) -> str:
     return data_dir
 
 
+def external_set_path(set_code):
+    return os.path.join(data_dir_path(DataDir.EXTERNAL), set_code)
+
+
+def data_file_path(set_code, dataset_type: str, event_type=EventType.PREMIER):
+    if dataset_type == "set_context":
+        return os.path.join(external_set_path(set_code), f"{set_code}_context.parquet")
+
+    if dataset_type == "card":
+        return os.path.join(external_set_path(set_code), f"{set_code}_card.parquet")
+
+    return os.path.join(
+        external_set_path(set_code), f"{set_code}_{event_type}_{dataset_type}.parquet"
+    )
+
+
 def cache_dir_for_set(set_code: str) -> str:
     return os.path.join(data_dir_path(DataDir.CACHE), set_code)
 

{spells_mtg-0.8.3 → spells_mtg-0.9.0}/spells/columns.py

@@ -68,10 +68,7 @@ _specs: dict[str, ColSpec] = {
     ColName.FORMAT_DAY: ColSpec(
         col_type=ColType.GROUP_BY,
         expr=lambda set_context: (
-            pl.col(ColName.DRAFT_DATE)
-            - pl.lit(set_context["release_time"])
-            .str.to_datetime("%Y-%m-%d %H:%M:%S")
-            .dt.date()
+            pl.col(ColName.DRAFT_DATE) - pl.lit(set_context["release_date"])
         ).dt.total_days()
         + 1,
     ),

{spells_mtg-0.8.3 → spells_mtg-0.9.0}/spells/draft_data.py

@@ -16,8 +16,7 @@ from typing import Callable, TypeVar, Any
 import polars as pl
 from polars.exceptions import ColumnNotFoundError
 
-from spells.external import data_file_path
-import spells.cache
+from spells import cache
 import spells.filter
 import spells.manifest
 from spells.columns import ColDef, ColSpec, get_specs
@@ -36,11 +35,11 @@ def _cache_key(args) -> str:
 
 @functools.lru_cache(maxsize=None)
 def get_names(set_code: str) -> list[str]:
-    card_fp = data_file_path(set_code, View.CARD)
+    card_fp = cache.data_file_path(set_code, View.CARD)
     card_view = pl.read_parquet(card_fp)
     card_names_set = frozenset(card_view.get_column("name").to_list())
 
-    draft_fp = data_file_path(set_code, View.DRAFT)
+    draft_fp = cache.data_file_path(set_code, View.DRAFT)
     draft_view = pl.scan_parquet(draft_fp)
     cols = draft_view.collect_schema().names()
 
@@ -78,7 +77,7 @@ def _get_card_context(
 
         columns = list(col_def_map.keys())
 
-        fp = data_file_path(set_code, View.CARD)
+        fp = cache.data_file_path(set_code, View.CARD)
         card_df = pl.read_parquet(fp)
         select_rows = _view_select(
             card_df, frozenset(columns), col_def_map, is_agg_view=False
@@ -223,7 +222,7 @@ def _infer_dependencies(
             ):
                 dependencies.add(split[0])
                 found = True
-        assert found, f"Could not locate column spec for root col {item}"
+        # fail silently here, so that columns can be passed in harmlessly
 
     return dependencies
 
@@ -231,23 +230,13 @@ def _infer_dependencies(
 def _get_set_context(
     set_code: str, set_context: pl.DataFrame | dict[str, Any] | None
 ) -> dict[str, Any]:
-    context_fp = data_file_path(set_code, "context")
-
-    report = functools.partial(
-        spells.cache.spells_print,
-        "report",
-        f"Set context for {set_code} invalid, please investigate!",
-    )
+    context_fp = cache.data_file_path(set_code, "context")
 
     context = {}
-    if not os.path.isfile(context_fp):
-        report()
-    else:
+    if os.path.isfile(context_fp):
         context_df = pl.read_parquet(context_fp)
         if len(context_df) == 1:
             context.update(context_df.to_dicts()[0])
-        else:
-            report()
 
     if isinstance(set_context, pl.DataFrame):
         assert len(set_context != 1), "Invalid set context provided"
@@ -276,7 +265,10 @@ def _hydrate_col_defs(
     assert len(names) > 0, "there should be names"
     hydrated = {}
     for col, spec in specs.items():
-        expr = _determine_expression(col, spec, names, card_context, set_context)
+        try:
+            expr = _determine_expression(col, spec, names, card_context, set_context)
+        except KeyError:
+            continue
         dependencies = _infer_dependencies(col, expr, specs, names)
 
         sig_expr = expr if isinstance(expr, pl.Expr) else expr[0]
@@ -355,13 +347,13 @@ def _fetch_or_cache(
     key = _cache_key(cache_args)
 
     if read_cache:
-        if spells.cache.cache_exists(set_code, key):
-            return spells.cache.read_cache(set_code, key)
+        if cache.cache_exists(set_code, key):
+            return cache.read_cache(set_code, key)
 
     df = calc_fn()
 
     if write_cache:
-        spells.cache.write_cache(set_code, key, df)
+        cache.write_cache(set_code, key, df)
 
     return df
 
@@ -380,7 +372,7 @@ def _base_agg_df(
     for view, cols_for_view in m.view_cols.items():
         if view == View.CARD:
             continue
-        df_path = data_file_path(set_code, view)
+        df_path = cache.data_file_path(set_code, view)
         base_view_df = pl.scan_parquet(df_path)
         base_df_prefilter = _view_select(
             base_view_df, cols_for_view, m.col_def_map, is_agg_view=False
@@ -411,14 +403,10 @@ def _base_agg_df(
             c for c in cols_for_view if m.col_def_map[c].col_type == ColType.NAME_SUM
         )
         for col in name_sum_cols:
-            cdef = m.col_def_map[col]
-            pattern = f"^{cdef.name}_"
-            name_map = functools.partial(
-                lambda patt, name: re.split(patt, name)[1], pattern
-            )
+            names = get_names(set_code)
+            expr = tuple(pl.col(f"{col}_{name}").alias(name) for name in names)
 
-            expr = pl.col(f"^{cdef.name}_.*$").name.map(name_map)
-            pre_agg_df = base_df.select((expr,) + nonname_gb)
+            pre_agg_df = base_df.select(expr + nonname_gb)
 
             if nonname_gb:
                 agg_df = pre_agg_df.group_by(nonname_gb).sum()
@@ -428,7 +416,7 @@ def _base_agg_df(
             index = nonname_gb if nonname_gb else None
             unpivoted = agg_df.unpivot(
                 index=index,
-                value_name=m.col_def_map[col].name,
+                value_name=col,
                 variable_name=ColName.NAME,
             )
 
@@ -523,7 +511,7 @@ def summon(
 
         if View.CARD in m.view_cols:
             card_cols = m.view_cols[View.CARD].union({ColName.NAME})
-            fp = data_file_path(code, View.CARD)
+            fp = cache.data_file_path(code, View.CARD)
             card_df = pl.read_parquet(fp)
             select_df = _view_select(
                 card_df, card_cols, m.col_def_map, is_agg_view=False
@@ -578,7 +566,7 @@ def view_select(
 
     col_def_map = _hydrate_col_defs(set_code, specs, card_context, set_context)
 
-    df_path = data_file_path(set_code, view)
+    df_path = cache.data_file_path(set_code, view)
     base_view_df = pl.scan_parquet(df_path)
 
     select_cols = frozenset(columns)

{spells_mtg-0.8.3 → spells_mtg-0.9.0}/spells/extension.py

@@ -26,24 +26,29 @@ def context_cols(attr, silent: bool = False) -> dict[str, ColSpec]:
                 .otherwise(None)
             ),
         ),
-        f"pick_{attr}": ColSpec(
+        f"pick_{attr}_sum": ColSpec(
             col_type=ColType.PICK_SUM,
             expr=lambda name, card_context: pl.lit(None)
             if card_context[name][attr] is None or math.isnan(card_context[name][attr])
             else card_context[name][attr],
         ),
+        f"pick_{attr}": ColSpec(
+            col_type=ColType.AGG, expr=pl.col(f"pick_{attr}_sum") / pl.col("num_taken")
+        ),
         f"seen_{attr}_is_greatest": ColSpec(
             col_type=ColType.NAME_SUM,
             expr=lambda name: pl.col(f"seen_{attr}_{name}")
-            == pl.col(f"greatest_{attr}_seen_{name}"),
+            == pl.col(f"greatest_{attr}_seen"),
         ),
         f"seen_{attr}_greater": ColSpec(
             col_type=ColType.NAME_SUM,
-            expr=lambda name: pl.col(f"seen_{attr}_{name}") > pl.col(f"pick_{attr}"),
+            expr=lambda name: pl.col(f"seen_{attr}_{name}")
+            > pl.col(f"pick_{attr}_sum"),
         ),
         f"seen_{attr}_less": ColSpec(
             col_type=ColType.NAME_SUM,
-            expr=lambda name: pl.col(f"seen_{attr}_{name}") < pl.col(f"pick_{attr}"),
+            expr=lambda name: pl.col(f"seen_{attr}_{name}")
+            < pl.col(f"pick_{attr}_sum"),
         ),
         f"greatest_{attr}_seen": ColSpec(
             col_type=ColType.PICK_SUM,
@@ -79,11 +84,11 @@ def context_cols(attr, silent: bool = False) -> dict[str, ColSpec]:
         ),
         f"pick_{attr}_vs_least": ColSpec(
             col_type=ColType.PICK_SUM,
-            expr=pl.col(f"pick_{attr}") - pl.col(f"least_{attr}_seen"),
+            expr=pl.col(f"pick_{attr}_sum") - pl.col(f"least_{attr}_seen"),
         ),
         f"pick_{attr}_vs_greatest": ColSpec(
             col_type=ColType.PICK_SUM,
-            expr=pl.col(f"pick_{attr}") - pl.col(f"greatest_{attr}_seen"),
+            expr=pl.col(f"pick_{attr}_sum") - pl.col(f"greatest_{attr}_seen"),
         ),
         f"pick_{attr}_vs_least_mean": ColSpec(
             col_type=ColType.AGG,
@@ -95,7 +100,7 @@ def context_cols(attr, silent: bool = False) -> dict[str, ColSpec]:
         ),
         f"least_{attr}_taken": ColSpec(
             col_type=ColType.PICK_SUM,
-            expr=pl.col(f"pick_{attr}") <= pl.col(f"least_{attr}_seen"),
+            expr=pl.col(f"pick_{attr}_sum") <= pl.col(f"least_{attr}_seen"),
         ),
         f"least_{attr}_taken_rate": ColSpec(
             col_type=ColType.AGG,
@@ -103,7 +108,7 @@ def context_cols(attr, silent: bool = False) -> dict[str, ColSpec]:
         ),
         f"greatest_{attr}_taken": ColSpec(
             col_type=ColType.PICK_SUM,
-            expr=pl.col(f"pick_{attr}") >= pl.col(f"greatest_{attr}_seen"),
+            expr=pl.col(f"pick_{attr}_sum") >= pl.col(f"greatest_{attr}_seen"),
         ),
         f"greatest_{attr}_taken_rate": ColSpec(
             col_type=ColType.AGG,
@@ -111,7 +116,7 @@ def context_cols(attr, silent: bool = False) -> dict[str, ColSpec]:
         ),
         f"pick_{attr}_mean": ColSpec(
             col_type=ColType.AGG,
-            expr=pl.col(f"pick_{attr}") / pl.col(ColName.NUM_TAKEN),
+            expr=pl.col(f"pick_{attr}_sum") / pl.col(ColName.NUM_TAKEN),
         ),
     }
 

{spells_mtg-0.8.3 → spells_mtg-0.9.0}/spells/external.py

@@ -19,8 +19,9 @@ from polars.exceptions import ComputeError
 
 from spells import cards
 from spells import cache
-from spells.enums import View
+from spells.enums import View, ColName
 from spells.schema import schema
+from spells.draft_data import summon
 
 
 DATASET_TEMPLATE = "{dataset_type}_data_public.{set_code}.{event_type}.csv.gz"
@@ -28,17 +29,10 @@ RESOURCE_TEMPLATE = (
     "https://17lands-public.s3.amazonaws.com/analysis_data/{dataset_type}_data/"
 )
 
-
 class FileFormat(StrEnum):
     CSV = "csv"
     PARQUET = "parquet"
 
-
-class EventType(StrEnum):
-    PREMIER = "PremierDraft"
-    TRADITIONAL = "TradDraft"
-
-
 # Fred Cirera via https://stackoverflow.com/questions/1094841/get-a-human-readable-version-of-a-file-size
 def sizeof_fmt(num, suffix="B"):
     for unit in ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"):
@@ -64,7 +58,7 @@ def cli() -> int:
         e.g. $ spells add OTJ
 
     refresh: Force download and overwrite of existing files (for new data drops, use sparingly!). Clear 
-        local cache.
+        local 
 
     remove: Delete the [data home]/external/[set code] and [data home]/local/[set code] directories and their contents
 
@@ -115,7 +109,7 @@ def _refresh(set_code: str):
 
 def _remove(set_code: str):
     mode = "remove"
-    dir_path = _external_set_path(set_code)
+    dir_path = cache.external_set_path(set_code)
     if os.path.isdir(dir_path):
         with os.scandir(dir_path) as set_dir:
             count = 0
@@ -135,7 +129,7 @@ def _remove(set_code: str):
     else:
         cache.spells_print(mode, f"No external cache found for set {set_code}")
 
-    return cache.clear(set_code)
+    return cache.clean(set_code)
 
 
 def _info():
@@ -207,22 +201,6 @@ def _info():
     return 0
 
 
-def _external_set_path(set_code):
-    return os.path.join(cache.data_dir_path(cache.DataDir.EXTERNAL), set_code)
-
-
-def data_file_path(set_code, dataset_type: str, event_type=EventType.PREMIER):
-    if dataset_type == "set_context":
-        return os.path.join(_external_set_path(set_code), f"{set_code}_context.parquet")
-
-    if dataset_type == "card":
-        return os.path.join(_external_set_path(set_code), f"{set_code}_card.parquet")
-
-    return os.path.join(
-        _external_set_path(set_code), f"{set_code}_{event_type}_{dataset_type}.parquet"
-    )
-
-
 def _process_zipped_file(gzip_path, target_path):
     csv_path = gzip_path[:-3]
     # if polars supports streaming from file obj, we can just stream straight
@@ -252,17 +230,17 @@ def _process_zipped_file(gzip_path, target_path):
 def download_data_set(
     set_code,
     dataset_type: View,
-    event_type=EventType.PREMIER,
+    event_type=cache.EventType.PREMIER,
     force_download=False,
     clear_set_cache=True,
 ):
     mode = "refresh" if force_download else "add"
     cache.spells_print(mode, f"Downloading {dataset_type} dataset from 17Lands.com")
 
-    if not os.path.isdir(set_dir := _external_set_path(set_code)):
+    if not os.path.isdir(set_dir := cache.external_set_path(set_code)):
         os.makedirs(set_dir)
 
-    target_path = data_file_path(set_code, dataset_type)
+    target_path = cache.data_file_path(set_code, dataset_type)
 
     if os.path.isfile(target_path) and not force_download:
         cache.spells_print(
@@ -274,7 +252,7 @@ def download_data_set(
     dataset_file = DATASET_TEMPLATE.format(
         set_code=set_code, dataset_type=dataset_type, event_type=event_type
     )
-    dataset_path = os.path.join(_external_set_path(set_code), dataset_file)
+    dataset_path = os.path.join(cache.external_set_path(set_code), dataset_file)
     wget.download(
         RESOURCE_TEMPLATE.format(dataset_type=dataset_type) + dataset_file,
         out=dataset_path,
@@ -287,7 +265,7 @@ def download_data_set(
     _process_zipped_file(dataset_path, target_path)
     cache.spells_print(mode, f"Wrote file {target_path}")
     if clear_set_cache:
-        cache.clear(set_code)
+        cache.clean(set_code)
 
     return 0
 
@@ -302,7 +280,7 @@ def write_card_file(draft_set_code: str, force_download=False) -> int:
     cache.spells_print(
         mode, "Fetching card data from mtgjson.com and writing card file"
     )
-    card_filepath = data_file_path(draft_set_code, View.CARD)
+    card_filepath = cache.data_file_path(draft_set_code, View.CARD)
     if os.path.isfile(card_filepath) and not force_download:
         cache.spells_print(
             mode,
@@ -310,7 +288,7 @@ def write_card_file(draft_set_code: str, force_download=False) -> int:
         )
         return 1
 
-    draft_filepath = data_file_path(draft_set_code, View.DRAFT)
+    draft_filepath = cache.data_file_path(draft_set_code, View.DRAFT)
 
     if not os.path.isfile(draft_filepath):
         cache.spells_print(mode, f"Error: No draft file for set {draft_set_code}")
@@ -336,7 +314,7 @@ def write_card_file(draft_set_code: str, force_download=False) -> int:
 def get_set_context(set_code: str, force_download=False) -> int:
     mode = "refresh" if force_download else "add"
 
-    context_fp = data_file_path(set_code, "context")
+    context_fp = cache.data_file_path(set_code, "context")
     cache.spells_print(mode, "Calculating set context")
     if os.path.isfile(context_fp) and not force_download:
         cache.spells_print(
@@ -345,15 +323,14 @@ def get_set_context(set_code: str, force_download=False) -> int:
         )
         return 1
 
-    draft_fp = data_file_path(set_code, View.DRAFT)
-    draft_view = pl.scan_parquet(draft_fp)
+    df = summon(set_code, columns=[ColName.NUM_DRAFTS], group_by=[ColName.DRAFT_DATE, ColName.PICK_NUM])
 
-    context_df = draft_view.select(
+    context_df = df.filter(pl.col(ColName.NUM_DRAFTS) > 1000).select(
         [
-            pl.max("pick_number").alias("picks_per_pack") + 1,
-            pl.min("draft_time").alias("release_time"),
+            pl.col(ColName.DRAFT_DATE).min().alias("release_date"),
+            pl.col(ColName.PICK_NUM).max().alias("picks_per_pack"),
         ]
-    ).collect()
+    )
 
     context_df.write_parquet(context_fp)