spells-mtg 0.8.4__py3-none-any.whl → 0.9.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of spells-mtg might be problematic. Click here for more details.
- spells/__init__.py +3 -0
- spells/cache.py +21 -0
- spells/columns.py +3 -13
- spells/draft_data.py +75 -72
- spells/extension.py +16 -10
- spells/external.py +22 -39
- spells/log.py +68 -0
- {spells_mtg-0.8.4.dist-info → spells_mtg-0.9.1.dist-info}/METADATA +1 -1
- spells_mtg-0.9.1.dist-info/RECORD +18 -0
- spells_mtg-0.8.4.dist-info/RECORD +0 -17
- {spells_mtg-0.8.4.dist-info → spells_mtg-0.9.1.dist-info}/WHEEL +0 -0
- {spells_mtg-0.8.4.dist-info → spells_mtg-0.9.1.dist-info}/entry_points.txt +0 -0
- {spells_mtg-0.8.4.dist-info → spells_mtg-0.9.1.dist-info}/licenses/LICENSE +0 -0
spells/__init__.py
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
from spells.columns import ColSpec
|
|
2
2
|
from spells.enums import ColType, ColName
|
|
3
3
|
from spells.draft_data import summon, view_select, get_names
|
|
4
|
+
from spells.log import setup_logging
|
|
5
|
+
|
|
6
|
+
setup_logging()
|
|
4
7
|
|
|
5
8
|
__all__ = ["summon", "view_select", "get_names", "ColSpec", "ColType", "ColName"]
|
spells/cache.py
CHANGED
|
@@ -14,6 +14,11 @@ import sys
|
|
|
14
14
|
import polars as pl
|
|
15
15
|
|
|
16
16
|
|
|
17
|
+
class EventType(StrEnum):
|
|
18
|
+
PREMIER = "PremierDraft"
|
|
19
|
+
TRADITIONAL = "TradDraft"
|
|
20
|
+
|
|
21
|
+
|
|
17
22
|
class DataDir(StrEnum):
|
|
18
23
|
CACHE = "cache"
|
|
19
24
|
EXTERNAL = "external"
|
|
@@ -52,6 +57,22 @@ def data_dir_path(cache_dir: DataDir) -> str:
|
|
|
52
57
|
return data_dir
|
|
53
58
|
|
|
54
59
|
|
|
60
|
+
def external_set_path(set_code):
|
|
61
|
+
return os.path.join(data_dir_path(DataDir.EXTERNAL), set_code)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def data_file_path(set_code, dataset_type: str, event_type=EventType.PREMIER):
|
|
65
|
+
if dataset_type == "set_context":
|
|
66
|
+
return os.path.join(external_set_path(set_code), f"{set_code}_context.parquet")
|
|
67
|
+
|
|
68
|
+
if dataset_type == "card":
|
|
69
|
+
return os.path.join(external_set_path(set_code), f"{set_code}_card.parquet")
|
|
70
|
+
|
|
71
|
+
return os.path.join(
|
|
72
|
+
external_set_path(set_code), f"{set_code}_{event_type}_{dataset_type}.parquet"
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
55
76
|
def cache_dir_for_set(set_code: str) -> str:
|
|
56
77
|
return os.path.join(data_dir_path(DataDir.CACHE), set_code)
|
|
57
78
|
|
spells/columns.py
CHANGED
|
@@ -68,10 +68,7 @@ _specs: dict[str, ColSpec] = {
|
|
|
68
68
|
ColName.FORMAT_DAY: ColSpec(
|
|
69
69
|
col_type=ColType.GROUP_BY,
|
|
70
70
|
expr=lambda set_context: (
|
|
71
|
-
pl.col(ColName.DRAFT_DATE)
|
|
72
|
-
- pl.lit(set_context["release_time"])
|
|
73
|
-
.str.to_datetime("%Y-%m-%d %H:%M:%S")
|
|
74
|
-
.dt.date()
|
|
71
|
+
pl.col(ColName.DRAFT_DATE) - pl.lit(set_context["release_date"])
|
|
75
72
|
).dt.total_days()
|
|
76
73
|
+ 1,
|
|
77
74
|
),
|
|
@@ -583,12 +580,5 @@ for item in ColName:
|
|
|
583
580
|
assert item in _specs, f"column {item} enumerated but not specified"
|
|
584
581
|
|
|
585
582
|
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
self._specs = spec_dict
|
|
589
|
-
|
|
590
|
-
def __call__(self):
|
|
591
|
-
return dict(self._specs)
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
get_specs = GetSpecs(_specs)
|
|
583
|
+
def get_specs():
|
|
584
|
+
return dict(_specs)
|
spells/draft_data.py
CHANGED
|
@@ -9,6 +9,7 @@ for performance.
|
|
|
9
9
|
import functools
|
|
10
10
|
import hashlib
|
|
11
11
|
import re
|
|
12
|
+
import logging
|
|
12
13
|
from inspect import signature
|
|
13
14
|
import os
|
|
14
15
|
from typing import Callable, TypeVar, Any
|
|
@@ -16,13 +17,12 @@ from typing import Callable, TypeVar, Any
|
|
|
16
17
|
import polars as pl
|
|
17
18
|
from polars.exceptions import ColumnNotFoundError
|
|
18
19
|
|
|
19
|
-
from spells
|
|
20
|
-
import spells.cache
|
|
20
|
+
from spells import cache
|
|
21
21
|
import spells.filter
|
|
22
22
|
import spells.manifest
|
|
23
23
|
from spells.columns import ColDef, ColSpec, get_specs
|
|
24
24
|
from spells.enums import View, ColName, ColType
|
|
25
|
-
|
|
25
|
+
from spells.log import make_verbose
|
|
26
26
|
|
|
27
27
|
DF = TypeVar("DF", pl.LazyFrame, pl.DataFrame)
|
|
28
28
|
|
|
@@ -36,11 +36,11 @@ def _cache_key(args) -> str:
|
|
|
36
36
|
|
|
37
37
|
@functools.lru_cache(maxsize=None)
|
|
38
38
|
def get_names(set_code: str) -> list[str]:
|
|
39
|
-
card_fp = data_file_path(set_code, View.CARD)
|
|
39
|
+
card_fp = cache.data_file_path(set_code, View.CARD)
|
|
40
40
|
card_view = pl.read_parquet(card_fp)
|
|
41
41
|
card_names_set = frozenset(card_view.get_column("name").to_list())
|
|
42
42
|
|
|
43
|
-
draft_fp = data_file_path(set_code, View.DRAFT)
|
|
43
|
+
draft_fp = cache.data_file_path(set_code, View.DRAFT)
|
|
44
44
|
draft_view = pl.scan_parquet(draft_fp)
|
|
45
45
|
cols = draft_view.collect_schema().names()
|
|
46
46
|
|
|
@@ -78,13 +78,17 @@ def _get_card_context(
|
|
|
78
78
|
|
|
79
79
|
columns = list(col_def_map.keys())
|
|
80
80
|
|
|
81
|
-
fp = data_file_path(set_code, View.CARD)
|
|
81
|
+
fp = cache.data_file_path(set_code, View.CARD)
|
|
82
82
|
card_df = pl.read_parquet(fp)
|
|
83
83
|
select_rows = _view_select(
|
|
84
84
|
card_df, frozenset(columns), col_def_map, is_agg_view=False
|
|
85
85
|
).to_dicts()
|
|
86
86
|
|
|
87
|
+
names = get_names(set_code)
|
|
87
88
|
loaded_context = {row[ColName.NAME]: row for row in select_rows}
|
|
89
|
+
|
|
90
|
+
for name in names:
|
|
91
|
+
loaded_context[name] = loaded_context.get(name, {})
|
|
88
92
|
else:
|
|
89
93
|
names = get_names(set_code)
|
|
90
94
|
loaded_context = {name: {} for name in names}
|
|
@@ -100,10 +104,7 @@ def _get_card_context(
|
|
|
100
104
|
|
|
101
105
|
names = list(loaded_context.keys())
|
|
102
106
|
for name in names:
|
|
103
|
-
|
|
104
|
-
name in card_context
|
|
105
|
-
), f"card_context must include a row for each card name. {name} missing."
|
|
106
|
-
for col, value in card_context[name].items():
|
|
107
|
+
for col, value in card_context.get(name, {}).items():
|
|
107
108
|
loaded_context[name][col] = value
|
|
108
109
|
|
|
109
110
|
return loaded_context
|
|
@@ -130,46 +131,55 @@ def _determine_expression(
|
|
|
130
131
|
|
|
131
132
|
if spec.col_type == ColType.NAME_SUM:
|
|
132
133
|
if spec.expr is not None:
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
134
|
+
try:
|
|
135
|
+
assert isinstance(
|
|
136
|
+
spec.expr, Callable
|
|
137
|
+
), f"NAME_SUM column {col} must have a callable `expr` accepting a `name` argument"
|
|
138
|
+
unnamed_exprs = [
|
|
139
|
+
spec.expr(**{"name": name, **seed_params(spec.expr)})
|
|
140
|
+
for name in names
|
|
141
|
+
]
|
|
142
|
+
|
|
143
|
+
expr = tuple(
|
|
144
|
+
map(
|
|
145
|
+
lambda ex, name: ex.alias(f"{col}_{name}"),
|
|
146
|
+
unnamed_exprs,
|
|
147
|
+
names,
|
|
148
|
+
)
|
|
145
149
|
)
|
|
146
|
-
|
|
150
|
+
except KeyError:
|
|
151
|
+
expr = tuple(pl.lit(None).alias(f"{col}_{name}") for name in names)
|
|
147
152
|
else:
|
|
148
153
|
expr = tuple(map(lambda name: pl.col(f"{col}_{name}"), names))
|
|
149
154
|
|
|
150
155
|
elif spec.expr is not None:
|
|
151
156
|
if isinstance(spec.expr, Callable):
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
name_params = {"name": name, **params}
|
|
166
|
-
expr = (
|
|
167
|
-
pl.when(pl.col(condition_col) == name)
|
|
168
|
-
.then(spec.expr(**name_params))
|
|
169
|
-
.otherwise(expr)
|
|
157
|
+
try:
|
|
158
|
+
assert (
|
|
159
|
+
not spec.col_type == ColType.AGG
|
|
160
|
+
), f"AGG column {col} must be a pure spells expression"
|
|
161
|
+
params = seed_params(spec.expr)
|
|
162
|
+
if (
|
|
163
|
+
spec.col_type in (ColType.PICK_SUM, ColType.CARD_ATTR)
|
|
164
|
+
and "name" in signature(spec.expr).parameters
|
|
165
|
+
):
|
|
166
|
+
condition_col = (
|
|
167
|
+
ColName.PICK
|
|
168
|
+
if spec.col_type == ColType.PICK_SUM
|
|
169
|
+
else ColName.NAME
|
|
170
170
|
)
|
|
171
|
-
|
|
172
|
-
|
|
171
|
+
expr = pl.lit(None)
|
|
172
|
+
for name in names:
|
|
173
|
+
name_params = {"name": name, **params}
|
|
174
|
+
expr = (
|
|
175
|
+
pl.when(pl.col(condition_col) == name)
|
|
176
|
+
.then(spec.expr(**name_params))
|
|
177
|
+
.otherwise(expr)
|
|
178
|
+
)
|
|
179
|
+
else:
|
|
180
|
+
expr = spec.expr(**params)
|
|
181
|
+
except KeyError:
|
|
182
|
+
expr = pl.lit(None)
|
|
173
183
|
else:
|
|
174
184
|
expr = spec.expr
|
|
175
185
|
expr = expr.alias(col)
|
|
@@ -223,7 +233,7 @@ def _infer_dependencies(
|
|
|
223
233
|
):
|
|
224
234
|
dependencies.add(split[0])
|
|
225
235
|
found = True
|
|
226
|
-
|
|
236
|
+
# fail silently here, so that columns can be passed in harmlessly
|
|
227
237
|
|
|
228
238
|
return dependencies
|
|
229
239
|
|
|
@@ -231,23 +241,13 @@ def _infer_dependencies(
|
|
|
231
241
|
def _get_set_context(
|
|
232
242
|
set_code: str, set_context: pl.DataFrame | dict[str, Any] | None
|
|
233
243
|
) -> dict[str, Any]:
|
|
234
|
-
context_fp = data_file_path(set_code, "context")
|
|
235
|
-
|
|
236
|
-
report = functools.partial(
|
|
237
|
-
spells.cache.spells_print,
|
|
238
|
-
"report",
|
|
239
|
-
f"Set context for {set_code} invalid, please investigate!",
|
|
240
|
-
)
|
|
244
|
+
context_fp = cache.data_file_path(set_code, "context")
|
|
241
245
|
|
|
242
246
|
context = {}
|
|
243
|
-
if
|
|
244
|
-
report()
|
|
245
|
-
else:
|
|
247
|
+
if os.path.isfile(context_fp):
|
|
246
248
|
context_df = pl.read_parquet(context_fp)
|
|
247
249
|
if len(context_df) == 1:
|
|
248
250
|
context.update(context_df.to_dicts()[0])
|
|
249
|
-
else:
|
|
250
|
-
report()
|
|
251
251
|
|
|
252
252
|
if isinstance(set_context, pl.DataFrame):
|
|
253
253
|
assert len(set_context != 1), "Invalid set context provided"
|
|
@@ -355,13 +355,15 @@ def _fetch_or_cache(
|
|
|
355
355
|
key = _cache_key(cache_args)
|
|
356
356
|
|
|
357
357
|
if read_cache:
|
|
358
|
-
if
|
|
359
|
-
|
|
358
|
+
if cache.cache_exists(set_code, key):
|
|
359
|
+
logging.debug(f"Cache {key} found")
|
|
360
|
+
return cache.read_cache(set_code, key)
|
|
360
361
|
|
|
362
|
+
logging.debug(f"Cache not found, calculating with signature {cache_args}")
|
|
361
363
|
df = calc_fn()
|
|
362
364
|
|
|
363
365
|
if write_cache:
|
|
364
|
-
|
|
366
|
+
cache.write_cache(set_code, key, df)
|
|
365
367
|
|
|
366
368
|
return df
|
|
367
369
|
|
|
@@ -380,7 +382,7 @@ def _base_agg_df(
|
|
|
380
382
|
for view, cols_for_view in m.view_cols.items():
|
|
381
383
|
if view == View.CARD:
|
|
382
384
|
continue
|
|
383
|
-
df_path = data_file_path(set_code, view)
|
|
385
|
+
df_path = cache.data_file_path(set_code, view)
|
|
384
386
|
base_view_df = pl.scan_parquet(df_path)
|
|
385
387
|
base_df_prefilter = _view_select(
|
|
386
388
|
base_view_df, cols_for_view, m.col_def_map, is_agg_view=False
|
|
@@ -411,14 +413,10 @@ def _base_agg_df(
|
|
|
411
413
|
c for c in cols_for_view if m.col_def_map[c].col_type == ColType.NAME_SUM
|
|
412
414
|
)
|
|
413
415
|
for col in name_sum_cols:
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
name_map = functools.partial(
|
|
417
|
-
lambda patt, name: re.split(patt, name)[1], pattern
|
|
418
|
-
)
|
|
416
|
+
names = get_names(set_code)
|
|
417
|
+
expr = tuple(pl.col(f"{col}_{name}").alias(name) for name in names)
|
|
419
418
|
|
|
420
|
-
|
|
421
|
-
pre_agg_df = base_df.select((expr,) + nonname_gb)
|
|
419
|
+
pre_agg_df = base_df.select(expr + nonname_gb)
|
|
422
420
|
|
|
423
421
|
if nonname_gb:
|
|
424
422
|
agg_df = pre_agg_df.group_by(nonname_gb).sum()
|
|
@@ -428,7 +426,7 @@ def _base_agg_df(
|
|
|
428
426
|
index = nonname_gb if nonname_gb else None
|
|
429
427
|
unpivoted = agg_df.unpivot(
|
|
430
428
|
index=index,
|
|
431
|
-
value_name=
|
|
429
|
+
value_name=col,
|
|
432
430
|
variable_name=ColName.NAME,
|
|
433
431
|
)
|
|
434
432
|
|
|
@@ -452,9 +450,11 @@ def _base_agg_df(
|
|
|
452
450
|
else:
|
|
453
451
|
joined_df = pl.concat(join_dfs, how="horizontal")
|
|
454
452
|
|
|
453
|
+
joined_df = joined_df.select(sorted(joined_df.schema.names()))
|
|
455
454
|
return joined_df
|
|
456
455
|
|
|
457
456
|
|
|
457
|
+
@make_verbose
|
|
458
458
|
def summon(
|
|
459
459
|
set_code: str | list[str],
|
|
460
460
|
columns: list[str] | None = None,
|
|
@@ -476,8 +476,10 @@ def summon(
|
|
|
476
476
|
specs.update(ext)
|
|
477
477
|
|
|
478
478
|
if isinstance(set_code, str):
|
|
479
|
-
card_context
|
|
480
|
-
|
|
479
|
+
if not (isinstance(card_context, dict) and set_code in card_context):
|
|
480
|
+
card_context = {set_code: card_context}
|
|
481
|
+
if not (isinstance(set_context, dict) and set_code in set_context):
|
|
482
|
+
set_context = {set_code: set_context}
|
|
481
483
|
codes = [set_code]
|
|
482
484
|
else:
|
|
483
485
|
codes = set_code
|
|
@@ -488,6 +490,7 @@ def summon(
|
|
|
488
490
|
|
|
489
491
|
concat_dfs = []
|
|
490
492
|
for code in codes:
|
|
493
|
+
logging.debug(f"Calculating agg df for {code}")
|
|
491
494
|
if isinstance(card_context, pl.DataFrame):
|
|
492
495
|
set_card_context = card_context.filter(pl.col("expansion") == code)
|
|
493
496
|
elif isinstance(card_context, dict):
|
|
@@ -523,7 +526,7 @@ def summon(
|
|
|
523
526
|
|
|
524
527
|
if View.CARD in m.view_cols:
|
|
525
528
|
card_cols = m.view_cols[View.CARD].union({ColName.NAME})
|
|
526
|
-
fp = data_file_path(code, View.CARD)
|
|
529
|
+
fp = cache.data_file_path(code, View.CARD)
|
|
527
530
|
card_df = pl.read_parquet(fp)
|
|
528
531
|
select_df = _view_select(
|
|
529
532
|
card_df, card_cols, m.col_def_map, is_agg_view=False
|
|
@@ -578,7 +581,7 @@ def view_select(
|
|
|
578
581
|
|
|
579
582
|
col_def_map = _hydrate_col_defs(set_code, specs, card_context, set_context)
|
|
580
583
|
|
|
581
|
-
df_path = data_file_path(set_code, view)
|
|
584
|
+
df_path = cache.data_file_path(set_code, view)
|
|
582
585
|
base_view_df = pl.scan_parquet(df_path)
|
|
583
586
|
|
|
584
587
|
select_cols = frozenset(columns)
|
spells/extension.py
CHANGED
|
@@ -19,19 +19,23 @@ def context_cols(attr, silent: bool = False) -> dict[str, ColSpec]:
|
|
|
19
19
|
col_type=ColType.NAME_SUM,
|
|
20
20
|
expr=(
|
|
21
21
|
lambda name, card_context: pl.lit(None)
|
|
22
|
-
if card_context[name]
|
|
22
|
+
if card_context[name].get(attr) is None
|
|
23
23
|
or math.isnan(card_context[name][attr])
|
|
24
24
|
else pl.when(pl.col(f"pack_card_{name}") > 0)
|
|
25
25
|
.then(card_context[name][attr])
|
|
26
26
|
.otherwise(None)
|
|
27
27
|
),
|
|
28
28
|
),
|
|
29
|
-
f"pick_{attr}": ColSpec(
|
|
29
|
+
f"pick_{attr}_sum": ColSpec(
|
|
30
30
|
col_type=ColType.PICK_SUM,
|
|
31
31
|
expr=lambda name, card_context: pl.lit(None)
|
|
32
|
-
if card_context[name]
|
|
32
|
+
if card_context[name].get(attr) is None
|
|
33
|
+
or math.isnan(card_context[name][attr])
|
|
33
34
|
else card_context[name][attr],
|
|
34
35
|
),
|
|
36
|
+
f"pick_{attr}": ColSpec(
|
|
37
|
+
col_type=ColType.AGG, expr=pl.col(f"pick_{attr}_sum") / pl.col("num_taken")
|
|
38
|
+
),
|
|
35
39
|
f"seen_{attr}_is_greatest": ColSpec(
|
|
36
40
|
col_type=ColType.NAME_SUM,
|
|
37
41
|
expr=lambda name: pl.col(f"seen_{attr}_{name}")
|
|
@@ -39,11 +43,13 @@ def context_cols(attr, silent: bool = False) -> dict[str, ColSpec]:
|
|
|
39
43
|
),
|
|
40
44
|
f"seen_{attr}_greater": ColSpec(
|
|
41
45
|
col_type=ColType.NAME_SUM,
|
|
42
|
-
expr=lambda name: pl.col(f"seen_{attr}_{name}")
|
|
46
|
+
expr=lambda name: pl.col(f"seen_{attr}_{name}")
|
|
47
|
+
> pl.col(f"pick_{attr}_sum"),
|
|
43
48
|
),
|
|
44
49
|
f"seen_{attr}_less": ColSpec(
|
|
45
50
|
col_type=ColType.NAME_SUM,
|
|
46
|
-
expr=lambda name: pl.col(f"seen_{attr}_{name}")
|
|
51
|
+
expr=lambda name: pl.col(f"seen_{attr}_{name}")
|
|
52
|
+
< pl.col(f"pick_{attr}_sum"),
|
|
47
53
|
),
|
|
48
54
|
f"greatest_{attr}_seen": ColSpec(
|
|
49
55
|
col_type=ColType.PICK_SUM,
|
|
@@ -79,11 +85,11 @@ def context_cols(attr, silent: bool = False) -> dict[str, ColSpec]:
|
|
|
79
85
|
),
|
|
80
86
|
f"pick_{attr}_vs_least": ColSpec(
|
|
81
87
|
col_type=ColType.PICK_SUM,
|
|
82
|
-
expr=pl.col(f"pick_{attr}") - pl.col(f"least_{attr}_seen"),
|
|
88
|
+
expr=pl.col(f"pick_{attr}_sum") - pl.col(f"least_{attr}_seen"),
|
|
83
89
|
),
|
|
84
90
|
f"pick_{attr}_vs_greatest": ColSpec(
|
|
85
91
|
col_type=ColType.PICK_SUM,
|
|
86
|
-
expr=pl.col(f"pick_{attr}") - pl.col(f"greatest_{attr}_seen"),
|
|
92
|
+
expr=pl.col(f"pick_{attr}_sum") - pl.col(f"greatest_{attr}_seen"),
|
|
87
93
|
),
|
|
88
94
|
f"pick_{attr}_vs_least_mean": ColSpec(
|
|
89
95
|
col_type=ColType.AGG,
|
|
@@ -95,7 +101,7 @@ def context_cols(attr, silent: bool = False) -> dict[str, ColSpec]:
|
|
|
95
101
|
),
|
|
96
102
|
f"least_{attr}_taken": ColSpec(
|
|
97
103
|
col_type=ColType.PICK_SUM,
|
|
98
|
-
expr=pl.col(f"pick_{attr}") <= pl.col(f"least_{attr}_seen"),
|
|
104
|
+
expr=pl.col(f"pick_{attr}_sum") <= pl.col(f"least_{attr}_seen"),
|
|
99
105
|
),
|
|
100
106
|
f"least_{attr}_taken_rate": ColSpec(
|
|
101
107
|
col_type=ColType.AGG,
|
|
@@ -103,7 +109,7 @@ def context_cols(attr, silent: bool = False) -> dict[str, ColSpec]:
|
|
|
103
109
|
),
|
|
104
110
|
f"greatest_{attr}_taken": ColSpec(
|
|
105
111
|
col_type=ColType.PICK_SUM,
|
|
106
|
-
expr=pl.col(f"pick_{attr}") >= pl.col(f"greatest_{attr}_seen"),
|
|
112
|
+
expr=pl.col(f"pick_{attr}_sum") >= pl.col(f"greatest_{attr}_seen"),
|
|
107
113
|
),
|
|
108
114
|
f"greatest_{attr}_taken_rate": ColSpec(
|
|
109
115
|
col_type=ColType.AGG,
|
|
@@ -111,7 +117,7 @@ def context_cols(attr, silent: bool = False) -> dict[str, ColSpec]:
|
|
|
111
117
|
),
|
|
112
118
|
f"pick_{attr}_mean": ColSpec(
|
|
113
119
|
col_type=ColType.AGG,
|
|
114
|
-
expr=pl.col(f"pick_{attr}") / pl.col(ColName.NUM_TAKEN),
|
|
120
|
+
expr=pl.col(f"pick_{attr}_sum") / pl.col(ColName.NUM_TAKEN),
|
|
115
121
|
),
|
|
116
122
|
}
|
|
117
123
|
|
spells/external.py
CHANGED
|
@@ -19,8 +19,9 @@ from polars.exceptions import ComputeError
|
|
|
19
19
|
|
|
20
20
|
from spells import cards
|
|
21
21
|
from spells import cache
|
|
22
|
-
from spells.enums import View
|
|
22
|
+
from spells.enums import View, ColName
|
|
23
23
|
from spells.schema import schema
|
|
24
|
+
from spells.draft_data import summon
|
|
24
25
|
|
|
25
26
|
|
|
26
27
|
DATASET_TEMPLATE = "{dataset_type}_data_public.{set_code}.{event_type}.csv.gz"
|
|
@@ -34,11 +35,6 @@ class FileFormat(StrEnum):
|
|
|
34
35
|
PARQUET = "parquet"
|
|
35
36
|
|
|
36
37
|
|
|
37
|
-
class EventType(StrEnum):
|
|
38
|
-
PREMIER = "PremierDraft"
|
|
39
|
-
TRADITIONAL = "TradDraft"
|
|
40
|
-
|
|
41
|
-
|
|
42
38
|
# Fred Cirera via https://stackoverflow.com/questions/1094841/get-a-human-readable-version-of-a-file-size
|
|
43
39
|
def sizeof_fmt(num, suffix="B"):
|
|
44
40
|
for unit in ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"):
|
|
@@ -64,7 +60,7 @@ def cli() -> int:
|
|
|
64
60
|
e.g. $ spells add OTJ
|
|
65
61
|
|
|
66
62
|
refresh: Force download and overwrite of existing files (for new data drops, use sparingly!). Clear
|
|
67
|
-
local
|
|
63
|
+
local
|
|
68
64
|
|
|
69
65
|
remove: Delete the [data home]/external/[set code] and [data home]/local/[set code] directories and their contents
|
|
70
66
|
|
|
@@ -115,7 +111,7 @@ def _refresh(set_code: str):
|
|
|
115
111
|
|
|
116
112
|
def _remove(set_code: str):
|
|
117
113
|
mode = "remove"
|
|
118
|
-
dir_path =
|
|
114
|
+
dir_path = cache.external_set_path(set_code)
|
|
119
115
|
if os.path.isdir(dir_path):
|
|
120
116
|
with os.scandir(dir_path) as set_dir:
|
|
121
117
|
count = 0
|
|
@@ -135,7 +131,7 @@ def _remove(set_code: str):
|
|
|
135
131
|
else:
|
|
136
132
|
cache.spells_print(mode, f"No external cache found for set {set_code}")
|
|
137
133
|
|
|
138
|
-
return cache.
|
|
134
|
+
return cache.clean(set_code)
|
|
139
135
|
|
|
140
136
|
|
|
141
137
|
def _info():
|
|
@@ -207,22 +203,6 @@ def _info():
|
|
|
207
203
|
return 0
|
|
208
204
|
|
|
209
205
|
|
|
210
|
-
def _external_set_path(set_code):
|
|
211
|
-
return os.path.join(cache.data_dir_path(cache.DataDir.EXTERNAL), set_code)
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
def data_file_path(set_code, dataset_type: str, event_type=EventType.PREMIER):
|
|
215
|
-
if dataset_type == "set_context":
|
|
216
|
-
return os.path.join(_external_set_path(set_code), f"{set_code}_context.parquet")
|
|
217
|
-
|
|
218
|
-
if dataset_type == "card":
|
|
219
|
-
return os.path.join(_external_set_path(set_code), f"{set_code}_card.parquet")
|
|
220
|
-
|
|
221
|
-
return os.path.join(
|
|
222
|
-
_external_set_path(set_code), f"{set_code}_{event_type}_{dataset_type}.parquet"
|
|
223
|
-
)
|
|
224
|
-
|
|
225
|
-
|
|
226
206
|
def _process_zipped_file(gzip_path, target_path):
|
|
227
207
|
csv_path = gzip_path[:-3]
|
|
228
208
|
# if polars supports streaming from file obj, we can just stream straight
|
|
@@ -252,17 +232,17 @@ def _process_zipped_file(gzip_path, target_path):
|
|
|
252
232
|
def download_data_set(
|
|
253
233
|
set_code,
|
|
254
234
|
dataset_type: View,
|
|
255
|
-
event_type=EventType.PREMIER,
|
|
235
|
+
event_type=cache.EventType.PREMIER,
|
|
256
236
|
force_download=False,
|
|
257
237
|
clear_set_cache=True,
|
|
258
238
|
):
|
|
259
239
|
mode = "refresh" if force_download else "add"
|
|
260
240
|
cache.spells_print(mode, f"Downloading {dataset_type} dataset from 17Lands.com")
|
|
261
241
|
|
|
262
|
-
if not os.path.isdir(set_dir :=
|
|
242
|
+
if not os.path.isdir(set_dir := cache.external_set_path(set_code)):
|
|
263
243
|
os.makedirs(set_dir)
|
|
264
244
|
|
|
265
|
-
target_path = data_file_path(set_code, dataset_type)
|
|
245
|
+
target_path = cache.data_file_path(set_code, dataset_type)
|
|
266
246
|
|
|
267
247
|
if os.path.isfile(target_path) and not force_download:
|
|
268
248
|
cache.spells_print(
|
|
@@ -274,7 +254,7 @@ def download_data_set(
|
|
|
274
254
|
dataset_file = DATASET_TEMPLATE.format(
|
|
275
255
|
set_code=set_code, dataset_type=dataset_type, event_type=event_type
|
|
276
256
|
)
|
|
277
|
-
dataset_path = os.path.join(
|
|
257
|
+
dataset_path = os.path.join(cache.external_set_path(set_code), dataset_file)
|
|
278
258
|
wget.download(
|
|
279
259
|
RESOURCE_TEMPLATE.format(dataset_type=dataset_type) + dataset_file,
|
|
280
260
|
out=dataset_path,
|
|
@@ -287,7 +267,7 @@ def download_data_set(
|
|
|
287
267
|
_process_zipped_file(dataset_path, target_path)
|
|
288
268
|
cache.spells_print(mode, f"Wrote file {target_path}")
|
|
289
269
|
if clear_set_cache:
|
|
290
|
-
cache.
|
|
270
|
+
cache.clean(set_code)
|
|
291
271
|
|
|
292
272
|
return 0
|
|
293
273
|
|
|
@@ -302,7 +282,7 @@ def write_card_file(draft_set_code: str, force_download=False) -> int:
|
|
|
302
282
|
cache.spells_print(
|
|
303
283
|
mode, "Fetching card data from mtgjson.com and writing card file"
|
|
304
284
|
)
|
|
305
|
-
card_filepath = data_file_path(draft_set_code, View.CARD)
|
|
285
|
+
card_filepath = cache.data_file_path(draft_set_code, View.CARD)
|
|
306
286
|
if os.path.isfile(card_filepath) and not force_download:
|
|
307
287
|
cache.spells_print(
|
|
308
288
|
mode,
|
|
@@ -310,7 +290,7 @@ def write_card_file(draft_set_code: str, force_download=False) -> int:
|
|
|
310
290
|
)
|
|
311
291
|
return 1
|
|
312
292
|
|
|
313
|
-
draft_filepath = data_file_path(draft_set_code, View.DRAFT)
|
|
293
|
+
draft_filepath = cache.data_file_path(draft_set_code, View.DRAFT)
|
|
314
294
|
|
|
315
295
|
if not os.path.isfile(draft_filepath):
|
|
316
296
|
cache.spells_print(mode, f"Error: No draft file for set {draft_set_code}")
|
|
@@ -336,7 +316,7 @@ def write_card_file(draft_set_code: str, force_download=False) -> int:
|
|
|
336
316
|
def get_set_context(set_code: str, force_download=False) -> int:
|
|
337
317
|
mode = "refresh" if force_download else "add"
|
|
338
318
|
|
|
339
|
-
context_fp = data_file_path(set_code, "context")
|
|
319
|
+
context_fp = cache.data_file_path(set_code, "context")
|
|
340
320
|
cache.spells_print(mode, "Calculating set context")
|
|
341
321
|
if os.path.isfile(context_fp) and not force_download:
|
|
342
322
|
cache.spells_print(
|
|
@@ -345,15 +325,18 @@ def get_set_context(set_code: str, force_download=False) -> int:
|
|
|
345
325
|
)
|
|
346
326
|
return 1
|
|
347
327
|
|
|
348
|
-
|
|
349
|
-
|
|
328
|
+
df = summon(
|
|
329
|
+
set_code,
|
|
330
|
+
columns=[ColName.NUM_DRAFTS],
|
|
331
|
+
group_by=[ColName.DRAFT_DATE, ColName.PICK_NUM],
|
|
332
|
+
)
|
|
350
333
|
|
|
351
|
-
context_df =
|
|
334
|
+
context_df = df.filter(pl.col(ColName.NUM_DRAFTS) > 1000).select(
|
|
352
335
|
[
|
|
353
|
-
pl.
|
|
354
|
-
pl.
|
|
336
|
+
pl.col(ColName.DRAFT_DATE).min().alias("release_date"),
|
|
337
|
+
pl.col(ColName.PICK_NUM).max().alias("picks_per_pack"),
|
|
355
338
|
]
|
|
356
|
-
)
|
|
339
|
+
)
|
|
357
340
|
|
|
358
341
|
context_df.write_parquet(context_fp)
|
|
359
342
|
|
spells/log.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
from contextlib import contextmanager
|
|
2
|
+
import logging
|
|
3
|
+
import logging.handlers
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import sys
|
|
6
|
+
from typing import Callable
|
|
7
|
+
|
|
8
|
+
from spells.cache import data_home
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def setup_logging(
|
|
12
|
+
log_level=logging.DEBUG,
|
|
13
|
+
log_file_name="spells.log",
|
|
14
|
+
max_bytes=5_000_000, # 5MB
|
|
15
|
+
backup_count=3,
|
|
16
|
+
log_format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
|
17
|
+
):
|
|
18
|
+
log_dir = Path(data_home()) / ".logs"
|
|
19
|
+
log_dir.mkdir(exist_ok=True)
|
|
20
|
+
log_file_path = log_dir / log_file_name
|
|
21
|
+
|
|
22
|
+
handler = logging.handlers.RotatingFileHandler(
|
|
23
|
+
filename=log_file_path, maxBytes=max_bytes, backupCount=backup_count
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
formatter = logging.Formatter(log_format)
|
|
27
|
+
handler.setFormatter(formatter)
|
|
28
|
+
|
|
29
|
+
root_logger = logging.getLogger()
|
|
30
|
+
root_logger.setLevel(log_level)
|
|
31
|
+
|
|
32
|
+
root_logger.handlers.clear()
|
|
33
|
+
root_logger.addHandler(handler)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def rotate_logs():
|
|
37
|
+
root_logger = logging.getLogger()
|
|
38
|
+
for handler in root_logger.handlers:
|
|
39
|
+
if isinstance(handler, logging.handlers.RotatingFileHandler):
|
|
40
|
+
handler.doRollover()
|
|
41
|
+
logging.debug("Log file manually rotated")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@contextmanager
|
|
45
|
+
def add_console_logging():
|
|
46
|
+
console_handler = logging.StreamHandler(sys.stdout)
|
|
47
|
+
formatter = logging.Formatter(
|
|
48
|
+
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
49
|
+
)
|
|
50
|
+
console_handler.setFormatter(formatter)
|
|
51
|
+
logger = logging.getLogger()
|
|
52
|
+
logger.addHandler(console_handler)
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
yield
|
|
56
|
+
finally:
|
|
57
|
+
logger.removeHandler(console_handler)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def make_verbose(callable: Callable) -> Callable:
|
|
61
|
+
def wrapped(*args, verbose=False, **kwargs):
|
|
62
|
+
if verbose:
|
|
63
|
+
with add_console_logging():
|
|
64
|
+
return callable(*args, **kwargs)
|
|
65
|
+
else:
|
|
66
|
+
return callable(*args, **kwargs)
|
|
67
|
+
|
|
68
|
+
return wrapped
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
spells/__init__.py,sha256=bpxhRijyI-wxLPRVNathM9N9Cpq3E8uBSqWyYpDGFAI,275
|
|
2
|
+
spells/cache.py,sha256=JTFA31quGUL8pxy04NLWjEhQd5GEp37Q4PWUMnMuUJU,3797
|
|
3
|
+
spells/cards.py,sha256=EOXAB_F2yedjf6KquCERCIHl0TSIJIoOe1jv8g4JzOc,3601
|
|
4
|
+
spells/columns.py,sha256=PztPNLtqKIqIQdiCHYykk7bcYlx0_QRypu3WOsSZb-A,17856
|
|
5
|
+
spells/config.py,sha256=HDelOKrEbgMIAJfSGqzJbZdf-sInMZXRD-BI2j0QnZI,180
|
|
6
|
+
spells/draft_data.py,sha256=_gzxlpnKa5WlxTfwdnxJhvFvGsTY69bCl0cOe4VRlaI,19271
|
|
7
|
+
spells/enums.py,sha256=DL7e1xDEvrsTMbA7vJB_Et1DaYkyO4rIEzvIQDz3MZk,4871
|
|
8
|
+
spells/extension.py,sha256=DVzIedggeGfkD6BD5g-dko9l9BoPgmXWvcQ3NWdEG0U,6991
|
|
9
|
+
spells/external.py,sha256=T34oqhyM5cIFvMAMJwlltdFS4CDeouVMbU-6My60OLg,11305
|
|
10
|
+
spells/filter.py,sha256=J-YTOOAzOQpvIX29tviYL04RVoOUlfsbjBXoQBDCEdQ,3380
|
|
11
|
+
spells/log.py,sha256=-GnocAZ7K-032QohHCY5vC2myRhumuUQCTuq1erTmL4,1819
|
|
12
|
+
spells/manifest.py,sha256=dOUmj2uZZ17vCWpFwv7B5F6wOIWnoQdZkEB9SDKdx9M,8310
|
|
13
|
+
spells/schema.py,sha256=DbMvV8PIThJTp0Xzp_XIorlW6JhE1ud1kWRGf5SQ4_c,6406
|
|
14
|
+
spells_mtg-0.9.1.dist-info/METADATA,sha256=VcOjIXJ0q-qvg5aVOW-cwO3PfmV4b3DBhKuC5vY_xLc,46731
|
|
15
|
+
spells_mtg-0.9.1.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
|
|
16
|
+
spells_mtg-0.9.1.dist-info/entry_points.txt,sha256=a9Y1omdl9MdnKuIj3aOodgrp-zZII6OCdvqwgP6BFvI,63
|
|
17
|
+
spells_mtg-0.9.1.dist-info/licenses/LICENSE,sha256=tS54XYbJSgmq5zuHhbsQGbNQLJPVgXqhF5nu2CSRMig,1068
|
|
18
|
+
spells_mtg-0.9.1.dist-info/RECORD,,
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
spells/__init__.py,sha256=dWkRW298GOYHFiG84S3cUrp4R-vydgA-v95CFPaWu2M,221
|
|
2
|
-
spells/cache.py,sha256=YpTmlW-U2EosvGCH9gRQdQc8gRjjcGbgQovn5gfA_WE,3165
|
|
3
|
-
spells/cards.py,sha256=EOXAB_F2yedjf6KquCERCIHl0TSIJIoOe1jv8g4JzOc,3601
|
|
4
|
-
spells/columns.py,sha256=SdNw51lA1ox5rD2M7F8M2yqo-m0Dal5sKzS4Rp4q9MQ,18092
|
|
5
|
-
spells/config.py,sha256=HDelOKrEbgMIAJfSGqzJbZdf-sInMZXRD-BI2j0QnZI,180
|
|
6
|
-
spells/draft_data.py,sha256=R44E8eb7bMOT0AxnkWcOAyuVCLDUxbvYmlYCLEtqi-s,18800
|
|
7
|
-
spells/enums.py,sha256=DL7e1xDEvrsTMbA7vJB_Et1DaYkyO4rIEzvIQDz3MZk,4871
|
|
8
|
-
spells/extension.py,sha256=o0IeCrafo7HhNBT7fSEMF-y0Vi1zu-8uUu_0EIj9-5U,6783
|
|
9
|
-
spells/external.py,sha256=PN0PkwdKnvx-4HkApFhI1_ZwRvBUDMUd3Vfbky0xSJA,11786
|
|
10
|
-
spells/filter.py,sha256=J-YTOOAzOQpvIX29tviYL04RVoOUlfsbjBXoQBDCEdQ,3380
|
|
11
|
-
spells/manifest.py,sha256=dOUmj2uZZ17vCWpFwv7B5F6wOIWnoQdZkEB9SDKdx9M,8310
|
|
12
|
-
spells/schema.py,sha256=DbMvV8PIThJTp0Xzp_XIorlW6JhE1ud1kWRGf5SQ4_c,6406
|
|
13
|
-
spells_mtg-0.8.4.dist-info/METADATA,sha256=9szmimCDBYLOQcy4wfdi8E6tAoxe1Bt_KLjF_coyXkY,46731
|
|
14
|
-
spells_mtg-0.8.4.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
|
|
15
|
-
spells_mtg-0.8.4.dist-info/entry_points.txt,sha256=a9Y1omdl9MdnKuIj3aOodgrp-zZII6OCdvqwgP6BFvI,63
|
|
16
|
-
spells_mtg-0.8.4.dist-info/licenses/LICENSE,sha256=tS54XYbJSgmq5zuHhbsQGbNQLJPVgXqhF5nu2CSRMig,1068
|
|
17
|
-
spells_mtg-0.8.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|