spells-mtg 0.9.4__tar.gz → 0.9.6__tar.gz

{spells_mtg-0.9.4 → spells_mtg-0.9.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spells-mtg
-Version: 0.9.4
+Version: 0.9.6
 Summary: analaysis of 17Lands.com public datasets
 Author-Email: Joel Barnes <oelarnes@gmail.com>
 License: MIT
@@ -15,21 +15,23 @@ Description-Content-Type: text/markdown
 
 ```
 $ spells add DSK
-🪄 spells ✨ [data home]=/Users/joel/.local/share/spells/
+  🪄 spells ✨ [data home]=/home/joel/.local/share/spells/
 
-🪄 add ✨ Downloading draft dataset from 17Lands.com
+  🪄 add ✨ Downloading draft dataset from 17Lands.com
 100% [......................................................................] 250466473 / 250466473
-🪄 add ✨ Unzipping and transforming to parquet (this might take a few minutes)...
-🪄 add ✨ Wrote file /Users/joel/.local/share/spells/external/DSK/DSK_PremierDraft_draft.parquet
-🪄 clean ✨ No local cache found for set DSK
-🪄 add ✨ Fetching card data from mtgjson.com and writing card parquet file
-🪄 add ✨ Wrote file /Users/joel/.local/share/spells/external/DSK/DSK_card.parquet
-🪄 add ✨ Downloading game dataset from 17Lands.com
+  🪄 add ✨ Unzipping and transforming to parquet (this might take a few minutes)...
+  🪄 add ✨ Wrote file /home/joel/.local/share/spells/external/DSK/DSK_PremierDraft_draft.parquet
+  🪄 clean ✨ No local cache found for set DSK
+  🪄 add ✨ Fetching card data from mtgjson.com and writing card file
+  🪄 add ✨ Wrote file /home/joel/.local/share/spells/external/DSK/DSK_card.parquet
+  🪄 add ✨ Calculating set context
+  🪄 add ✨ Wrote file /home/joel/.local/share/spells/external/DSK/DSK_PremierDraft_context.parquet
+  🪄 add ✨ Downloading game dataset from 17Lands.com
 100% [........................................................................] 77145600 / 77145600
-🪄 add ✨ Unzipping and transforming to parquet (this might take a few minutes)...
-🪄 add ✨ Wrote file /Users/joel/.local/share/spells/external/DSK/DSK_PremierDraft_game.parquet
-🪄 clean ✨ No local cache found for set DSK
-$ ipython
+  🪄 add ✨ Unzipping and transforming to parquet (this might take a few minutes)...
+  🪄 add ✨ Wrote file /home/joel/.local/share/spells/external/DSK/DSK_PremierDraft_game.parquet
+  🪄 clean ✨ Removed 1 files from local cache for set DSK
+  🪄 clean ✨ Removed local cache dir /home/joel/.local/share/spells/cache/DSK
 ```
 
 ```python
@@ -70,6 +72,7 @@ Spells is not affiliated with 17Lands. Please review the [Usage Guidelines](http
 
 - Uses [Polars](https://docs.pola.rs/) for high-performance, multi-threaded aggregations of large datasets
 - Uses Polars to power an expressive query language for specifying custom extensions
+- Analyzes larger-than-memory datasets using Polars streaming mode
 - Converts csv datasets to parquet for 10x faster calculations and 20x smaller file sizes
 - Supports calculating the standard aggregations and measures out of the box with no arguments (ALSA, GIH WR, etc)
 - Caches aggregate DataFrames in the local file system automatically for instantaneous reproduction of previous analysis
@@ -249,7 +252,7 @@ Spells caches the results of expensive aggregations in the local file system as
 
 ### Memory Usage
 
-One of my goals in creating Spells was to eliminate issues with memory pressure by exclusively using the map-reduce paradigm and a technology that supports partitioned/streaming aggregation of larget-than-memory datasets. By default, Polars loads the entire dataset in memory, but the API exposes a parameter `streaming` which I have exposed as `use_streaming`. Unfortunately, that feature does not seem to work for my queries and the memory performance can be quite poor. The one feature that may assist in memory management is the local caching, since you can restart the kernel without losing all of your progress. In particular, be careful about opening multiple Jupyter tabs unless you have at least 32 GB. In general I have not run into issues on my 16 GB MacBook Air except with running multiple kernels at once. Supporting larger-than memory computations is on my roadmap, so check back periodically to see if I've made any progress.
+One of my goals in creating Spells was to eliminate issues with memory pressure by exclusively using the map-reduce paradigm and a technology that supports partitioned/streaming aggregation of larget-than-memory datasets. By default, Polars loads the entire dataset in memory, but the API exposes a parameter `streaming` which I have exposed as `use_streaming`. Further testing is needed to determine the performance impacts, but this is the first thing you should try if you run into memory issues.
 
 When refreshing a given set's data files from 17Lands using the provided cli, the cache for that set is automatically cleared. The `spells` CLI gives additional tools for managing the local and external caches.
 
@@ -289,9 +292,10 @@ To use `spells`, make sure Spells is installed in your environment using pip or
 ### Summon
 
 ```python
-from spell import summon
+from spells import summon
 
 summon(
+    set_code: list[str] | str,
     columns: list[str] | None = None,
     group_by: list[str] | None = None,
     filter_spec: dict | None = None,
@@ -300,11 +304,16 @@ summon(
     set_context: pl.DataFrame | dict[str, Any] | None = None,
     read_cache: bool = True,
     write_cache: bool = True,
+    use_streaming: bool = False,
+    log_to_console: int = logging.ERROR,
 ) -> polars.DataFrame
 ```
 
 #### parameters
 
+- `set_code`: a set code or list of set codes among those that you have added using `spells add`.
+You can use "expansion" as a group_by to separate results from multiple sets, or you can aggregate them together.
+
 - `columns`: a list of string or `ColName` values to select as non-grouped columns. Valid `ColTypes` are `PICK_SUM`, `NAME_SUM`, `GAME_SUM`, and `AGG`. Min/Max/Unique 
 aggregations of non-numeric (or numeric) data types are not supported. If `None`, use a set of columns modeled on the commonly used values on 17Lands.com/card_data.
 
@@ -326,6 +335,8 @@ aggregations of non-numeric (or numeric) data types are not supported. If `None`
 
 - `read_cache`/`write_cache`: Use the local file system to cache and retrieve aggregations to minimize expensive reads of the large datasets. You shouldn't need to touch these arguments unless you are debugging.
 
+- 'log_to_console': Set to `logging.INFO` to see useful messages on the progress of your aggregation, or `logging.WARNING` to see warning messages about potentially invalid column definitions.
+
 ### Enums
 
 ```python
@@ -505,13 +516,10 @@ A table of all included columns. Columns can be referenced by enum or by string
 # Roadmap to 1.0
 
 - [ ] Support Traditional and Premier datasets (currently only Premier is supported)
-- [ ] Group by all
 - [ ] Enable configuration using $XDG_CONFIG_HOME/cfg.toml
-- [ ] Support min and max aggregations over base views
 - [ ] Enhanced profiling
 - [ ] Optimized caching strategy
 - [ ] Organize and analyze daily downloads from 17Lands (not a scraper!)
 - [ ] Helper functions to generate second-order analysis by card name
 - [ ] Helper functions for common plotting paradigms
-- [ ] Example notebooks
 - [ ] Scientific workflows: regression, MLE, etc

{spells_mtg-0.9.4 → spells_mtg-0.9.6}/README.md

@@ -4,21 +4,23 @@
 
 ```
 $ spells add DSK
-🪄 spells ✨ [data home]=/Users/joel/.local/share/spells/
+  🪄 spells ✨ [data home]=/home/joel/.local/share/spells/
 
-🪄 add ✨ Downloading draft dataset from 17Lands.com
+  🪄 add ✨ Downloading draft dataset from 17Lands.com
 100% [......................................................................] 250466473 / 250466473
-🪄 add ✨ Unzipping and transforming to parquet (this might take a few minutes)...
-🪄 add ✨ Wrote file /Users/joel/.local/share/spells/external/DSK/DSK_PremierDraft_draft.parquet
-🪄 clean ✨ No local cache found for set DSK
-🪄 add ✨ Fetching card data from mtgjson.com and writing card parquet file
-🪄 add ✨ Wrote file /Users/joel/.local/share/spells/external/DSK/DSK_card.parquet
-🪄 add ✨ Downloading game dataset from 17Lands.com
+  🪄 add ✨ Unzipping and transforming to parquet (this might take a few minutes)...
+  🪄 add ✨ Wrote file /home/joel/.local/share/spells/external/DSK/DSK_PremierDraft_draft.parquet
+  🪄 clean ✨ No local cache found for set DSK
+  🪄 add ✨ Fetching card data from mtgjson.com and writing card file
+  🪄 add ✨ Wrote file /home/joel/.local/share/spells/external/DSK/DSK_card.parquet
+  🪄 add ✨ Calculating set context
+  🪄 add ✨ Wrote file /home/joel/.local/share/spells/external/DSK/DSK_PremierDraft_context.parquet
+  🪄 add ✨ Downloading game dataset from 17Lands.com
 100% [........................................................................] 77145600 / 77145600
-🪄 add ✨ Unzipping and transforming to parquet (this might take a few minutes)...
-🪄 add ✨ Wrote file /Users/joel/.local/share/spells/external/DSK/DSK_PremierDraft_game.parquet
-🪄 clean ✨ No local cache found for set DSK
-$ ipython
+  🪄 add ✨ Unzipping and transforming to parquet (this might take a few minutes)...
+  🪄 add ✨ Wrote file /home/joel/.local/share/spells/external/DSK/DSK_PremierDraft_game.parquet
+  🪄 clean ✨ Removed 1 files from local cache for set DSK
+  🪄 clean ✨ Removed local cache dir /home/joel/.local/share/spells/cache/DSK
 ```
 
 ```python
@@ -59,6 +61,7 @@ Spells is not affiliated with 17Lands. Please review the [Usage Guidelines](http
 
 - Uses [Polars](https://docs.pola.rs/) for high-performance, multi-threaded aggregations of large datasets
 - Uses Polars to power an expressive query language for specifying custom extensions
+- Analyzes larger-than-memory datasets using Polars streaming mode
 - Converts csv datasets to parquet for 10x faster calculations and 20x smaller file sizes
 - Supports calculating the standard aggregations and measures out of the box with no arguments (ALSA, GIH WR, etc)
 - Caches aggregate DataFrames in the local file system automatically for instantaneous reproduction of previous analysis
@@ -238,7 +241,7 @@ Spells caches the results of expensive aggregations in the local file system as
 
 ### Memory Usage
 
-One of my goals in creating Spells was to eliminate issues with memory pressure by exclusively using the map-reduce paradigm and a technology that supports partitioned/streaming aggregation of larget-than-memory datasets. By default, Polars loads the entire dataset in memory, but the API exposes a parameter `streaming` which I have exposed as `use_streaming`. Unfortunately, that feature does not seem to work for my queries and the memory performance can be quite poor. The one feature that may assist in memory management is the local caching, since you can restart the kernel without losing all of your progress. In particular, be careful about opening multiple Jupyter tabs unless you have at least 32 GB. In general I have not run into issues on my 16 GB MacBook Air except with running multiple kernels at once. Supporting larger-than memory computations is on my roadmap, so check back periodically to see if I've made any progress.
+One of my goals in creating Spells was to eliminate issues with memory pressure by exclusively using the map-reduce paradigm and a technology that supports partitioned/streaming aggregation of larget-than-memory datasets. By default, Polars loads the entire dataset in memory, but the API exposes a parameter `streaming` which I have exposed as `use_streaming`. Further testing is needed to determine the performance impacts, but this is the first thing you should try if you run into memory issues.
 
 When refreshing a given set's data files from 17Lands using the provided cli, the cache for that set is automatically cleared. The `spells` CLI gives additional tools for managing the local and external caches.
 
@@ -278,9 +281,10 @@ To use `spells`, make sure Spells is installed in your environment using pip or
 ### Summon
 
 ```python
-from spell import summon
+from spells import summon
 
 summon(
+    set_code: list[str] | str,
     columns: list[str] | None = None,
     group_by: list[str] | None = None,
     filter_spec: dict | None = None,
@@ -289,11 +293,16 @@ summon(
     set_context: pl.DataFrame | dict[str, Any] | None = None,
     read_cache: bool = True,
     write_cache: bool = True,
+    use_streaming: bool = False,
+    log_to_console: int = logging.ERROR,
 ) -> polars.DataFrame
 ```
 
 #### parameters
 
+- `set_code`: a set code or list of set codes among those that you have added using `spells add`.
+You can use "expansion" as a group_by to separate results from multiple sets, or you can aggregate them together.
+
 - `columns`: a list of string or `ColName` values to select as non-grouped columns. Valid `ColTypes` are `PICK_SUM`, `NAME_SUM`, `GAME_SUM`, and `AGG`. Min/Max/Unique 
 aggregations of non-numeric (or numeric) data types are not supported. If `None`, use a set of columns modeled on the commonly used values on 17Lands.com/card_data.
 
@@ -315,6 +324,8 @@ aggregations of non-numeric (or numeric) data types are not supported. If `None`
 
 - `read_cache`/`write_cache`: Use the local file system to cache and retrieve aggregations to minimize expensive reads of the large datasets. You shouldn't need to touch these arguments unless you are debugging.
 
+- 'log_to_console': Set to `logging.INFO` to see useful messages on the progress of your aggregation, or `logging.WARNING` to see warning messages about potentially invalid column definitions.
+
 ### Enums
 
 ```python
@@ -494,13 +505,10 @@ A table of all included columns. Columns can be referenced by enum or by string
 # Roadmap to 1.0
 
 - [ ] Support Traditional and Premier datasets (currently only Premier is supported)
-- [ ] Group by all
 - [ ] Enable configuration using $XDG_CONFIG_HOME/cfg.toml
-- [ ] Support min and max aggregations over base views
 - [ ] Enhanced profiling
 - [ ] Optimized caching strategy
 - [ ] Organize and analyze daily downloads from 17Lands (not a scraper!)
 - [ ] Helper functions to generate second-order analysis by card name
 - [ ] Helper functions for common plotting paradigms
-- [ ] Example notebooks
 - [ ] Scientific workflows: regression, MLE, etc

{spells_mtg-0.9.4 → spells_mtg-0.9.6}/pyproject.toml

@@ -11,7 +11,7 @@ dependencies = [
 ]
 requires-python = ">=3.11"
 readme = "README.md"
-version = "0.9.4"
+version = "0.9.6"
 
 [project.license]
 text = "MIT"
@@ -28,6 +28,9 @@ build-backend = "pdm.backend"
 [tool.pdm]
 distribution = true
 
+[tool.pdm.scripts]
+post_install = "scripts/post_install.py"
+
 [tool.pdm.publish.upload]
 env_file = "$HOME/.pypienv"
 

{spells_mtg-0.9.4 → spells_mtg-0.9.6}/spells/cache.py

@@ -25,7 +25,7 @@ class DataDir(StrEnum):
 
 
 def spells_print(mode, content):
-    print(f"🪄 {mode} ✨ {content}")
+    print(f"  🪄 {mode} ✨ {content}")
 
 
 def data_home() -> str:

{spells_mtg-0.9.4 → spells_mtg-0.9.6}/spells/cards.py

@@ -78,7 +78,7 @@ def card_df(draft_set_code: str, names: list[str]) -> pl.DataFrame:
     draft_set_json = _fetch_mtg_json(draft_set_code)
     booster_info = draft_set_json["data"]["booster"]
 
-    booster_type = "play" if "play" in booster_info else "draft"
+    booster_type = "play" if "play" in booster_info else "draft" if "draft" in booster_info else list(booster_info.keys())[0]
     set_codes = booster_info[booster_type]["sourceSetCodes"]
     set_codes.reverse()
 

{spells_mtg-0.9.4 → spells_mtg-0.9.6}/spells/draft_data.py

@@ -455,7 +455,7 @@ def _base_agg_df(
     return joined_df
 
 
-@make_verbose
+@make_verbose()
 def summon(
     set_code: str | list[str],
     columns: list[str] | None = None,

{spells_mtg-0.9.4 → spells_mtg-0.9.6}/spells/log.py

@@ -59,10 +59,12 @@ def console_logging(log_level):
         logger.removeHandler(console_handler)
 
 
-def make_verbose(func: Callable) -> Callable:
-    @wraps(func)
-    def wrapped(*args, logging: int=logging.ERROR, **kwargs):
-        with console_logging(logging):
-            return func(*args, **kwargs)
-    return wrapped
+def make_verbose(level: int=logging.ERROR) -> Callable:
+    def decorator(func: Callable) -> Callable:
+        @wraps(func)
+        def wrapped(*args, log_to_console: int=level, **kwargs):
+            with console_logging(log_to_console):
+                return func(*args, **kwargs)
+        return wrapped
+    return decorator