goad-toolkit 0.2.4__tar.gz → 0.2.5__tar.gz

{goad_toolkit-0.2.4 → goad_toolkit-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: goad-toolkit
-Version: 0.2.4
+Version: 0.2.5
 Summary: An extensible toolkit for Goal Oriented Analysis of Data
 Project-URL: Github, https://github.com/raoulg/goad_toolkit
 Author-email: raoul grouls <Raoul.Grouls@han.nl>

{goad_toolkit-0.2.4 → goad_toolkit-0.2.5}/docs/02-pipelines.md

@@ -59,6 +59,7 @@ input frame is safe, and the steps stay cheap.
 | `RollingAvg` | rolling mean, drops the leading NaNs | `column`, `window`, `rename` |
 | `ZScaler` | standardise to mean 0, std 1 | `column`, `rename` |
 | `TimeFeatures` | derive calendar columns from a timestamp | `column`, `features` |
+| `RegexFeature` | count / flag / extract a pattern in a text column | `column`, `pattern`, `feature`, `mode` |
 
 `rename=True` writes to a new column (`deaths_shifted`, `deaths_zscore`, …) instead of
 overwriting. Prefer it. An overwritten column is a step you cannot debug, and the whole
@@ -104,39 +105,39 @@ Two rules for your `transform`:
 - **Name your parameters explicitly** in the signature. `def transform(self, data, column,
   window)` documents itself; `**kwargs` does not, and the validation cannot help you.
 
-## 2.5 A transform for feature enrichment
+## 2.5 `RegexFeature`: feature enrichment from text
 
-The single most useful custom transform for text data, and a good template:
+The single most useful transform for text data, and the reason `TransformBase` is worth
+subclassing at all. It ships:
 
 ```python
-class RegexFeature(TransformBase):
-    """Add a feature extracted from a text column with a regular expression."""
-
-    def transform(
-        self,
-        data: pd.DataFrame,
-        column: str,
-        pattern: str,
-        feature: str,
-        mode: str = "count",
-    ) -> pd.DataFrame:
-        text = data[column].fillna("")
-        if mode == "count":
-            data[feature] = text.str.count(pattern)
-        elif mode == "has":
-            data[feature] = text.str.contains(pattern, regex=True)
-        elif mode == "extract":
-            data[feature] = text.str.extract(pattern, expand=False)
-        else:
-            raise ValueError(f"mode must be count/has/extract, got {mode!r}")
-        return data
-```
+from goad_toolkit.datatransforms import RegexFeature
 
-```python
 pipeline.add(RegexFeature, name="url_flag",
              column="message", pattern=r"https?://\S+", feature="has_url", mode="has")
 ```
 
+Three modes, each writing one new column named by `feature`:
+
+| `mode` | writes | use for |
+|---|---|---|
+| `"count"` | how many times the pattern occurs (int) | how many URLs, how many question marks |
+| `"has"` | whether it occurs at all (bool) | flags you will group or filter on |
+| `"extract"` | the first capture group, NaN where nothing matched | pulling a value *out* of the text |
+
+`"extract"` needs exactly one capture group in `pattern`, and it is the mode worth being
+careful with. `count` and `has` fail visibly when a pattern is wrong — a column of all zeros
+or all `False` is hard to miss. Extraction fails *silently*, filling with NaN, so it reports
+its own coverage through `loguru`:
+
+```
+mentions: extracted 'addressed_to' from 92,415/627,172 rows (14.7%); 534,757 rows had no match
+```
+
+That number is the point. A pattern that matches 15% of rows may be exactly right — on IRC,
+most messages do not address anyone — or it may be silently broken. The log line makes you
+decide which, instead of finding out four notebooks later.
+
 Note the two different `name`s: `Pipeline.add(name=...)` names the *step*, and
 `TransformBase.__init__` consumes it. So the new-column parameter has to be called something
 else — `feature` here. Any transform that wants to name an output column hits this.

{goad_toolkit-0.2.4 → goad_toolkit-0.2.5}/docs/08-api-reference.md

@@ -84,6 +84,7 @@ class TransformBase(ABC):
 | `RollingAvg` | `column: str, window: int, rename: bool = False` |
 | `ZScaler` | `column: str, rename: bool = False` |
 | `TimeFeatures` | `column: str, features: Optional[List[str]] = None` |
+| `RegexFeature` | `column: str, pattern: str, feature: str, mode: str = "count"` |
 
 `rename=True` writes to `{column}_shifted` / `_diff` / `_rolling_avg` / `_zscore` instead of
 overwriting.
@@ -92,6 +93,12 @@ overwriting.
 `TimeFeatures.DEFAULT` — used when `features` is omitted — is `("day_name", "isoweek",
 "year_week")`. A feature named the same as `column` overwrites it.
 
+`RegexFeature.mode` is one of `"count"` (int), `"has"` (bool) or `"extract"` (the first
+capture group, NaN where nothing matched); anything else raises `ValueError`. `extract`
+requires exactly one capture group in `pattern` and logs its match rate through `loguru`.
+The output column is `feature`, not `name` — `Pipeline.add(name=...)` already claims `name`
+for the step.
+
 ```python
 class Pipeline:
     def add(self, transform_class: Type[T], name: Optional[str] = None, **kwargs) -> "Pipeline"

{goad_toolkit-0.2.4 → goad_toolkit-0.2.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "goad-toolkit"
-version = "0.2.4"
+version = "0.2.5"
 description = "An extensible toolkit for Goal Oriented Analysis of Data"
 readme = "README.md"
 authors = [

{goad_toolkit-0.2.4 → goad_toolkit-0.2.5}/src/goad_toolkit/datatransforms.py

@@ -2,6 +2,7 @@ from abc import ABC, abstractmethod
 from typing import Any, Dict, List, Optional, Type, TypeVar
 
 import pandas as pd
+from loguru import logger
 from tqdm import tqdm
 
 T = TypeVar("T", bound="TransformBase")
@@ -161,6 +162,52 @@ class TimeFeatures(TransformBase):
         return data
 
 
+class RegexFeature(TransformBase):
+    """Add a feature extracted from a text column with a regular expression.
+
+    Three modes, all writing a single new column named by `feature`:
+
+    - `"count"` — how many times the pattern occurs per row (int);
+    - `"has"` — whether it occurs at all (bool);
+    - `"extract"` — the first capture group's text, or NaN where nothing matched.
+      `pattern` must contain exactly one capture group in this mode.
+
+    The new column is `feature`, not `name`: `Pipeline.add(name=...)` already claims
+    `name` for the step, and `TransformBase.__init__` consumes it. Any transform that
+    names an output column has to spell that parameter something else.
+
+    `mode="extract"` reports its own coverage through `loguru`. `count` and `has` fail
+    visibly when a pattern is wrong — a column of all zeros or all False. Extraction
+    fails silently, filling with NaN, so it says out loud how much it matched.
+    """
+
+    def transform(
+        self,
+        data: pd.DataFrame,
+        column: str,
+        pattern: str,
+        feature: str,
+        mode: str = "count",
+    ) -> pd.DataFrame:
+        text = data[column].fillna("")
+        if mode == "count":
+            data[feature] = text.str.count(pattern)
+        elif mode == "has":
+            data[feature] = text.str.contains(pattern, regex=True)
+        elif mode == "extract":
+            data[feature] = text.str.extract(pattern, expand=False)
+            matched = data[feature].notna().sum()
+            total = len(data)
+            share = f"{matched / total:.1%}" if total else "n/a"
+            logger.info(
+                f"{self.name}: extracted '{feature}' from {matched:,}/{total:,} rows "
+                f"({share}); {total - matched:,} rows had no match"
+            )
+        else:
+            raise ValueError(f"mode must be count/has/extract, got {mode!r}")
+        return data
+
+
 class Pipeline:
     """Pipeline for chaining data transformations."""
 

goad_toolkit-0.2.5/tests/test_datatransforms.py

@@ -0,0 +1,167 @@
+import pandas as pd
+import pytest
+
+from goad_toolkit.datatransforms import Pipeline, RegexFeature, TimeFeatures
+
+
+@pytest.fixture
+def data() -> pd.DataFrame:
+    # 2024-01-01 is a Monday
+    timestamps = pd.date_range("2024-01-01 08:00", periods=10, freq="6h")
+    return pd.DataFrame({"timestamp": timestamps, "value": range(10)})
+
+
+def test_default_features_match_lesson_one(data):
+    result = TimeFeatures(column="timestamp")(data.copy())
+
+    assert "day_name" in result.columns
+    assert "isoweek" in result.columns
+    assert "year_week" in result.columns
+    assert "date" not in result.columns
+    assert "hour" not in result.columns
+
+
+def test_selected_features_only(data):
+    result = TimeFeatures(column="timestamp", features=["date", "hour"])(data.copy())
+
+    assert "date" in result.columns
+    assert "hour" in result.columns
+    assert "day_name" not in result.columns
+    assert result["hour"].tolist() == [8, 14, 20, 2, 8, 14, 20, 2, 8, 14]
+
+
+def test_all_features(data):
+    result = TimeFeatures(
+        column="timestamp",
+        features=list(TimeFeatures.AVAILABLE),
+    )(data.copy())
+
+    for feature in TimeFeatures.AVAILABLE:
+        assert feature in result.columns
+    assert result["day_name"].iloc[0] == "Monday"
+    assert result["isoweek"].iloc[0] == 1
+
+
+def test_unknown_feature_raises(data):
+    with pytest.raises(ValueError, match="Unknown feature"):
+        TimeFeatures(column="timestamp", features=["not_a_feature"])(data.copy())
+
+
+def test_feature_overwrites_same_named_source_column():
+    # mirrors lesson 1's own pipeline: source column is called "date"
+    df = pd.DataFrame({"date": pd.date_range("2024-01-01", periods=3, freq="D")})
+
+    result = TimeFeatures(column="date", features=["date"])(df)
+
+    assert result["date"].iloc[0] == pd.Timestamp("2024-01-01").date()
+
+
+def test_works_in_a_pipeline(data):
+    pipeline = Pipeline()
+    pipeline.add(TimeFeatures, column="timestamp", features=["day_name", "hour"])
+
+    result = pipeline.apply(data)
+
+    assert "day_name" in result.columns
+    assert "hour" in result.columns
+
+
+@pytest.fixture
+def messages() -> pd.DataFrame:
+    return pd.DataFrame(
+        {
+            "message": [
+                "daftykins: try rebooting?",
+                "see https://example.com and https://other.example",
+                "no url, no mention, no question",
+                None,
+            ]
+        }
+    )
+
+
+def test_count_mode(messages):
+    result = RegexFeature(
+        column="message", pattern=r"https?://\S+", feature="n_url", mode="count"
+    )(messages.copy())
+
+    assert result["n_url"].tolist() == [0, 2, 0, 0]
+
+
+def test_has_mode(messages):
+    result = RegexFeature(
+        column="message", pattern=r"\?", feature="has_question", mode="has"
+    )(messages.copy())
+
+    assert result["has_question"].tolist() == [True, False, False, False]
+
+
+def test_extract_mode_leaves_nan_where_nothing_matched(messages):
+    result = RegexFeature(
+        column="message",
+        pattern=r"^(\S+)[:,]\s",
+        feature="addressed_to",
+        mode="extract",
+    )(messages.copy())
+
+    assert result["addressed_to"].iloc[0] == "daftykins"
+    assert result["addressed_to"].isna().tolist() == [False, True, True, True]
+
+
+def test_count_mode_defaults(messages):
+    # mode is the one parameter with a default; it counts
+    result = RegexFeature(column="message", pattern=r"\?", feature="n_question")(
+        messages.copy()
+    )
+
+    assert result["n_question"].tolist() == [1, 0, 0, 0]
+
+
+def test_na_rows_are_treated_as_empty_text(messages):
+    result = RegexFeature(
+        column="message", pattern=r"\?", feature="has_question", mode="has"
+    )(messages.copy())
+
+    # the None row must not propagate NaN into a boolean column
+    assert result["has_question"].dtype == bool
+
+
+def test_unknown_mode_raises(messages):
+    with pytest.raises(ValueError, match="mode must be count/has/extract"):
+        RegexFeature(column="message", pattern=r"\?", feature="x", mode="extractt")(
+            messages.copy()
+        )
+
+
+def test_missing_column_raises(messages):
+    with pytest.raises(ValueError, match="does not exist"):
+        RegexFeature(column="nope", pattern=r"\?", feature="x")(messages.copy())
+
+
+def test_extract_on_an_empty_frame_does_not_divide_by_zero():
+    empty = pd.DataFrame({"message": pd.Series([], dtype="object")})
+
+    result = RegexFeature(
+        column="message", pattern=r"^(\S+):", feature="addressed_to", mode="extract"
+    )(empty)
+
+    assert len(result) == 0
+    assert "addressed_to" in result.columns
+
+
+def test_feature_is_the_column_name_and_name_is_the_step(messages):
+    pipeline = Pipeline()
+    pipeline.add(
+        RegexFeature,
+        name="mentions",
+        column="message",
+        pattern=r"^(\S+)[:,]\s",
+        feature="addressed_to",
+        mode="extract",
+    )
+
+    result = pipeline.apply(messages)
+
+    assert "addressed_to" in result.columns
+    assert "mentions" not in result.columns
+    assert "mentions" in repr(pipeline)

{goad_toolkit-0.2.4 → goad_toolkit-0.2.5}/uv.lock

@@ -383,7 +383,7 @@ wheels = [
 
 [[package]]
 name = "goad-toolkit"
-version = "0.2.4"
+version = "0.2.5"
 source = { editable = "." }
 dependencies = [
     { name = "loguru" },

goad_toolkit-0.2.4/tests/test_datatransforms.py

@@ -1,66 +0,0 @@
-import pandas as pd
-import pytest
-
-from goad_toolkit.datatransforms import Pipeline, TimeFeatures
-
-
-@pytest.fixture
-def data() -> pd.DataFrame:
-    # 2024-01-01 is a Monday
-    timestamps = pd.date_range("2024-01-01 08:00", periods=10, freq="6h")
-    return pd.DataFrame({"timestamp": timestamps, "value": range(10)})
-
-
-def test_default_features_match_lesson_one(data):
-    result = TimeFeatures(column="timestamp")(data.copy())
-
-    assert "day_name" in result.columns
-    assert "isoweek" in result.columns
-    assert "year_week" in result.columns
-    assert "date" not in result.columns
-    assert "hour" not in result.columns
-
-
-def test_selected_features_only(data):
-    result = TimeFeatures(column="timestamp", features=["date", "hour"])(data.copy())
-
-    assert "date" in result.columns
-    assert "hour" in result.columns
-    assert "day_name" not in result.columns
-    assert result["hour"].tolist() == [8, 14, 20, 2, 8, 14, 20, 2, 8, 14]
-
-
-def test_all_features(data):
-    result = TimeFeatures(
-        column="timestamp",
-        features=list(TimeFeatures.AVAILABLE),
-    )(data.copy())
-
-    for feature in TimeFeatures.AVAILABLE:
-        assert feature in result.columns
-    assert result["day_name"].iloc[0] == "Monday"
-    assert result["isoweek"].iloc[0] == 1
-
-
-def test_unknown_feature_raises(data):
-    with pytest.raises(ValueError, match="Unknown feature"):
-        TimeFeatures(column="timestamp", features=["not_a_feature"])(data.copy())
-
-
-def test_feature_overwrites_same_named_source_column():
-    # mirrors lesson 1's own pipeline: source column is called "date"
-    df = pd.DataFrame({"date": pd.date_range("2024-01-01", periods=3, freq="D")})
-
-    result = TimeFeatures(column="date", features=["date"])(df)
-
-    assert result["date"].iloc[0] == pd.Timestamp("2024-01-01").date()
-
-
-def test_works_in_a_pipeline(data):
-    pipeline = Pipeline()
-    pipeline.add(TimeFeatures, column="timestamp", features=["day_name", "hour"])
-
-    result = pipeline.apply(data)
-
-    assert "day_name" in result.columns
-    assert "hour" in result.columns