rsc-universal 0.1.1

package/skills/dashboard/scripts/verify.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+# verify.sh — check a dashboard.yaml for SHAPE and DISCIPLINE, not correctness.
+# Read-only. Exits 0 on a clean/empty target (no false failures).
+#
+# Checks:
+#   1. file parses as YAML
+#   2. tiles: exists and holds 1-9 entries (fail if >9 — the budget rule)
+#   3. exactly one tile has north_star: true
+#   4. every tile has non-empty: metric, chart, comparison, owner, refresh, decision
+#   5. banned charts: no pie with >5 categories; no gauge/dial
+#
+# Does NOT judge metric choice or numeric values.
+#
+# Usage: verify.sh [path/to/dashboard.yaml]   (default: ./dashboard.yaml)
+
+set -euo pipefail
+
+TARGET="${1:-./dashboard.yaml}"
+
+if [[ ! -e "$TARGET" ]]; then
+  echo "verify: no dashboard.yaml at '$TARGET' — nothing to check. PASS (empty target)."
+  exit 0
+fi
+
+if [[ ! -s "$TARGET" ]]; then
+  echo "verify: '$TARGET' is empty — nothing to check. PASS (empty target)."
+  exit 0
+fi
+
+# Need a YAML parser. Prefer python3; fall back gracefully if absent.
+if ! command -v python3 >/dev/null 2>&1; then
+  echo "verify: python3 not found — cannot parse YAML on this machine."
+  echo "        Install python3 (with PyYAML) to run shape checks. SKIP (no false failure)."
+  exit 0
+fi
+
+python3 - "$TARGET" <<'PY'
+import sys
+
+path = sys.argv[1]
+
+try:
+    import yaml
+except ModuleNotFoundError:
+    print("verify: PyYAML not installed — cannot parse YAML.")
+    print("        Run: python3 -m pip install pyyaml  then re-run. SKIP (no false failure).")
+    sys.exit(0)
+
+errors = []
+warnings = []
+
+try:
+    with open(path) as fh:
+        doc = yaml.safe_load(fh)
+except yaml.YAMLError as e:
+    print(f"verify: FAIL — '{path}' is not valid YAML:\n{e}")
+    sys.exit(1)
+
+if doc is None:
+    print(f"verify: '{path}' parsed as empty. PASS (empty target).")
+    sys.exit(0)
+
+if not isinstance(doc, dict):
+    print("verify: FAIL — top level must be a mapping with a 'tiles:' key.")
+    sys.exit(1)
+
+tiles = doc.get("tiles")
+if not isinstance(tiles, list) or not tiles:
+    print("verify: FAIL — 'tiles:' must be a non-empty list.")
+    sys.exit(1)
+
+# Tile budget
+n = len(tiles)
+if n > 9:
+    errors.append(f"tile budget: {n} tiles exceeds the 9-tile ceiling — tier the extras to a drill-down.")
+elif n < 5:
+    warnings.append(f"tile budget: only {n} tiles — fine, but exec dashboards usually carry 5-9.")
+
+REQUIRED = ["metric", "chart", "comparison", "owner", "refresh", "decision"]
+BANNED_CHARTS = {"gauge", "dial"}
+
+north_star_count = 0
+for i, tile in enumerate(tiles):
+    label = f"tile[{i}]"
+    if not isinstance(tile, dict):
+        errors.append(f"{label}: not a mapping.")
+        continue
+    tid = tile.get("id", label)
+    label = f"tile '{tid}'"
+
+    if tile.get("north_star") is True:
+        north_star_count += 1
+
+    for key in REQUIRED:
+        val = tile.get(key)
+        if val is None or (isinstance(val, str) and not val.strip()) or val == {} or val == []:
+            errors.append(f"{label}: missing or empty required field '{key}'.")
+
+    chart = tile.get("chart")
+    if isinstance(chart, str):
+        c = chart.strip().lower()
+        if c in BANNED_CHARTS:
+            errors.append(f"{label}: chart '{chart}' is banned — use a bullet chart instead.")
+        if c == "pie":
+            cats = tile.get("categories")
+            if isinstance(cats, int) and cats > 5:
+                errors.append(f"{label}: pie with {cats} categories — over 5 is unreadable; use a sorted bar.")
+            elif isinstance(cats, list) and len(cats) > 5:
+                errors.append(f"{label}: pie with {len(cats)} categories — over 5 is unreadable; use a sorted bar.")
+            else:
+                warnings.append(f"{label}: pie chart — keep it to 3-5 slices or switch to a sorted bar.")
+
+if north_star_count == 0:
+    errors.append("north_star: no tile flagged 'north_star: true' — a dashboard needs one entry point.")
+elif north_star_count > 1:
+    errors.append(f"north_star: {north_star_count} tiles flagged 'north_star: true' — there must be exactly one.")
+
+for w in warnings:
+    print(f"verify: warn — {w}")
+
+if errors:
+    print()
+    for e in errors:
+        print(f"verify: FAIL — {e}")
+    print(f"\nverify: {len(errors)} problem(s) found in '{path}'.")
+    sys.exit(1)
+
+print(f"verify: PASS — '{path}' has {n} tile(s), one north-star, all required fields, no banned charts.")
+sys.exit(0)
+PY

package/skills/data-cleaning/SKILL.md

@@ -0,0 +1,285 @@
+---
+name: data-cleaning
+description: "Use when a raw dataset is too dirty to trust — nulls and sentinels (-1, 999, \"N/A\"), duplicate rows, inconsistent categories (USA / U.S. / United States), mixed types in one column, junk whitespace/encoding, out-of-range numbers or malformed dates — and you need it cleaned the same way every time, with a schema that fails loud on bad input. Triggers: 'clean and dedupe this CSV', 'normalize these messy categories', 'why did my integer column turn into floats', 'build a repeatable cleaning pipeline', 'add a validation gate before this feeds my model', 'quarantine the bad rows instead of dropping them', 'limpiar y deduplicar este dataset', 'normalitzar dades brutes abans d''analitzar', 'mis fechas se parsean mal y unas filas se duplican'. NOT producing or repairing a spreadsheet/formula/.xlsx (that is spreadsheet-ops), NOT scraping rows off a website (that is data-scraper), NOT parsing fields out of PDFs/HTML (that is structured-extraction)."
+tags: [data-cleaning, data-quality, pandas, pandera, validation, deduplication, reproducibility, polars]
+recommends: [duckdb, spreadsheet-ops, structured-extraction, data-scraper, analytics, business-intelligence, forecasting, python]
+profiles: []
+origin: risco
+---
+
+# Data cleaning — make dirty data trustworthy, and make the cleaning auditable
+
+A clean table is **typed + deduped + normalized + validated + reproducible**. The deliverable here is
+never "I opened a notebook and fixed some rows by hand." It is a re-runnable function `clean(raw) -> df`
+plus a **schema gate** that fails loud when next month's file violates the contract. Reproducible means
+the same input always yields the same output: versions pinned, sorts deterministic, nothing random without
+a seed. If you can't re-run it tomorrow and get the identical result, you haven't cleaned the data — you've
+edited a snapshot.
+
+Current stack (verified 2026-06-02): **pandas 3.0.x** (3.0.0 shipped 2026-01-21). Two 3.0 facts shape how you
+read and assert: the default `str` dtype is **PyArrow-backed when PyArrow is installed, NumPy-object-backed
+otherwise** (PDEP-14 deliberately kept the object fallback to avoid a hard PyArrow dependency — PyArrow is
+*recommended, not required*), and the new string dtype uses **`NaN` missing-value semantics** like every other
+default dtype, so test for null with `pd.isna()`, not by comparing against a specific token. Install PyArrow
+to get the faster backed path. **pandera 0.31.1** (supports pandas ≥ 3) for in-pipeline schema validation.
+**Polars** and **DuckDB** when pandas runs out of RAM. Pin them: `pandas==3.0.3`, `pandera==0.31.1`.
+
+## Do you need this skill? (decide first)
+
+| Your situation | Reach for |
+| --- | --- |
+| Rows are dirty (nulls, dupes, mixed types, bad dates) and you want a re-runnable clean + a gate | **data-cleaning** (this skill) |
+| You need an `.xlsx` / formula / styled sheet / chart *out* | [`spreadsheet-ops`](../spreadsheet-ops/SKILL.md) |
+| The data is still on a website — you have to *acquire* it | [`data-scraper`](../data-scraper/SKILL.md) |
+| Fields are trapped in PDF/HTML/free text and must be *parsed into rows* | `structured-extraction` |
+| Multi-GB analytical SQL / heavy groupby-join is the actual bottleneck | [`duckdb`](../duckdb/SKILL.md) |
+| The table is already clean and you want charts / KPIs / a model | [`analytics`](../analytics/SKILL.md), `business-intelligence`, `forecasting` |
+
+Cleaning **starts** once you hold tabular rows and **ends** at a validated table/DataFrame/Parquet. The
+boundary with every consumer sibling is that validated table. (Some routed siblings may not be built in
+this collection yet; the routing decision still holds.)
+
+## The pipeline shape
+
+One canonical order. Each step is positioned for a reason, not by habit.
+
+```python
+import pandas as pd
+
+def clean(raw_path: str) -> pd.DataFrame:
+    df = read_typed(raw_path)     # 1. read with explicit dtypes — never let pandas guess
+    df = normalize(df)            # 2. strings/categories/numbers/dates — collapse invisible variance
+    df = dedupe(df)               # 3. AFTER normalize+type, so "1"/1 and "US "/"US" actually collapse
+    df = handle_missing(df)       # 4. decide per column: drop / impute+flag / leave NA / quarantine
+    df = Schema.validate(df, lazy=True)  # 5. the GATE — fail loud, surface every violation at once
+    return df
+```
+
+- **Type before dedupe** — otherwise `"1"` (string) and `1` (int) survive as two distinct keys.
+- **Normalize before dedupe** — `"US "` and `"US"` are the same customer; dedupe can't see that until
+  whitespace/case are collapsed.
+- **Validate last** — it is the gate, not a cleaning step. It asserts the contract holds *after* all fixes.
+- **Write to a NEW artifact** — the raw file is read-only; you never overwrite your only source.
+
+## Read it right
+
+The single most common reproducibility footgun: pandas' legacy numpy path silently casts an integer column
+containing one NaN to `float64`, so your `id` becomes `1001.0`. Control the dtype on read.
+
+```python
+# BAD — pandas guesses: ids become floats, "N/A" stays a string, "" is sometimes NaN sometimes ""
+df = pd.read_csv("raw.csv")
+
+# GOOD — explicit, deterministic, real nullable types
+df = pd.read_csv(
+    "raw.csv",
+    dtype_backend="pyarrow",          # real nullable ints/strings; no silent float-cast — REQUIRES pyarrow installed; use "numpy_nullable" if it isn't
+    na_values=["", "N/A", "NA", "null", "-1", "999"],  # YOUR sentinels become real NA
+    keep_default_na=True,             # keep pandas' default NA tokens too
+    encoding="utf-8",                 # state it; don't let locale decide
+)
+```
+
+pandas 3.0 notes that matter here: `dtype_backend="pyarrow"` only works if pyarrow is actually installed —
+it's recommended but *not* a hard pandas dependency, so install it (`pip install pyarrow`) or fall back to
+`dtype_backend="numpy_nullable"` when it's absent. The default `str` dtype is PyArrow-backed when pyarrow is
+present and NumPy-object-backed otherwise, but either way it uses `NaN` missing-value semantics — so assert
+nullability with `pd.isna()`, never by comparing against which null token appeared.
+
+## Profile before you fix
+
+Let the numbers drive the plan, not a glance at `df.head()`. Run this first, every time.
+
+```python
+def profile(df: pd.DataFrame) -> pd.DataFrame:
+    return pd.DataFrame({
+        "dtype":     df.dtypes.astype(str),
+        "null_pct":  (df.isna().mean() * 100).round(1),
+        "n_unique":  df.nunique(dropna=True),       # cardinality — catches category sprawl
+        "sample":    df.apply(lambda s: s.dropna().unique()[:3].tolist()),
+    })
+
+print(profile(df))
+print("rows:", len(df), "exact dupes:", df.duplicated().sum())
+```
+
+A column at 90% null is a drop candidate; one with 400 distinct "countries" needs a mapping table; an
+"age" with min `-1`/max `999` has sentinels to map. The profile is your TODO list.
+
+## Normalize
+
+Each fix below: **Bad → Good**, with a one-line why.
+
+**Strings** — invisible variance (trailing space, mixed case, lookalike unicode) silently breaks joins
+and dedupe.
+
+```python
+# BAD: "US ", "us", "ｕｓ" all look different to a join
+# GOOD:
+s = df["country"].str.strip().str.casefold().str.normalize("NFKC")
+```
+
+**Categories** — use a **mapping table**, never a tower of regex. A dict is auditable and an *unmapped*
+value gets quarantined instead of silently passing through.
+
+```python
+COUNTRY = {"usa": "US", "u.s.": "US", "united states": "US", "u.s.a.": "US", "es": "ES", "españa": "ES"}
+key = df["country"].str.strip().str.casefold()
+df["country"] = key.map(COUNTRY)            # unmapped -> NA, which the gate below will catch (no silent pass)
+```
+
+**Numbers** — turn sentinels into NA, then choose a range policy explicitly: *clip* (cap to bound) when
+out-of-range is plausibly a recording cap, *reject* (→ NA / quarantine) when it is impossible.
+
+```python
+df["age"] = df["age"].mask(df["age"].isin([-1, 999]))   # sentinels -> NA
+df["age"] = df["age"].clip(lower=0, upper=120)          # clip policy; or .mask(~df["age"].between(0,120)) to reject
+```
+
+**Dates** — state the `format`, coerce, then **count the casualties**. Never trust dayfirst inference;
+`03/04/2026` is ambiguous and pandas will pick silently.
+
+```python
+parsed = pd.to_datetime(df["signup"], format="%Y-%m-%d", utc=True, errors="coerce")
+bad = parsed.isna() & df["signup"].notna()
+assert bad.sum() == 0, f"{bad.sum()} dates failed the expected format — inspect before proceeding"
+df["signup"] = parsed
+```
+
+## Dedupe
+
+`drop_duplicates(keep="first")` is meaningless without a defined key and a stable sort — "first" of what
+order? Define both.
+
+```python
+key = ["customer_id"]                                   # the BUSINESS key, stated explicitly
+df = (df.sort_values(["customer_id", "updated_at"], ascending=[True, False], kind="stable")
+        .drop_duplicates(subset=key, keep="first"))     # keep most-recent per customer, deterministically
+```
+
+Near-duplicates (`"Acme Inc"` vs `"Acme, Inc."`) are a *normalization* problem — collapse them in the
+normalize step first; only then does exact dedupe catch them. Fuzzy matching is a separate, riskier
+decision — make it visible, never automatic.
+
+## Missing values — decide per column
+
+No silent `fillna(0)`: a zero is a value, and treating "unknown" as zero poisons every mean, sum, and model
+downstream. Pick deliberately.
+
+| Situation | Action | Why |
+| --- | --- | --- |
+| Column is mostly null (e.g. >70%) and not load-bearing | Drop the **column** | Imputing it invents signal that isn't there |
+| A few rows missing a *required* key (id, date) | Drop the **row** (and log/quarantine) | Can't dedupe or join without the key |
+| Numeric gap you must fill for a model | Impute **and add a `_was_missing` flag** | The model can learn "was missing"; you keep the audit trail |
+| Genuinely optional field | **Leave NA** | NA is information; don't fabricate a value |
+| Value is present but *invalid* (unmapped category, bad date) | **Quarantine the row** | Don't drop silently and don't let it pass the gate |
+
+```python
+df["income_was_missing"] = df["income"].isna()
+df["income"] = df["income"].fillna(df["income"].median())   # impute + flag, never bare fillna(0)
+```
+
+## Validate — the gate
+
+This is where cleaning becomes *trustworthy*. Declare the contract as a pandera `DataFrameModel`, validate
+**output** (and input expectations where they exist), and split valid rows from failures instead of
+crashing — the failures become your quarantine.
+
+```python
+import pandera.pandas as pa
+from pandera.typing import Series
+
+class CustomerSchema(pa.DataFrameModel):
+    customer_id: Series[int]   = pa.Field(unique=True, ge=1)
+    country:     Series[str]   = pa.Field(isin=["US", "ES", "FR"])     # only mapped categories survive
+    age:         Series[float] = pa.Field(ge=0, le=120, nullable=True)
+    signup:      Series[pa.DateTime] = pa.Field(nullable=False)
+
+    class Config:
+        strict = True       # reject unexpected columns
+        coerce = True       # coerce to declared dtype, fail loud if impossible
+
+# lazy=True collects EVERY violation at once instead of dying on the first
+try:
+    valid = CustomerSchema.validate(df, lazy=True)
+except pa.errors.SchemaErrors as e:
+    failures = e.failure_cases          # dataframe of exactly which rows/checks failed
+    failures.to_parquet("quarantine.parquet")   # keep, don't drop — someone investigates these
+    valid = df.drop(index=e.failure_cases["index"].dropna().unique())  # proceed with the clean subset
+```
+
+`coerce=True` fixes types the contract expects; `nullable` states which columns may hold NA; field
+`Check`s (`ge`, `le`, `isin`, `unique`) are the allowed-value rules. `strict` catches columns that
+shouldn't be there. Together they are the data contract in code.
+
+When to escalate beyond pandera: reach for **GX Core 1.0** (Great Expectations' rebranded OSS — Data
+Context → Data Source → Expectation Suite → Validation Definition → Checkpoint) when you need a *shared
+data-quality platform* across many datasets and teams with a results store and docs. Use **dbt model
+contracts** (enforced at build) plus **dbt tests** (post-materialization) when the cleaning lives in a
+SQL warehouse, not Python. Full pandera patterns, the GX checkpoint sketch, the dbt YAML, and the
+"which validator" chooser are in [references/validation-patterns.md](references/validation-patterns.md).
+
+## Scale — when pandas hurts
+
+Heuristic: pandas is fine while the data fits comfortably in RAM (roughly ≤ 1–2 GB working set). Beyond
+that, or when a groupby/join dominates the runtime, switch the *mechanics* (not the principles):
+
+- **Polars** for clean-at-scale: `pl.scan_csv(...)` (lazy, parallel, Rust), then `.unique()`,
+  `.drop_nulls()`, `.fill_null(...)`, `.str.*` — the same profile→normalize→dedupe→validate shape, faster.
+  pandera validates Polars frames too.
+- **DuckDB** when the bottleneck is analytical SQL over multi-GB files — point heavy joins/aggregations
+  there: [`duckdb`](../duckdb/SKILL.md). It is an *engine* choice; correctness/normalization is still this
+  skill's job.
+
+Polars equivalents for every recipe above are in
+[references/normalization-recipes.md](references/normalization-recipes.md).
+
+## Reproducibility checklist
+
+- [ ] Library versions pinned (`pandas==3.0.3`, `pandera==0.31.1`) — same input → same output.
+- [ ] Read with explicit `dtype_backend` + `na_values`; no silent int→float cast.
+- [ ] Dedupe has an explicit `subset` key AND a deterministic `sort_values(..., kind="stable")` before it.
+- [ ] No `random` / sampling without a fixed seed; stable tie-breaks everywhere.
+- [ ] Raw source untouched; output written to a NEW artifact.
+- [ ] Schema validated on output; `lazy=True` so all violations surface together.
+- [ ] Failing rows **quarantined**, not silently dropped.
+- [ ] Row-count diff logged (`in`, `out`, coerced, quarantined) — an auditable record of what changed.
+
+## Anti-patterns
+
+| Rationalization | Reality → STOP |
+| --- | --- |
+| "`fillna(0)` to get rid of the nulls" | Zero is a value; it distorts every mean/sum/model. Impute deliberately and add a `_was_missing` flag. |
+| "`drop_duplicates()` — done" | No `subset`, no sort → which row survives is nondeterministic. Define the key, `sort_values(kind="stable")`, set `keep`. |
+| "`pd.read_csv(path)` and start cleaning" | pandas guesses: ids become floats, dates become strings. Pass `dtype_backend` + `na_values`. |
+| "I fixed the rows in a notebook cell" | Not reproducible — next month's file gets nothing. Wrap it in `clean(raw) -> df`. |
+| "Drop the rows that look wrong" | Silent data loss with no audit trail. Quarantine to a file; someone investigates. |
+| "A few regexes will normalize the countries" | Unmaintainable and silent on new values. Use a mapping dict; unmapped → NA → caught by the gate. |
+| "`pd.to_datetime` figures out the format" | Ambiguous dates parse silently wrong. State `format=`, `errors="coerce"`, then assert the NaT count. |
+| "Validation passed, so we're good" | A gate that never fails is a no-op. Feed it a known-bad row and confirm it *rejects*. |
+| "It's slow, rewrite everything in Polars" | Switch the engine, not the discipline — profile→normalize→dedupe→validate still applies. |
+
+## References
+
+- [references/validation-patterns.md](references/validation-patterns.md) — full pandera `DataFrameModel`
+  (coerce, custom `@pa.check`, lazy `SchemaErrors` report, valid/quarantine split helper), GX Core 1.0
+  checkpoint sketch, dbt model-contract + `data_tests` YAML, and the "which validator" chooser.
+- [references/normalization-recipes.md](references/normalization-recipes.md) — copy-paste recipes: category
+  mapping with unmapped→quarantine, robust date parser, unicode/encoding repair, sentinel→NA table, numeric
+  clip-vs-reject, and Polars equivalents (`unique`, `drop_nulls`, `fill_null`, `str.*`) for scale.
+
+## Verify
+
+Run `scripts/verify.sh` from anywhere. It does static structure checks on this skill (frontmatter keys,
+references present) always, and — when pandas + pandera are installed — extracts the documented pattern,
+feeds it one clearly-good row and one clearly-bad row, and asserts the good row PASSES validation while the
+bad row is FLAGGED/quarantined, proving the gate is not a no-op. If pandas/pandera aren't installed it
+prints SKIP for the runtime check and still passes the static checks. No network.
+
+## Project grounding (02-DOCS + CLAUDE.md)
+
+In a project with a `02-DOCS/` layer (the [`harness`](../harness/SKILL.md) wiki), record this dataset's
+cleaning decisions — the schema/contract, the category mapping tables, the dedupe key, the quarantine
+location, version pins — in `02-DOCS/wiki/data/<dataset>.md` and link it from the root `CLAUDE.md`
+`## Knowledge map`. Read it first on every re-run so the contract stays consistent. No `02-DOCS/`? Skip
+silently. Conventions are recorded, never gated.

package/skills/data-cleaning/evals/README.md

@@ -0,0 +1,16 @@
+# Eval harness — `data-cleaning`
+
+`cases.yaml` is a human/LLM-graded rubric, not an automated runner. It grades two things: **triggering**
+(does the skill fire on real messy-data asks and stay quiet on near-misses that belong to a sibling?) and
+**capability** (does loading the skill produce a measurably better cleaning pipeline?). The mechanical
+check — that the documented validation gate actually rejects a bad row — lives separately in
+`scripts/verify.sh`, which these cases do not duplicate.
+
+To run triggering: start a fresh agent with only `data-cleaning` discoverable, paste each `should_trigger`
+prompt verbatim, and confirm it reaches for this skill (3–5 trials each, since the decision is stochastic;
+pass if it fires in the majority). For each `should_not_trigger`, confirm it does **not** fire and would
+plausibly route to the named `route_to` sibling (some siblings may not be built yet — the routing intent is
+what's graded). To run capability: give the scenario to a clean agent WITHOUT the skill and again WITH it,
+then grade each transcript against the `must_include` list, one point per bullet that is specifically and
+correctly covered (not merely name-dropped). WITH should clearly beat WITHOUT. Record trial counts and
+per-bullet verdicts so a reviewer can audit; don't report a bare pass/fail.

package/skills/data-cleaning/evals/cases.yaml

@@ -0,0 +1,57 @@
+skill: data-cleaning
+
+should_trigger:
+  - prompt: "Clean and dedupe this customer CSV and give me something I can re-run every month with the same result."
+    why: Core ask — reproducible cleaning pipeline plus dedupe; the headline use case.
+  - prompt: "My integer ID column came in as 1001.0 floats and a bunch of rows are duplicated. Fix it."
+    why: Non-obvious dtype footgun — only this skill explains read-typing (dtype_backend) before deterministic dedupe.
+  - prompt: "Normalize these country values — USA, U.S., United States all mean the same thing."
+    why: Category normalization via a mapping table; the unmapped→quarantine pattern is this skill's.
+  - prompt: "Add a validation gate so bad rows fail loudly before they ever reach my model."
+    why: Schema-contract / pandera framing — declare types, ranges, allowed values; fail loud.
+  - prompt: "Quarantine the rows that don't pass instead of silently dropping them."
+    why: Non-obvious quarantine pattern unique to this skill; not a generic cleaning ask.
+  - prompt: "Limpiar y deduplicar este dataset y dejar un pipeline repetible."
+    why: Spanish trigger — same reproducible clean+dedupe intent, no English keyword present.
+  - prompt: "Why do my dates parse wrong sometimes and a few rows look duplicated after import?"
+    why: Symptom-led, never says 'clean' — ambiguous date format + dedupe both live here.
+
+should_not_trigger:
+  - prompt: "Export this cleaned DataFrame to Excel with a summary chart and conditional formatting."
+    route_to: spreadsheet-ops
+    why: Output is a styled sheet/formulas/chart; cleaning already ended at the clean table.
+  - prompt: "Scrape all the product prices off this catalog website."
+    route_to: data-scraper
+    why: Acquisition — the rows aren't held yet; cleaning only starts once you have tabular rows.
+  - prompt: "Pull the line items out of these invoice PDFs into a table."
+    route_to: structured-extraction
+    why: Parsing fields out of unstructured blobs into rows, not cleaning rows you already hold.
+  - prompt: "Aggregate this 5GB CSV with SQL fast on my laptop."
+    route_to: duckdb
+    why: OLAP speed over multi-GB files is the bottleneck, not correctness/normalization.
+  - prompt: "Build the KPI dashboard from this already-clean table."
+    route_to: business-intelligence
+    why: Consumes the clean output; the boundary is the validated table.
+  - prompt: "Forecast next quarter's revenue from this monthly clean series."
+    route_to: forecasting
+    why: Modeling on top of clean data, not producing the clean data.
+
+capability:
+  - scenario: >
+      Given a messy customers CSV — an id column read as floats with .0, "N/A" and 999 sentinels in age,
+      duplicate rows per customer, country values USA / U.S. / United States, and malformed dates — produce
+      a reproducible clean() pipeline plus a pandera schema that types, normalizes, dedupes deterministically,
+      validates, and quarantines failures rather than dropping them.
+    must_include:
+      - Reads with explicit dtype control (dtype_backend="pyarrow" or numpy_nullable + na_values) so ids stay int, not float
+      - Profiles first (nulls %, dup count, per-column dtype, cardinality) to drive the plan
+      - Normalizes strings with strip/casefold/NFKC before dedupe
+      - Maps categories with a mapping table (dict), unmapped → NA/quarantine, not a regex pile
+      - Turns sentinels (-1/999/"N/A") into NA, with an explicit clip-vs-reject range decision
+      - Parses dates with an explicit format and errors="coerce", then checks the coerced-NaT count
+      - Deduplicates deterministically — explicit subset key + stable sort_values + keep
+      - Handles missing per column; no bare fillna(0); impute adds a _was_missing flag
+      - Defines a pandera DataFrameModel with coerce, nullable, Field checks (isin/ge/le/unique), and lazy=True
+      - Quarantines failing rows to an artifact instead of silently dropping them
+      - Writes to a NEW output artifact, leaving the raw source untouched
+      - Logs a row-count in/out/coerced/quarantined diff and pins library versions

package/skills/data-cleaning/references/normalization-recipes.md

@@ -0,0 +1,136 @@
+# Normalization recipes — copy-paste
+
+Each recipe is deterministic and re-runnable. Drop them into your `normalize(df)` step. Polars equivalents
+at the end for when pandas runs out of RAM.
+
+## Category mapping (unmapped → quarantine, never silent pass)
+
+A dict is auditable; a regex tower is not. Anything not in the map becomes NA so the schema gate catches it.
+
+```python
+COUNTRY = {
+    "usa": "US", "u.s.": "US", "u.s.a.": "US", "united states": "US", "us": "US",
+    "españa": "ES", "espana": "ES", "spain": "ES", "es": "ES",
+    "france": "FR", "fr": "FR",
+}
+
+def map_category(s, mapping):
+    key = s.str.strip().str.casefold().str.normalize("NFKC")
+    mapped = key.map(mapping)
+    unmapped = key.notna() & mapped.isna()
+    if unmapped.any():
+        # surface the new values so you extend the map deliberately, not silently
+        print("UNMAPPED categories:", sorted(key[unmapped].unique()))
+    return mapped     # unmapped rows are NA -> the schema's isin/nullable=False check rejects them
+
+df["country"] = map_category(df["country"], COUNTRY)
+```
+
+## Robust date parsing (explicit format, count the casualties)
+
+```python
+def parse_dates(s, fmt="%Y-%m-%d", utc=True):
+    parsed = pd.to_datetime(s, format=fmt, utc=utc, errors="coerce")
+    failed = parsed.isna() & s.notna()        # coerced to NaT but original wasn't null = parse failure
+    if failed.any():
+        raise ValueError(f"{failed.sum()} values did not match {fmt!r}: {s[failed].head().tolist()}")
+    return parsed
+```
+
+If a single source genuinely mixes formats, parse each known format and `combine_first`, then assert no NaT
+remains. Never fall back to dayfirst inference — `03/04/2026` is ambiguous and pandas picks silently.
+
+```python
+def parse_mixed(s, formats=("%Y-%m-%d", "%d/%m/%Y")):
+    out = pd.Series(pd.NaT, index=s.index, dtype="datetime64[ns, UTC]")
+    for fmt in formats:
+        try_ = pd.to_datetime(s, format=fmt, utc=True, errors="coerce")
+        out = out.fillna(try_)
+    remaining = out.isna() & s.notna()
+    assert not remaining.any(), f"{remaining.sum()} dates matched none of {formats}"
+    return out
+```
+
+## Unicode / encoding repair
+
+```python
+# 1. Read with the RIGHT encoding (don't let pandas/locale guess). If you see mojibake like "Rodríguez",
+#    the file is utf-8 misread as latin-1 — re-read as utf-8.
+df = pd.read_csv("raw.csv", encoding="utf-8")
+
+# 2. Normalize text: NFKC folds full-width/compatibility forms; strip control chars and zero-width junk.
+import re
+ZERO_WIDTH = re.compile(r"[​‌‍]")
+def clean_text(s):
+    return (s.str.normalize("NFKC")
+             .str.replace(ZERO_WIDTH, "", regex=True)
+             .str.strip())
+
+# 3. Last-resort repair of already-corrupted text (needs `pip install ftfy`):
+#    from ftfy import fix_text; df["name"] = df["name"].map(fix_text)
+```
+
+## Sentinel → NA
+
+Map the per-column sentinels to real NA *before* any numeric check, so means and ranges aren't poisoned.
+
+```python
+SENTINELS = {
+    "age":     [-1, 999],
+    "income":  [-1, 0, 999999],     # 0 is a sentinel here only if 0 income is impossible in this dataset
+    "rating":  [-1],
+}
+for col, vals in SENTINELS.items():
+    df[col] = df[col].mask(df[col].isin(vals))     # -> NA; decide drop/impute/quarantine in handle_missing
+```
+
+## Numeric range — clip vs reject
+
+```python
+# CLIP: out-of-range is plausibly a recording cap → pull to the bound (keeps the row).
+df["age"] = df["age"].clip(lower=0, upper=120)
+
+# REJECT: out-of-range is impossible → set NA so the gate quarantines the row (don't fabricate a value).
+df["age"] = df["age"].mask(~df["age"].between(0, 120))
+```
+
+Decide per column and write the choice down — clipping a fraud-amount silently hides outliers; rejecting a
+sensor reading silently loses data. The right call depends on the field's meaning.
+
+## Polars equivalents (clean-at-scale)
+
+Same profile→normalize→dedupe→validate shape, lazy and parallel. Pin Polars and stay deterministic.
+
+```python
+import polars as pl
+
+lf = pl.scan_csv("raw.csv", infer_schema_length=10_000)     # lazy: nothing reads until .collect()
+
+clean = (
+    lf
+    # strings: strip + lowercase + (NFKC via map if needed)
+    .with_columns(pl.col("country").str.strip_chars().str.to_lowercase().alias("country"))
+    # category mapping table
+    .with_columns(pl.col("country").replace_strict(COUNTRY, default=None))
+    # sentinels -> null
+    .with_columns(pl.when(pl.col("age").is_in([-1, 999])).then(None).otherwise(pl.col("age")).alias("age"))
+    # dates: explicit format, strict=False coerces failures to null (then check the count)
+    .with_columns(pl.col("signup").str.to_datetime("%Y-%m-%d", strict=False, time_zone="UTC"))
+    # missing: drop rows missing the required key
+    .drop_nulls(subset=["customer_id"])
+    # dedupe: deterministic — sort then keep first per key
+    .sort(["customer_id", "updated_at"], descending=[False, True])
+    .unique(subset=["customer_id"], keep="first")
+    .collect()                                              # execute the lazy plan
+)
+
+# fill_null deliberately (with a flag column), never a blind 0:
+clean = clean.with_columns(
+    pl.col("income").is_null().alias("income_was_missing"),
+    pl.col("income").fill_null(strategy="mean"),
+)
+```
+
+pandera validates Polars frames too (`import pandera.polars as pa`), so the gate from
+[validation-patterns.md](validation-patterns.md) carries over. Hand multi-GB analytical SQL to
+[`duckdb`](../duckdb/SKILL.md) — the engine changes, the discipline does not.