acteval 0.2.0__tar.gz → 0.3.0__tar.gz

acteval-0.3.0/.github/workflows/benchmark.yml

@@ -0,0 +1,71 @@
+name: Benchmark
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: astral-sh/setup-uv@v4
+
+      - name: Install Python 3.13
+        run: uv python install 3.13
+
+      # Baseline and candidate run on the same runner, so the comparison is not
+      # thrown off by hardware differences between GitHub-hosted machines.
+      # The baseline also runs the candidate's benchmark files, so a commit that
+      # changes the benchmark workload compares like with like. Benchmarks the
+      # baseline can't run (e.g. new API) aren't saved, so they're not compared.
+      - name: Run baseline benchmarks (previous commit)
+        env:
+          BASE_SHA: ${{ github.event.before }}
+        run: |
+          if ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then
+            BASE_SHA=$(git rev-parse HEAD~1)
+          fi
+          echo "Baseline: $BASE_SHA"
+          git worktree add "$RUNNER_TEMP/baseline" "$BASE_SHA"
+          cd "$RUNNER_TEMP/baseline"
+          cp "$GITHUB_WORKSPACE"/tests/test_bench_*.py tests/
+          uv sync
+          uv run pytest tests/test_bench_evaluate.py tests/test_bench_pairwise.py \
+            --benchmark-only \
+            --benchmark-save=baseline \
+            --continue-on-collection-errors \
+            || echo "Some baseline benchmarks failed; they are skipped in the comparison"
+          cp -r .benchmarks "$GITHUB_WORKSPACE/.benchmarks"
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Run candidate benchmarks and compare
+        run: >
+          uv run pytest tests/test_bench_evaluate.py tests/test_bench_pairwise.py
+          --benchmark-only
+          --benchmark-compare=0001
+          --benchmark-compare-fail=min:50%
+          --benchmark-json output.json
+
+      # History for the gh-pages chart only; regressions are gated by the
+      # same-runner comparison above, as cross-runner alerts are too noisy.
+      - name: Store benchmark results
+        if: ${{ !cancelled() && hashFiles('output.json') != '' }}
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: pytest
+          output-file-path: output.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          alert-threshold: "150%"
+          comment-on-alert: false
+          fail-on-alert: false
+          benchmark-data-dir-path: dev/bench

{acteval-0.2.0 → acteval-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: acteval
-Version: 0.2.0
+Version: 0.3.0
 Summary: Standalone library for evaluating activity sequences
 License-Expression: MIT
 License-File: LICENSE

{acteval-0.2.0 → acteval-0.3.0}/README.md

@@ -126,23 +126,23 @@ import pandas as pd
 from acteval import compare
 
 observed = pd.DataFrame([
-    {"pid": 0, "act": "home", "start": 0,  "end": 8},
-    {"pid": 0, "act": "work", "start": 8,  "end": 16},
-    {"pid": 0, "act": "home", "start": 16, "end": 24},
-    {"pid": 1, "act": "home", "start": 0,  "end": 12},
-    {"pid": 1, "act": "shop", "start": 12, "end": 13},
-    {"pid": 1, "act": "home", "start": 13, "end": 24},
+    {"pid": 0, "act": "home", "start": 0,    "end": 480},
+    {"pid": 0, "act": "work", "start": 480,  "end": 960},
+    {"pid": 0, "act": "home", "start": 960,  "end": 1440},
+    {"pid": 1, "act": "home", "start": 0,    "end": 720},
+    {"pid": 1, "act": "shop", "start": 720,  "end": 780},
+    {"pid": 1, "act": "home", "start": 780,  "end": 1440},
 ])
 
 synthetic = pd.DataFrame([
-    {"pid": 0, "act": "home", "start": 0,  "end": 9},
-    {"pid": 0, "act": "work", "start": 9,  "end": 17},
-    {"pid": 0, "act": "home", "start": 17, "end": 24},
-    {"pid": 1, "act": "home", "start": 0,  "end": 8},
-    {"pid": 1, "act": "work", "start": 8,  "end": 16},
-    {"pid": 1, "act": "home", "start": 16, "end": 24},
-    {"pid": 2, "act": "home", "start": 0,  "end": 8},
-    {"pid": 2, "act": "home", "start": 8, "end": 24},
+    {"pid": 0, "act": "home", "start": 0,    "end": 540},
+    {"pid": 0, "act": "work", "start": 540,  "end": 1020},
+    {"pid": 0, "act": "home", "start": 1020, "end": 1440},
+    {"pid": 1, "act": "home", "start": 0,    "end": 480},
+    {"pid": 1, "act": "work", "start": 480,  "end": 960},
+    {"pid": 1, "act": "home", "start": 960,  "end": 1440},
+    {"pid": 2, "act": "home", "start": 0,    "end": 480},
+    {"pid": 2, "act": "home", "start": 480,  "end": 1440},
 ])
 
 result = compare(observed, synthetic)
@@ -151,9 +151,9 @@ print(result.summary())
 # domain
 # creativity      0.166667
 # feasibility     0.333333
-# participations  0.162037
-# timing          0.082728
-# transitions     0.380952
+# participations  0.333333
+# timing          0.261429
+# transitions     0.416667
 ```
 
 The `compare()` function also supports **batch** comparisons using `{name: DataFrame}` dict to compare several samples:
@@ -215,11 +215,13 @@ A DataFrame with one row per activity:
 |--------|------|-------------|
 | `pid` | int/str | Person identifier |
 | `act` | str | Activity label (e.g. `"home"`, `"work"`, `"shop"`) |
-| `start` | numeric | Start time (any consistent unit, e.g. hours) |
-| `end` | numeric | End time |
-| `duration` | numeric | Duration (`end - start`) |
+| `start` | numeric | Start time in minutes since midnight (e.g. `480` = 08:00) |
+| `end` | numeric | End time in minutes |
+| `duration` | numeric | Duration in minutes (`end - start`) |
 
-Any two of `start`, `end`, and `duration` are sufficient — the third is derived automatically. A polars DataFrame in the same shape works anywhere a pandas one does; it's converted internally.
+Any two of `start`, `end`, and `duration` are sufficient — the third is derived automatically.
+
+> **Times must be in minutes.** Timing features bin values into 15-minute buckets and normalise by a 1440-minute day, and `time_consistency` checks that schedules end at 1440. Other units (e.g. hours) won't raise an error but give distorted timing scores, so convert first (e.g. `df[["start", "end", "duration"]] *= 60`). A polars DataFrame in the same shape works anywhere a pandas one does; it's converted internally.
 
 ### Attributes
 
@@ -276,6 +278,30 @@ This makes the `by_attribute`/`by_category` splits available at every level via
 > categorical/string beforehand (e.g. your own age bands) to control the
 > buckets yourself and suppress the warning.
 
+**Interaction slices (`split_order`).** By default each `split_on` attribute is
+sliced on its own (`gender=M`, `income=high`). Pass `split_order=n` to slice on
+every combination of `n` attributes instead, e.g. `split_order=2` scores every
+`gender & income` cell (labelled `label="gender&income"`, `cat="M&high"`):
+
+```python
+result = compare(
+    observed, {"my_model": synthetic},
+    target_attributes=target_attrs,
+    synthetic_attributes={"my_model": synthetic_attributes},
+    split_on=["gender", "income", "car"],
+    split_order=2,
+    min_split_size=30,  # skip cells with < 30 target persons (default above order 1)
+)
+result.at("domains", "by_category")           # one row per gender&income cell, etc.
+result.at("domains", "by_category", order=1)  # gender=M pooled over every gender=M & * cell
+```
+
+Only order `n` is computed; lower-order views (`order=k`) pool the order-`n`
+cells that contain each `k`-attribute assignment. Note that `combined` pools
+different slices at different orders, so its values change with `split_order`.
+Both settings can also be set in `config.toml` under `[splits]`. Slice counts
+grow combinatorially, so keep `n` small (2 is usually plenty).
+
 
 #### `Evaluator`
 
@@ -450,7 +476,7 @@ We generally therefore do not aggregate them further into a "meta score". Howeve
 
 Save all (low-level features, via groups to top-level domains) to CSV with `result.save("output_dir/")`.
 
-#### `EvalResult.at(level, split)`
+#### `EvalResult.at(level, split, order=None)`
 
 For anything more detailed than the summary table — a specific aggregation level, or a specific split — `result.at(...)`:
 
@@ -463,12 +489,13 @@ result.at("domains", "by_category")      # domains × by_category   (requires sp
 
 - `level` is one of `"features"`, `"groups"`, `"domains"`
 - `split` is one of `"combined"`, `"by_attribute"`, `"by_category"` 
+- `order` (split views only) pools results down to `k`-attribute slices, for `1 <= k <= split_order`; it defaults to `split_order`
 
 Each call returns an `AggregatedResult` with `.distances` and `.descriptions` DataFrames.
 
 #### `EvalResult.raw`
 
-`EvalResult.raw` exposes the pre-aggregation data (one `ResultFrame` each for descriptions and distances) that every level above is aggregated from — `distances` covers models only (there's no such thing as the target's distance to itself); pair it with `result.target_distance_weights` if you're building custom distance aggregations of your own.
+`EvalResult.raw` exposes the pre-aggregation data (one `ResultFrame` each for descriptions and distances) that every level above is aggregated from — `distances` covers models only (there's no such thing as the target's distance to itself); pair it with `result.target_distance_weights` if you're building custom distance aggregations of your own. Model distance weights are rescaled to the target's size within each slice, and each segment is weighted by the larger of the model and target weights when aggregating, so a segment only one side has keeps its full weight.
 
 > **Note on timing features:** A distance of `1.0` for a timing feature means the activity is *entirely absent* from the synthetic population — not just timed differently. This is treated as a maximum-penalty missing feature rather than a distributional difference.
 
@@ -513,11 +540,17 @@ acteval compare observed.csv --target-attrs target_attrs.csv \
   -m model_a synthetic_a.csv synth_attrs_a.csv \
   -m model_b synthetic_b.csv synth_attrs_b.csv \
   -s gender age
+
+# Score every gender & age combination (also prints the pooled single-attribute view)
+acteval compare observed.csv --target-attrs target_attrs.csv \
+  -m model_a synthetic_a.csv synth_attrs_a.csv \
+  -s gender age income --split-order 2 --min-split-size 30
 ```
 
 **Attribute rules:**
 - Attributes must be provided for **all** inputs (target + every model) or **none**. Partial specification raises an error.
 - `--split-on` (`-s`) and `--target-attrs` must be specified together.
+- `--split-order` and `--min-split-size` require `--split-on`. With `-o` and order > 1, extra `*_by_attribute_order1`/`*_by_category_order1` directories hold the pooled single-attribute views.
 - In batch mode, if any model subdirectory contains an attributes file, all subdirectories must contain one.
 
 For the lazy - there is also an auto-discovery mode for batch experiments, but use with care:

{acteval-0.2.0 → acteval-0.3.0}/llms.txt

@@ -11,7 +11,7 @@ pip install acteval
 pandas (or polars) DataFrame, one row per activity episode:
 - pid — person identifier (int or str)
 - act — activity label (str, e.g. "home", "work", "shop")
-- start — episode start time (numeric, any consistent unit)
+- start — episode start time (numeric, minutes since midnight; other units distort timing scores)
 - end — episode end time (numeric)
 - duration — episode duration (numeric)
 
@@ -23,6 +23,9 @@ Any two of start/end/duration are enough — the third is derived.
   Compare one or more synthetic DataFrames to observed.
   synthetic can be a single DataFrame (result column "synthetic") or {"name": df, ...} dict.
   kwargs: attributes, target_attributes, split_on (attribute-based splitting),
+  split_order (combine n split_on attributes per slice, e.g. 2 → "gender&income"
+  labels with "m&high" cats; default 1 or config [splits] order), min_split_size
+  (skip slices with fewer target persons; default none at order 1, 30 above),
   config_path (custom config.toml), disable (dotted config.toml keys to switch
   off, applied on top of config_path), progress (tqdm bars), verbose.
 
@@ -69,9 +72,14 @@ Three aggregation levels, each further split three ways:
 Dispatcher: result.at(level="domains", split="combined") reaches any of the 9
 combinations; result.at() == result.at("domains", "combined") == result.summary()
 as a distances-only shortcut.
+result.at(level, split, order=k) (split views only, 1 <= k <= result.split_order)
+pools the computed order-n slices down to k-attribute slices; e.g. at n=2,
+order=1 by_category "gender=m" pools every gender=m & * cell.
+result.slice_assignments maps (label, cat) → ((attr, value), ...).
 
 Convenience: result.summary(), result.rank_models(), result.best_model,
-result.save(path) (writes every level/split to CSV).
+result.save(path) (writes every level/split to CSV; plus *_order1 split
+views when split_order > 1).
 
 Advanced: result.raw → {"descriptions": ResultFrame, "distances": ResultFrame}
 (pre-aggregation values/weights/units, before any tier collapse); pair with
@@ -94,7 +102,7 @@ list_disable_keys() enumerates every dotted disable=[...] path (e.g. "jobs.creat
 ## CLI
 
 acteval compare TARGET -m NAME SCHEDULE [ATTRS] [-m ...] [--target-attrs PATH
---split-on COL] [-c CONFIG] [--disable KEY] [-o DIR] [-l features|groups|domains] [--batch DIR]
+--split-on COL [--split-order N] [--min-split-size N]] [-c CONFIG] [--disable KEY] [-o DIR] [-l features|groups|domains] [--batch DIR]
 
 acteval filter non-home-based|consecutive SCHEDULE [-o OUT] — spot-check a
 synthetic population for structural issues before running compare.

{acteval-0.2.0 → acteval-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "acteval"
-version = "0.2.0"
+version = "0.3.0"
 description = "Standalone library for evaluating activity sequences"
 requires-python = ">=3.10"
 license = "MIT"
@@ -18,8 +18,8 @@ dependencies = [
     "tqdm",
 ]
 
-[tool.uv]
-dev-dependencies = [
+[dependency-groups]
+dev = [
     "pytest",
     "pytest-cov",
     "pytest-benchmark",

{acteval-0.2.0 → acteval-0.3.0}/src/acteval/_aggregation.py

@@ -126,16 +126,19 @@ def distances_to_group_level(
 
     Args:
         distances: ``ResultFrame`` at the segment level.
-        target_weights: Raw per-row target weight, for blending (see
-            ``ResultFrame.aggregate_distances``).
+        target_weights: Raw per-row target weight, in the same row order as
+            ``distances`` (see ``ResultFrame.aggregate_distances``).
         extra: Additional index levels to preserve (e.g. ``["label"]``).
         drop: Index prefix tuples to exclude before aggregating.
               Defaults to ``DEFAULT_REMOVE_FEATURES``.  Pass ``None`` or ``[]``
               to skip filtering.
     """
     grouper = ["domain", "feature"] + extra
-    rf = distances.drop_rows(drop) if drop else distances
-    return rf.aggregate_distances(grouper, target_weights=target_weights)
+    if drop:
+        # Drop the same rows from target_weights: they're combined by position.
+        target_weights = target_weights.iloc[distances.kept_positions(drop)]
+        distances = distances.drop_rows(drop)
+    return distances.aggregate_distances(grouper, target_weights=target_weights)
 
 
 def descriptions_to_domain_level(

{acteval-0.2.0 → acteval-0.3.0}/src/acteval/_jobs.py

@@ -62,11 +62,18 @@ class StructuralConfig:
         return self.home_based_novel or self.consecutive_novel
 
 
+@dataclass(frozen=True)
+class SplitConfig:
+    order: int = 1  # number of split_on attributes combined per slice
+    min_size: int | None = None  # min target persons per slice; None = auto
+
+
 @dataclass(frozen=True)
 class EvalConfig:
     density: list[JobSpec]
     creativity: CreativityConfig
     structural: StructuralConfig
+    splits: SplitConfig = SplitConfig()
 
 
 def load_config(path=None) -> dict:
@@ -361,6 +368,11 @@ def build_structural_config(cfg: dict) -> StructuralConfig:
     )
 
 
+def build_split_config(cfg: dict) -> SplitConfig:
+    s = cfg.get("splits", {})
+    return SplitConfig(order=s.get("order", 1), min_size=s.get("min_size"))
+
+
 def get_jobs(config_path=None, disable: list[str] | None = None) -> EvalConfig:
     """Load config and return active job specification.
 
@@ -377,4 +389,5 @@ def get_jobs(config_path=None, disable: list[str] | None = None) -> EvalConfig:
         density=build_density_jobs(cfg),
         creativity=build_creativity_config(cfg),
         structural=build_structural_config(cfg),
+        splits=build_split_config(cfg),
     )

{acteval-0.2.0 → acteval-0.3.0}/src/acteval/_pipeline.py

@@ -25,7 +25,7 @@ same MultiIndex ``(domain, feature, segment)`` and column set, plus a
 ``units`` Series. ``descriptions`` has one column per model plus a ``"target"``
 column; ``distances`` has one column per model only (there's no such thing as
 the target's distance to itself) — the target's distance-side weight, used
-only to blend into each model's own weight before aggregating, is tracked
+only to combine with each model's own weight (their max) before aggregating, is tracked
 separately (see ``Evaluator._target_distance_weights`` / ``EvalResult``).
 
 ## ``missing_distance``
@@ -36,7 +36,6 @@ Each ``JobSpec`` carries a ``missing_distance`` value:
 - ``None`` (participation, transitions): EMD is computed on whatever data exists.
 """
 
-from concurrent.futures import ThreadPoolExecutor
 from typing import Callable
 
 import numpy as np
@@ -77,9 +76,6 @@ def _aggregate_features(
     return feat_desc, feat_dist
 
 
-_PARALLEL_THRESHOLD = 50
-
-
 # ---------------------------------------------------------------------------
 # Low-level helpers: density features
 # ---------------------------------------------------------------------------
@@ -349,12 +345,9 @@ def _score_features(
             return missing_distance
         return distance(_get_or_default(a, k, default), _get_or_default(b, k, default))
 
-    if len(index) > _PARALLEL_THRESHOLD:
-        with ThreadPoolExecutor() as executor:
-            values = list(executor.map(_compute, index))
-        metrics = Series(dict(zip(index, values)), name=model)
-    else:
-        metrics = Series({k: _compute(k) for k in index}, name=model)
+    # Serial on purpose: individual EMDs are small and GIL-bound, so a thread
+    # pool costs more in scheduling overhead than it saves.
+    metrics = Series({k: _compute(k) for k in index}, name=model)
     metrics = metrics.fillna(0)
     return metrics
 

{acteval-0.2.0 → acteval-0.3.0}/src/acteval/_result_frame.py

@@ -18,6 +18,7 @@ from __future__ import annotations
 from dataclasses import dataclass
 from typing import Sequence
 
+import numpy as np
 from pandas import DataFrame, Series
 
 
@@ -50,16 +51,22 @@ class ResultFrame:
         """
         if not keys:
             return self
-        n = len(keys[0])
-        key_set = set(keys)
-        mask = [idx[:n] not in key_set for idx in self.values.index]
-        iloc = [i for i, m in enumerate(mask) if m]
+        iloc = self.kept_positions(keys)
         return ResultFrame(
             values=self.values.iloc[iloc],
             weights=self.weights.iloc[iloc],
             units=self.units.iloc[iloc] if self.units is not None else None,
         )
 
+    def kept_positions(self, keys: Sequence[tuple]) -> list[int]:
+        """Row positions ``drop_rows(keys)`` keeps, for dropping the same rows
+        from a Series aligned to this frame (e.g. target weights)."""
+        if not keys:
+            return list(range(len(self.values)))
+        n = len(keys[0])
+        key_set = set(keys)
+        return [i for i, idx in enumerate(self.values.index) if idx[:n] not in key_set]
+
     # ------------------------------------------------------------------
     # Aggregation
     # ------------------------------------------------------------------
@@ -82,23 +89,37 @@ class ResultFrame:
     def aggregate_distances(
         self, groupby: list[str], target_weights: Series
     ) -> "ResultFrame":
-        """Asymmetric weighted average aggregation (for distances).
+        """Weighted average aggregation for distances, using both sides' weights.
 
-        Averages each model's weight with the target's weight before computing
-        the weighted mean. This handles asymmetric feature coverage: a feature
-        present in only one side gets half-weight rather than zero-weight.
+        Each row is weighted by the larger of the model's and the target's
+        weight, so a segment present on only one side (e.g. an activity the
+        model never produces, or one it invents) keeps that side's full weight.
+        Model weights are already rescaled to the target's slice size (see
+        ``Evaluator.compare_population``), so neither side dominates because
+        of its sample size.
 
-            ``combined_w = (weights[c] + target_weights) / 2``
+            ``combined_w = max(weights[c], target_weights)``
             ``agg[c] = sum(values[c] * combined_w) / sum(combined_w)``
 
         Args:
-            target_weights: Raw per-row target weight, aligned to this
-                ``ResultFrame``'s index.
+            target_weights: Raw per-row target weight, in the same row order
+                as this ``ResultFrame``. Combined by position, not by label:
+                lower-order views repeat index labels, which label alignment
+                would cross-multiply.
         """
+        if len(target_weights) != len(self.values):
+            raise ValueError(
+                f"target_weights has {len(target_weights)} rows, expected "
+                f"{len(self.values)} (one per row, in the same order)"
+            )
+        target = target_weights.to_numpy(dtype=float)
         agg_values: dict[str, Series] = {}
         agg_weights: dict[str, Series] = {}
         for col in self.values.columns:
-            combined_w = (self.weights[col] + target_weights) / 2
+            combined_w = Series(
+                np.maximum(self.weights[col].to_numpy(dtype=float), target),
+                index=self.values.index,
+            )
             total = combined_w.groupby(groupby).sum()
             weighted = (self.values[col].mul(combined_w)).groupby(groupby).sum()
             agg_values[col] = (weighted / total).fillna(0.0)

acteval-0.3.0/src/acteval/_splits.py

@@ -0,0 +1,133 @@
+import itertools
+import warnings
+from dataclasses import dataclass
+
+import numpy as np
+from pandas import DataFrame
+
+# Default minimum target persons per slice for order >= 2 (order 1 keeps every
+# observed category, matching pre-``split_order`` behaviour).
+DEFAULT_HIGHER_ORDER_MIN_SIZE = 30
+
+# Warn when a run would evaluate more slices than this.
+_MANY_SLICES = 1000
+
+LABEL_SEP = "&"
+
+
+@dataclass(frozen=True)
+class Slice:
+    """One population slice: a joint assignment of ``order`` attributes.
+
+    ``attrs`` follow ``split_on`` order; ``values`` are aligned with ``attrs``.
+    """
+
+    attrs: tuple[str, ...]
+    values: tuple
+
+    @property
+    def order(self) -> int:
+        return len(self.attrs)
+
+    @property
+    def label(self) -> str:
+        """Index ``label`` string, e.g. ``"gender"`` or ``"gender&income"``."""
+        return LABEL_SEP.join(self.attrs)
+
+    @property
+    def cat(self):
+        """Index ``cat`` value: the raw value at order 1 (unchanged from
+        pre-``split_order`` output), else the joined string, e.g. ``"male&high"``."""
+        if self.order == 1:
+            return self.values[0]
+        return LABEL_SEP.join(str(v) for v in self.values)
+
+    @property
+    def key(self) -> tuple[str, object]:
+        return (self.label, self.cat)
+
+    @property
+    def assignment(self) -> tuple[tuple[str, object], ...]:
+        """``((attr, value), ...)`` — what containment views expand from."""
+        return tuple(zip(self.attrs, self.values))
+
+
+def resolve_min_size(order: int, min_size: int | None) -> int:
+    """``None`` means auto: no minimum at order 1, else
+    ``DEFAULT_HIGHER_ORDER_MIN_SIZE``."""
+    if min_size is not None:
+        return min_size
+    return 0 if order == 1 else DEFAULT_HIGHER_ORDER_MIN_SIZE
+
+
+def group_pids(
+    attributes: DataFrame, attrs: tuple[str, ...]
+) -> dict[tuple, np.ndarray]:
+    """``{values_tuple: original_pids}`` for every observed combination of
+    ``attrs``, in order of first appearance."""
+    grouped = attributes.groupby(list(attrs), sort=False, dropna=True).pid
+    return {(k if isinstance(k, tuple) else (k,)): pids.values for k, pids in grouped}
+
+
+def build_slices(
+    attributes: DataFrame,
+    split_on: list[str],
+    order: int,
+    min_size: int | None = None,
+) -> list[tuple[Slice, np.ndarray]]:
+    """Build every order-``order`` slice observed in *attributes*.
+
+    Args:
+        attributes: Target attributes with a ``pid`` column and every
+            ``split_on`` column.
+        split_on: Attribute columns to combine.
+        order: Number of attributes per slice (``1 <= order <= len(split_on)``).
+        min_size: Minimum target persons per slice; ``None`` for the default
+            (see ``resolve_min_size``).
+
+    Returns:
+        ``[(slice, target_original_pids), ...]``, grouped by attribute
+        combination in ``itertools.combinations(split_on, order)`` order.
+
+    Raises:
+        ValueError: If ``order`` is out of range, or every slice is smaller
+            than ``min_size``.
+    """
+    if not 1 <= order <= len(split_on):
+        raise ValueError(
+            f"split_order must be between 1 and len(split_on)={len(split_on)}, "
+            f"got {order}"
+        )
+    min_size = resolve_min_size(order, min_size)
+    slices: list[tuple[Slice, np.ndarray]] = []
+    dropped: list[str] = []
+    for attrs in itertools.combinations(split_on, order):
+        for values, pids in group_pids(attributes, attrs).items():
+            s = Slice(attrs=attrs, values=values)
+            if len(np.unique(pids)) < min_size:
+                dropped.append(f"{s.label}={s.cat}")
+                continue
+            slices.append((s, pids))
+    if dropped:
+        if not slices:
+            raise ValueError(
+                f"every order-{order} slice has fewer than min_split_size={min_size} "
+                "target persons; lower split_order or min_split_size"
+            )
+        preview = ", ".join(dropped[:5]) + (", ..." if len(dropped) > 5 else "")
+        warnings.warn(
+            f"skipped {len(dropped)} of {len(dropped) + len(slices)} order-{order} "
+            f"slices with fewer than min_split_size={min_size} target persons: "
+            f"{preview}",
+            UserWarning,
+            stacklevel=3,
+        )
+    if len(slices) > _MANY_SLICES:
+        warnings.warn(
+            f"split_order={order} produces {len(slices)} slices; expect roughly "
+            "0.1-0.25s per slice for the target and again per model. Consider a "
+            "lower split_order, fewer split_on columns, or a larger min_split_size.",
+            UserWarning,
+            stacklevel=3,
+        )
+    return slices

{acteval-0.2.0 → acteval-0.3.0}/src/acteval/cli.py

@@ -264,6 +264,22 @@ def _build_parser() -> argparse.ArgumentParser:
         action="extend",
         help="Attribute columns to split evaluation on (requires --target-attrs and per-model attrs)",
     )
+    compare.add_argument(
+        "--split-order",
+        type=int,
+        metavar="N",
+        default=None,
+        help="Combine N split-on attributes per slice, e.g. 2 for gender & income "
+        "(requires --split-on; default from config, 1)",
+    )
+    compare.add_argument(
+        "--min-split-size",
+        type=int,
+        metavar="N",
+        default=None,
+        help="Skip slices with fewer than N target persons "
+        "(default from config; none at order 1, 30 above)",
+    )
     compare.add_argument(
         "--config", "-c", metavar="PATH", help="Path to custom config.toml"
     )
@@ -345,6 +361,10 @@ def _run(args: argparse.Namespace) -> None:
     # --- split-on requires attrs (and vice versa) ---
     if bool(args.split_on) and not bool(args.target_attrs):
         sys.exit("--split-on requires --target-attrs to be specified")
+    if (
+        args.split_order is not None or args.min_split_size is not None
+    ) and not args.split_on:
+        sys.exit("--split-order and --min-split-size require --split-on")
 
     # --- load target attributes ---
     target_attributes: pd.DataFrame | None = None
@@ -418,6 +438,8 @@ def _run(args: argparse.Namespace) -> None:
         config_path=args.config,
         progress=not args.no_progress,
         disable=args.disable,
+        split_order=args.split_order,
+        min_split_size=args.min_split_size,
     )
     # Attrs are only meaningful for splitting; don't pass them when split_on is absent
     attrs_for_evaluator = attributes if args.split_on else None
@@ -450,6 +472,12 @@ def _run(args: argparse.Namespace) -> None:
         if result.has_splits:
             print("\nDomain distances by attribute:")
             print_markdown(result.domains.by_attribute.distances)
+            if result.split_order > 1:
+                print(
+                    f"\nDomain distances by single attribute "
+                    f"(pooled over order-{result.split_order} slices):"
+                )
+                print_markdown(result.at("domains", "by_attribute", order=1).distances)
         print("\nDomain distances (lower is better):")
         print_markdown(result.domains.combined.distances)