acteval 0.1.0__py3-none-any.whl

acteval/density/features/transitions.py

@@ -0,0 +1,241 @@
+import numpy as np
+from numpy import ndarray
+from pandas import DataFrame, MultiIndex, Series
+
+from acteval.density.features.pid_features import PidFeatures
+from acteval.density.features.utils import weighted_features
+from acteval.population import Population
+
+
+def _build_ngrams(
+    population: Population, n: int, min_count: int = 0
+) -> dict[str, tuple[ndarray, ndarray]]:
+    """Build n-gram transition features from a Population.
+
+    Uses integer encoding to avoid string operations in the inner loop.
+    Activity codes are packed into a single integer per n-gram, then
+    converted back to string labels at the end.
+    """
+    codes = population.act_codes
+    pids = population.pids
+    base = population.n_act_types
+
+    # Build integer-encoded n-grams using vectorised arithmetic
+    # Each n-gram is encoded as code[0]*base^(n-1) + code[1]*base^(n-2) + ... + code[n-1]
+    powers = base ** np.arange(n - 1, -1, -1)
+    ngram_codes = np.zeros(len(codes), dtype=np.int64)
+    for i in range(n):
+        shifted = np.roll(codes, -i)
+        ngram_codes += shifted * powers[i]
+
+    # Identify valid positions (not crossing pid boundaries)
+    valid_mask = np.ones(len(codes), dtype=bool)
+    # Exclude trailing positions within last pid
+    if n > 1:
+        valid_mask[-(n - 1) :] = False
+    # Mark positions near pid boundaries
+    pid_boundaries = population.pid_boundaries
+    if len(pid_boundaries) > 0:
+        for offset in range(-(n - 1), 0):
+            positions = pid_boundaries + offset
+            positions = positions[(positions >= 0) & (positions < len(codes))]
+            valid_mask[positions] = False
+
+    valid_ngrams = ngram_codes[valid_mask]
+    valid_pids = pids[valid_mask]
+
+    # Early return if no valid n-grams
+    if len(valid_ngrams) == 0:
+        return {}
+
+    # Count n-grams per pid using numpy
+    unique_ngrams, ngram_indices = np.unique(valid_ngrams, return_inverse=True)
+    unique_pids, pid_indices = np.unique(valid_pids, return_inverse=True)
+
+    # Build count matrix (pids x ngrams)
+    count_matrix = np.zeros((len(unique_pids), len(unique_ngrams)), dtype=int)
+    np.add.at(count_matrix, (pid_indices, ngram_indices), 1)
+
+    # Filter rare n-grams
+    if min_count > 0:
+        col_totals = count_matrix.sum(axis=0)
+        keep = col_totals >= min_count
+        count_matrix = count_matrix[:, keep]
+        unique_ngrams = unique_ngrams[keep]
+
+    # Decode integer n-grams back to string labels
+    int_to_act = population.int_to_act
+
+    def _decode_ngram(code):
+        labels = []
+        for i in range(n):
+            labels.append(int_to_act[code // powers[i]])
+            code %= powers[i]
+        return ">".join(str(label) for label in labels)
+
+    # Build result dict with pid counts as lists (matching original format)
+    # todo: maybe don't need these by pid?
+    result = {}
+    for j, ng_code in enumerate(unique_ngrams):
+        label = _decode_ngram(ng_code)
+        result[label] = count_matrix[:, j].tolist()
+
+    return weighted_features(result)
+
+
+def transitions_by_act(
+    population: Population, min_count: int = 0
+) -> dict[str, tuple[ndarray, ndarray]]:
+    return _build_ngrams(population, 2, min_count=min_count)
+
+
+def transition_3s_by_act(
+    population: Population, min_count: int = 0
+) -> dict[str, tuple[ndarray, ndarray]]:
+    return _build_ngrams(population, 3, min_count=min_count)
+
+
+def transition_4s_by_act(
+    population: Population, min_count: int = 0
+) -> dict[str, tuple[ndarray, ndarray]]:
+    return _build_ngrams(population, 4, min_count=min_count)
+
+
+def _build_ngrams_per_pid(
+    population: Population, n: int, min_count: int = 0
+) -> PidFeatures:
+    """Build per-pid n-gram transition features returning PidFeatures.
+
+    Uses the same integer-encoding logic as ``_build_ngrams`` but keeps
+    per-person counts with pid tracking instead of compressing immediately.
+    ``min_count`` filtering is applied on the full population to keep a
+    consistent feature space across subsets.
+    """
+    codes = population.act_codes
+    pids = population.pids
+    base = population.n_act_types
+
+    powers = base ** np.arange(n - 1, -1, -1)
+    ngram_codes = np.zeros(len(codes), dtype=np.int64)
+    for i in range(n):
+        shifted = np.roll(codes, -i)
+        ngram_codes += shifted * powers[i]
+
+    valid_mask = np.ones(len(codes), dtype=bool)
+    if n > 1:
+        valid_mask[-(n - 1) :] = False
+    pid_boundaries = population.pid_boundaries
+    if len(pid_boundaries) > 0:
+        for offset in range(-(n - 1), 0):
+            positions = pid_boundaries + offset
+            positions = positions[(positions >= 0) & (positions < len(codes))]
+            valid_mask[positions] = False
+
+    valid_ngrams = ngram_codes[valid_mask]
+    valid_pids = pids[valid_mask]
+
+    if len(valid_ngrams) == 0:
+        return PidFeatures(data={}, bin_size=None, factor=1)
+
+    unique_ngrams, ngram_indices = np.unique(valid_ngrams, return_inverse=True)
+    unique_pids, pid_indices = np.unique(valid_pids, return_inverse=True)
+
+    count_matrix = np.zeros((len(unique_pids), len(unique_ngrams)), dtype=int)
+    np.add.at(count_matrix, (pid_indices, ngram_indices), 1)
+
+    if min_count > 0:
+        col_totals = count_matrix.sum(axis=0)
+        keep = col_totals >= min_count
+        count_matrix = count_matrix[:, keep]
+        unique_ngrams = unique_ngrams[keep]
+
+    int_to_act = population.int_to_act
+
+    def _decode_ngram(code):
+        labels = []
+        for i in range(n):
+            labels.append(int_to_act[code // powers[i]])
+            code %= powers[i]
+        return ">".join(str(label) for label in labels)
+
+    # Build PidFeatures: each key maps to (per-person counts, pids)
+    # All keys share the same pid array for efficient mask caching.
+    result_data: dict[str, tuple[ndarray, ndarray]] = {}
+    for j, ng_code in enumerate(unique_ngrams):
+        label = _decode_ngram(ng_code)
+        result_data[label] = (count_matrix[:, j], unique_pids)
+
+    return PidFeatures(data=result_data, bin_size=None, factor=1)
+
+
+def transitions_by_act_per_pid(
+    population: Population, min_count: int = 0
+) -> PidFeatures:
+    return _build_ngrams_per_pid(population, 2, min_count=min_count)
+
+
+def transition_3s_by_act_per_pid(
+    population: Population, min_count: int = 0
+) -> PidFeatures:
+    return _build_ngrams_per_pid(population, 3, min_count=min_count)
+
+
+def transition_4s_by_act_per_pid(
+    population: Population, min_count: int = 0
+) -> PidFeatures:
+    return _build_ngrams_per_pid(population, 4, min_count=min_count)
+
+
+def tour(acts: Series) -> str:
+    """
+    Extracts the tour from the given Series of activities.
+
+    Args:
+        acts (Series): A Series containing the activities.
+
+    Returns:
+        str: A string representation of the tour.
+    """
+    return ">".join(acts.str[0])
+
+
+def full_sequences(population: DataFrame) -> dict[str, tuple[ndarray, ndarray]]:
+    transitions = population.reset_index()
+    transitions = transitions.set_index(["index", "pid"])
+    transitions.act = transitions.act.astype(str)
+    transitions = transitions.groupby("pid").act.apply(tour)
+    transitions = (
+        transitions.groupby("pid")
+        .value_counts()
+        .unstack()
+        .fillna(0)
+        .astype(int)
+        .to_dict(orient="list")
+    )
+    return weighted_features(transitions)
+
+
+def collect_sequence(acts: Series) -> str:
+    return ">".join(acts)
+
+
+def sequence_probs(population: DataFrame) -> DataFrame:
+    """
+    Calculates the sequence probabilities in the given population DataFrame.
+
+    Args:
+        population (DataFrame): A DataFrame containing the population data.
+
+    Returns:
+        DataFrame: A DataFrame containing the probability of each sequence.
+    """
+    metrics = (
+        population.groupby("pid")
+        .act.apply(collect_sequence)
+        .value_counts(normalize=True)
+    )
+    metrics = metrics.sort_values(ascending=False)
+    metrics.index = MultiIndex.from_tuples(
+        [("sequence rate", acts) for acts in metrics.index]
+    )
+    return metrics

acteval/density/features/utils.py

@@ -0,0 +1,197 @@
+from typing import Optional, Union
+
+import numpy as np
+from numpy import array, ndarray, unique
+
+
+def _collect_by_group(keys, values):
+    """Group values by keys into a dict of arrays.
+
+    Args:
+        keys: 1D array of group keys.
+        values: 1D or 2D array of values to group.
+
+    Returns:
+        dict mapping each unique key to the corresponding array of values.
+    """
+    order = np.argsort(keys, kind="stable")
+    sorted_keys = keys[order]
+    sorted_vals = values[order]
+    unique_keys, start_idx = np.unique(sorted_keys, return_index=True)
+    splits = np.split(sorted_vals, start_idx[1:])
+    return {k: v for k, v in zip(unique_keys, splits)}
+
+
+def _collect_by_group_with_pids(keys, values, pids):
+    """Group values by keys, tracking which pid each value belongs to.
+
+    Args:
+        keys: 1D array of group keys.
+        values: 1D or 2D array of values to group.
+        pids: 1D array of pid ints (same length as keys/values).
+
+    Returns:
+        dict mapping each unique key to ``(grouped_values, grouped_pids)``.
+    """
+    order = np.argsort(keys, kind="stable")
+    sorted_keys = keys[order]
+    sorted_vals = values[order]
+    sorted_pids = pids[order]
+    unique_keys, start_idx = np.unique(sorted_keys, return_index=True)
+    val_splits = np.split(sorted_vals, start_idx[1:])
+    pid_splits = np.split(sorted_pids, start_idx[1:])
+    return {k: (v, p) for k, v, p in zip(unique_keys, val_splits, pid_splits)}
+
+
+def _count_matrix(row_keys, col_keys):
+    """Build a count matrix (crosstab) from row and column key arrays.
+
+    Args:
+        row_keys: 1D array of row group keys.
+        col_keys: 1D array of column group keys.
+
+    Returns:
+        tuple of (matrix, unique_rows, unique_cols).
+    """
+    unique_rows, row_codes = np.unique(row_keys, return_inverse=True)
+    unique_cols, col_codes = np.unique(col_keys, return_inverse=True)
+    matrix = np.zeros((len(unique_rows), len(unique_cols)), dtype=np.int64)
+    np.add.at(matrix, (row_codes, col_codes), 1)
+    return matrix, unique_rows, unique_cols
+
+
+def _cumcount(group_keys):
+    """Compute cumulative count within each group (like groupby().cumcount()).
+
+    Args:
+        group_keys: 1D array of group keys.
+
+    Returns:
+        1D int array with the within-group ordinal for each element.
+    """
+    n = len(group_keys)
+    order = np.argsort(group_keys, kind="stable")
+    sorted_keys = group_keys[order]
+    # Find group start positions in the sorted array
+    change_positions = np.concatenate(
+        [[0], np.where(sorted_keys[1:] != sorted_keys[:-1])[0] + 1]
+    )
+    counts = np.diff(np.concatenate([change_positions, [n]]))
+    group_starts = np.repeat(change_positions, counts)
+    sorted_cumcounts = np.arange(n) - group_starts
+    # Map back to original order
+    result = np.empty(n, dtype=np.int64)
+    result[order] = sorted_cumcounts
+    return result
+
+
+def _grouped_sum(keys, values):
+    """Sum values per group key.
+
+    Args:
+        keys: 1D array of group keys.
+        values: 1D numeric array.
+
+    Returns:
+        tuple of (unique_keys, sums).
+    """
+    order = np.argsort(keys, kind="stable")
+    sorted_keys = keys[order]
+    sorted_vals = values[order]
+    unique_keys, start_idx = np.unique(sorted_keys, return_index=True)
+    sums = np.add.reduceat(sorted_vals, start_idx)
+    return unique_keys, sums
+
+
+def _first_last_per_group(keys):
+    """Find the original indices of the first and last element per group.
+
+    Args:
+        keys: 1D array of group keys.
+
+    Returns:
+        tuple of (unique_keys, first_indices, last_indices) where indices
+        refer to positions in the original array.
+    """
+    order = np.argsort(keys, kind="stable")
+    sorted_keys = keys[order]
+    unique_keys, start_idx = np.unique(sorted_keys, return_index=True)
+    end_idx = np.empty_like(start_idx)
+    end_idx[:-1] = start_idx[1:] - 1
+    end_idx[-1] = len(keys) - 1
+    return unique_keys, order[start_idx], order[end_idx]
+
+
+def equals(
+    a: dict[str, tuple[ndarray, ndarray]], b: dict[str, tuple[ndarray, ndarray]]
+) -> bool:
+    if set(a.keys()) != set(b.keys()):
+        return False
+    for k in a.keys():
+        if not len(a[k][0]) == len(b[k][0]):
+            return False
+        if not len(a[k][1]) == len(b[k][1]):
+            return False
+        if not (a[k][0] == b[k][0]).all():
+            return False
+        if not (a[k][1] == b[k][1]).all():
+            return False
+    return True
+
+
+def bin_values(values: array, bin_size: Union[int, float]) -> ndarray:
+    """
+    Bins the input values based on the given bin size.
+
+    Args:
+        values (array): Input values to be binned.
+        bin_size (int, float): Size of each bin.
+
+    Returns:
+        array: Binned values.
+    """
+    return (values // bin_size * bin_size) + (bin_size / 2)
+
+
+def compress_feature(
+    feature: list, bin_size: Optional[int] = None, factor: int = 1440
+) -> tuple[ndarray, ndarray]:
+    """
+    Compresses a feature by optionally binning its values and returning unique values with counts.
+
+    Args:
+        feature (list): The feature to compress.
+        bin_size (int, optional): The size of each bin. If None, no binning is performed.
+        factor (int): Factor to apply to convert output values.
+
+    Returns:
+        tuple: A tuple containing two arrays and the total weight. The first array contains the unique
+            values, and the second  array contains the counts of each value.
+    """
+    s = array(feature)
+    if bin_size is not None:
+        s = bin_values(s, bin_size)
+    ks, ws = unique(s, axis=0, return_counts=True)
+    ks = ks / factor
+    return ks, ws
+
+
+def weighted_features(
+    features: dict[str, ndarray],
+    bin_size: Optional[int] = None,
+    factor: int = 1,
+) -> dict[str, tuple[ndarray, ndarray]]:
+    """
+    Apply optional binning and value counting to dictionary of features.
+
+    Args:
+        features (dict[array): A dictionary of features to compress.
+        bin_size (Optional[int]): The size of the bin to use for compression. Defaults to None.
+        factor (int): Factor to apply to convert output values.
+
+    Returns:
+        dict[str, tuple[array, array[int]]]: A dictionary of features and weights.
+    """
+    return {
+        k: compress_feature(values, bin_size, factor) for k, values in features.items()
+    }

acteval/distance/__init__.py

@@ -0,0 +1,16 @@
+from acteval.distance.scalar import (
+    abs_av_diff as abs_av_diff,
+)
+from acteval.distance.scalar import (
+    mae as mae,
+)
+from acteval.distance.scalar import (
+    mape as mape,
+)
+from acteval.distance.scalar import (
+    mape_scalar as mape_scalar,
+)
+from acteval.distance.scalar import (
+    mse as mse,
+)
+from acteval.distance.wasserstein import emd as emd

acteval/distance/scalar.py

@@ -0,0 +1,68 @@
+import numpy as np
+
+
+def mape(a: tuple[np.ndarray, np.ndarray], b: tuple[np.ndarray, np.ndarray]) -> float:
+    """Calculate mean average percentage error between distributions a and b.
+
+    Clipped at 1.0.
+
+    Args:
+        a (tuple[np.ndarray, np.ndarray]): Distribution a.
+        b (tuple[np.ndarray, np.ndarray]): Distribution b.
+
+    Returns:
+        float: MAPE.
+    """
+    # TODO test this
+    # unpack
+    ak, aw = a
+    bk, bw = b
+    # calc weighted average
+    akw = (ak * aw).sum() / aw.sum()
+    bkw = (bk * bw).sum() / bw.sum()
+    diff = np.abs(akw - bkw)
+    if diff == 0:
+        return 0.0
+    if bkw == 0:
+        return clamp(diff / akw)
+    return clamp(diff / bkw)
+
+
+def clamp(x):
+    if x > 1.0:
+        return 1.0
+    return x
+
+
+def mape_scalar(a, b):
+    return np.abs((a - b) / a).mean()
+
+
+def mse(a: tuple[np.ndarray, np.ndarray], b: tuple[np.ndarray, np.ndarray]) -> float:
+    # requires and b have same support.
+    # unpack
+    _, aw = a
+    _, bw = b
+    return ((aw - bw) ** 2).mean()
+
+
+def mae(a: tuple[np.ndarray, np.ndarray], b: tuple[np.ndarray, np.ndarray]) -> float:
+    # TODO test this
+    # requires and b have same support.
+    # unpack
+    _, aw = a
+    _, bw = b
+    return (np.abs(aw - bw)).mean()
+
+
+def abs_av_diff(
+    a: tuple[np.ndarray, np.ndarray], b: tuple[np.ndarray, np.ndarray]
+) -> float:
+    # TODO test this
+    # unpack
+    ak, aw = a
+    bk, bw = b
+    a_average = (ak * aw).sum() / aw.sum()
+    b_average = (bk * bw).sum() / bw.sum()
+
+    return np.abs(a_average - b_average)