activitymodel 0.1.0__py3-none-any.whl

activitymodel/matching.py

@@ -0,0 +1,189 @@
+"""Assign each occupant a cluster per day type from their socio-demographics.
+
+The previous implementation drew a *survey diary* per occupant through
+``buildingmodel.join_random_by_columns`` (repeating every diary row
+``pond_carnet`` times, i.e. tens of millions of rows) and matched weekday and
+weekend independently, losing the fact that the same respondent filled both.
+
+Here the draw is over the library's aggregate ``patterns`` table:
+
+1. Occupants are grouped by the values of the keys of the current ladder
+   level; the pattern rows are summed to the same keys.
+2. Inside each cell one pattern row (a triple ``cluster_weekday``,
+   ``cluster_saturday``, ``cluster_sunday``, some possibly null) is drawn per
+   occupant with probability proportional to the summed person weight. Drawing
+   a whole row keeps the weekday/weekend coupling observed on real people.
+3. Occupants whose cell has no pattern fall through to the next, coarser level.
+4. Day types left null by the drawn pattern are filled from the marginal
+   distribution of that day type, again walking the ladder from the finest
+   level with support.
+
+Everything is driven by one ``numpy.random.Generator``; cells are visited in
+sorted order, so a seed gives the same assignment on every machine.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import numpy as np
+import polars as pl
+
+from .exceptions import MatchingError, VocabularyError
+from .library import DiaryLibrary, cluster_column
+from .vocabulary import DEFAULT_MATCHING_LADDER
+
+MATCH_LEVEL_COLUMN = "match_level"
+_CELL = "_cell"
+_NULL_TOKEN = "\x00"
+
+
+def _cell_expr(keys: Sequence[str]) -> pl.Expr:
+    if not keys:
+        return pl.lit("")
+    return pl.concat_str(
+        [pl.col(k).cast(pl.String).fill_null(_NULL_TOKEN) for k in keys], separator="|"
+    )
+
+
+def _augment(frame: pl.DataFrame) -> pl.DataFrame:
+    """Derive the coarser ``activity_status`` when only ``employment_status`` exists."""
+    if "activity_status" not in frame.columns and "employment_status" in frame.columns:
+        frame = frame.with_columns(
+            activity_status=pl.when(pl.col("employment_status").str.starts_with("employed"))
+            .then(pl.lit("employed"))
+            .otherwise(pl.col("employment_status"))
+        )
+    return frame
+
+
+def _draw_rows(
+    occupants: pl.DataFrame,
+    table: pl.DataFrame,
+    keys: Sequence[str],
+    value_columns: Sequence[str],
+    rng: np.random.Generator,
+) -> pl.DataFrame:
+    """Draw one ``table`` row per occupant inside its cell; unmatched occupants are dropped.
+
+    Returns ``occupants`` columns ``_occ`` plus ``value_columns``.
+    """
+    grouped = (
+        table.group_by(list(keys) + list(value_columns), maintain_order=False)
+        .agg(pl.col("weight").sum())
+        .with_columns(_cell_expr(keys).alias(_CELL))
+        .sort([_CELL] + list(value_columns), nulls_last=True)
+    )
+    occ = occupants.with_columns(_cell_expr(keys).alias(_CELL))
+    pools = grouped.partition_by(_CELL, as_dict=True, maintain_order=True)
+    pieces = []
+    for (cell,), members in sorted(occ.partition_by(_CELL, as_dict=True).items()):
+        pool = pools.get((cell,))
+        if pool is None:
+            continue
+        weights = pool.get_column("weight").to_numpy()
+        picks = rng.choice(pool.height, size=members.height, p=weights / weights.sum())
+        pieces.append(
+            members.select("_occ").with_columns(
+                pool.select(list(value_columns))[picks].get_columns()
+            )
+        )
+    if not pieces:
+        return (
+            occupants.select("_occ")
+            .head(0)
+            .with_columns([pl.lit(None, dtype=table.schema[c]).alias(c) for c in value_columns])
+        )
+    return pl.concat(pieces)
+
+
+def assign_clusters(
+    library: DiaryLibrary,
+    occupants: pl.DataFrame,
+    seed: int | np.random.Generator | None = 0,
+    ladder: Sequence[Sequence[str]] = DEFAULT_MATCHING_LADDER,
+    occupant_id: str = "occupant_id",
+) -> pl.DataFrame:
+    """Assign one cluster per day type to every occupant.
+
+    Args:
+        library: the diary library.
+        occupants: one row per occupant with ``occupant_id`` and the harmonised
+            key columns used by ``ladder``.
+        seed: integer seed or a ``numpy.random.Generator``.
+        ladder: key subsets tried from finest to coarsest; every key must be a
+            library matching key or ``activity_status``. The last level should
+            be empty so nobody is left unmatched.
+        occupant_id: name of the occupant identifier column.
+
+    Returns:
+        Frame with ``occupant_id``, ``cluster_<day_type>`` (``Int16``) for each
+        library day type and ``match_level`` (``Int8``, index of the ladder
+        level that matched, 0 = finest).
+
+    Raises:
+        VocabularyError: an occupant column required by the ladder is missing.
+        MatchingError: occupants remain unmatched after the last level.
+    """
+    rng = seed if isinstance(seed, np.random.Generator) else np.random.default_rng(seed)
+    if occupant_id not in occupants.columns:
+        raise VocabularyError(f"occupants lack identifier column {occupant_id!r}")
+    if occupants.get_column(occupant_id).n_unique() != occupants.height:
+        raise VocabularyError(f"{occupant_id!r} is not unique")
+
+    patterns = _augment(library.patterns)
+    allowed = set(patterns.columns)
+    needed = {k for level in ladder for k in level}
+    unknown = needed - allowed
+    if unknown:
+        raise VocabularyError(
+            f"ladder uses keys absent from the library patterns: {sorted(unknown)}"
+        )
+    occ = _augment(occupants)
+    missing = needed - set(occ.columns)
+    if missing:
+        raise VocabularyError(f"occupants lack columns required by the ladder: {sorted(missing)}")
+    value_columns = library.cluster_columns
+    occ = occ.with_row_index("_occ")
+
+    # Pass 1: joint draw of a pattern row, walking the ladder.
+    remaining = occ
+    assigned = []
+    for level, keys in enumerate(ladder):
+        if remaining.height == 0:
+            break
+        drawn = _draw_rows(remaining, patterns, keys, value_columns, rng)
+        if drawn.height:
+            assigned.append(
+                drawn.with_columns(pl.lit(level, dtype=pl.Int8).alias(MATCH_LEVEL_COLUMN))
+            )
+            remaining = remaining.join(drawn.select("_occ"), on="_occ", how="anti")
+    if remaining.height:
+        raise MatchingError(
+            f"{remaining.height} occupants matched no pattern even at the coarsest level; "
+            "end the ladder with an empty key set"
+        )
+    result = pl.concat(assigned).join(occ, on="_occ", how="left")
+
+    # Pass 2: fill day types the drawn pattern left null from the marginals.
+    for day_type in library.day_types:
+        column = cluster_column(day_type)
+        marginal = patterns.filter(pl.col(column).is_not_null())
+        for keys in ladder:
+            holes = result.filter(pl.col(column).is_null())
+            if holes.height == 0:
+                break
+            filled = _draw_rows(holes, marginal, keys, [column], rng).rename({column: "_fill"})
+            result = (
+                result.join(filled, on="_occ", how="left")
+                .with_columns(pl.coalesce([pl.col(column), pl.col("_fill")]).alias(column))
+                .drop("_fill")
+            )
+        if result.get_column(column).null_count():
+            raise MatchingError(f"no pattern provides a {day_type} cluster at any ladder level")
+
+    return (
+        result.sort("_occ")
+        .select([occupant_id] + value_columns + [MATCH_LEVEL_COLUMN])
+        .with_columns([pl.col(c).cast(pl.Int16) for c in value_columns])
+    )

activitymodel/pipeline.py

@@ -0,0 +1,239 @@
+"""Build a :class:`~activitymodel.library.DiaryLibrary` from the survey.
+
+This is the entry point the buildingdata Snakemake rule calls::
+
+    from activitymodel.pipeline import LibraryBuildOptions, build_library
+    library = build_library(survey_directory, LibraryBuildOptions())
+    library.write(output_directory)
+
+It can also be run by hand::
+
+    python -m activitymodel.pipeline /path/to/edt2010 /path/to/output --weekday 16 --saturday 10 --sunday 10
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+import numpy as np
+import polars as pl
+
+from . import __version__
+from .clustering import ClusteringOptions, cluster_codes
+from .library import (
+    CLOCK,
+    SLOT_COLUMNS,
+    SLOT_MINUTES,
+    SLOTS_PER_DAY,
+    DiaryLibrary,
+    cluster_column,
+)
+from .survey.diaries import DAY_TYPES, DiaryOptions, DiaryTable, build_diaries
+from .survey.io import SurveyFiles, load_survey_tables
+from .taxonomy import DETAILED_ACTIVITIES
+from .vocabulary import DEFAULT_MATCHING_KEYS
+
+DEFAULT_SOURCE = {
+    "name": "INSEE Enquete Emploi du temps 2009-2010",
+    "distributor": "PROGEDO-ADISP",
+    "doi": "10.13144/lil-0695",
+    "url": "https://data.progedo.fr/studies/doi/10.13144/lil-0695",
+    "access": "on request, not redistributable; the library holds no microdata row",
+}
+
+
+@dataclass(frozen=True)
+class LibraryBuildOptions:
+    """Everything that shapes a library build, recorded in its manifest."""
+
+    diaries: DiaryOptions = field(default_factory=DiaryOptions)
+    clustering: ClusteringOptions = field(default_factory=ClusteringOptions)
+    matching_keys: tuple[str, ...] = DEFAULT_MATCHING_KEYS
+    source: dict = field(default_factory=lambda: dict(DEFAULT_SOURCE))
+
+
+def build_library(
+    survey: str | Path | SurveyFiles | tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame],
+    options: LibraryBuildOptions | None = None,
+) -> DiaryLibrary:
+    """Read the survey, code and cluster the diaries, assemble the library.
+
+    Args:
+        survey: a directory holding the ADISP files, a :class:`SurveyFiles`, or
+            the three raw frames ``(diaries, individuals, households)``.
+        options: build options (defaults to :class:`LibraryBuildOptions`).
+    """
+    options = LibraryBuildOptions() if options is None else options
+    if isinstance(survey, tuple):
+        diaries, individuals, households = survey
+    else:
+        files = survey if isinstance(survey, SurveyFiles) else SurveyFiles.from_directory(survey)
+        diaries, individuals, households = load_survey_tables(files)
+    table = build_diaries(diaries, individuals, households, options.diaries)
+    labels = cluster_diaries(table, options.clustering)
+    return assemble_library(table, labels, options)
+
+
+def cluster_diaries(table: DiaryTable, options: ClusteringOptions) -> dict[str, np.ndarray]:
+    """Cluster each day type's pool. Returns ``day_type -> labels`` (aligned with the table)."""
+    rng = np.random.default_rng(options.seed)
+    day_types = table.frame.get_column("day_type").to_numpy().astype(object)
+    weights = table.frame.get_column("weight").to_numpy()
+    labels = {}
+    for day_type in DAY_TYPES:
+        rows = np.flatnonzero(day_types == day_type)
+        if rows.size == 0:
+            continue
+        k = options.n_clusters.get(day_type)
+        if k is None:
+            raise ValueError(f"no cluster count configured for day type {day_type!r}")
+        out = np.full(table.n_diaries, -1, dtype=np.int32)
+        out[rows] = cluster_codes(table.codes[rows], k, options, weights=weights[rows], rng=rng)
+        labels[day_type] = out
+    return labels
+
+
+def assemble_library(
+    table: DiaryTable, labels: dict[str, np.ndarray], options: LibraryBuildOptions
+) -> DiaryLibrary:
+    """Turn a coded diary table and its cluster labels into the published tables."""
+    day_types = tuple(d for d in DAY_TYPES if d in labels)
+    frame = table.frame.with_row_index("_row")
+    keys = list(options.matching_keys)
+
+    # Global cluster ids: contiguous blocks per day type, in DAY_TYPES order.
+    cluster_index = np.full(table.n_diaries, -1, dtype=np.int32)
+    cluster_id = np.full(table.n_diaries, -1, dtype=np.int32)
+    offset = 0
+    cluster_rows = []
+    for day_type in day_types:
+        mask = labels[day_type] >= 0
+        k = int(labels[day_type][mask].max()) + 1
+        cluster_index[mask] = labels[day_type][mask]
+        cluster_id[mask] = offset + labels[day_type][mask]
+        for i in range(k):
+            cluster_rows.append(
+                {"cluster_id": offset + i, "day_type": day_type, "cluster_index": i}
+            )
+        offset += k
+    frame = frame.with_columns(
+        pl.Series("cluster_id", cluster_id, dtype=pl.Int16),
+    )
+
+    slots = pl.DataFrame(table.codes, schema=list(SLOT_COLUMNS)).cast(pl.UInt8)
+    days = frame.select(
+        pl.col("_row").cast(pl.UInt32).alias("day_id"),
+        pl.col("day_type").cast(pl.String),
+        pl.col("cluster_id"),
+        pl.col("weight").cast(pl.Float64),
+    ).with_columns(slots.get_columns())
+
+    clusters = (
+        pl.DataFrame(cluster_rows)
+        .cast({"cluster_id": pl.Int16, "cluster_index": pl.Int16})
+        .join(
+            days.group_by("cluster_id").agg(
+                pl.len().cast(pl.UInt32).alias("n_days"), pl.col("weight").sum().alias("weight_sum")
+            ),
+            on="cluster_id",
+            how="left",
+        )
+        .sort("cluster_id")
+        .select("cluster_id", "day_type", "cluster_index", "n_days", "weight_sum")
+    )
+
+    # One row per person: cluster per day type (first diary of that type),
+    # person weight (questionnaire weight, else mean diary weight), keys.
+    per_person = (
+        frame.sort("diary_id")
+        .group_by("person_id", maintain_order=True)
+        .agg(
+            [pl.col(k).first() for k in keys]
+            + [
+                pl.coalesce([pl.col("person_weight").first(), pl.col("weight").mean()]).alias(
+                    "weight"
+                ),
+            ]
+            + [
+                pl.col("cluster_id")
+                .filter(pl.col("day_type") == d)
+                .first()
+                .alias(cluster_column(d))
+                for d in day_types
+            ]
+        )
+    )
+    patterns = (
+        per_person.group_by(keys + [cluster_column(d) for d in day_types])
+        .agg(pl.col("weight").sum(), pl.len().cast(pl.UInt32).alias("n_persons"))
+        .with_columns([pl.col(k).cast(pl.String) for k in keys])
+        .sort(keys + [cluster_column(d) for d in day_types], nulls_last=True)
+    )
+
+    manifest = {
+        "activitymodel_version": __version__,
+        "created_at": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
+        "source": dict(options.source),
+        "taxonomy_level": "detailed",
+        "activities": list(DETAILED_ACTIVITIES),
+        "slot_minutes": SLOT_MINUTES,
+        "slots_per_day": SLOTS_PER_DAY,
+        "clock": CLOCK,
+        "day_types": list(day_types),
+        "matching_keys": keys,
+        "age_class_bounds": list(options.diaries.age_classes.bounds),
+        "diary_options": _jsonable(asdict(options.diaries)),
+        "clustering": _jsonable(asdict(options.clustering)),
+        "n_days": int(days.height),
+        "n_persons": int(per_person.height),
+    }
+    return DiaryLibrary.from_frames(days, clusters, patterns, manifest)
+
+
+def _jsonable(value):
+    if isinstance(value, dict):
+        return {str(k): _jsonable(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [_jsonable(v) for v in value]
+    if isinstance(value, (str, int, float, bool)) or value is None:
+        return value
+    if hasattr(value, "bounds"):  # AgeClasses
+        return list(value.bounds)
+    return str(value)
+
+
+def main(argv=None) -> None:
+    parser = argparse.ArgumentParser(
+        description="Build an activitymodel diary library from the EDT 2010 files."
+    )
+    parser.add_argument(
+        "survey_dir", type=Path, help="directory holding carnet/individu/menage (.parquet or .dta)"
+    )
+    parser.add_argument("output_dir", type=Path, help="directory receiving the library tables")
+    parser.add_argument("--weekday", type=int, default=16)
+    parser.add_argument("--saturday", type=int, default=10)
+    parser.add_argument("--sunday", type=int, default=10)
+    parser.add_argument("--method", choices=("ward", "kmeans"), default="ward")
+    parser.add_argument("--level", default="detailed", help="taxonomy level used for clustering")
+    parser.add_argument("--seed", type=int, default=0)
+    args = parser.parse_args(argv)
+    options = LibraryBuildOptions(
+        clustering=ClusteringOptions(
+            n_clusters={"weekday": args.weekday, "saturday": args.saturday, "sunday": args.sunday},
+            method=args.method,
+            level=args.level,
+            seed=args.seed,
+        )
+    )
+    library = build_library(args.survey_dir, options)
+    paths = library.write(args.output_dir)
+    print(library.summary())
+    for name, path in paths.items():
+        print(f"  wrote {name:9s} -> {path}")
+
+
+if __name__ == "__main__":
+    main()

activitymodel/survey/__init__.py

@@ -0,0 +1,18 @@
+"""Reading and preparing the Enquete Emploi du temps 2009-2010 (pipeline side).
+
+Nothing in this sub-package is imported by the matching or calendar modules:
+building_eload never needs it, and it is the only part of activitymodel that
+touches restricted survey microdata.
+"""
+
+from .diaries import DiaryOptions, DiaryTable, build_diaries
+from .io import SurveyFiles, load_survey_tables, read_table
+
+__all__ = [
+    "DiaryOptions",
+    "DiaryTable",
+    "SurveyFiles",
+    "build_diaries",
+    "load_survey_tables",
+    "read_table",
+]