activitymodel 0.1.0__py3-none-any.whl

activitymodel/survey/diaries.py

@@ -0,0 +1,285 @@
+"""Turn raw diary rows into the coded diary table.
+
+A processed diary is:
+
+* one metadata row - ``diary_id``, ``person_id``, ``household_id``,
+  ``day_type``, ``survey_day_kind``, ``weight``, ``quality`` plus the
+  harmonised person attributes; and
+* one ``uint8`` vector of 144 detailed activity codes (10-minute slots from
+  00:00, survey local time).
+
+Two things the prototypes ignored are used here:
+
+* ``LIEUDET1-144`` records *where* each slot happened. Presence at home is
+  what the building model needs, so activities are reconciled with the
+  location (working "at work" while at home becomes ``Home_working``, watching
+  television at a friend's becomes ``Other_outside``).
+* ``TYPJ`` records the kind of day. A public holiday falling on a weekday is
+  a Sunday for the building, and days of leave are excluded by default
+  because building_eload models absences with its own vacation schedule.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+import polars as pl
+
+from ..exceptions import VocabularyError
+from ..harmonise import harmonise_survey_individuals
+from ..taxonomy import DETAILED_ACTIVITIES, DETAILED_INDEX, UNKNOWN_CODE, encode_actpr
+from ..vocabulary import DEFAULT_AGE_CLASSES, HARMONISED_COLUMNS, AgeClasses
+
+SLOTS_PER_DAY = 144
+SLOT_MINUTES = 10
+
+#: Per-slot column families read from ``carnet``.
+SLOT_COLUMN_FAMILIES = ("actpr", "lieudet")
+
+#: Scalar diary columns read from ``carnet``.
+DIARY_COLUMNS = ("idcar", "idind", "idmen", "jour", "typj", "pond_carnet", "qualicar", "imput_act")
+
+DAY_TYPES = ("weekday", "saturday", "sunday")
+
+_JOUR_TO_DAY_TYPE = {
+    "lundi": "weekday",
+    "mardi": "weekday",
+    "mercredi": "weekday",
+    "jeudi": "weekday",
+    "vendredi": "weekday",
+    "samedi": "saturday",
+    "dimanche": "sunday",
+}
+
+#: ``TYPJ`` codebook labels, kept on the diary row as ``survey_day_kind``.
+SURVEY_DAY_KINDS = {
+    "1": "working_day",
+    "2": "half_working_day",
+    "3": "public_holiday_not_worked",
+    "4": "public_holiday_worked",
+    "5": "sick_leave",
+    "6": "leave",
+    "7": "not_in_employment",
+}
+
+#: ``LIEUDET`` code for "at home (or secondary residence)".
+LIEUDET_HOME = "11"
+#: ``LIEUDET`` codes with no usable location.
+LIEUDET_UNKNOWN = ("", "19", "99")
+#: ``LIEUDET`` code for the workplace or place of study.
+LIEUDET_WORK = "12"
+
+#: Activity reported while the person is at home although the activity is an
+#: out-of-home one -> what it becomes.
+AT_HOME_RECODE = {
+    "At_work": "Home_working",
+    "At_school": "Other_at_home",
+    "Shopping": "Other_at_home",
+    "Other_outside": "Other_at_home",
+    "Meal_outside": "Meal_at_home",
+    "Work_journey": "Other_at_home",
+    "Other_journey": "Other_at_home",
+}
+
+#: Activity reported at a known location that is not home although the
+#: activity is a home one -> what it becomes. Sleeping elsewhere is an absence
+#: for the dwelling, so it is recoded too.
+AWAY_RECODE = {
+    name: "Other_outside"
+    for name in DETAILED_ACTIVITIES
+    if name
+    not in (
+        "Other_outside",
+        "Meal_outside",
+        "At_work",
+        "At_school",
+        "Shopping",
+        "Work_journey",
+        "Other_journey",
+        "Meal_at_home",
+    )
+}
+AWAY_RECODE["Meal_at_home"] = "Meal_outside"
+#: ...except home working at the workplace, which is plain work.
+AWAY_RECODE_AT_WORK = {"Home_working": "At_work"}
+
+
+@dataclass(frozen=True)
+class DiaryOptions:
+    """Filters and recodes applied when building the diary table."""
+
+    #: Drop diaries whose activity description is worse than this ``QUALICAR``
+    #: grade (1 very precise ... 4 unusable).
+    max_quality: int = 3
+    #: Drop diaries where at least ten minutes had to be imputed.
+    drop_imputed: bool = False
+    #: ``TYPJ`` kinds excluded from the pool. Leave is excluded because the
+    #: building model schedules vacations itself.
+    excluded_day_kinds: tuple[str, ...] = ("leave",)
+    #: Day type given to a public holiday not worked that falls on a weekday
+    #: (``None`` keeps the calendar day type).
+    holiday_day_type: str | None = "sunday"
+    #: Reconcile activities with the recorded location.
+    reconcile_location: bool = True
+    #: Detailed activity given to slots with an empty or unknown code.
+    unknown_activity: str = "Other_at_home"
+    age_classes: AgeClasses = field(default=DEFAULT_AGE_CLASSES)
+
+
+@dataclass
+class DiaryTable:
+    """Coded diaries: metadata frame + aligned ``uint8`` code matrix."""
+
+    frame: pl.DataFrame
+    codes: np.ndarray
+
+    def __post_init__(self):
+        if self.codes.shape != (self.frame.height, SLOTS_PER_DAY):
+            raise ValueError(
+                f"codes shape {self.codes.shape} does not match {self.frame.height} diaries x {SLOTS_PER_DAY}"
+            )
+        if self.codes.dtype != np.uint8:
+            raise ValueError("codes must be uint8")
+
+    @property
+    def n_diaries(self) -> int:
+        return self.frame.height
+
+
+def _slot_matrix(diaries: pl.DataFrame, family: str, dtype) -> np.ndarray:
+    columns = [f"{family}{i}" for i in range(1, SLOTS_PER_DAY + 1)]
+    missing = [c for c in columns if c not in diaries.columns]
+    if missing:
+        raise VocabularyError(f"diary table is missing {len(missing)} {family!r} slot columns")
+    return diaries.select(columns).to_numpy().astype(dtype)
+
+
+def decode_activity_codes(diaries: pl.DataFrame, unknown_activity: str) -> np.ndarray:
+    """``actpr1..144`` (strings or ints) -> detailed ``uint8`` matrix."""
+    raw = diaries.select(
+        [
+            pl.col(f"actpr{i}").cast(pl.String).str.strip_chars().cast(pl.Int64, strict=False)
+            for i in range(1, SLOTS_PER_DAY + 1)
+        ]
+    ).fill_null(-1)
+    return encode_actpr(raw.to_numpy(), unknown=unknown_activity)
+
+
+def reconcile_location(codes: np.ndarray, location: np.ndarray) -> np.ndarray:
+    """Apply the location recodes; ``location`` is the ``lieudet`` string matrix."""
+    out = codes.copy()
+    at_home = location == LIEUDET_HOME
+    at_work = location == LIEUDET_WORK
+    known_away = ~at_home & ~np.isin(location, LIEUDET_UNKNOWN)
+    for source, target in AT_HOME_RECODE.items():
+        out[at_home & (codes == DETAILED_INDEX[source])] = DETAILED_INDEX[target]
+    for source, target in AWAY_RECODE_AT_WORK.items():
+        out[at_work & (codes == DETAILED_INDEX[source])] = DETAILED_INDEX[target]
+    for source, target in AWAY_RECODE.items():
+        if source in AWAY_RECODE_AT_WORK:
+            mask = known_away & ~at_work
+        else:
+            mask = known_away
+        out[mask & (codes == DETAILED_INDEX[source])] = DETAILED_INDEX[target]
+    return out
+
+
+def build_diaries(
+    diaries: pl.DataFrame,
+    individuals: pl.DataFrame,
+    households: pl.DataFrame,
+    options: DiaryOptions | None = None,
+) -> DiaryTable:
+    """Build the coded diary table from the three raw survey tables.
+
+    Args:
+        diaries: ``carnet`` rows with :data:`DIARY_COLUMNS` and the
+            ``actpr``/``lieudet`` slot columns.
+        individuals: ``individu`` rows (see
+            :func:`~activitymodel.harmonise.harmonise_survey_individuals`).
+        households: ``menage`` rows.
+        options: filters and recodes (defaults to :class:`DiaryOptions`).
+
+    Returns:
+        A :class:`DiaryTable`. Rows are sorted by ``diary_id`` so the build is
+        reproducible whatever the input order.
+    """
+    options = DiaryOptions() if options is None else options
+    people = harmonise_survey_individuals(individuals, households, options.age_classes)
+    person_columns = ["person_id", "household_id", "weight"] + list(HARMONISED_COLUMNS)
+    people = people.select(person_columns).rename({"weight": "person_weight"}).unique("person_id")
+
+    meta = (
+        diaries.select(list(DIARY_COLUMNS))
+        .with_columns(
+            diary_id=pl.col("idcar").cast(pl.String),
+            person_id=pl.col("idind").cast(pl.String),
+            household_id=pl.col("idmen").cast(pl.String),
+            weight=pl.col("pond_carnet").cast(pl.Float64),
+            quality=pl.col("qualicar")
+            .cast(pl.String)
+            .str.strip_chars()
+            .cast(pl.Int64, strict=False),
+            imputed=pl.col("imput_act").cast(pl.Float64).fill_null(0) > 0,
+            survey_day_kind=pl.col("typj")
+            .cast(pl.String)
+            .str.strip_chars()
+            .replace_strict(SURVEY_DAY_KINDS, default="unknown"),
+            calendar_day_type=pl.col("jour")
+            .cast(pl.String)
+            .str.strip_chars()
+            .str.to_lowercase()
+            .replace_strict(_JOUR_TO_DAY_TYPE, default=None),
+        )
+        .with_row_index("_row")
+    )
+    if meta.get_column("calendar_day_type").null_count():
+        raise VocabularyError("diary column 'jour' holds values other than the French day names")
+
+    day_type = pl.col("calendar_day_type")
+    if options.holiday_day_type is not None:
+        day_type = (
+            pl.when(
+                (pl.col("survey_day_kind") == "public_holiday_not_worked")
+                & (pl.col("calendar_day_type") == "weekday")
+            )
+            .then(pl.lit(options.holiday_day_type))
+            .otherwise(day_type)
+        )
+    meta = meta.with_columns(day_type=day_type)
+
+    keep = (
+        (pl.col("quality").fill_null(options.max_quality) <= options.max_quality)
+        & ~pl.col("survey_day_kind").is_in(list(options.excluded_day_kinds))
+        & pl.col("weight").is_not_null()
+    )
+    if options.drop_imputed:
+        keep = keep & ~pl.col("imputed")
+    meta = meta.filter(keep)
+
+    meta = meta.join(people, on="person_id", how="inner")
+    meta = meta.sort("diary_id")
+    rows = meta.get_column("_row").to_numpy()
+
+    codes = decode_activity_codes(diaries, options.unknown_activity)
+    if options.reconcile_location:
+        location = _slot_matrix(
+            diaries.select(
+                [
+                    pl.col(f"lieudet{i}").cast(pl.String).str.strip_chars()
+                    for i in range(1, SLOTS_PER_DAY + 1)
+                ]
+            ),
+            "lieudet",
+            object,
+        )
+        codes = reconcile_location(codes, location)
+    codes = np.ascontiguousarray(codes[rows])
+    if (codes == UNKNOWN_CODE).any():
+        raise VocabularyError("unknown activity codes survived decoding")
+
+    frame = meta.drop(
+        ["_row", "idcar", "idind", "idmen", "jour", "typj", "pond_carnet", "qualicar", "imput_act"]
+    )
+    return DiaryTable(frame=frame, codes=codes)

activitymodel/survey/io.py

@@ -0,0 +1,98 @@
+"""Locate and read the ADISP delivery of the survey.
+
+The delivery (doi:10.13144/lil-0695, access on request, not redistributable)
+ships Stata files: ``carnet.dta`` (one row per diary day), ``individu.dta``
+(one row per respondent) and ``menage.dta`` (one row per household). Parquet
+conversions with the same stem are accepted so a converted copy can be reused.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import polars as pl
+
+_STEMS = {"diaries": "carnet", "individuals": "individu", "households": "menage"}
+_SUFFIXES = (".parquet", ".dta")
+
+
+@dataclass(frozen=True)
+class SurveyFiles:
+    """Paths of the three survey tables."""
+
+    diaries: Path
+    individuals: Path
+    households: Path
+
+    @classmethod
+    def from_directory(cls, directory: str | Path) -> SurveyFiles:
+        """Find ``carnet``, ``individu`` and ``menage`` (parquet preferred, else dta).
+
+        Raises:
+            FileNotFoundError: a table is missing; the message names the directory
+                and the file stems expected, which is the operator's cue to copy
+                the ADISP delivery there.
+        """
+        directory = Path(directory)
+        found = {}
+        for key, stem in _STEMS.items():
+            for suffix in _SUFFIXES:
+                candidate = directory / f"{stem}{suffix}"
+                if candidate.is_file():
+                    found[key] = candidate
+                    break
+            else:
+                raise FileNotFoundError(
+                    f"survey table {stem!r} not found in {directory} "
+                    f"(expected {stem}.parquet or {stem}.dta from the ADISP delivery)"
+                )
+        return cls(**found)
+
+
+def read_table(path: str | Path, columns: list[str] | None = None) -> pl.DataFrame:
+    """Read a survey table as a polars frame.
+
+    Stata files are read through pandas (``survey`` extra) with value labels
+    left as codes, because the codes are what the harmonisation maps.
+    """
+    path = Path(path)
+    if path.suffix == ".parquet":
+        return pl.read_parquet(path, columns=columns)
+    if path.suffix == ".dta":
+        import pandas as pd  # optional dependency, pipeline only
+
+        frame = pd.read_stata(path, columns=columns, convert_categoricals=False)
+        return pl.from_pandas(frame)
+    raise ValueError(f"unsupported survey table format {path.suffix!r} for {path}")
+
+
+def load_survey_tables(files: SurveyFiles):
+    """Read the three tables, restricted to the columns the pipeline uses.
+
+    Returns:
+        ``(diaries, individuals, households)`` polars frames.
+    """
+    from .diaries import DIARY_COLUMNS, SLOT_COLUMN_FAMILIES
+
+    diary_columns = list(DIARY_COLUMNS) + [
+        f"{family}{i}" for family in SLOT_COLUMN_FAMILIES for i in range(1, 145)
+    ]
+    diaries = read_table(files.diaries, columns=diary_columns)
+    individuals = read_table(
+        files.individuals,
+        columns=[
+            "idind",
+            "idmen",
+            "sexe",
+            "age",
+            "situa",
+            "cs24",
+            "couple",
+            "lienpref",
+            "pond_qi",
+            "nb_carnet_rempli",
+        ],
+    )
+    households = read_table(files.households, columns=["idmen", "npers", "typmen5", "tuu", "zau"])
+    return diaries, individuals, households