buildingdata 0.3.0__tar.gz → 0.4.0__tar.gz

{buildingdata-0.3.0 → buildingdata-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: buildingdata
-Version: 0.3.0
+Version: 0.4.0
 Summary: Data management layer for buildingmodel — reference data download, BDTOPO retrieval, ERA5 weather
 License-Expression: MIT
 Requires-Python: >=3.10

buildingdata-0.4.0/buildingdata/reference/diagnosis.py

@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+import polars as pl
+
+from ..gcs import download_blob, ensure_blob_cached, get_blob
+from ..validation import require_columns
+
+
+_BLOB_NAME = "energy_performance_diagnosis_latest.parquet"
+
+
+# Heating/DHW energies excluded from inference (no meaningful DPE data for coal)
+_EXCLUDED_ENERGIES = ["Charbon"]
+
+# Columns this accessor operates on directly (coal filter + dtype casts). If any
+# is absent the raw polars error is opaque; validating up front names the DPE
+# schema drift explicitly. ``heating_system`` + ``region`` are the join keys
+# ``buildingmodel``'s energy-system inference keys on; ``backup_heating_energy``
+# / ``dhw_energy`` carry the fuel labels the coal filter reads.
+_REQUIRED_COLUMNS = (
+    "heating_system",
+    "region",
+    "backup_heating_energy",
+    "dhw_energy",
+)
+
+
+def get_diagnosis(refresh=False):
+    """Return the cleaned DPE energy performance diagnosis DataFrame.
+
+    Downloads energy_performance_diagnosis_latest.parquet from GCS on first
+    call. Applies the filtering and type casts that previously lived in
+    buildingmodel/io/diagnosis.py so that buildingmodel receives a clean frame.
+
+    One row is one post-reform DPE record (issued on or after 1 July 2022), not
+    one dwelling of the stock: records are matched to buildings by
+    ``buildingmodel``, so every column is intensive — a ratio, a U-value, a rate
+    or a per-m² quantity — and carries over regardless of the building's size.
+
+    Args:
+        refresh (bool): force re-download even if the cache is warm.
+            Defaults to False.
+
+    Returns:
+        polars.DataFrame: DPE records. Beyond the matching keys
+            (``construction_year_class``, ``residential_type``,
+            ``heating_system``, and the ``district``/``city``/``city_group``/
+            ``department``/``region`` geography) the columns group as:
+
+            * envelope — ``wall_u_value``, ``roof_u_value``, ``floor_u_value``,
+              ``wall_window_u_value``, ``wall_window_share``,
+              ``envelope_u_value`` (whole-envelope Ubat),
+              ``thermal_bridge_linear_loss``, ``thermal_bridge_loss_share``,
+              ``air_change_rate``, ``air_permeability``, ``storey_height``,
+              ``inertia_class``, ``{wall,roof,floor}_insulation_type``;
+            * systems — ``main_heating_energy``, ``backup_heating_energy``,
+              ``dhw_energy``, ``heating_mode``/``dhw_mode`` (individual /
+              collective / mixed), the ``*_efficiency`` and ``*_scop`` pairs,
+              ``intermittency_factor``, ``backup_heating_share``,
+              ``dhw_storage_volume``;
+            * observed performance — ``energy_class``, ``ghg_class`` and the
+              ``annual_*_per_area`` intensities, for calibrating simulated
+              output against the diagnosis itself.
+
+            ``main_heating_system_efficiency`` is a combustion efficiency (≤ 1)
+            and is **null for heat pumps**, which have no such value; their
+            seasonal performance is in ``main_heating_system_scop`` instead
+            (null for every other generator). The same split applies to the
+            ``backup_heating_*`` pair. Treating a SCOP as an efficiency
+            understates heat-pump performance roughly threefold, hence the two
+            columns.
+
+    Raises:
+        RemoteNotAvailableError: if the blob is not found in the GCS bucket.
+        SchemaValidationError: if the fetched frame is missing one of the
+            columns this accessor operates on (see ``_REQUIRED_COLUMNS``).
+    """
+    dest = ensure_blob_cached(
+        _BLOB_NAME, refresh=refresh, get_blob_fn=get_blob, download_blob_fn=download_blob
+    )
+
+
+    df = pl.read_parquet(dest)
+
+
+    require_columns(df, _REQUIRED_COLUMNS, "DPE/EPC diagnosis")
+
+    # Remove records with coal heating/DHW — no useful inference data.
+    # ``fill_null(False)`` is load-bearing: ``backup_heating_energy`` is null for
+    # the ~76% of records with no secondary generator, ``is_in`` returns null for
+    # those, and ``filter`` drops null rows. Without it this filter keeps only
+    # dwellings that happen to own a backup system — a heavily biased subsample —
+    # instead of dropping the few hundred coal records it is meant to remove.
+    df = df.filter(
+        ~pl.col("backup_heating_energy").is_in(_EXCLUDED_ENERGIES).fill_null(False)
+        & ~pl.col("dhw_energy").is_in(_EXCLUDED_ENERGIES).fill_null(False)
+    )
+
+    df = df.with_columns([
+        pl.col("heating_system").cast(pl.Categorical),
+        pl.col("region").cast(pl.Int64),
+    ])
+
+    if "living_area" in df.columns:
+        df = df.drop(["living_area"])
+    if "living_area_class" in df.columns:
+        df = df.drop(["living_area_class"])
+
+    return df

buildingdata-0.4.0/buildingdata/tests/test_pipeline_diagnosis.py

@@ -0,0 +1,283 @@
+# -*- coding: utf-8 -*-
+"""Contracts of pipeline/rules/scripts/process_diagnosis.py.
+
+The script is not part of the installed distribution, so it is loaded from the
+repo by path and skipped when absent (e.g. running the suite from a wheel).
+
+What is pinned here is what silently drifts: the value vocabularies that must
+stay identical to process_census.py for buildingmodel's join to hit, the
+ordering-sensitive generator classification, and the efficiency/SCOP split.
+"""
+import ast
+import importlib.util
+from pathlib import Path
+
+import polars as pl
+import pytest
+
+_SCRIPT = (
+    Path(__file__).resolve().parents[2]
+    / "pipeline"
+    / "rules"
+    / "scripts"
+    / "process_diagnosis.py"
+)
+_CENSUS_SCRIPT = _SCRIPT.with_name("process_census.py")
+
+pytestmark = pytest.mark.skipif(
+    not _SCRIPT.exists(), reason="pipeline/ is not shipped in the distribution"
+)
+
+
+def _load(path):
+    spec = importlib.util.spec_from_file_location(path.stem, path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _read_literal(path, name):
+    """Read a module-level literal without importing the module.
+
+    process_census.py reads ``snakemake`` and imports buildingmodel at module
+    level, so it cannot be imported here.
+    """
+    tree = ast.parse(path.read_text(encoding="utf-8"))
+    for node in tree.body:
+        targets = getattr(node, "targets", [])
+        if any(getattr(t, "id", None) == name for t in targets):
+            return ast.literal_eval(node.value)
+    raise AssertionError(f"{name} not found in {path}")
+
+
+@pytest.fixture(scope="module")
+def script():
+    return _load(_SCRIPT)
+
+
+# One coherent upstream record. Tests override only the fields they exercise, so
+# a new column in COLUMN_SELECTION shows up as a KeyError here rather than as a
+# silently-null column in the published dataset.
+UPSTREAM_DEFAULTS = {
+    # identity / matching keys
+    "type_energie_chauffage_principal": "Gaz naturel",
+    "periode_construction_recensement": "1946 à 1970",
+    "type_logement": "appartement",
+    "surface_habitable_logement": 100.0,
+    "SURF": "de 80 à 100 m²",
+    "zone_climatique": "H1a",
+    "classe_altitude": "inférieur à 400m",
+    # envelope
+    "classe_inertie": "Moyenne",
+    "u_mur": 1.2,
+    "u_ph": 0.5,
+    "u_pb": 1.7,
+    "uw_baie": 2.6,
+    "ubat_calcule": 0.95,
+    "k_pont_thermique": 0.5,
+    "q4pa_conv": 1.9,
+    "hsp": 2.5,
+    "qvarep_conv": 1.35,
+    "surface_paroi_opaque_mur": 85.0,
+    "surface_baie_vertical_Nord": 3.0,
+    "surface_baie_vertical_Sud": 4.0,
+    "surface_baie_vertical_Est": 2.0,
+    "surface_baie_vertical_Ouest": 2.0,
+    "deperdition_pont_thermique": 20.0,
+    "deperdition_enveloppe": 200.0,
+    "type_isolation_majoritaire_mur": "ITI",
+    "type_isolation_majoritaire_ph": "Non isolé",
+    "type_isolation_majoritaire_pb": "inconnu",
+    # heating system 1
+    "type_generateur_ch_systeme_1": "Chaudière gaz à condensation après 2015",
+    "mode_chauffage": "individuel",
+    "surface_chauffee_systeme_1": 100.0,
+    "rendement_generation_systeme_1": 0.9,
+    "scop_systeme_1": None,
+    "rendement_emission_systeme_1": 0.95,
+    "rendement_distribution_systeme_1": 0.95,
+    "rendement_regulation_systeme_1": 0.99,
+    "i0_systeme_1": 0.9,
+    "conso_ch_systeme_1": 8000.0,
+    # heating system 2 (a wood stove backing up the boiler)
+    "type_energie_systeme_2": "Bois – Bûches",
+    "rendement_generation_systeme_2": 0.5,
+    "scop_systeme_2": None,
+    "rendement_emission_systeme_2": 0.95,
+    "rendement_distribution_systeme_2": 1.0,
+    "rendement_regulation_systeme_2": 0.99,
+    "conso_ch_systeme_2": 2000.0,
+    # DHW
+    "type_energie_ecs": "Électricité",
+    "mode_ecs": "individuel",
+    "rendement_generation_ecs": 0.8,
+    "rendement_generation_stockage_ecs": 0.0,
+    "rendement_stockage_ecs": 0.0,
+    "volume_stockage_ecs": 150.0,
+    # observed performance
+    "classe_bilan_dpe": "D",
+    "classe_emission_ges": "C",
+    "besoin_ch": 10000.0,
+    "besoin_ecs": 1700.0,
+    "conso_5_usages_m2": 145.0,
+    "ep_conso_5_usages_m2": 230.0,
+    "emission_ges_5_usages_m2": 18.0,
+    # geography
+    "code_iris": "751124705",
+    "code_insee": "75112",
+    "DEP": "75",
+    "EPCI": "200054781",
+    "REG": "11",
+}
+
+
+def _upstream(*overrides):
+    """Build an upstream frame, one row per override dict."""
+    rows = [{**UPSTREAM_DEFAULTS, **o} for o in (overrides or [{}])]
+    return pl.DataFrame(
+        {key: [row[key] for row in rows] for key in UPSTREAM_DEFAULTS},
+        schema_overrides={
+            key: pl.Float64
+            for key, value in UPSTREAM_DEFAULTS.items()
+            if isinstance(value, float)
+        },
+    ).lazy()
+
+
+def _run(script, *overrides):
+    return script.transform(_upstream(*overrides)).collect()
+
+
+def test_a_fuel_generator_gets_a_chained_efficiency_and_no_scop(script):
+    out = _run(script)
+
+    assert out["main_heating_system_efficiency"][0] == pytest.approx(
+        0.9 * 0.95 * 0.95 * 0.99
+    )
+    assert out["main_heating_system_scop"][0] is None
+
+
+def test_a_heat_pump_gets_a_scop_and_no_efficiency(script):
+    # ADEME leaves rendement_generation null and fills scop for every heat pump.
+    # Publishing the SCOP as an efficiency would understate it ~3x, so the two
+    # live in separate columns.
+    out = _run(
+        script,
+        {
+            "type_generateur_ch_systeme_1": "PAC air/eau installée après 2017",
+            "rendement_generation_systeme_1": None,
+            "scop_systeme_1": 3.0,
+        },
+    )
+
+    assert out["heating_system"][0] == "electric heat pump"
+    assert out["main_heating_system_efficiency"][0] is None
+    assert out["main_heating_system_scop"][0] == pytest.approx(3.0 * 0.95 * 0.95 * 0.99)
+
+
+def test_a_multi_building_gas_plant_is_classified_as_a_heat_network(script):
+    # "Chaudière(s) gaz multi bâtiment modélisée comme un réseau de chaleur"
+    # matches both a gas and a heat-network pattern. ADEME reports the fuel as
+    # "Réseau de Chauffage urbain" for these, so the heat-network reading is the
+    # one that keeps main_heating_energy and heating_system coherent — which is
+    # only true because "réseau de chaleur" is tested first.
+    out = _run(
+        script,
+        {
+            "type_generateur_ch_systeme_1": (
+                "Chaudière(s) gaz multi bâtiment modélisée comme un réseau de chaleur"
+            ),
+            "type_energie_chauffage_principal": "Réseau de Chauffage urbain",
+        },
+    )
+
+    assert out["heating_system"][0] == "urban heat network"
+    assert out["main_heating_energy"][0] == "district_network"
+
+
+def test_an_unclassifiable_generator_is_dropped(script):
+    # Coal boilers and cooking ranges used as main heating: buildingmodel has no
+    # category for them, so they must not reach the join.
+    assert _run(script, {"type_generateur_ch_systeme_1": "Cuisinière installé avant 1990"}).height == 0
+
+
+def test_the_generator_vocabulary_matches_the_census_one(script):
+    # buildingmodel joins buildings (census heating_system) to diagnosis records
+    # on this column: a label present on one side only silently loses those rows.
+    census_labels = set(
+        _read_literal(_CENSUS_SCRIPT, "HEATING_SYSTEM_LABELS").values()
+    ) - {"unknown"}
+
+    assert set(script.HEATING_SYSTEM_PATTERNS.values()) == census_labels
+
+
+def test_coal_survives_the_fuel_relabelling(script):
+    # get_diagnosis() drops coal records by matching the literal "Charbon", so
+    # this script must not translate it away.
+    out = _run(script, {"type_energie_ecs": "Charbon"})
+
+    assert out["dhw_energy"][0] == "Charbon"
+
+
+def test_houses_get_the_system_mode_upstream_leaves_null(script):
+    # methode_application_dpe_log only spells out a mode for multi-dwelling
+    # buildings, so every house arrives null.
+    individual, networked = _run(
+        script,
+        {"type_logement": "maison", "mode_chauffage": None, "mode_ecs": None},
+        {
+            "type_logement": "maison",
+            "mode_chauffage": None,
+            "mode_ecs": None,
+            "type_energie_chauffage_principal": "Réseau de Chauffage urbain",
+            "type_generateur_ch_systeme_1": "Réseau de chaleur isolé",
+        },
+    )["heating_mode"]
+
+    assert individual == "individual"
+    assert networked == "collective"
+
+
+def test_an_apartment_keeps_its_upstream_system_mode(script):
+    out = _run(script, {"mode_chauffage": "collectif", "mode_ecs": "mixte"})
+
+    assert out["heating_mode"][0] == "collective"
+    assert out["dhw_mode"][0] == "mixed"
+
+
+def test_an_out_of_range_energy_intensity_is_nulled_not_clipped(script):
+    # These are calibration targets: clipping would pull a fit toward the bound,
+    # nulling just drops the record from the comparison.
+    out = _run(script, {"besoin_ch": 10_000_000.0})
+
+    assert out["annual_heating_need_per_area"][0] is None
+
+
+def test_an_out_of_range_physical_attribute_is_clipped_not_nulled(script):
+    out = _run(script, {"hsp": 3018.0, "k_pont_thermique": 275.0})
+
+    assert out["storey_height"][0] == script.STOREY_HEIGHT_BOUNDS[1]
+    assert out["thermal_bridge_linear_loss"][0] == (
+        script.THERMAL_BRIDGE_LINEAR_LOSS_BOUNDS[1]
+    )
+
+
+def test_a_record_with_an_implausible_glazing_share_is_dropped(script):
+    # A wall reported as almost entirely glass is a DPE geometry-coding error.
+    assert _run(script, {"surface_paroi_opaque_mur": 1.0}).height == 0
+
+
+def test_every_published_column_is_populated(script):
+    # Guards COLUMN_SELECTION against a source rename: a missing source column
+    # would otherwise surface as an all-null column in the published dataset.
+    out = _run(script)
+
+    assert set(out.columns) == set(script.COLUMN_SELECTION)
+    # The SCOP columns are null by construction for a non-heat-pump record.
+    scop_columns = {"main_heating_system_scop", "backup_heating_system_scop"}
+    unexpected_nulls = [
+        column
+        for column in out.columns
+        if column not in scop_columns and out[column].null_count()
+    ]
+    assert unexpected_nulls == []

{buildingdata-0.3.0 → buildingdata-0.4.0}/buildingdata/tests/test_reference.py

@@ -130,15 +130,17 @@ def test_get_census_refresh_redownloads(monkeypatch):
 # --- get_diagnosis ----------------------------------------------------------
 
 def _diagnosis_df():
+    # Row 3 has no secondary generator, which is the majority case in the real
+    # dataset (~76% of DPE records): backup_heating_energy is null there.
     return pl.DataFrame(
         {
-            "heating_system": ["gas_boiler", "heat_pump", "gas_boiler"],
-            "region": [11, 84, 11],
-            "backup_heating_energy": ["Gaz", "Charbon", "Electricite"],
-            "dhw_energy": ["Gaz", "Gaz", "Charbon"],
-            "living_area": [80.0, 95.0, 60.0],
-            "living_area_class": ["A", "B", "A"],
-            "u_wall": [0.3, 0.4, 0.5],
+            "heating_system": ["gas_boiler", "heat_pump", "gas_boiler", "gas_boiler"],
+            "region": [11, 84, 11, 11],
+            "backup_heating_energy": ["Gaz", "Charbon", "Electricite", None],
+            "dhw_energy": ["Gaz", "Gaz", "Charbon", "Gaz"],
+            "living_area": [80.0, 95.0, 60.0, 70.0],
+            "living_area_class": ["A", "B", "A", "B"],
+            "u_wall": [0.3, 0.4, 0.5, 0.6],
         }
     )
 
@@ -149,8 +151,8 @@ def test_get_diagnosis_filters_coal_and_casts(monkeypatch):
 
     out = diagnosis_mod.get_diagnosis()
 
-    # Two of three rows reference coal (backup or dhw) and must be dropped.
-    assert out.height == 1
+    # Two of four rows reference coal (backup or dhw) and must be dropped.
+    assert out.height == 2
     assert out["heating_system"].dtype == pl.Categorical
     assert out["region"].dtype == pl.Int64
     # living_area / living_area_class are dropped.
@@ -160,6 +162,21 @@ def test_get_diagnosis_filters_coal_and_casts(monkeypatch):
     assert "u_wall" in out.columns
 
 
+def test_get_diagnosis_keeps_records_without_a_backup_system(monkeypatch):
+    # Regression: `~col.is_in([...])` is null when the column is null, and
+    # `filter` drops null rows — so the coal filter used to discard every record
+    # with no secondary generator (80% of the dataset, and a biased subsample:
+    # only dwellings owning a backup wood stove or similar survived).
+    df = _diagnosis_df().with_columns(
+        pl.lit(None, dtype=pl.String).alias("backup_heating_energy"),
+        pl.lit("Gaz").alias("dhw_energy"),
+    )
+    blob = FakeBlob(diagnosis_mod._BLOB_NAME, generation=2)
+    _stub_gcs(monkeypatch, diagnosis_mod, blob, lambda dest: df.write_parquet(dest))
+
+    assert diagnosis_mod.get_diagnosis().height == df.height
+
+
 def test_get_diagnosis_raises_when_blob_absent(monkeypatch):
     monkeypatch.setattr(diagnosis_mod, "get_blob", lambda name: None)
     with pytest.raises(RemoteNotAvailableError):

{buildingdata-0.3.0 → buildingdata-0.4.0}/buildingdata.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: buildingdata
-Version: 0.3.0
+Version: 0.4.0
 Summary: Data management layer for buildingmodel — reference data download, BDTOPO retrieval, ERA5 weather
 License-Expression: MIT
 Requires-Python: >=3.10

{buildingdata-0.3.0 → buildingdata-0.4.0}/buildingdata.egg-info/SOURCES.txt

@@ -35,6 +35,7 @@ buildingdata/tests/conftest.py
 buildingdata/tests/test_bulk.py
 buildingdata/tests/test_cache.py
 buildingdata/tests/test_config.py
+buildingdata/tests/test_pipeline_diagnosis.py
 buildingdata/tests/test_public_api.py
 buildingdata/tests/test_reference.py
 buildingdata/tests/test_simulation.py

{buildingdata-0.3.0 → buildingdata-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "buildingdata"
-version = "0.3.0"
+version = "0.4.0"
 description = "Data management layer for buildingmodel — reference data download, BDTOPO retrieval, ERA5 weather"
 
 readme = "README.md"

buildingdata-0.3.0/buildingdata/reference/diagnosis.py

@@ -1,74 +0,0 @@
-# -*- coding: utf-8 -*-
-import polars as pl
-
-from ..gcs import download_blob, ensure_blob_cached, get_blob
-from ..validation import require_columns
-
-
-_BLOB_NAME = "energy_performance_diagnosis_latest.parquet"
-
-
-# Heating/DHW energies excluded from inference (no meaningful DPE data for coal)
-_EXCLUDED_ENERGIES = ["Charbon"]
-
-# Columns this accessor operates on directly (coal filter + dtype casts). If any
-# is absent the raw polars error is opaque; validating up front names the DPE
-# schema drift explicitly. ``heating_system`` + ``region`` are the join keys
-# ``buildingmodel``'s energy-system inference keys on; ``backup_heating_energy``
-# / ``dhw_energy`` carry the fuel labels the coal filter reads.
-_REQUIRED_COLUMNS = (
-    "heating_system",
-    "region",
-    "backup_heating_energy",
-    "dhw_energy",
-)
-
-
-def get_diagnosis(refresh=False):
-    """Return the cleaned DPE energy performance diagnosis DataFrame.
-
-    Downloads energy_performance_diagnosis_latest.parquet from GCS on first
-    call. Applies the filtering and type casts that previously lived in
-    buildingmodel/io/diagnosis.py so that buildingmodel receives a clean frame.
-
-    Args:
-        refresh (bool): force re-download even if the cache is warm.
-            Defaults to False.
-
-    Returns:
-        polars.DataFrame: DPE records with columns heating_system (Categorical),
-            region (Int64), backup_heating_energy, dhw_energy, and all U-value
-            and efficiency columns used by inference/building_attributes.py.
-
-    Raises:
-        RemoteNotAvailableError: if the blob is not found in the GCS bucket.
-        SchemaValidationError: if the fetched frame is missing one of the
-            columns this accessor operates on (see ``_REQUIRED_COLUMNS``).
-    """
-    dest = ensure_blob_cached(
-        _BLOB_NAME, refresh=refresh, get_blob_fn=get_blob, download_blob_fn=download_blob
-    )
-
-
-    df = pl.read_parquet(dest)
-
-
-    require_columns(df, _REQUIRED_COLUMNS, "DPE/EPC diagnosis")
-
-    # Remove records with coal heating/DHW — no useful inference data
-    df = df.filter(
-        ~pl.col("backup_heating_energy").is_in(_EXCLUDED_ENERGIES)
-        & ~pl.col("dhw_energy").is_in(_EXCLUDED_ENERGIES)
-    )
-
-    df = df.with_columns([
-        pl.col("heating_system").cast(pl.Categorical),
-        pl.col("region").cast(pl.Int64),
-    ])
-
-    if "living_area" in df.columns:
-        df = df.drop(["living_area"])
-    if "living_area_class" in df.columns:
-        df = df.drop(["living_area_class"])
-
-    return df