buildingdata 0.2.1__tar.gz → 0.2.2__tar.gz

{buildingdata-0.2.1 → buildingdata-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: buildingdata
-Version: 0.2.1
+Version: 0.2.2
 Summary: Data management layer for buildingmodel — reference data download, BDTOPO retrieval, ERA5 weather
 License-Expression: MIT
 Requires-Python: >=3.10

{buildingdata-0.2.1 → buildingdata-0.2.2}/buildingdata/simulation/bdtopo.py

@@ -24,7 +24,19 @@ _REQUIRED_COLUMNS = (
     "usage_1",
 )
 
-# IGN Géoplateforme WFS endpoint for BDTOPO buildings
+# IGN Géoplateforme WFS endpoint for BDTOPO buildings.
+#
+# WFS 2.0.0 paging contract (confirmed empirically against the live server on
+# 2026-08-04, feature type BDTOPO_V3:batiment, OUTPUTFORMAT json):
+#   * The FeatureCollection response carries top-level integer attributes
+#     ``numberMatched`` (total features matching the query/BBOX) and
+#     ``numberReturned`` (features in *this* page).
+#   * With no ``COUNT`` the server silently caps the page at 5000 features
+#     (numberReturned=5000 while numberMatched can be far larger) -- the exact
+#     silent-truncation trap this loader now pages around.
+#   * ``COUNT`` (page size) and ``STARTINDEX`` (0-based offset) are both
+#     honoured; consecutive 5000-feature pages have zero id overlap.
+# 5000 is the server's maximum page size, so we page in 5000-feature strides.
 _IGN_BASE_URL = (
     "https://data.geopf.fr/wfs/ows"
     "?SERVICE=WFS"
@@ -34,10 +46,88 @@ _IGN_BASE_URL = (
     "&OUTPUTFORMAT=application/json"
     "&SRSNAME=EPSG:2154"
     "&BBOX={minx},{miny},{maxx},{maxy},urn:ogc:def:crs:EPSG::2154"
+    "&COUNT={count}"
+    "&STARTINDEX={startindex}"
 )
 
 _TIMEOUT = 120  # seconds
 
+# Server-side maximum WFS page size. Requesting more is silently capped here.
+_PAGE_SIZE = 5000
+
+
+def _fetch_all_wfs_features(base_url, iris_code):
+    """Page a BDTOPO WFS GetFeature query to completion.
+
+    Issues ``GetFeature`` requests with ``COUNT``/``STARTINDEX`` and walks the
+    result set until every matching feature has been collected, so a BBOX with
+    more than one page (>5000) of buildings is no longer silently truncated.
+
+    Args:
+        base_url (str): URL template already formatted with the BBOX; must still
+            contain ``{count}`` and ``{startindex}`` placeholders.
+        iris_code (str): IRIS code, for error messages only.
+
+    Returns:
+        list[dict]: all GeoJSON features across every page.
+
+    Raises:
+        RemoteNotAvailableError: on a non-200 status, or if the server reports
+            more matches than we could retrieve (paging could not complete) --
+            truncation must never again pass silently.
+    """
+    features = []
+    startindex = 0
+    number_matched = None
+
+    # Hard iteration ceiling so a WFS that ignores STARTINDEX (and would return
+    # the same first page forever) fails fast instead of looping indefinitely.
+    max_pages = 10_000
+    for _ in range(max_pages):
+        url = base_url.format(count=_PAGE_SIZE, startindex=startindex)
+        response = requests.get(url, timeout=_TIMEOUT)
+        if response.status_code != 200:
+            raise RemoteNotAvailableError(
+                f"IGN Géoplateforme returned HTTP {response.status_code} "
+                f"for IRIS {iris_code!r} (startindex={startindex})."
+            )
+
+        payload = response.json()
+        page = payload.get("features", [])
+        features.extend(page)
+
+        # numberReturned/numberMatched are the WFS 2.0 paging cursor. Fall back
+        # to the page length when a (mocked/legacy) response omits them.
+        number_returned = payload.get("numberReturned", len(page))
+        if payload.get("numberMatched") is not None:
+            number_matched = payload["numberMatched"]
+
+        # Stop once we've drained the result set: a short page, an empty page,
+        # or having collected everything the server said it had.
+        if number_returned < _PAGE_SIZE or number_returned == 0:
+            break
+        if number_matched is not None and len(features) >= number_matched:
+            break
+
+        startindex += _PAGE_SIZE
+    else:
+        raise RemoteNotAvailableError(
+            f"IGN Géoplateforme paging for IRIS {iris_code!r} exceeded "
+            f"{max_pages} pages; the WFS may be ignoring STARTINDEX."
+        )
+
+    # Defensive truncation guard: if the server advertised more matches than we
+    # actually retrieved, surface it loudly rather than returning a partial,
+    # under-populated building stock (the original silent-drop bug).
+    if number_matched is not None and len(features) < number_matched:
+        raise RemoteNotAvailableError(
+            f"IGN Géoplateforme reported numberMatched={number_matched} for "
+            f"IRIS {iris_code!r} but paging only retrieved {len(features)} "
+            f"features -- refusing to return a truncated building stock."
+        )
+
+    return features
+
 
 def _cache_path(iris_code):
     subdir = ensure_subdir("bdtopo")
@@ -94,14 +184,13 @@ def get_bdtopo(iris_code, refresh=False):
         district_geom = iris_geom.iloc[0].geometry
         bbox = district_geom.bounds  # (minx, miny, maxx, maxy)
 
-        url = _IGN_BASE_URL.format(minx=bbox[0], miny=bbox[1], maxx=bbox[2], maxy=bbox[3])
-        response = requests.get(url, timeout=_TIMEOUT)
-        if response.status_code != 200:
-            raise RemoteNotAvailableError(
-                f"IGN Géoplateforme returned HTTP {response.status_code} for IRIS {iris_code!r}."
-            )
-
-        features = response.json()["features"]
+        # Note ``{count}``/``{startindex}`` are left unformatted here; the
+        # paginator fills them in per page.
+        base_url = _IGN_BASE_URL.format(
+            minx=bbox[0], miny=bbox[1], maxx=bbox[2], maxy=bbox[3],
+            count="{count}", startindex="{startindex}",
+        )
+        features = _fetch_all_wfs_features(base_url, iris_code)
         if not features:
             raise RemoteNotAvailableError(
                 f"IGN Géoplateforme returned no buildings for IRIS {iris_code!r}."

{buildingdata-0.2.1 → buildingdata-0.2.2}/buildingdata/tests/test_simulation.py

@@ -157,6 +157,103 @@ def test_get_bdtopo_raises_when_no_features(monkeypatch):
         bdtopo_mod.get_bdtopo(iris_code)
 
 
+def test_get_bdtopo_paginates_past_the_5000_feature_cap(monkeypatch):
+    """A dense BBOX returns every page, not the WFS's silent first-page cap.
+
+    Regression for the silent-truncation bug: the IGN WFS caps an un-paged
+    ``GetFeature`` at 5000 features while reporting the true total in
+    ``numberMatched``. We simulate a 12 001-feature result over four pages and
+    assert the loader stitches them all together instead of stopping at 5000.
+    """
+    iris_code = "641320000"
+    total = 12_001  # spans 3 full 5000-pages + a 2001-feature tail
+    page_size = bdtopo_mod._PAGE_SIZE
+
+    def _page(startindex):
+        n = max(0, min(page_size, total - startindex))
+        feats = [
+            {
+                "type": "Feature",
+                # Inside the _mock_districts box so it survives the
+                # downstream polygon-intersection filter.
+                "geometry": {"type": "Point", "coordinates": [652000.0, 6862200.0]},
+                "properties": {
+                    "hauteur": 9.0,
+                    "nombre_de_logements": 4,
+                    "usage_1": "Résidentiel",
+                },
+            }
+            for _ in range(n)
+        ]
+        return {"numberMatched": total, "numberReturned": n, "features": feats}
+
+    calls = {"startindices": []}
+
+    def fake_get(url, timeout=None):
+        # STARTINDEX is the last query param in the template.
+        startindex = int(url.rsplit("STARTINDEX=", 1)[1])
+        calls["startindices"].append(startindex)
+        return FakeResponse(200, _page(startindex))
+
+    monkeypatch.setattr(
+        "buildingdata.simulation.bdtopo.get_districts",
+        lambda: _mock_districts(iris_code),
+    )
+    monkeypatch.setattr(bdtopo_mod.requests, "get", fake_get)
+
+    gdf = bdtopo_mod.get_bdtopo(iris_code)
+
+    # All pages were walked (offsets 0, 5000, 10000) and every feature kept.
+    assert calls["startindices"] == [0, 5000, 10000]
+    assert len(gdf) == total  # not truncated to page_size
+
+
+def test_fetch_all_wfs_features_raises_when_paging_cannot_complete(monkeypatch):
+    """If the server advertises more than we can retrieve, fail loud."""
+    def short_page(url, timeout=None):
+        # Claims 8000 matches but only ever yields a 5000-feature page, and a
+        # short second page -- i.e. paging is broken and truncated.
+        startindex = int(url.rsplit("STARTINDEX=", 1)[1])
+        n = 5000 if startindex == 0 else 100
+        return FakeResponse(
+            200,
+            {"numberMatched": 8000, "numberReturned": n, "features": [{}] * n},
+        )
+
+    monkeypatch.setattr(bdtopo_mod.requests, "get", short_page)
+    with pytest.raises(RemoteNotAvailableError, match="truncated building stock"):
+        bdtopo_mod._fetch_all_wfs_features(
+            "http://x?BBOX=0,0,1,1&COUNT={count}&STARTINDEX={startindex}", "641320000"
+        )
+
+
+@pytest.mark.network
+def test_get_bdtopo_live_wfs_returns_full_numbermatched():
+    """Live IGN WFS: a dense IRIS BBOX must return its full ``numberMatched``.
+
+    Opt-in (needs network): run with ``-m network``. Probes the true match
+    count with a COUNT=1 request, then asserts the paginator retrieves all of
+    them (>5000, i.e. past the silent cap) for the same BBOX.
+    """
+    # A dense BBOX in Lambert-93 (EPSG:2154) known to exceed one 5000-page.
+    bbox = "417000,6420000,420000,6423000"
+    base = (
+        "https://data.geopf.fr/wfs/ows?SERVICE=WFS&VERSION=2.0.0"
+        "&REQUEST=GetFeature&TYPENAMES=BDTOPO_V3:batiment"
+        "&OUTPUTFORMAT=application/json&SRSNAME=EPSG:2154"
+        f"&BBOX={bbox},urn:ogc:def:crs:EPSG::2154"
+    )
+    import requests as _rq
+
+    probe = _rq.get(base + "&COUNT=1&STARTINDEX=0", timeout=120).json()
+    number_matched = probe["numberMatched"]
+    assert number_matched > bdtopo_mod._PAGE_SIZE, "pick a denser BBOX"
+
+    tmpl = base + "&COUNT={count}&STARTINDEX={startindex}"
+    features = bdtopo_mod._fetch_all_wfs_features(tmpl, "dense-bbox")
+    assert len(features) == number_matched
+
+
 # --- get_era5_climate -------------------------------------------------------
 
 def test_era5_snap_to_grid_and_coordinate_formatting():

{buildingdata-0.2.1 → buildingdata-0.2.2}/buildingdata.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: buildingdata
-Version: 0.2.1
+Version: 0.2.2
 Summary: Data management layer for buildingmodel — reference data download, BDTOPO retrieval, ERA5 weather
 License-Expression: MIT
 Requires-Python: >=3.10

{buildingdata-0.2.1 → buildingdata-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "buildingdata"
-version = "0.2.1"
+version = "0.2.2"
 description = "Data management layer for buildingmodel — reference data download, BDTOPO retrieval, ERA5 weather"
 
 readme = "README.md"
@@ -53,3 +53,6 @@ exclude = ["pipeline*"]
 
 [tool.pytest.ini_options]
 testpaths = ["buildingdata/tests"]
+markers = [
+    "network: test reaches a live external endpoint (e.g. the IGN WFS); opt-in, deselect with -m 'not network'.",
+]