udata-hydra 2.2.2.dev7633__tar.gz → 2.2.2.dev7667__tar.gz

{udata_hydra-2.2.2.dev7633 → udata_hydra-2.2.2.dev7667}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: udata-hydra
-Version: 2.2.2.dev7633
+Version: 2.2.2.dev7667
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team
@@ -38,7 +38,7 @@ Requires-Dist: python-magic (>=0.4.25)
 Requires-Dist: python-slugify (>=8.0.4)
 Requires-Dist: redis (>=4.1.4)
 Requires-Dist: rq (>=1.11.1)
-Requires-Dist: ruff (>=0.5.7) ; extra == "dev"
+Requires-Dist: ruff (>=0.9.3) ; extra == "dev"
 Requires-Dist: sentry-sdk (>=2.10.0)
 Requires-Dist: setuptools (>=70.3.0)
 Requires-Dist: sqlalchemy (>=1.4.46)

{udata_hydra-2.2.2.dev7633 → udata_hydra-2.2.2.dev7667}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "udata-hydra"
-version = "2.2.2.dev7633"
+version = "2.2.2.dev7667"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = [{ name = "Opendata Team", email = "opendatateam@data.gouv.fr" }]
 dependencies = [
@@ -45,7 +45,7 @@ dev = [
     "pytest-asyncio>=0.18.3",
     "pytest-cov>=5.0.0",
     "pytest-mock>=3.7.0",
-    "ruff>=0.5.7",
+    "ruff>=0.9.3",
 ]
 
 [tool.mypy]

{udata_hydra-2.2.2.dev7633 → udata_hydra-2.2.2.dev7667}/udata_hydra/analysis/csv.py

@@ -5,7 +5,6 @@ import logging
 import os
 import sys
 from datetime import datetime, timezone
-from math import isnan
 from typing import Iterator
 
 import pandas as pd
@@ -33,6 +32,7 @@ from sqlalchemy.schema import CreateIndex, CreateTable, Index
 
 from udata_hydra import config, context
 from udata_hydra.analysis import helpers
+from udata_hydra.analysis.geojson import csv_to_geojson_and_pmtiles
 from udata_hydra.db import compute_insert_query
 from udata_hydra.db.check import Check
 from udata_hydra.db.resource import Resource
@@ -43,6 +43,7 @@ from udata_hydra.utils import (
     Timer,
     detect_tabular_from_headers,
     handle_parse_exception,
+    remove_remainders,
 )
 from udata_hydra.utils.minio import MinIOClient
 from udata_hydra.utils.parquet import save_as_parquet
@@ -160,10 +161,24 @@ async def analyse_csv(
             )
             timer.mark("csv-to-parquet")
         except Exception as e:
+            remove_remainders(resource_id, ["parquet"])
             raise ParseException(
                 step="parquet_export", resource_id=resource_id, url=url, check_id=check["id"]
             ) from e
 
+        try:
+            geojson_args: tuple[str, int, str, int] | None = await csv_to_geojson_and_pmtiles(
+                df=df,
+                inspection=csv_inspection,
+                resource_id=resource_id,
+            )
+            timer.mark("csv-to-geojson-pmtiles")
+        except Exception as e:
+            remove_remainders(resource_id, ["geojson", "pmtiles", "pmtiles-journal"])
+            raise ParseException(
+                step="geojson_export", resource_id=resource_id, url=url, check_id=check["id"]
+            ) from e
+
         check = await Check.update(
             check["id"],
             {
@@ -171,6 +186,10 @@ async def analyse_csv(
                 "parsing_finished_at": datetime.now(timezone.utc),
                 "parquet_url": parquet_args[0] if parquet_args else None,
                 "parquet_size": parquet_args[1] if parquet_args else None,
+                "geojson_url": geojson_args[0] if geojson_args else None,
+                "geojson_size": geojson_args[1] if geojson_args else None,
+                "pmtiles_url": geojson_args[2] if geojson_args else None,
+                "pmtiles_size": geojson_args[3] if geojson_args else None,
             },
         )
         await csv_to_db_index(table_name, csv_inspection, check)

udata_hydra-2.2.2.dev7667/udata_hydra/analysis/geojson.py

@@ -0,0 +1,250 @@
+import json
+import logging
+import os
+from datetime import datetime, timezone
+
+import pandas as pd
+import tippecanoe
+from asyncpg import Record
+
+from udata_hydra import config
+from udata_hydra.analysis import helpers
+from udata_hydra.db.check import Check
+from udata_hydra.db.resource import Resource
+from udata_hydra.db.resource_exception import ResourceException
+from udata_hydra.utils import (
+    IOException,
+    ParseException,
+    Timer,
+    handle_parse_exception,
+)
+from udata_hydra.utils.minio import MinIOClient
+
+log = logging.getLogger("udata-hydra")
+minio_client_pmtiles = MinIOClient(
+    bucket=config.MINIO_PMTILES_BUCKET, folder=config.MINIO_PMTILES_FOLDER
+)
+minio_client_geojson = MinIOClient(
+    bucket=config.MINIO_GEOJSON_BUCKET, folder=config.MINIO_GEOJSON_FOLDER
+)
+
+
+async def analyse_geojson(
+    check: dict,
+    file_path: str | None = None,
+) -> None:
+    """Launch GeoJSON analysis from a check or an URL (debug), using previously downloaded file at file_path if any"""
+    if not config.GEOJSON_TO_PMTILES:
+        log.debug("GEOJSON_TO_PMTILES turned off, skipping.")
+        return
+
+    resource_id: str = str(check["resource_id"])
+    url = check["url"]
+
+    # Update resource status to ANALYSING_GEOJSON
+    resource: Record | None = await Resource.update(resource_id, {"status": "ANALYSING_GEOJSON"})
+
+    # Check if the resource is in the exceptions table
+    exception: Record | None = await ResourceException.get_by_resource_id(resource_id)
+
+    timer = Timer("analyse-geojson")
+    assert any(_ is not None for _ in (check["id"], url))
+
+    tmp_file = None
+    try:
+        tmp_file = await helpers.read_or_download_file(
+            check=check,
+            file_path=file_path,
+            file_format="geojson",
+            exception=exception,
+        )
+        timer.mark("download-file")
+
+        check = await Check.update(check["id"], {"parsing_started_at": datetime.now(timezone.utc)})
+
+        # Convert to PMTiles
+        try:
+            pmtiles_url, pmtiles_size = await geojson_to_pmtiles(
+                file_path=tmp_file.name,
+                resource_id=resource_id,
+            )
+            timer.mark("geojson-to-pmtiles")
+        except Exception as e:
+            raise ParseException(
+                step="pmtiles_export", resource_id=resource_id, url=url, check_id=check["id"]
+            ) from e
+
+        check = await Check.update(
+            check["id"],
+            {
+                "parsing_finished_at": datetime.now(timezone.utc),
+                "pmtiles_url": pmtiles_url,
+                "pmtiles_size": pmtiles_size,
+            },
+        )
+
+    except (ParseException, IOException) as e:
+        await handle_parse_exception(e, None, check)
+    finally:
+        await helpers.notify_udata(resource, check)
+        timer.stop()
+        if tmp_file is not None:
+            tmp_file.close()
+            os.remove(tmp_file.name)
+
+        # Reset resource status to None
+        await Resource.update(resource_id, {"status": None})
+
+
+async def geojson_to_pmtiles(
+    file_path: str,
+    resource_id: str | None = None,
+) -> tuple[str, int]:
+    """
+    Convert a GeoJSON file to PMTiles format.
+
+    Args:
+        file_path: GeoJSON file path to convert.
+        resource_id: Optional resource ID for status updates.
+
+    Returns:
+        pmtiles_url: URL of the PMTiles file.
+        pmtiles_size: size of the PMTiles file.
+    """
+
+    log.debug(f"Converting GeoJSON to PMTiles for {file_path}")
+
+    if resource_id:
+        await Resource.update(resource_id, {"status": "CONVERTING_TO_PMTILES"})
+
+    output_pmtiles = f"{resource_id}.pmtiles"
+
+    command = [
+        "--maximum-zoom=g",  # guess
+        "-o",
+        output_pmtiles,
+        "--coalesce-densest-as-needed",
+        "--extend-zooms-if-still-dropping",
+        file_path,
+    ]
+    exit_code = tippecanoe._program("tippecanoe", *command)
+    if exit_code:
+        raise ValueError(f"GeoJSON to PMTiles conversion failed for {file_path}")
+    log.debug(f"Successfully converted {file_path} to {output_pmtiles}")
+
+    pmtiles_size = os.path.getsize(output_pmtiles)
+    pmtiles_url: str = minio_client_pmtiles.send_file(output_pmtiles)
+
+    return pmtiles_url, pmtiles_size
+
+
+async def csv_to_geojson_and_pmtiles(
+    df: pd.DataFrame,
+    inspection: dict,
+    resource_id: str | None = None,
+) -> tuple[str, int, str, int] | None:
+    def cast_latlon(latlon: str) -> list[float, float]:
+        # we can safely do this as the detection was successful
+        lat, lon = latlon.replace(" ", "").split(",")
+        # using the geojson standard: longitude before latitude
+        return [float(lon), float(lat)]
+
+    def prevent_nan(value):
+        # convenience to prevent downstream crash (NaN in json or PMtiles)
+        if pd.isna(value):
+            return None
+        return value
+
+    if not config.CSV_TO_GEOJSON:
+        log.debug("CSV_TO_GEOJSON turned off, skipping geojson/PMtiles export.")
+        return
+
+    log.debug(
+        f"Converting to geojson and PMtiles if relevant for {resource_id} and sending to Minio."
+    )
+
+    geo = {}
+    for column, detection in inspection["columns"].items():
+        # see csv-detective's geo formats:
+        # https://github.com/datagouv/csv-detective/tree/master/csv_detective/detect_fields/geo
+        if "geojson" in detection["format"]:
+            geo["geometry"] = column
+            break
+        if "latlon" in detection["format"]:
+            geo["latlon"] = column
+            break
+        if "latitude" in detection["format"]:
+            geo["lat"] = column
+        if "longitude" in detection["format"]:
+            geo["lon"] = column
+    # priority is given to geometry, then latlon, then latitude + longitude
+    if "geometry" in geo:
+        geo = {"geometry": geo["geometry"]}
+    if "latlon" in geo:
+        geo = {"latlon": geo["latlon"]}
+    if not geo or (("lat" in geo and "lon" not in geo) or ("lon" in geo and "lat" not in geo)):
+        log.debug("No geographical columns found, skipping")
+        return None
+
+    if resource_id:
+        await Resource.update(resource_id, {"status": "CONVERTING_TO_GEOJSON"})
+
+    template = {"type": "FeatureCollection", "features": []}
+    for _, row in df.iterrows():
+        if "geometry" in geo:
+            template["features"].append(
+                {
+                    "type": "Feature",
+                    # json is not pre-cast by csv-detective
+                    "geometry": json.loads(row[geo["geometry"]]),
+                    "properties": {
+                        col: prevent_nan(row[col]) for col in df.columns if col != geo["geometry"]
+                    },
+                }
+            )
+        elif "latlon" in geo:
+            # ending up here means we either have the exact lat,lon format, or NaN
+            # skipping row if NaN
+            if pd.isna(row[geo["latlon"]]):
+                continue
+            template["features"].append(
+                {
+                    "type": "Feature",
+                    "geometry": {
+                        "type": "Point",
+                        "coordinates": cast_latlon(row[geo["latlon"]]),
+                    },
+                    "properties": {
+                        col: prevent_nan(row[col]) for col in df.columns if col != geo["latlon"]
+                    },
+                }
+            )
+        else:
+            # skipping row if lat or lon is NaN
+            if any(pd.isna(coord) for coord in (row[geo["lon"]], row[geo["lat"]])):
+                continue
+            template["features"].append(
+                {
+                    "type": "Feature",
+                    "geometry": {
+                        "type": "Point",
+                        # these columns are precast by csv-detective
+                        "coordinates": [row[geo["lon"]], row[geo["lat"]]],
+                    },
+                    "properties": {
+                        col: prevent_nan(row[col])
+                        for col in df.columns
+                        if col not in [geo["lon"], geo["lat"]]
+                    },
+                }
+            )
+    geojson_file = f"{resource_id}.geojson"
+    with open(geojson_file, "w") as f:
+        json.dump(template, f, indent=4, ensure_ascii=False, default=str)
+    geojson_size = os.path.getsize(geojson_file)
+
+    pmtiles_url, pmtiles_size = await geojson_to_pmtiles(geojson_file, resource_id)
+
+    geojson_url: str = minio_client_geojson.send_file(geojson_file)
+
+    return geojson_url, geojson_size, pmtiles_url, pmtiles_size

{udata_hydra-2.2.2.dev7633 → udata_hydra-2.2.2.dev7667}/udata_hydra/config_default.toml

@@ -87,3 +87,8 @@ MINIO_PARQUET_FOLDER = "" # no trailing slash
 GEOJSON_TO_PMTILES = false
 MINIO_PMTILES_BUCKET = ""
 MINIO_PMTILES_FOLDER = "" # no trailing slash
+
+# -- Geojson conversion settings -- #
+CSV_TO_GEOJSON = false
+MINIO_GEOJSON_BUCKET = ""
+MINIO_GEOJSON_FOLDER = "" # no trailing slash

udata_hydra-2.2.2.dev7667/udata_hydra/migrations/main/20250615_add_geojson_fields.sql

@@ -0,0 +1,5 @@
+-- Add PMTiles fields to checks table
+
+ALTER TABLE checks
+    ADD COLUMN geojson_url VARCHAR,
+    ADD COLUMN geojson_size BIGINT;

{udata_hydra-2.2.2.dev7633 → udata_hydra-2.2.2.dev7667}/udata_hydra/routes/status.py

@@ -147,5 +147,10 @@ async def get_health(request: web.Request) -> web.Response:
         {
             "version": config.APP_VERSION,
             "environment": config.ENVIRONMENT or "unknown",
+            "csv_analysis": config.CSV_ANALYSIS,
+            "csv_to_db": config.CSV_TO_DB,
+            "csv_to_parquet": config.CSV_TO_PARQUET,
+            "geojson_to_pmtiles": config.GEOJSON_TO_PMTILES,
+            "csv_to_geojson": config.CSV_TO_GEOJSON,
         }
     )

{udata_hydra-2.2.2.dev7633 → udata_hydra-2.2.2.dev7667}/udata_hydra/utils/__init__.py

@@ -1,7 +1,7 @@
 from .auth import token_auth_middleware
 from .csv import detect_tabular_from_headers
 from .errors import IOException, ParseException, handle_parse_exception
-from .file import compute_checksum_from_file, download_resource
+from .file import compute_checksum_from_file, download_resource, read_csv_gz, remove_remainders
 from .geojson import detect_geojson_from_headers_or_catalog
 from .http import UdataPayload, get_request_params, send
 from .queue import enqueue
@@ -15,6 +15,8 @@ __all__ = [
     "handle_parse_exception",
     "compute_checksum_from_file",
     "download_resource",
+    "read_csv_gz",
+    "remove_remainders",
     "detect_geojson_from_headers_or_catalog",
     "UdataPayload",
     "get_request_params",

{udata_hydra-2.2.2.dev7633 → udata_hydra-2.2.2.dev7667}/udata_hydra/utils/file.py

@@ -80,3 +80,10 @@ async def download_resource(
         ]:
             tmp_file = read_csv_gz(tmp_file.name)
         return tmp_file
+
+
+def remove_remainders(resource_id: str, extensions: list[str]) -> None:
+    """Delete potential remainders from process that crashed"""
+    for ext in extensions:
+        if os.path.exists(f"{resource_id}.{ext}"):
+            os.remove(f"{resource_id}.{ext}")

udata_hydra-2.2.2.dev7633/udata_hydra/analysis/geojson.py

@@ -1,131 +0,0 @@
-import logging
-import os
-from datetime import datetime, timezone
-
-import tippecanoe
-from asyncpg import Record
-
-from udata_hydra import config
-from udata_hydra.analysis import helpers
-from udata_hydra.db.check import Check
-from udata_hydra.db.resource import Resource
-from udata_hydra.db.resource_exception import ResourceException
-from udata_hydra.utils import (
-    IOException,
-    ParseException,
-    Timer,
-    handle_parse_exception,
-)
-from udata_hydra.utils.minio import MinIOClient
-
-log = logging.getLogger("udata-hydra")
-minio_client = MinIOClient(bucket=config.MINIO_PMTILES_BUCKET, folder=config.MINIO_PMTILES_FOLDER)
-
-
-async def analyse_geojson(
-    check: dict,
-    file_path: str | None = None,
-) -> None:
-    """Launch GeoJSON analysis from a check or an URL (debug), using previously downloaded file at file_path if any"""
-    if not config.GEOJSON_TO_PMTILES:
-        log.debug("GEOJSON_TO_PMTILES turned off, skipping.")
-        return
-
-    resource_id: str = str(check["resource_id"])
-    url = check["url"]
-
-    # Update resource status to ANALYSING_GEOJSON
-    resource: Record | None = await Resource.update(resource_id, {"status": "ANALYSING_GEOJSON"})
-
-    # Check if the resource is in the exceptions table
-    exception: Record | None = await ResourceException.get_by_resource_id(resource_id)
-
-    timer = Timer("analyse-geojson")
-    assert any(_ is not None for _ in (check["id"], url))
-
-    tmp_file = None
-    try:
-        tmp_file = await helpers.read_or_download_file(
-            check=check,
-            file_path=file_path,
-            file_format="geojson",
-            exception=exception,
-        )
-        timer.mark("download-file")
-
-        check = await Check.update(check["id"], {"parsing_started_at": datetime.now(timezone.utc)})
-
-        # Convert to PMTiles
-        try:
-            pmtiles_url, pmtiles_size = await geojson_to_pmtiles(
-                file_path=tmp_file.name,
-                resource_id=resource_id,
-            )
-            timer.mark("geojson-to-pmtiles")
-        except Exception as e:
-            raise ParseException(
-                step="pmtiles_export", resource_id=resource_id, url=url, check_id=check["id"]
-            ) from e
-
-        check = await Check.update(
-            check["id"],
-            {
-                "parsing_finished_at": datetime.now(timezone.utc),
-                "pmtiles_url": pmtiles_url,
-                "pmtiles_size": pmtiles_size,
-            },
-        )
-
-    except (ParseException, IOException) as e:
-        await handle_parse_exception(e, None, check)
-    finally:
-        await helpers.notify_udata(resource, check)
-        timer.stop()
-        if tmp_file is not None:
-            tmp_file.close()
-            os.remove(tmp_file.name)
-
-        # Reset resource status to None
-        await Resource.update(resource_id, {"status": None})
-
-
-async def geojson_to_pmtiles(
-    file_path: str,
-    resource_id: str | None = None,
-) -> tuple[str, int]:
-    """
-    Convert a GeoJSON file to PMTiles format.
-
-    Args:
-        file_path: GeoJSON file path to convert.
-        resource_id: Optional resource ID for status updates.
-
-    Returns:
-        pmtiles_url: URL of the PMTiles file.
-        pmtiles_size: size of the PMTiles file.
-    """
-
-    log.debug(f"Converting GeoJSON to PMTiles for {file_path}")
-
-    if resource_id:
-        await Resource.update(resource_id, {"status": "CONVERTING_TO_PMTILES"})
-
-    output_pmtiles = f"{resource_id}.pmtiles"
-
-    command = [
-        "--maximum-zoom=g",  # guess
-        "-o",
-        output_pmtiles,
-        "--coalesce-densest-as-needed",
-        "--extend-zooms-if-still-dropping",
-        file_path,
-    ]
-    exit_code = tippecanoe._program("tippecanoe", *command)
-    if exit_code:
-        raise ValueError(f"GeoJSON to PMTiles conversion failed for {file_path}")
-    log.debug(f"Successfully converted {file_path} to {output_pmtiles}")
-
-    pmtiles_size = os.path.getsize(output_pmtiles)
-    pmtiles_url: str = minio_client.send_file(output_pmtiles)
-
-    return pmtiles_url, pmtiles_size