udata-hydra 2.2.2.dev7533__tar.gz → 2.2.2.dev7551__tar.gz

{udata_hydra-2.2.2.dev7533 → udata_hydra-2.2.2.dev7551}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: udata-hydra
-Version: 2.2.2.dev7533
+Version: 2.2.2.dev7551
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team
@@ -18,7 +18,7 @@ Requires-Dist: aioresponses (>=0.7.3) ; extra == "dev"
 Requires-Dist: asyncpg (>=0.29.0)
 Requires-Dist: bumpx (>=0.3.10) ; extra == "dev"
 Requires-Dist: coloredlogs (>=15.0.1)
-Requires-Dist: csv-detective (==0.8.0)
+Requires-Dist: csv-detective (==0.8.1.dev1549)
 Requires-Dist: dateparser (>=1.1.7)
 Requires-Dist: gunicorn (>=20.1.0) ; extra == "dev"
 Requires-Dist: humanfriendly (>=10.0)

{udata_hydra-2.2.2.dev7533 → udata_hydra-2.2.2.dev7551}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "udata-hydra"
-version = "2.2.2.dev7533"
+version = "2.2.2.dev7551"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = [{ name = "Opendata Team", email = "opendatateam@data.gouv.fr" }]
 dependencies = [
@@ -8,7 +8,7 @@ dependencies = [
     "aiohttp>=3.10.3",
     "asyncpg>=0.29.0",
     "coloredlogs>=15.0.1",
-    "csv-detective==0.8.0",
+    "csv-detective==0.8.1.dev1549",
     "dateparser>=1.1.7",
     "humanfriendly>=10.0",
     "marshmallow>=3.14.1",

{udata_hydra-2.2.2.dev7533 → udata_hydra-2.2.2.dev7551}/udata_hydra/analysis/csv.py

@@ -68,6 +68,7 @@ PYTHON_TYPE_TO_PG = {
     "json": JSON,
     "date": Date,
     "datetime": DateTime,
+    "datetime_aware": DateTime(timezone=True),
 }
 
 PYTHON_TYPE_TO_PY = {
@@ -78,6 +79,7 @@ PYTHON_TYPE_TO_PY = {
     "json": helpers.to_json,
     "date": helpers.to_date,
     "datetime": helpers.to_datetime,
+    "datetime_aware": helpers.to_datetime,
 }
 
 RESERVED_COLS = ("__id", "cmin", "cmax", "collation", "ctid", "tableoid", "xmin", "xmax")
@@ -376,7 +378,7 @@ async def csv_to_db(
 
     # build a `column_name: type` mapping and explicitely rename reserved column names
     columns = {
-        f"{c}__hydra_renamed" if c.lower() in RESERVED_COLS else c: v["python_type"]
+        f"{c}__hydra_renamed" if c.lower() in RESERVED_COLS else c: helpers.get_python_type(v)
         for c, v in inspection["columns"].items()
     }
 

{udata_hydra-2.2.2.dev7533 → udata_hydra-2.2.2.dev7551}/udata_hydra/analysis/helpers.py

@@ -16,7 +16,7 @@ def to_json(value: str) -> str:
     return value
 
 
-def _parse_dt(value: str) -> datetime | None:
+def to_datetime(value: str) -> datetime | None:
     """For performance reasons, we try first with dateutil and fallback on dateparser"""
     try:
         return dateutil_parser(value)
@@ -25,12 +25,17 @@ def _parse_dt(value: str) -> datetime | None:
 
 
 def to_date(value: str) -> date | None:
-    parsed = _parse_dt(value)
+    parsed = to_datetime(value)
     return parsed.date() if parsed else None
 
 
-def to_datetime(value: str) -> datetime | None:
-    return _parse_dt(value)
+def get_python_type(column: dict) -> str:
+    """Outsourcing the distinction of aware datetimes"""
+    return (
+        "datetime_aware"
+        if column["format"] in {"datetime_aware", "datetime_rfc822"}
+        else column["python_type"]
+    )
 
 
 async def read_or_download_file(

udata_hydra-2.2.2.dev7551/udata_hydra/migrations/csv/20250626_delete_datetime_iso_references.sql

@@ -0,0 +1,8 @@
+-- the `datetime_iso` format is not in csv-detective anymore (https://github.com/datagouv/csv-detective/pull/132)
+-- to prevent crashes, we replace all references to this format with `datetime_aware`
+-- NB: it doesn't matter if it's the right format, if it's not, validation will fail
+-- and a new analysis will find the right format
+
+UPDATE tables_index
+SET csv_detective = replace(csv_detective::text, 'datetime_iso', 'datetime_aware')::jsonb
+WHERE csv_detective::text LIKE '%datetime_iso%';