udata-hydra 2.2.1.dev7337__tar.gz → 2.2.1.dev7367__tar.gz

{udata_hydra-2.2.1.dev7337 → udata_hydra-2.2.1.dev7367}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: udata-hydra
-Version: 2.2.1.dev7337
+Version: 2.2.1.dev7367
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team

{udata_hydra-2.2.1.dev7337 → udata_hydra-2.2.1.dev7367}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "udata-hydra"
-version = "2.2.1.dev7337"
+version = "2.2.1.dev7367"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = [{ name = "Opendata Team", email = "opendatateam@data.gouv.fr" }]
 dependencies = [

{udata_hydra-2.2.1.dev7337 → udata_hydra-2.2.1.dev7367}/udata_hydra/analysis/geojson.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import subprocess
 from datetime import datetime, timezone
 
 import tippecanoe

{udata_hydra-2.2.1.dev7337 → udata_hydra-2.2.1.dev7367}/udata_hydra/cli.py

@@ -1,7 +1,7 @@
 import csv
 import logging
 import os
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 
@@ -15,6 +15,7 @@ from progressist import ProgressBar
 from udata_hydra import config
 from udata_hydra.analysis.csv import analyse_csv
 from udata_hydra.analysis.geojson import analyse_geojson
+from udata_hydra.analysis.resource import analyse_resource
 from udata_hydra.crawl.check_resources import check_resource as crawl_check_resource
 from udata_hydra.db.check import Check
 from udata_hydra.db.resource import Resource
@@ -159,6 +160,16 @@ async def check_resource(resource_id: str, method: str = "get", force_analysis:
         )
 
 
+@cli(name="analyse-resource")
+async def analyse_resource_cli(resource_id: str):
+    """Trigger a resource analysis, mainly useful for local debug (with breakpoints)"""
+    check: Record | None = await Check.get_by_resource_id(resource_id)
+    if not check:
+        log.error("Could not find a check linked to the specified resource ID")
+        return
+    await analyse_resource(check=check, last_check=None, force_analysis=True)
+
+
 @cli(name="analyse-csv")
 async def analyse_csv_cli(
     check_id: str | None = None,
@@ -349,23 +360,27 @@ async def purge_csv_tables(quiet: bool = False) -> None:
     ON checks.parsing_table = md5(c.url)
     WHERE checks.parsing_table IS NOT NULL AND (c.id IS NULL OR c.deleted = TRUE);
     """
-    conn = await connection()
-    res: list[Record] = await conn.fetch(q)
+    conn_main = await connection()
+    res: list[Record] = await conn_main.fetch(q)
     tables_to_delete: list[str] = [r["parsing_table"] for r in res]
 
     success_count = 0
     error_count = 0
 
+    conn_csv = await connection(db_name="csv")
     for table in tables_to_delete:
         try:
-            async with conn.transaction():
-                log.debug(f'Deleting table "{table}"')
-                await conn.execute(f'DROP TABLE IF EXISTS "{table}"')
-                await conn.execute("DELETE FROM tables_index WHERE parsing_table = $1", table)
-                await conn.execute(
-                    "UPDATE checks SET parsing_table = NULL WHERE parsing_table = $1", table
-                )
-                success_count += 1
+            async with conn_main.transaction():
+                async with conn_csv.transaction():
+                    log.debug(f'Deleting table "{table}"')
+                    await conn_csv.execute(f'DROP TABLE IF EXISTS "{table}"')
+                    await conn_main.execute(
+                        "DELETE FROM tables_index WHERE parsing_table = $1", table
+                    )
+                    await conn_main.execute(
+                        "UPDATE checks SET parsing_table = NULL WHERE parsing_table = $1", table
+                    )
+                    success_count += 1
         except Exception as e:
             error_count += 1
             log.error(f'Failed to delete table "{table}": {str(e)}')
@@ -417,7 +432,10 @@ async def insert_resource_into_catalog(resource_id: str):
             datetime.fromisoformat(resource["resource"]["harvest"]["modified_at"]).replace(
                 tzinfo=timezone.utc
             )
-            if resource["resource"].get("harvest")
+            if (
+                resource["resource"].get("harvest") is not None
+                and resource["resource"]["harvest"].get("modified_at")
+            )
             else None,
         )
         log.info(f"Resource {resource_id} successfully {action}ed into DB.")
@@ -425,6 +443,60 @@ async def insert_resource_into_catalog(resource_id: str):
         raise e
 
 
+@cli
+async def purge_selected_csv_tables(
+    nb_days_to_keep: int | None = None,
+    nb_tables_to_keep: int | None = None,
+    quiet: bool = False,
+) -> None:
+    """Delete converted CSV tables either:
+    - if they're more than nb_days_to_keep days old
+    - if they're not in the top nb_table_to_keep most recent
+    """
+    if quiet:
+        log.setLevel(logging.ERROR)
+
+    assert nb_days_to_keep is not None or nb_tables_to_keep is not None
+    conn_csv = await connection(db_name="csv")
+    if nb_days_to_keep is not None:
+        threshold = datetime.now(timezone.utc) - timedelta(days=int(nb_days_to_keep))
+        q = """SELECT parsing_table FROM tables_index WHERE created_at <= $1"""
+        res: list[Record] = await conn_csv.fetch(q, threshold)
+    elif nb_tables_to_keep is not None:
+        q = """SELECT parsing_table FROM tables_index ORDER BY created_at DESC OFFSET $1"""
+        res: list[Record] = await conn_csv.fetch(q, int(nb_tables_to_keep))
+
+    tables_to_delete: list[str] = [r["parsing_table"] for r in res]
+
+    success_count = 0
+    error_count = 0
+    conn_main = await connection()
+    for table in tables_to_delete:
+        try:
+            async with conn_main.transaction():
+                async with conn_csv.transaction():
+                    log.debug(f'Deleting table "{table}"')
+                    await conn_csv.execute(f'DROP TABLE IF EXISTS "{table}"')
+                    await conn_csv.execute(
+                        "DELETE FROM tables_index WHERE parsing_table = $1", table
+                    )
+                    await conn_main.execute(
+                        "UPDATE checks SET parsing_table = NULL WHERE parsing_table = $1", table
+                    )
+                    success_count += 1
+        except Exception as e:
+            error_count += 1
+            log.error(f'Failed to delete table "{table}": {str(e)}')
+            continue
+
+    if success_count:
+        log.info(f"Successfully deleted {success_count} table(s).")
+    if error_count:
+        log.warning(f"Failed to delete {error_count} table(s). Check logs for details.")
+    if not (success_count or error_count):
+        log.info("Nothing to delete.")
+
+
 @wrap
 async def cli_wrapper():
     context["conn"] = {}