udata-hydra 2.1.3.dev7106__tar.gz → 2.1.3.dev7204__tar.gz

{udata_hydra-2.1.3.dev7106 → udata_hydra-2.1.3.dev7204}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: udata-hydra
-Version: 2.1.3.dev7106
+Version: 2.1.3.dev7204
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team
@@ -18,7 +18,7 @@ Requires-Dist: aioresponses (>=0.7.3) ; extra == "dev"
 Requires-Dist: asyncpg (>=0.29.0)
 Requires-Dist: bumpx (>=0.3.10) ; extra == "dev"
 Requires-Dist: coloredlogs (>=15.0.1)
-Requires-Dist: csv-detective (==0.7.3)
+Requires-Dist: csv-detective (==0.7.4)
 Requires-Dist: dateparser (>=1.1.7)
 Requires-Dist: gunicorn (>=20.1.0) ; extra == "dev"
 Requires-Dist: humanfriendly (>=10.0)

{udata_hydra-2.1.3.dev7106 → udata_hydra-2.1.3.dev7204}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "udata-hydra"
-version = "2.1.3.dev7106"
+version = "2.1.3.dev7204"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = [{ name = "Opendata Team", email = "opendatateam@data.gouv.fr" }]
 dependencies = [
@@ -8,7 +8,7 @@ dependencies = [
     "aiohttp>=3.10.3",
     "asyncpg>=0.29.0",
     "coloredlogs>=15.0.1",
-    "csv-detective==0.7.3",
+    "csv-detective==0.7.4",
     "dateparser>=1.1.7",
     "humanfriendly>=10.0",
     "marshmallow>=3.14.1",

{udata_hydra-2.1.3.dev7106 → udata_hydra-2.1.3.dev7204}/udata_hydra/analysis/csv.py

@@ -183,7 +183,6 @@ async def analyse_csv(
             parquet_args: tuple[str, int] | None = await csv_to_parquet(
                 file_path=tmp_file.name,
                 inspection=csv_inspection,
-                table_name=table_name,
                 resource_id=resource_id,
             )
             timer.mark("csv-to-parquet")
@@ -294,7 +293,6 @@ def generate_records(file_path: str, inspection: dict, columns: dict) -> Iterato
 async def csv_to_parquet(
     file_path: str,
     inspection: dict,
-    table_name: str,
     resource_id: str | None = None,
 ) -> tuple[str, int] | None:
     """
@@ -315,13 +313,13 @@ async def csv_to_parquet(
 
     if int(inspection.get("total_lines", 0)) < config.MIN_LINES_FOR_PARQUET:
         log.debug(
-            f"Skipping parquet export for {table_name} because it has less than {config.MIN_LINES_FOR_PARQUET} lines."
+            f"Skipping parquet export for {resource_id} because it has less than {config.MIN_LINES_FOR_PARQUET} lines."
         )
         return
 
     log.debug(
         f"Converting from {engine_to_file.get(inspection.get('engine', ''), 'CSV')} "
-        f"to parquet for {table_name} and sending to Minio."
+        f"to parquet for {resource_id} and sending to Minio."
     )
 
     if resource_id:
@@ -333,7 +331,7 @@ async def csv_to_parquet(
     parquet_file, _ = save_as_parquet(
         records=generate_records(file_path, inspection, columns),
         columns=columns,
-        output_filename=table_name,
+        output_filename=resource_id,
     )
     parquet_size: int = os.path.getsize(parquet_file)
     parquet_url: str = minio_client.send_file(parquet_file)

{udata_hydra-2.1.3.dev7106 → udata_hydra-2.1.3.dev7204}/udata_hydra/analysis/resource.py

@@ -187,11 +187,18 @@ async def detect_resource_change_from_checksum(
         "analysis:last-modified-detection": "computed-checksum",
     }
     """
-    if last_check and last_check.get("checksum") != new_checksum:
-        return Change.HAS_CHANGED, {
-            "analysis:last-modified-at": datetime.now(timezone.utc).isoformat(),
-            "analysis:last-modified-detection": "computed-checksum",
-        }
+    if last_check and last_check.get("checksum"):
+        if last_check.get("checksum") != new_checksum:
+            return Change.HAS_CHANGED, {
+                "analysis:last-modified-at": datetime.now(timezone.utc).isoformat(),
+                "analysis:last-modified-detection": "computed-checksum",
+            }
+        elif last_check.get("detected_last_modified_at"):
+            return Change.HAS_NOT_CHANGED, {
+                "analysis:last-modified-at": last_check["detected_last_modified_at"].isoformat(),
+                "analysis:last-modified-detection": "previous-check-detection",
+            }
+    # if the previous check did not have the info, we investigate further
     return Change.NO_GUESS, None
 
 
@@ -224,16 +231,24 @@ async def detect_resource_change_from_content_length_header(
     data: dict,
 ) -> tuple[Change, dict | None]:
     # content-length variation between current and last check
-    if len(data) <= 1 or not data[0]["content_length"]:
+    if len(data) <= 1 or not data[0].get("content_length"):
         return Change.NO_GUESS, None
-    changed_at = data[0]["created_at"]
-    payload = {
-        "analysis:last-modified-at": changed_at.isoformat(),
-        "analysis:last-modified-detection": "content-length-header",
-    }
-    if data[0]["content_length"] != data[1]["content_length"]:
-        return Change.HAS_CHANGED, payload
-    return Change.HAS_NOT_CHANGED, payload
+    if data[0].get("content_length") and data[1].get("content_length"):
+        if data[0]["content_length"] != data[1]["content_length"]:
+            return Change.HAS_CHANGED, {
+                # if resource has changed, set last-modified to the current check's creation
+                "analysis:last-modified-at": data[0]["created_at"].isoformat(),
+                "analysis:last-modified-detection": "content-length-header",
+            }
+        # same content_length is not 100% certainly no change, but a good tradeoff to prevent many downloads
+        elif data[1].get("detected_last_modified_at"):
+            return Change.HAS_NOT_CHANGED, {
+                # no change, using the last-modified from the previous check (passed on from check to check)
+                "analysis:last-modified-at": data[1]["detected_last_modified_at"].isoformat(),
+                "analysis:last-modified-detection": "previous-check-detection",
+            }
+    # if the previous check did not have the info, we investigate further
+    return Change.NO_GUESS, None
 
 
 async def detect_resource_change_on_early_hints(

{udata_hydra-2.1.3.dev7106 → udata_hydra-2.1.3.dev7204}/udata_hydra/utils/minio.py

@@ -10,12 +10,11 @@ log = logging.getLogger("udata-hydra")
 
 class MinIOClient:
     def __init__(self, bucket=config.MINIO_BUCKET):
-        self.url = config.MINIO_URL
         self.user = config.MINIO_USER
         self.password = config.MINIO_PWD
         self.bucket = bucket
         self.client = Minio(
-            self.url or "test",
+            config.MINIO_URL or "test",
             access_key=self.user or "test",
             secret_key=self.password or "test",
             secure=True,
@@ -40,6 +39,6 @@ class MinIOClient:
             )
             if delete_source:
                 os.remove(file_name)
-            return f"https://{self.url}/{self.bucket}/{config.MINIO_FOLDER}/{file_name}"
+            return f"https://{config.MINIO_URL}/{self.bucket}/{config.MINIO_FOLDER}/{file_name}"
         else:
             raise Exception(f"file '{file_name}' does not exists")