udata-hydra 2.0.0.dev2497__tar.gz → 2.0.0.dev2547__tar.gz

{udata_hydra-2.0.0.dev2497 → udata_hydra-2.0.0.dev2547}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata-hydra
-Version: 2.0.0.dev2497
+Version: 2.0.0.dev2547
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team
@@ -15,14 +15,15 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: aiocontextvars (>=0.2.2,<0.3.0)
 Requires-Dist: aiohttp (>=3.8.1,<4.0.0)
 Requires-Dist: asyncpg (>=0.27.0,<0.28.0)
-Requires-Dist: boto3 (>=1.21.21,<2.0.0)
 Requires-Dist: coloredlogs (>=15.0.1,<16.0.0)
 Requires-Dist: csv-detective (==0.7.1)
 Requires-Dist: dateparser (>=1.1.7,<2.0.0)
 Requires-Dist: humanfriendly (>=10.0,<11.0)
 Requires-Dist: marshmallow (>=3.14.1,<4.0.0)
-Requires-Dist: minicli (>=0.5.0,<0.6.0)
+Requires-Dist: minicli (>=0.5.3,<0.6.0)
+Requires-Dist: minio (==7.2.7)
 Requires-Dist: progressist (>=0.1.0,<0.2.0)
+Requires-Dist: pyarrow (==16.1.0)
 Requires-Dist: python-dateutil (>=2.8.2,<3.0.0)
 Requires-Dist: python-magic (>=0.4.25,<0.5.0)
 Requires-Dist: redis (>=4.1.4,<5.0.0)

{udata_hydra-2.0.0.dev2497 → udata_hydra-2.0.0.dev2547}/pyproject.toml

@@ -9,7 +9,7 @@ readme = "README.md"
 
 [tool.poetry]
 name = "udata-hydra"
-version = "2.0.0.dev2497"
+version = "2.0.0.dev2547"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = ["Opendata Team <opendatateam@data.gouv.fr>"]
 license = "MIT"
@@ -19,10 +19,9 @@ readme = "README.md"
 python = "^3.9"
 aiohttp = "^3.8.1"
 asyncpg = "^0.27.0"
-boto3 = "^1.21.21"
 humanfriendly = "^10.0"
 marshmallow = "^3.14.1"
-minicli = "^0.5.0"
+minicli = "^0.5.3"
 progressist = "^0.1.0"
 python-magic = "^0.4.25"
 redis = "^4.1.4"
@@ -37,6 +36,8 @@ sqlalchemy = "^1.4.46"
 dateparser = "^1.1.7"
 python-dateutil = "^2.8.2"
 csv-detective = "0.7.1"
+pyarrow = "16.1.0"
+minio = "7.2.7"
 
 [tool.poetry.group.dev.dependencies]
 aiohttp-devtools = "^1.0.post0"

{udata_hydra-2.0.0.dev2497 → udata_hydra-2.0.0.dev2547}/udata_hydra/analysis/csv.py

@@ -5,7 +5,7 @@ import logging
 import os
 import sys
 from datetime import datetime, timezone
-from typing import Any, Union
+from typing import Any, Iterator, Union
 
 import sentry_sdk
 from csv_detective.detection import engine_to_file
@@ -35,6 +35,8 @@ from udata_hydra.analysis.errors import ParseException
 from udata_hydra.db import compute_insert_query
 from udata_hydra.db.check import Check
 from udata_hydra.utils import Reader, Timer, download_resource, queue, send
+from udata_hydra.utils.minio import MinIOClient
+from udata_hydra.utils.parquet import save_as_parquet
 
 log = logging.getLogger("udata-hydra")
 
@@ -69,9 +71,10 @@ PYTHON_TYPE_TO_PY = {
 }
 
 RESERVED_COLS = ("__id", "tableoid", "xmin", "cmin", "xmax", "cmax", "ctid")
+minio_client = MinIOClient()
 
 
-async def notify_udata(check_id: int) -> None:
+async def notify_udata(check_id: str, table_name: str) -> None:
     """Notify udata of the result of a parsing"""
     check = await Check.get(check_id)
     resource_id = check["resource_id"]
@@ -91,6 +94,8 @@ async def notify_udata(check_id: int) -> None:
                 else None,
             },
         }
+        if config.CSV_TO_PARQUET:
+            payload["parquet_id"] = table_name
         queue.enqueue(send, _priority="high", **payload)
 
 
@@ -101,8 +106,8 @@ async def analyse_csv(
     debug_insert: bool = False,
 ) -> None:
     """Launch csv analysis from a check or an URL (debug), using previsously downloaded file at file_path if any"""
-    if not config.CSV_ANALYSIS_ENABLED:
-        log.debug("CSV_ANALYSIS_ENABLED turned off, skipping.")
+    if not config.CSV_ANALYSIS:
+        log.debug("CSV_ANALYSIS turned off, skipping.")
         return
 
     exceptions = config.LARGE_RESOURCES_EXCEPTIONS
@@ -133,6 +138,8 @@ async def analyse_csv(
         timer.mark("csv-inspection")
         await csv_to_db(tmp_file.name, csv_inspection, table_name, debug_insert=debug_insert)
         timer.mark("csv-to-db")
+        await csv_to_parquet(tmp_file.name, csv_inspection, table_name)
+        timer.mark("csv-to-parquet")
         if check_id:
             await Check.update(
                 check_id,
@@ -146,7 +153,7 @@ async def analyse_csv(
         await handle_parse_exception(e, check_id, table_name)
     finally:
         if check_id:
-            await notify_udata(check_id)
+            await notify_udata(check_id, table_name)
         timer.stop()
         tmp_file.close()
         os.remove(tmp_file.name)
@@ -185,6 +192,40 @@ def compute_create_table_query(table_name: str, columns: list) -> str:
     return compiled.string.replace("%%", "%")
 
 
+def generate_records(file_path: str, inspection: dict, columns: dict) -> Iterator[list]:
+    # because we need the iterator twice, not possible to
+    # handle parquet and db through the same iteration
+    with Reader(file_path, inspection) as reader:
+        for line in reader:
+            if line:
+                yield [smart_cast(t, v, failsafe=True) for t, v in zip(columns.values(), line)]
+
+
+async def csv_to_parquet(file_path: str, inspection: dict, table_name: str) -> None:
+    """
+    Convert a csv file to parquet using inspection data.
+
+    :file_path: CSV file path to convert
+    :inspection: CSV detective report
+    :table_name: used to name the parquet file
+    """
+    if not config.CSV_TO_PARQUET:
+        log.debug("CSV_TO_PARQUET turned off, skipping parquet export.")
+        return
+    log.debug(
+        f"Converting from {engine_to_file.get(inspection.get('engine', ''), 'CSV')} "
+        f"to parquet for {table_name} and sending to Minio."
+    )
+    columns = {c: v["python_type"] for c, v in inspection["columns"].items()}
+    # save the file as parquet and store it on Minio instance
+    parquet_file, _ = save_as_parquet(
+        records=generate_records(file_path, inspection, columns),
+        columns=columns,
+        output_name=table_name,
+    )
+    minio_client.send_file(parquet_file)
+
+
 async def csv_to_db(
     file_path: str, inspection: dict, table_name: str, debug_insert: bool = False
 ) -> None:
@@ -197,42 +238,42 @@ async def csv_to_db(
     :table_name: used to create tables
     :debug_insert: insert record one by one instead of using postgresql COPY
     """
+    if not config.CSV_TO_DB:
+        log.debug("CSV_TO_DB turned off, skipping.")
+        return
     log.debug(
         f"Converting from {engine_to_file.get(inspection.get('engine', ''), 'CSV')} "
         f"to db for {table_name}"
     )
-    columns = inspection["columns"]
     # build a `column_name: type` mapping and explicitely rename reserved column names
     columns = {
         f"{c}__hydra_renamed" if c.lower() in RESERVED_COLS else c: v["python_type"]
-        for c, v in columns.items()
+        for c, v in inspection["columns"].items()
     }
     q = f'DROP TABLE IF EXISTS "{table_name}"'
     db = await context.pool("csv")
     await db.execute(q)
     q = compute_create_table_query(table_name, columns)
     await db.execute(q)
-    with Reader(file_path, inspection) as reader:
-        records = (
-            [smart_cast(t, v, failsafe=True) for t, v in zip(columns.values(), line)]
-            for line in reader
-            if line
-        )
-        # this use postgresql COPY from an iterator, it's fast but might be difficult to debug
-        if not debug_insert:
-            # NB: also see copy_to_table for a file source
-            try:
-                await db.copy_records_to_table(table_name, records=records, columns=columns.keys())
-            except Exception as e:  # I know what I'm doing, pinky swear
-                raise ParseException("copy_records_to_table") from e
-        # this inserts rows from iterator one by one, slow but useful for debugging
-        else:
-            bar = ProgressBar(total=inspection["total_lines"])
-            for r in bar.iter(records):
-                data = {k: v for k, v in zip(columns.keys(), r)}
-                # NB: possible sql injection here, but should not be used in prod
-                q = compute_insert_query(table_name=table_name, data=data, returning="__id")
-                await db.execute(q, *data.values())
+    # this use postgresql COPY from an iterator, it's fast but might be difficult to debug
+    if not debug_insert:
+        # NB: also see copy_to_table for a file source
+        try:
+            await db.copy_records_to_table(
+                table_name,
+                records=generate_records(file_path, inspection, columns),
+                columns=columns.keys(),
+            )
+        except Exception as e:  # I know what I'm doing, pinky swear
+            raise ParseException("copy_records_to_table") from e
+    # this inserts rows from iterator one by one, slow but useful for debugging
+    else:
+        bar = ProgressBar(total=inspection["total_lines"])
+        for r in bar.iter(generate_records(file_path, inspection, columns)):
+            data = {k: v for k, v in zip(columns.keys(), r)}
+            # NB: possible sql injection here, but should not be used in prod
+            q = compute_insert_query(table_name=table_name, data=data, returning="__id")
+            await db.execute(q, *data.values())
 
 
 async def csv_to_db_index(table_name: str, inspection: dict, check: dict) -> None:

{udata_hydra-2.0.0.dev2497 → udata_hydra-2.0.0.dev2547}/udata_hydra/config_default.toml

@@ -55,7 +55,8 @@ MAX_FILESIZE_ALLOWED.xlsx = 13107200 # /8
 MAX_FILESIZE_ALLOWED.ods = 10485760 # /10
 
 
-CSV_ANALYSIS_ENABLED = true
+CSV_ANALYSIS = true
+CSV_TO_DB = true
 TEMPORARY_DOWNLOAD_FOLDER = ""
 
 # -- Worker settings -- #
@@ -67,7 +68,7 @@ UDATA_URI = ""
 UDATA_URI_API_KEY = ""
 
 # -- Minio / datalake settings -- #
-SAVE_TO_MINIO = false
+CSV_TO_PARQUET = false
 MINIO_FOLDER = ""
 MINIO_URL = ""
 MINIO_BUCKET = ""

{udata_hydra-2.0.0.dev2497 → udata_hydra-2.0.0.dev2547}/udata_hydra/crawl.py

@@ -204,7 +204,7 @@ def fix_surrogates(value):
     """FIX Unicode low surrogate must follow a high surrogate.
     eg in 'TREMI_2017-R\xe9sultats enqu\xeate bruts.csv'
     """
-    if type(value) is not str:
+    if not isinstance(value, str):
         value = str(value)
     return value.encode("utf-8", "surrogateescape").decode("utf-8", "replace")
 

{udata_hydra-2.0.0.dev2497 → udata_hydra-2.0.0.dev2547}/udata_hydra/db/check.py

@@ -24,7 +24,9 @@ class Check:
         return check
 
     @classmethod
-    async def get_latest(cls, url: Union[str, None], resource_id: Union[str, None]) -> dict | None:
+    async def get_latest(
+        cls, url: Union[str, None], resource_id: Union[str, None]
+    ) -> Union[dict, None]:
         column: str = "url" if url else "resource_id"
         pool = await context.pool()
         async with pool.acquire() as connection:
@@ -38,7 +40,9 @@ class Check:
             return await connection.fetchrow(q, url or resource_id)
 
     @classmethod
-    async def get_all(cls, url: Union[str, None], resource_id: Union[str, None]) -> dict | None:
+    async def get_all(
+        cls, url: Union[str, None], resource_id: Union[str, None]
+    ) -> Union[dict, None]:
         column: str = "url" if url else "resource_id"
         pool = await context.pool()
         async with pool.acquire() as connection:

{udata_hydra-2.0.0.dev2497 → udata_hydra-2.0.0.dev2547}/udata_hydra/routes/resources.py

@@ -6,7 +6,7 @@ from marshmallow import ValidationError
 from udata_hydra import config
 from udata_hydra.db.resource import Resource
 from udata_hydra.schemas import ResourceSchema
-from udata_hydra.utils import delete_resource_from_minio, get_request_params
+from udata_hydra.utils import get_request_params
 
 
 async def get_resource(request: web.Request) -> web.Response:
@@ -82,13 +82,10 @@ async def delete_resource(request: web.Request) -> web.Response:
     except ValidationError as err:
         raise web.HTTPBadRequest(text=json.dumps(err.messages))
 
-    dataset_id = valid_payload["dataset_id"]
     resource_id = valid_payload["resource_id"]
 
     pool = request.app["pool"]
     async with pool.acquire() as connection:
-        if config.SAVE_TO_MINIO:
-            delete_resource_from_minio(dataset_id, resource_id)
         # Mark resource as deleted in catalog table
         q = f"""UPDATE catalog SET deleted = TRUE WHERE resource_id = '{resource_id}';"""
         await connection.execute(q)

{udata_hydra-2.0.0.dev2497 → udata_hydra-2.0.0.dev2547}/udata_hydra/utils/__init__.py

@@ -2,7 +2,6 @@ from .app_version import get_app_version
 from .csv import detect_tabular_from_headers
 from .file import compute_checksum_from_file, download_resource, read_csv_gz
 from .http import get_request_params, send
-from .minio import delete_resource_from_minio, get_resource_minio_url, save_resource_to_minio
 from .queue import enqueue
 from .reader import Reader, generate_dialect
 from .timer import Timer

udata_hydra-2.0.0.dev2547/udata_hydra/utils/minio.py

@@ -0,0 +1,46 @@
+import logging
+import os
+
+from minio import Minio
+
+from udata_hydra import config
+
+log = logging.getLogger("udata-hydra")
+
+
+class MinIOClient:
+    def __init__(self, bucket=config.MINIO_BUCKET):
+        self.url = config.MINIO_URL
+        self.user = config.MINIO_USER
+        self.password = config.MINIO_PWD
+        self.bucket = bucket
+        self.client = Minio(
+            self.url or "test",
+            access_key=self.user or "test",
+            secret_key=self.password or "test",
+            secure=True,
+        )
+        if self.bucket:
+            self.bucket_exists = self.client.bucket_exists(self.bucket)
+            if not self.bucket_exists:
+                raise ValueError(f"Bucket '{self.bucket}' does not exist.")
+
+    def send_file(
+        self,
+        file_path,
+        delete_source=True,
+    ):
+        if self.bucket is None:
+            raise AttributeError("A bucket has to be specified.")
+        is_file = os.path.isfile(os.path.join(file_path))
+        if is_file:
+            dest_path = f"{config.MINIO_FOLDER}{file_path.split('/')[-1]}"
+            self.client.fput_object(
+                self.bucket,
+                dest_path,
+                file_path,
+            )
+            if delete_source:
+                os.remove(file_path)
+        else:
+            raise Exception(f"file '{file_path}' does not exists")

udata_hydra-2.0.0.dev2547/udata_hydra/utils/parquet.py

@@ -0,0 +1,30 @@
+from typing import Generator
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+PYTHON_TYPE_TO_PA = {
+    "string": pa.string(),
+    "float": pa.float64(),
+    "int": pa.int64(),
+    "bool": pa.bool_(),
+    "json": pa.string(),
+    "date": pa.date32(),
+    "datetime": pa.date64(),
+}
+
+
+def save_as_parquet(
+    records: Generator,
+    columns: dict,
+    output_name: str,
+    save_output: bool = True,
+) -> tuple[str, pa.Table]:
+    # the "save_output" argument is only used in tests
+    table = pa.Table.from_pylist(
+        [{c: v for c, v in zip(columns, values)} for values in records],
+        schema=pa.schema([pa.field(c, PYTHON_TYPE_TO_PA[columns[c]]) for c in columns]),
+    )
+    if save_output:
+        pq.write_table(table, f"{output_name}.parquet")
+    return f"{output_name}.parquet", table

udata_hydra-2.0.0.dev2497/udata_hydra/utils/minio.py

@@ -1,56 +0,0 @@
-import logging
-from typing import BinaryIO
-
-import boto3
-from botocore.client import Config
-from botocore.exceptions import ClientError
-
-from udata_hydra import config
-
-log = logging.getLogger("udata-hydra")
-
-
-def get_resource_minio_url(key: str, resource_id: str) -> str:
-    """Returns location of given resource in minio once it is saved"""
-    return f"{config.MINIO_URL}/{config.MINIO_BUCKET}/{config.MINIO_FOLDER}/{key}/{resource_id}"
-
-
-def save_resource_to_minio(resource_file: BinaryIO, key: str, resource_id: str) -> None:
-    log.info("Saving to minio")
-    log.debug(f"Bucket is {config.MINIO_BUCKET}")
-    s3 = boto3.client(
-        "s3",
-        endpoint_url=config.MINIO_URL,
-        aws_access_key_id=config.MINIO_USER,
-        aws_secret_access_key=config.MINIO_PWD,
-        config=Config(signature_version="s3v4"),
-    )
-    try:
-        with open(resource_file.name, "rb") as f:
-            s3.upload_fileobj(
-                Fileobj=f,
-                Bucket=config.MINIO_BUCKET,
-                Key=f"{config.MINIO_FOLDER}/{key}/{resource_id}",
-            )
-        log.info(f"Resource saved into minio at {get_resource_minio_url(key, resource_id)}")
-    except ClientError as e:
-        log.error(e)
-
-
-def delete_resource_from_minio(key: str, resource_id: str) -> None:
-    log.info("Deleting from minio")
-    s3 = boto3.client(
-        "s3",
-        endpoint_url=config.MINIO_URL,
-        aws_access_key_id=config.MINIO_USER,
-        aws_secret_access_key=config.MINIO_PWD,
-        config=Config(signature_version="s3v4"),
-    )
-    try:
-        s3.delete_object(
-            Bucket=config.MINIO_BUCKET,
-            Key=f"{config.MINIO_FOLDER}/{key}/{resource_id}",
-        )
-        log.info(f"Resource deleted from minio at {get_resource_minio_url(key, resource_id)}")
-    except ClientError as e:
-        log.error(e)