udata-hydra 2.2.2.dev7611__tar.gz → 2.2.2.dev7633__tar.gz

{udata_hydra-2.2.2.dev7611 → udata_hydra-2.2.2.dev7633}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: udata-hydra
-Version: 2.2.2.dev7611
+Version: 2.2.2.dev7633
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team

{udata_hydra-2.2.2.dev7611 → udata_hydra-2.2.2.dev7633}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "udata-hydra"
-version = "2.2.2.dev7611"
+version = "2.2.2.dev7633"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = [{ name = "Opendata Team", email = "opendatateam@data.gouv.fr" }]
 dependencies = [

{udata_hydra-2.2.2.dev7611 → udata_hydra-2.2.2.dev7633}/udata_hydra/analysis/csv.py

@@ -5,12 +5,14 @@ import logging
 import os
 import sys
 from datetime import datetime, timezone
-from typing import Any, Iterator
+from math import isnan
+from typing import Iterator
 
+import pandas as pd
 from asyncpg import Record
+from csv_detective import routine as csv_detective_routine
+from csv_detective import validate_then_detect
 from csv_detective.detection.engine import engine_to_file
-from csv_detective.explore_csv import routine as csv_detective_routine
-from csv_detective.explore_csv import validate_then_detect
 from progressist import ProgressBar
 from slugify import slugify
 from sqlalchemy import (
@@ -28,8 +30,6 @@ from sqlalchemy import (
 )
 from sqlalchemy.dialects.postgresql import asyncpg
 from sqlalchemy.schema import CreateIndex, CreateTable, Index
-from str2bool import str2bool
-from str2float import str2float
 
 from udata_hydra import config, context
 from udata_hydra.analysis import helpers
@@ -40,7 +40,6 @@ from udata_hydra.db.resource_exception import ResourceException
 from udata_hydra.utils import (
     IOException,
     ParseException,
-    Reader,
     Timer,
     detect_tabular_from_headers,
     handle_parse_exception,
@@ -71,17 +70,6 @@ PYTHON_TYPE_TO_PG = {
     "datetime_aware": DateTime(timezone=True),
 }
 
-PYTHON_TYPE_TO_PY = {
-    "string": str,
-    "float": float,
-    "int": int,
-    "bool": bool,
-    "json": helpers.to_json,
-    "date": helpers.to_date,
-    "datetime": helpers.to_datetime,
-    "datetime_aware": helpers.to_datetime,
-}
-
 RESERVED_COLS = ("__id", "cmin", "cmax", "collation", "ctid", "tableoid", "xmin", "xmax")
 minio_client = MinIOClient(bucket=config.MINIO_PARQUET_BUCKET, folder=config.MINIO_PARQUET_FOLDER)
 
@@ -130,17 +118,21 @@ async def analyse_csv(
         try:
             previous_analysis: dict | None = await get_previous_analysis(resource_id=resource_id)
             if previous_analysis:
-                csv_inspection: dict = validate_then_detect(
+                csv_inspection, df = validate_then_detect(
                     file_path=tmp_file.name,
                     previous_analysis=previous_analysis,
                     output_profile=True,
+                    output_df=True,
+                    cast_json=False,
                     num_rows=-1,
                     save_results=False,
                 )
             else:
-                csv_inspection: dict | None = csv_detective_routine(
+                csv_inspection, df = csv_detective_routine(
                     file_path=tmp_file.name,
                     output_profile=True,
+                    output_df=True,
+                    cast_json=False,
                     num_rows=-1,
                     save_results=False,
                 )
@@ -151,7 +143,7 @@ async def analyse_csv(
         timer.mark("csv-inspection")
 
         await csv_to_db(
-            file_path=tmp_file.name,
+            df=df,
             inspection=csv_inspection,
             table_name=table_name,
             table_indexes=table_indexes,
@@ -162,7 +154,7 @@ async def analyse_csv(
 
         try:
             parquet_args: tuple[str, int] | None = await csv_to_parquet(
-                file_path=tmp_file.name,
+                df=df,
                 inspection=csv_inspection,
                 resource_id=resource_id,
             )
@@ -219,26 +211,6 @@ async def get_previous_analysis(resource_id: str) -> dict | None:
     return analysis
 
 
-def smart_cast(_type: str, value, failsafe: bool = False) -> Any:
-    try:
-        if value is None or value == "":
-            return None
-        if _type == "bool":
-            return str2bool(value)
-        return PYTHON_TYPE_TO_PY[_type](value)
-    except ValueError as e:
-        if _type == "int":
-            _value = str2float(value, default=None)
-            if _value:
-                return int(_value)
-        elif _type == "float":
-            return str2float(value, default=None)
-        if not failsafe:
-            raise e
-        log.warning(f'Could not convert "{value}" to {_type}, defaulting to null')
-        return None
-
-
 def compute_create_table_query(
     table_name: str, columns: dict, indexes: dict[str, str] | None = None
 ) -> str:
@@ -255,7 +227,8 @@ def compute_create_table_query(
         for col_name, index_type in indexes.items():
             if index_type not in config.SQL_INDEXES_TYPES_SUPPORTED:
                 log.error(
-                    f'Index type "{index_type}" is unknown or not supported yet! Index for column {col_name} was not created.'
+                    f'Index type "{index_type}" is unknown or not supported yet! '
+                    f"Index for column {col_name} was not created."
                 )
                 continue
 
@@ -267,7 +240,8 @@ def compute_create_table_query(
                         table.append_constraint(Index(index_name, col_name))
                     except KeyError:
                         raise KeyError(
-                            f'Error creating index "{index_name}" on column "{col_name}". Does the column "{col_name}" exist in the table?'
+                            f'Error creating index "{index_name}" on column "{col_name}". '
+                            f'Does the column "{col_name}" exist in the table?'
                         )
                 # TODO: other index types. Not easy with sqlalchemy, maybe use raw sql?
 
@@ -289,17 +263,15 @@ def compute_create_table_query(
     return query
 
 
-def generate_records(file_path: str, inspection: dict, columns: dict) -> Iterator[list]:
-    # because we need the iterator twice, not possible to
-    # handle parquet and db through the same iteration
-    with Reader(file_path, inspection) as reader:
-        for line in reader:
-            if line:
-                yield [smart_cast(t, v, failsafe=True) for t, v in zip(columns.values(), line)]
+def generate_records(df: pd.DataFrame) -> Iterator[list]:
+    # pandas cannot have None in columns typed as int so we have to cast
+    # NaN-int values to None for db insertion, and we also change NaN to None
+    for row in df.values:
+        yield tuple(cell if not pd.isna(cell) else None for cell in row)
 
 
 async def csv_to_parquet(
-    file_path: str,
+    df: pd.DataFrame,
     inspection: dict,
     resource_id: str | None = None,
 ) -> tuple[str, int] | None:
@@ -334,11 +306,9 @@ async def csv_to_parquet(
         # Update resource status to CONVERTING_TO_PARQUET
         await Resource.update(resource_id, {"status": "CONVERTING_TO_PARQUET"})
 
-    columns = {c: v["python_type"] for c, v in inspection["columns"].items()}
     # save the file as parquet and store it on Minio instance
     parquet_file, _ = save_as_parquet(
-        records=generate_records(file_path, inspection, columns),
-        columns=columns,
+        df=df,
         output_filename=resource_id,
     )
     parquet_size: int = os.path.getsize(parquet_file)
@@ -347,7 +317,7 @@ async def csv_to_parquet(
 
 
 async def csv_to_db(
-    file_path: str,
+    df: pd.DataFrame,
     inspection: dict,
     table_name: str,
     table_indexes: dict[str, str] | None = None,
@@ -401,8 +371,8 @@ async def csv_to_db(
         try:
             await db.copy_records_to_table(
                 table_name,
-                records=generate_records(file_path, inspection, columns),
-                columns=columns.keys(),
+                records=generate_records(df),
+                columns=list(columns.keys()),
             )
         except Exception as e:  # I know what I'm doing, pinky swear
             raise ParseException(
@@ -411,8 +381,8 @@ async def csv_to_db(
     # this inserts rows from iterator one by one, slow but useful for debugging
     else:
         bar = ProgressBar(total=inspection["total_lines"])
-        for r in bar.iter(generate_records(file_path, inspection, columns)):
-            data = {k: v for k, v in zip(columns.keys(), r)}
+        for r in bar.iter(generate_records(df)):
+            data = {k: v for k, v in zip(df.columns, r)}
             # NB: possible sql injection here, but should not be used in prod
             q = compute_insert_query(table_name=table_name, data=data, returning="__id")
             await db.execute(q, *data.values())

{udata_hydra-2.2.2.dev7611 → udata_hydra-2.2.2.dev7633}/udata_hydra/analysis/helpers.py

@@ -1,34 +1,12 @@
 import json
-from datetime import date, datetime
 from typing import IO
 
 from asyncpg import Record
-from dateparser import parse as date_parser
-from dateutil.parser import ParserError
-from dateutil.parser import parse as dateutil_parser
 
 from udata_hydra import config
 from udata_hydra.utils import UdataPayload, download_resource, queue, send
 
 
-def to_json(value: str) -> str:
-    """Convenience method, should be casted from string directly by postgres"""
-    return value
-
-
-def to_datetime(value: str) -> datetime | None:
-    """For performance reasons, we try first with dateutil and fallback on dateparser"""
-    try:
-        return dateutil_parser(value)
-    except ParserError:
-        return date_parser(value)
-
-
-def to_date(value: str) -> date | None:
-    parsed = to_datetime(value)
-    return parsed.date() if parsed else None
-
-
 def get_python_type(column: dict) -> str:
     """Outsourcing the distinction of aware datetimes"""
     return (

{udata_hydra-2.2.2.dev7611 → udata_hydra-2.2.2.dev7633}/udata_hydra/utils/__init__.py

@@ -5,7 +5,6 @@ from .file import compute_checksum_from_file, download_resource
 from .geojson import detect_geojson_from_headers_or_catalog
 from .http import UdataPayload, get_request_params, send
 from .queue import enqueue
-from .reader import Reader
 from .timer import Timer
 
 __all__ = [
@@ -21,6 +20,5 @@ __all__ = [
     "get_request_params",
     "send",
     "enqueue",
-    "Reader",
     "Timer",
 ]

udata_hydra-2.2.2.dev7633/udata_hydra/utils/parquet.py

@@ -0,0 +1,14 @@
+from io import BytesIO
+
+import pandas as pd
+
+
+def save_as_parquet(
+    df: pd.DataFrame,
+    output_filename: str | None = None,
+) -> tuple[str, BytesIO | None]:
+    bytes = df.to_parquet(
+        f"{output_filename}.parquet" if output_filename else None,
+        compression="zstd",  # best compression to date
+    )
+    return f"{output_filename}.parquet", bytes

udata_hydra-2.2.2.dev7611/udata_hydra/utils/parquet.py

@@ -1,29 +0,0 @@
-from typing import Iterator
-
-import pyarrow as pa
-import pyarrow.parquet as pq
-
-PYTHON_TYPE_TO_PA = {
-    "string": pa.string(),
-    "float": pa.float64(),
-    "int": pa.int64(),
-    "bool": pa.bool_(),
-    "json": pa.string(),
-    "date": pa.date32(),
-    "datetime": pa.date64(),
-}
-
-
-def save_as_parquet(
-    records: Iterator[list],
-    columns: dict,
-    output_filename: str | None = None,
-) -> tuple[str, pa.Table]:
-    # the "output_name = None" case is only used in tests
-    table = pa.Table.from_pylist(
-        [{c: v for c, v in zip(columns, values)} for values in records],
-        schema=pa.schema([pa.field(c, PYTHON_TYPE_TO_PA[columns[c]]) for c in columns]),
-    )
-    if output_filename:
-        pq.write_table(table, f"{output_filename}.parquet")
-    return f"{output_filename}.parquet", table

udata_hydra-2.2.2.dev7611/udata_hydra/utils/reader.py

@@ -1,69 +0,0 @@
-import csv as stdcsv
-from io import BytesIO
-from typing import Generator
-
-import openpyxl
-import xlrd
-
-
-def generate_dialect(inspection: dict) -> stdcsv.Dialect:
-    class CustomDialect(stdcsv.unix_dialect):
-        # TODO: it would be nice to have more info from csvdetective to feed the dialect
-        # in the meantime we might want to sniff the file a bit
-        delimiter = inspection["separator"]
-
-    return CustomDialect()
-
-
-class Reader:
-    def __init__(self, file_path, inspection):
-        self.file_path = file_path
-        self.inspection = inspection
-        self.nb_skip = self.inspection["header_row_idx"]
-        self.mapping = {
-            "openpyxl": "iter_rows",
-            "xlrd": "get_rows",
-        }
-        self.nb_columns = len(self.inspection["header"])
-        self.reader = None
-
-    def __enter__(self):
-        if self.inspection.get("engine") == "openpyxl":
-            with open(self.file_path, "rb") as f:
-                content = BytesIO(f.read())
-            self.file = openpyxl.load_workbook(content)
-            self.sheet = self.file[self.inspection["sheet_name"]]
-            self.reader = self._excel_reader()
-
-        elif self.inspection.get("engine") == "xlrd":
-            self.file = xlrd.open_workbook(self.file_path)
-            self.sheet = self.file[self.inspection["sheet_name"]]
-            self.reader = self._excel_reader()
-
-        else:
-            self.file = open(self.file_path, encoding=self.inspection["encoding"])
-            self.reader = stdcsv.reader(
-                self._skip_rows(), dialect=generate_dialect(self.inspection)
-            )
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        if self.file is not None and hasattr(self.file, "close"):
-            self.file.close()
-
-    def _skip_rows(self):
-        # skipping header
-        for _ in range(self.nb_skip + 1):
-            next(self.file)
-        return self.file
-
-    def _excel_reader(self) -> Generator:
-        _method = getattr(self.sheet, self.mapping[self.inspection["engine"]])
-        for idx, row in enumerate(_method()):
-            # skipping header
-            if idx <= self.nb_skip:
-                continue
-            yield [c.value for c in row]
-
-    def __iter__(self):
-        return self.reader