csv-detective 0.6.7__py3-none-any.whl → 0.9.3.dev2438__py3-none-any.whl

csv_detective/output/dataframe.py

@@ -0,0 +1,96 @@
+import json
+from datetime import date, datetime
+from time import time
+from typing import Iterator
+
+import pandas as pd
+
+from csv_detective.formats.binary import binary_casting
+from csv_detective.formats.booleen import bool_casting
+from csv_detective.formats.date import date_casting
+from csv_detective.formats.float import float_casting
+from csv_detective.parsing.csv import CHUNK_SIZE
+from csv_detective.utils import display_logs_depending_process_time
+
+
+def cast(value: str, _type: str) -> str | float | bool | date | datetime | bytes | None:
+    if not isinstance(value, str) or not value:
+        # None is the current default value in hydra, should we keep this?
+        return None
+    match _type:
+        case "float":
+            return float_casting(value)
+        case "bool":
+            return bool_casting(value)
+        case "json":
+            # in hydra json are given to postgres as strings, conversion is done by postgres
+            return json.loads(value)
+        case "date":
+            _date = date_casting(value)
+            return _date.date() if _date else None
+        case "datetime":
+            return date_casting(value)
+        case "binary":
+            return binary_casting(value)
+        case _:
+            raise ValueError(f"Unknown type `{_type}`")
+
+
+def cast_df(
+    df: pd.DataFrame, columns: dict, cast_json: bool = True, verbose: bool = False
+) -> pd.DataFrame:
+    # for efficiency this modifies the dataframe in place as we don't need it anymore afterwards
+    if verbose:
+        start = time()
+    for col_name, detection in columns.items():
+        if detection["python_type"] == "string" or (
+            detection["python_type"] == "json" and not cast_json
+        ):
+            # no change if detected type is string
+            continue
+        elif detection["python_type"] == "int":
+            # to allow having ints and NaN in the same column
+            df[col_name] = df[col_name].astype(pd.Int64Dtype())
+        else:
+            df[col_name] = df[col_name].apply(lambda col: cast(col, _type=detection["python_type"]))
+    if verbose:
+        display_logs_depending_process_time(
+            f"Casting columns completed in {round(time() - start, 3)}s",
+            time() - start,
+        )
+    return df
+
+
+def cast_df_chunks(
+    df: pd.DataFrame,
+    analysis: dict,
+    file_path: str,
+    cast_json: bool = True,
+    verbose: bool = False,
+) -> Iterator[pd.DataFrame]:
+    if analysis.get("engine") or analysis["total_lines"] <= CHUNK_SIZE:
+        # the file is loaded in one chunk, so returning the cast df
+        yield cast_df(
+            df=df,
+            columns=analysis["columns"],
+            cast_json=cast_json,
+            verbose=verbose,
+        )
+    else:
+        # loading the csv in chunks using the analysis
+        chunks = pd.read_csv(
+            file_path,
+            dtype=str,
+            sep=analysis["separator"],
+            encoding=analysis["encoding"],
+            skiprows=analysis["header_row_idx"],
+            compression=analysis.get("compression"),
+            chunksize=CHUNK_SIZE,
+        )
+        for chunk in chunks:
+            yield cast_df(
+                df=chunk,
+                columns=analysis["columns"],
+                cast_json=cast_json,
+                verbose=verbose,
+            )

csv_detective/output/example.py

@@ -0,0 +1,250 @@
+import json
+import random
+import string
+import uuid
+from datetime import datetime
+from typing import Any, Type
+
+import pandas as pd
+import requests
+import rstr
+from faker import Faker
+
+from csv_detective.utils import is_url
+
+fake = Faker()
+
+
+def create_example_csv_file(
+    fields: dict | None = None,
+    schema_path: str | None = None,
+    file_length: int = 10,
+    output_name: str | None = "example_file.csv",
+    output_sep: str = ";",
+    encoding: str = "utf-8",
+    ignore_required: bool = False,
+) -> pd.DataFrame:
+    """
+    Create an example file based on a list of dicts like follows:
+    fields = [
+        {
+            "name": "column_name",
+            "type": "column_type",
+            "args": {dict_of_args}  # optional
+        },
+        ...
+    ]
+    Or from a TableSchema
+    """
+    # need to make a CLI command
+
+    if not (fields or schema_path):
+        raise ValueError("At least fields or schema_path must be specified.")
+
+    def potential_skip(required: bool) -> bool:
+        if ignore_required:
+            return False
+        if not required:
+            # for now 30% chance to have an optional value, this could go as an argument
+            return random.randint(1, 100) <= 30
+
+    def _string(
+        length: int = 10,
+        required: bool = True,
+        pattern: str | None = None,
+        enum: str | None = None,
+    ) -> str:
+        if potential_skip(required):
+            return ""
+        if pattern is not None:
+            return rstr.xeger(pattern)
+        elif enum is not None:
+            return random.choice(enum)
+        else:
+            letters = string.ascii_lowercase
+            return "".join(random.choice(letters) for i in range(length))
+
+    def _id(
+        required: bool = True,
+    ) -> str:
+        if potential_skip(required):
+            return ""
+        return str(uuid.uuid4())
+
+    def _date(
+        date_range: list[str] | None = None,
+        format: str = "%Y-%m-%d",
+        required: bool = True,
+    ) -> str:
+        # the bounds specified in date_range are expected in the same format as the desired output format
+        assert all([k in format for k in ["%d", "%m", "%Y"]])
+        if potential_skip(required):
+            return ""
+        if date_range is None:
+            return fake.date(format)
+        else:
+            if len(date_range) != 2:
+                raise ValueError("'date_range' must have exactly two elements.")
+            return fake.date_between_dates(
+                datetime.strptime(date_range[0], format),
+                datetime.strptime(date_range[1], format),
+            ).strftime(format)
+
+    def _time(
+        format: str = "%H:%M:%S",
+        required: bool = True,
+    ) -> str:
+        assert all([k in format for k in ["%H", "%M", "%S"]])
+        if potential_skip(required):
+            return ""
+        # maybe add a time_range argument?
+        return fake.time(format)
+
+    def _datetime(
+        datetime_range: list[str] | None = None,
+        format: str = "%Y-%m-%d %H-%M-%S",
+        required: bool = True,
+    ) -> str:
+        # the bounds specified in datetime_range are expected in the same format as the desired output format
+        assert all([k in format for k in ["%d", "%m", "%Y", "%H", "%M", "%S"]])
+        if potential_skip(required):
+            return ""
+        if datetime_range is None:
+            return fake.date_time().strftime(format)
+        else:
+            if len(datetime_range) != 2:
+                raise ValueError("'date_range' must have exactly two elements.")
+            return fake.date_time_between(
+                datetime.strptime(datetime_range[0], format),
+                datetime.strptime(datetime_range[1], format),
+            ).strftime(format)
+
+    def _url(required: bool = True) -> str:
+        if potential_skip(required):
+            return ""
+        return f"http://{rstr.domainsafe()}.{rstr.letters(3)}/{rstr.urlsafe()}"
+
+    def _number(
+        num_type: Type[int | float] = int,
+        num_range: list[float] | None = None,
+        enum: list | None = None,
+        required: bool = True,
+    ) -> int | float:
+        assert num_range is None or len(num_range) == 2
+        if potential_skip(required):
+            return ""
+        if enum:
+            return random.choice(enum)
+        if num_range is None:
+            num_range = [0, 1000]
+        if num_type is int:
+            return random.randint(num_range[0], num_range[1])
+        else:
+            return round(random.uniform(num_range[0], num_range[1]), 1)
+
+    def _bool(required: bool = True) -> bool:
+        if potential_skip(required):
+            return ""
+        return random.randint(0, 1) == 0
+
+    def _array(enum: list[Any], required: bool = True) -> str:
+        if potential_skip(required):
+            return ""
+        return f"[{','.join(random.sample(enum, random.randint(1, len(enum))))}]"
+
+    def build_args_from_constraints(constraints: dict) -> dict:
+        args = {}
+        args["required"] = constraints.get("required", False)
+        for _ in ["pattern", "enum", "format"]:
+            if _ in constraints:
+                args[_] = constraints[_]
+        if "minimum" in constraints and "maximum" in constraints:
+            args["num_range"] = [constraints["minimum"], constraints["maximum"]]
+        # maybe there are better values than these?
+        elif "minimum" in constraints:
+            args["num_range"] = [constraints["minimum"], 10 + constraints["minimum"]]
+        elif "maximum" in constraints:
+            args["num_range"] = [constraints["maximum"] - 10, constraints["maximum"]]
+        if "minLength" in constraints:
+            args["length"] = constraints["minLength"]
+        if "maxLength" in constraints:
+            args["length"] = constraints["maxLength"]
+        return args
+
+    schema_types_to_python = {
+        "number": "float",
+        "integer": "int",
+        "string": "str",
+        "year": "year",
+        "boolean": "bool",
+        "date": "date",
+        "yearmonth": "date",
+        "time": "time",
+        "datetime": "datetime",
+        "array": "array",
+    }
+
+    if schema_path:
+        if is_url(schema_path):
+            schema = requests.get(schema_path).json()
+        else:
+            with open(schema_path, encoding=encoding) as jsonfile:
+                schema = json.load(jsonfile)
+        if "fields" not in schema.keys():
+            raise ValueError("The schema must have a 'fields' key.")
+        else:
+            fields = [
+                {
+                    "name": f["name"],
+                    "type": schema_types_to_python.get(f["type"], "str"),
+                    # when frformat is supported in TableSchema, we can build args for French standards
+                    # linked to https://github.com/datagouv/fr-format/issues/26
+                    "args": (
+                        build_args_from_constraints(f["constraints"])
+                        if "constraints" in f.keys()
+                        else build_args_from_constraints(f["arrayItem"]["constraints"])
+                        if "arrayItem" in f.keys() and "constraints" in f["arrayItem"].keys()
+                        else {}
+                    ),
+                }
+                for f in schema["fields"]
+            ]
+
+    for k in range(len(fields)):
+        if "args" not in fields[k]:
+            fields[k]["args"] = {}
+        if fields[k]["type"] == "float":
+            fields[k]["args"]["num_type"] = float
+        elif fields[k]["type"] == "int":
+            fields[k]["args"]["num_type"] = int
+        elif fields[k]["type"] == "year":
+            fields[k]["args"]["num_type"] = int
+            fields[k]["args"]["num_range"] = [1990, 2050]
+
+    types_to_func = {
+        "int": _number,
+        "float": _number,
+        "date": _date,
+        "time": _time,
+        "str": _string,
+        "url": _url,
+        "id": _id,
+        "year": _number,
+        "bool": _bool,
+        "datetime": _datetime,
+        "array": _array,
+    }
+
+    # would it be better to create by column or by row (as for now)?
+    output = pd.DataFrame(
+        [
+            [types_to_func.get(f["type"], "str")(**f["args"]) for f in fields]
+            for _ in range(file_length)
+        ],
+        columns=[f["name"] for f in fields],
+    )
+
+    if output_name:
+        output.to_csv(output_name, sep=output_sep, index=False)
+
+    return output

csv_detective/output/profile.py

@@ -0,0 +1,119 @@
+import logging
+from collections import defaultdict
+from time import time
+
+import numpy as np
+import pandas as pd
+
+from csv_detective.formats.float import float_casting
+from csv_detective.utils import cast_prevent_nan, display_logs_depending_process_time
+
+
+def create_profile(
+    table: pd.DataFrame,
+    columns: dict,
+    num_rows: int,
+    limited_output: bool = True,
+    cast_json: bool = True,
+    verbose: bool = False,
+    _col_values: dict[str, pd.Series] | None = None,
+) -> dict:
+    if verbose:
+        start = time()
+        logging.info("Creating profile")
+
+    if num_rows > 0:
+        raise ValueError("To create profiles num_rows has to be set to -1")
+    if not limited_output:
+        columns = {
+            k: v[0] if v else {"python_type": "string", "format": "string", "score": 1.0}
+            for k, v in columns.items()
+        }
+    # value_counts().reset_index() tries to insert a "count" column, and fails if it's already here
+    _count_col = "count"
+    while _count_col in table.columns:
+        _count_col = "_" + _count_col
+    profile = defaultdict(dict)
+    for c in table.columns:
+        # for numerical formats we want min, max, mean, std
+        if columns[c]["python_type"] in ["float", "int"]:
+            # if we have read the file in chunks we already have what we need
+            if _col_values is None:
+                # we locally cast the column to perform the operations,
+                # using the same method as in cast_df
+                cast_col = (
+                    table[c].astype(pd.Int64Dtype())
+                    if columns[c]["python_type"] == "int"
+                    else table[c].apply(lambda x: float_casting(x) if isinstance(x, str) else pd.NA)
+                )
+                stats = {
+                    "min": cast_prevent_nan(cast_col.min(), columns[c]["python_type"]),
+                    "mean": cast_prevent_nan(cast_col.mean(), columns[c]["python_type"]),
+                    "max": cast_prevent_nan(cast_col.max(), columns[c]["python_type"]),
+                    "std": cast_prevent_nan(cast_col.std(), columns[c]["python_type"]),
+                }
+            else:
+                cast_col = _col_values[c].reset_index()
+                cast_col = cast_col.loc[cast_col[c].notna()]
+                cast_col[c] = (
+                    cast_col[c].astype(pd.Int64Dtype())
+                    if columns[c]["python_type"] == "int"
+                    else cast_col[c].apply(
+                        lambda x: float_casting(x) if isinstance(x, str) else pd.NA
+                    )
+                )
+                stats = {
+                    "min": cast_prevent_nan(cast_col[c].min(), columns[c]["python_type"]),
+                    "mean": cast_prevent_nan(
+                        (cast_col[c] * cast_col["count"]).sum() / sum(cast_col["count"]),
+                        columns[c]["python_type"],
+                    ),
+                    "max": cast_prevent_nan(cast_col[c].max(), columns[c]["python_type"]),
+                }
+                stats["std"] = cast_prevent_nan(
+                    np.sqrt(
+                        sum(cast_col["count"] * (cast_col[c] - stats["mean"]) ** 2)
+                        / sum(cast_col["count"])
+                    ),
+                    columns[c]["python_type"],
+                )
+            profile[c].update(**stats)
+            del cast_col
+        # for all formats we want most frequent values, nb unique values and nb missing values
+        tops_bruts = (
+            (table[c].value_counts() if _col_values is None else _col_values[c].sort_values())
+            .reset_index(name=_count_col)
+            .iloc[:10]
+            .to_dict(orient="records")
+        )
+        profile[c].update(
+            tops=[
+                {
+                    "count": tb[_count_col],
+                    "value": tb[c],
+                }
+                for tb in tops_bruts
+            ],
+            nb_distinct=(
+                (
+                    table[c].nunique()
+                    if columns[c]["python_type"] != "json" or not cast_json
+                    # a column containing cast json is not serializable
+                    else table[c].astype(str).nunique()
+                )
+                if _col_values is None
+                else len(_col_values)
+            ),
+            nb_missing_values=(
+                len(table[c].loc[table[c].isna()])
+                if _col_values is None
+                else (_col_values[c].loc[pd.NA] if pd.NA in _col_values[c].index else 0)
+            ),
+        )
+    if verbose:
+        display_logs_depending_process_time(
+            f"Created profile in {round(time() - start, 3)}s",
+            time() - start,
+        )
+    del _col_values
+    return profile