csv-detective 0.6.7__py3-none-any.whl → 0.9.3.dev2438__py3-none-any.whl

csv_detective/parsing/text.py

@@ -0,0 +1,56 @@
+from re import finditer
+
+
+def camel_case_split(identifier: str):
+    matches = finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)", identifier)
+    return " ".join([m.group(0) for m in matches])
+
+
+translate_dict = {
+    " ": ["-", "_", "'", ",", "  "],
+    "a": ["à", "â"],
+    "c": ["ç"],
+    "e": ["é", "è", "ê", "é"],
+    "i": ["î", "ï"],
+    "o": ["ô", "ö"],
+    "u": ["ù", "û", "ü"],
+}
+
+
+# Process text
+def _process_text(val: str):
+    """Traitement des chaînes de caractères pour les standardiser.
+    Plusieurs alternatives ont été testées : .translate, unidecode.unidecode,
+    des méthodes hybrides, mais aucune ne s'est avérée plus performante."""
+    val = camel_case_split(val)
+    val = val.lower()
+    for target in translate_dict:
+        for source in translate_dict[target]:
+            val = val.replace(source, target)
+    val = val.strip()
+    return val
+
+
+def is_word_in_string(word: str, string: str):
+    # if the substring is too short, the test can become irrelevant
+    return len(word) > 2 and word in string
+
+
+def header_score(header: str, words_combinations_list: list[str]) -> float:
+    """Returns:
+    - 1 if the header is exactly in the specified list
+    - 0.5 if any of the words is within the header
+    - 0 otherwise"""
+    processed_header = _process_text(header)
+
+    header_matches_words_combination = float(
+        any(words_combination == processed_header for words_combination in words_combinations_list)
+    )
+    words_combination_in_header = 0.5 * (
+        any(
+            is_word_in_string(words_combination, processed_header)
+            for words_combination in words_combinations_list
+        )
+    )
+
+    return max(header_matches_words_combination, words_combination_in_header)

csv_detective/utils.py

@@ -1,209 +1,36 @@
-import pandas as pd
 import logging
-from time import time
+
+import pandas as pd
 
 logging.basicConfig(level=logging.INFO)
+logging.addLevelName(
+    logging.CRITICAL, "\033[1;41m%s\033[1;0m" % logging.getLevelName(logging.CRITICAL)
+)
+logging.addLevelName(logging.WARN, "\033[1;31m%s\033[1;0m" % logging.getLevelName(logging.WARN))
 
+THRESHOLD_WARN = 1
+THRESHOLD_CRITICAL = 3
 
-def display_logs_depending_process_time(prompt: str, duration: float):
-    '''
-    Print colored logs according to the time the operation took.
-    '''
-    logging.addLevelName(logging.CRITICAL, "\033[1;41m%s\033[1;0m" % logging.getLevelName(logging.CRITICAL))
-    logging.addLevelName(logging.WARN, "\033[1;31m%s\033[1;0m" % logging.getLevelName(logging.WARN))
 
-    threshold_warn = 1
-    threshold_critical = 3
-
-    if duration < threshold_warn:
-        logging.info(prompt)
-    elif duration < threshold_critical:
-        logging.warn(prompt)
-    else:
-        logging.critical(prompt)
-
-
-def test_col_val(
-    serie, test_func, proportion=0.9, skipna=True, output_mode="ALL", verbose=False
-):
-    """Tests values of the serie using test_func.
-         - skipna : if True indicates that NaNs are not counted as False
-         - proportion :  indicates the proportion of values that have to pass the test
-    for the serie to be detected as a certain format
+def display_logs_depending_process_time(prompt: str, duration: float) -> None:
     """
-    if verbose:
-        start = time()
-
-    # TODO : change for a cleaner method and only test columns in modules labels
-    def apply_test_func(serie, test_func, _range):
-        return serie.sample(n=_range).apply(test_func)
-    try:
-        if skipna:
-            serie = serie[serie.notnull()]
-        ser_len = len(serie)
-        if ser_len == 0:
-            return 0.0
-        if output_mode == "ALL":
-            result = apply_test_func(serie, test_func, ser_len).sum() / ser_len
-            return result if result >= proportion else 0.0
-        else:
-            if proportion == 1:  # Then try first 1 value, then 5, then all
-                for _range in [
-                    min(1, ser_len),
-                    min(5, ser_len),
-                    ser_len,
-                ]:  # Pour ne pas faire d'opérations inutiles, on commence par 1,
-                    # puis 5 valeurs puis la serie complète
-                    if all(apply_test_func(serie, test_func, _range)):
-                        # print(serie.name, ': check OK')
-                        pass
-                    else:
-                        return 0.0
-                return 1.0
-            else:
-                # if we have a proportion, statistically it's OK to analyse up to 10k rows
-                # (arbitrary number) and get a significant result
-                to_analyse = min(ser_len, 10000)
-                result = apply_test_func(serie, test_func, to_analyse).sum() / to_analyse
-                return result if result >= proportion else 0.0
-    finally:
-        if verbose and time() - start > 3:
-            display_logs_depending_process_time(
-                f"\t/!\\ Column '{serie.name}' took too long ({round(time() - start, 3)}s)",
-                time() - start
-            )
-
-
-def test_col_label(label, test_func, proportion=1, output_mode="ALL"):
-    """Tests label (from header) using test_func.
-    - proportion :  indicates the minimum score to pass the test for the serie
-    to be detected as a certain format
+    Print colored logs according to the time the operation took.
     """
-    if output_mode == "ALL":
-        return test_func(label)
+    if duration < THRESHOLD_WARN:
+        logging.info(prompt)
+    elif duration < THRESHOLD_CRITICAL:
+        logging.warning(prompt)
     else:
-        result = test_func(label)
-        return result if result >= proportion else 0
-
-
-def test_col(table, all_tests, output_mode, verbose: bool = False):
-    # Initialising dict for tests
-    if verbose:
-        start = time()
-        logging.info("Testing columns to get types")
-    test_funcs = dict()
-    for test in all_tests:
-        name = test.__name__.split(".")[-1]
-        test_funcs[name] = {"func": test._is, "prop": test.PROPORTION}
-    return_table = pd.DataFrame(columns=table.columns)
-    for idx, (key, value) in enumerate(test_funcs.items()):
-        if verbose:
-            start_type = time()
-            logging.info(f"\t- Starting with type '{key}'")
-        # improvement lead : put the longest tests behind and make them only if previous tests not satisfactory
-        # => the following needs to change, "apply" means all columns are tested for one type at once
-        return_table.loc[key] = table.apply(
-            lambda serie: test_col_val(
-                serie,
-                value["func"],
-                value["prop"],
-                output_mode=output_mode,
-                verbose=verbose,
-            )
-        )
-        if verbose:
-            display_logs_depending_process_time(
-                f'\t> Done with type "{key}" in {round(time() - start_type, 3)}s ({idx+1}/{len(test_funcs)})',
-                time() - start_type
-            )
-    if verbose:
-        display_logs_depending_process_time(f"Done testing columns in {round(time() - start, 3)}s", time() - start)
-    return return_table
-
-
-def test_label(table, all_tests, output_mode, verbose: bool = False):
-    # Initialising dict for tests
-    if verbose:
-        start = time()
-        logging.info("Testing labels to get types")
-    test_funcs = dict()
-    for test in all_tests:
-        name = test.__name__.split(".")[-1]
-        test_funcs[name] = {"func": test._is, "prop": test.PROPORTION}
-
-    return_table = pd.DataFrame(columns=table.columns)
-    for idx, (key, value) in enumerate(test_funcs.items()):
-        if verbose:
-            start_type = time()
-        return_table.loc[key] = [
-            test_col_label(
-                col_name, value["func"], value["prop"], output_mode=output_mode
-            )
-            for col_name in table.columns
-        ]
-        if verbose:
-            display_logs_depending_process_time(
-                f'\t- Done with type "{key}" in {round(time() - start_type, 3)}s ({idx+1}/{len(test_funcs)})',
-                time() - start_type
-            )
-    if verbose:
-        display_logs_depending_process_time(f"Done testing labels in {round(time() - start, 3)}s", time() - start)
-    return return_table
-
-
-def prepare_output_dict(return_table, output_mode):
-    return_dict_cols = return_table.to_dict("dict")
-    return_dict_cols_intermediary = {}
-    for column_name in return_dict_cols:
-        return_dict_cols_intermediary[column_name] = []
-        for detected_value_type in return_dict_cols[column_name]:
-            if return_dict_cols[column_name][detected_value_type] == 0:
-                continue
-            dict_tmp = {}
-            dict_tmp["format"] = detected_value_type
-            dict_tmp["score"] = return_dict_cols[column_name][detected_value_type]
-            return_dict_cols_intermediary[column_name].append(dict_tmp)
-
-        # Clean dict using priorities
-        formats_detected = {
-            x["format"] for x in return_dict_cols_intermediary[column_name]
-        }
-        formats_to_remove = set()
-        # Deprioritise float and int detection vs others
-        if len(formats_detected - {"float", "int"}) > 0:
-            formats_to_remove = formats_to_remove.union({"float", "int"})
-        if "int" in formats_detected:
-            formats_to_remove.add("float")
-        if "latitude_wgs_fr_metropole" in formats_detected:
-            formats_to_remove.add("latitude_l93")
-            formats_to_remove.add("latitude_wgs")
-        if "longitude_wgs_fr_metropole" in formats_detected:
-            formats_to_remove.add("longitude_l93")
-            formats_to_remove.add("longitude_wgs")
-        if "longitude_wgs" in formats_detected:
-            formats_to_remove.add("longitude_l93")
-        if "code_region" in formats_detected:
-            formats_to_remove.add("code_departement")
-
-        formats_to_keep = formats_detected - formats_to_remove
+        logging.critical(prompt)
 
-        detections = return_dict_cols_intermediary[column_name]
-        detections = [x for x in detections if x["format"] in formats_to_keep]
-        if output_mode == "ALL":
-            return_dict_cols_intermediary[column_name] = detections
-        if output_mode == "LIMITED":
-            return_dict_cols_intermediary[column_name] = (
-                max(detections, key=lambda x: x["score"])
-                if len(detections) > 0
-                else {"format": "string", "score": 1.0}
-            )
 
-    return return_dict_cols_intermediary
+def is_url(file_path: str) -> bool:
+    # could be more sophisticated if needed
+    # using the URL detection test was considered but too broad (schema required to use requests)
+    return file_path.startswith("http")
 
 
-def full_word_strictly_inside_string(word, string):
-    return (
-        (" " + word + " " in string)
-        or (string.startswith(word + " "))
-        or (string.endswith(" " + word))
-    )
+def cast_prevent_nan(value: float, _type: str) -> float | int | None:
+    if _type not in {"int", "float"}:
+        raise ValueError(f"Invalid type was passed: {_type}")
+    return None if pd.isna(value) else eval(_type)(value)

csv_detective/validate.py

@@ -0,0 +1,138 @@
+import logging
+
+import pandas as pd
+
+from csv_detective.format import FormatsManager
+from csv_detective.parsing.columns import MAX_NUMBER_CATEGORICAL_VALUES, test_col_val
+
+VALIDATION_CHUNK_SIZE = int(1e5)
+logging.basicConfig(level=logging.INFO)
+
+formats = FormatsManager().formats
+
+
+def validate(
+    file_path: str,
+    previous_analysis: dict,
+    verbose: bool = False,
+    skipna: bool = True,
+) -> tuple[bool, pd.DataFrame | None, dict | None, dict[str, pd.Series] | None]:
+    """
+    Verify is the given file has the same fields and types as in the given analysis.
+
+    Args:
+        file_path: the path of the file to validate
+        previous_analysis: the previous analysis to validate against (expected in the same structure as the output of the routine)
+        verbose: whether the code displays the steps it's going through
+        skipna: whether to ignore NaN values in the checks
+    """
+    try:
+        if previous_analysis.get("separator"):
+            # loading the table in chunks
+            chunks = pd.read_csv(
+                file_path,
+                dtype=str,
+                sep=previous_analysis["separator"],
+                encoding=previous_analysis["encoding"],
+                skiprows=previous_analysis["header_row_idx"],
+                compression=previous_analysis.get("compression"),
+                chunksize=VALIDATION_CHUNK_SIZE,
+            )
+            analysis = {
+                k: v
+                for k, v in previous_analysis.items()
+                if k
+                in ["encoding", "separator", "compression", "heading_columns", "trailing_columns"]
+                and v is not None
+            }
+        else:
+            # or chunks-like if not chunkable
+            chunks = iter(
+                [
+                    pd.read_excel(
+                        file_path,
+                        dtype=str,
+                        engine=previous_analysis["engine"],
+                        sheet_name=previous_analysis["sheet_name"],
+                    )
+                ]
+            )
+            analysis = {k: v for k, v in previous_analysis.items() if k in ["engine", "sheet_name"]}
+        first_chunk = next(chunks)
+        analysis.update(
+            {k: v for k, v in previous_analysis.items() if k in ["header_row_idx", "header"]}
+        )
+    except Exception as e:
+        if verbose:
+            logging.warning(f"> Could not load the file with previous analysis values: {e}")
+        return False, None, None, None
+    if verbose:
+        logging.info("Comparing table with the previous analysis")
+        logging.info("- Checking if all columns match")
+    if len(first_chunk.columns) != len(previous_analysis["header"]) or any(
+        list(first_chunk.columns)[k] != previous_analysis["header"][k]
+        for k in range(len(previous_analysis["header"]))
+    ):
+        if verbose:
+            logging.warning("> Columns do not match, proceeding with full analysis")
+        return False, None, None, None
+    if verbose:
+        logging.info(
+            f"Testing previously detected formats on chunks of {VALIDATION_CHUNK_SIZE} rows"
+        )
+
+    # hashing rows to get nb_duplicates
+    row_hashes_count = first_chunk.apply(lambda row: hash(tuple(row)), axis=1).value_counts()
+    # getting values for profile to read the file only once
+    col_values = {col: first_chunk[col].value_counts(dropna=False) for col in first_chunk.columns}
+    analysis["total_lines"] = 0
+    for idx, chunk in enumerate([first_chunk, *chunks]):
+        if verbose:
+            logging.info(f"> Testing chunk number {idx}")
+        analysis["total_lines"] += len(chunk)
+        row_hashes_count = row_hashes_count.add(
+            chunk.apply(lambda row: hash(tuple(row)), axis=1).value_counts(),
+            fill_value=0,
+        )
+        for col in chunk.columns:
+            col_values[col] = col_values[col].add(
+                chunk[col].value_counts(dropna=False),
+                fill_value=0,
+            )
+        for col_name, args in previous_analysis["columns"].items():
+            if verbose:
+                logging.info(f"- Testing {col_name} for {args['format']}")
+            if args["format"] == "string":
+                # no test for columns that have not been recognized as a specific format
+                continue
+            test_result: float = test_col_val(
+                serie=chunk[col_name],
+                format=formats[args["format"]],
+                skipna=skipna,
+            )
+            if not bool(test_result):
+                if verbose:
+                    logging.warning("> Test failed, proceeding with full analysis")
+                return False, first_chunk, analysis, None
+    if verbose:
+        logging.info("> All checks successful")
+    analysis["nb_duplicates"] = sum(row_hashes_count > 1)
+    analysis["categorical"] = [
+        col for col, values in col_values.items() if len(values) <= MAX_NUMBER_CATEGORICAL_VALUES
+    ]
+    return (
+        True,
+        first_chunk,
+        analysis
+        | {
+            k: previous_analysis[k]
+            for k in [
+                "categorical",
+                "columns",
+                "columns_fields",
+                "columns_labels",
+                "formats",
+            ]
+        },
+        col_values,
+    )