csv-detective 0.10.1.dev2669__py3-none-any.whl → 0.10.2.dev1__py3-none-any.whl

tests/test_file.py

@@ -0,0 +1,448 @@
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+import pytest
+import responses
+
+from csv_detective import routine
+from csv_detective.output.profile import create_profile
+from csv_detective.parsing.csv import CHUNK_SIZE
+
+
+@pytest.mark.parametrize(
+    "chunk_size",
+    (100, 404, int(1e5)),
+)
+def test_columns_output_on_file(chunk_size):
+    with (
+        # maybe we should refactor later to avoid having to patch everywhere
+        patch("csv_detective.parsing.csv.CHUNK_SIZE", chunk_size),
+        patch("csv_detective.parsing.columns.CHUNK_SIZE", chunk_size),
+    ):
+        output = routine(
+            file_path="tests/data/a_test_file.csv",
+            num_rows=-1,
+            output_profile=False,
+            save_results=False,
+        )
+        assert isinstance(output, dict)
+        assert output["separator"] == ";"
+        assert output["header_row_idx"] == 2
+        assert output["header"] == [
+            "NUMCOM",
+            "NOMCOM",
+            "NUMDEP",
+            "NOMDEP",
+            "NUMEPCI",
+            "NOMEPCI",
+            "TXCOUVGLO_COM_2014",
+            "TXCOUVGLO_DEP_2014",
+            "TXCOUVGLO_EPCI_2014",
+            "STRUCTURED_INFO",
+            "GEO_INFO",
+        ]
+        assert output["total_lines"] == 404
+        assert output["nb_duplicates"] == 7
+        assert output["columns"]["NOMCOM"]["format"] == "commune"
+        assert output["columns"]["NOMDEP"]["format"] == "departement"
+        assert output["columns"]["NUMEPCI"]["format"] == "siren"
+        assert output["columns"]["STRUCTURED_INFO"]["python_type"] == "json"
+        assert output["columns"]["STRUCTURED_INFO"]["format"] == "json"
+        assert output["columns"]["GEO_INFO"]["python_type"] == "json"
+        assert output["columns"]["GEO_INFO"]["format"] == "geojson"
+
+
+def test_profile_output_on_file():
+    output = routine(
+        file_path="tests/data/a_test_file.csv",
+        num_rows=-1,
+        output_profile=True,
+        save_results=False,
+    )
+    assert all(
+        [
+            c in list(output["profile"]["TXCOUVGLO_COM_2014"].keys())
+            for c in [
+                "min",
+                "max",
+                "mean",
+                "std",
+                "tops",
+                "nb_distinct",
+                "nb_missing_values",
+            ]
+        ]
+    )
+    assert not any(
+        [
+            c in list(output["profile"]["NUMCOM"].keys())
+            for c in [
+                "min",
+                "max",
+                "mean",
+                "std",
+            ]
+        ]
+    )
+    assert output["profile"]["TXCOUVGLO_COM_2014"]["min"] == 0.0
+    assert output["profile"]["TXCOUVGLO_COM_2014"]["max"] == 200.2
+    assert round(output["profile"]["TXCOUVGLO_COM_2014"]["mean"]) == 60
+    assert round(output["profile"]["TXCOUVGLO_COM_2014"]["std"]) == 36
+    assert output["profile"]["TXCOUVGLO_COM_2014"]["nb_distinct"] == 290
+    assert output["profile"]["TXCOUVGLO_COM_2014"]["nb_missing_values"] == 3
+    assert output["profile"]["GEO_INFO"]["nb_distinct"] == 1
+
+
+def test_profile_with_num_rows():
+    with pytest.raises(ValueError):
+        routine(
+            file_path="tests/data/a_test_file.csv",
+            num_rows=50,
+            output_profile=True,
+            save_results=False,
+        )
+
+
+@pytest.mark.parametrize(
+    "params",
+    (
+        (
+            True,
+            {
+                "int_with_nan": {"format": "int", "python_type": "int"},
+                "date": {"format": "date", "python_type": "date"},
+            },
+        ),
+        (
+            False,
+            {
+                "int_with_nan": [{"format": "int", "python_type": "int"}],
+                "date": [{"format": "date", "python_type": "date"}],
+            },
+        ),
+    ),
+)
+def test_profile_specific_cases(params):
+    limited_output, columns = params
+    table = pd.DataFrame(
+        {
+            "int_with_nan": ["1", pd.NA, pd.NA],
+            "date": ["1996-01-02", "1996-01-02", "2024-11-12"],
+        }
+    )
+    profile = create_profile(
+        table=table,
+        columns=columns,
+        limited_output=limited_output,
+        num_rows=-1,
+    )
+    assert profile["int_with_nan"] == {
+        "min": 1,
+        "max": 1,
+        "mean": 1,
+        "std": None,
+        "tops": [{"count": 1, "value": "1"}],
+        "nb_distinct": 1,
+        "nb_missing_values": 2,
+    }
+    assert profile["date"] == {
+        "tops": [{"count": 2, "value": "1996-01-02"}, {"count": 1, "value": "2024-11-12"}],
+        "nb_distinct": 2,
+        "nb_missing_values": 0,
+    }
+
+
+def test_exception_different_number_of_columns():
+    """
+    A ValueError should be raised if the number of columns differs between the first rows
+    """
+    with pytest.raises(ValueError):
+        routine(
+            file_path="tests/data/c_test_file.csv",
+            num_rows=-1,
+            output_profile=True,
+            save_results=False,
+        )
+
+
+def test_code_dep_reg_on_file():
+    output = routine(
+        file_path="tests/data/b_test_file.csv",
+        num_rows=-1,
+        output_profile=False,
+        save_results=False,
+    )
+    assert isinstance(output, dict)
+    assert output["columns"]["code_departement"]["format"] == "code_departement"
+    assert output["columns"]["code_region"]["format"] == "code_region"
+
+
+def test_schema_on_file():
+    output = routine(
+        file_path="tests/data/b_test_file.csv",
+        num_rows=-1,
+        output_schema=True,
+        save_results=False,
+    )
+    assert isinstance(output, dict)
+    is_column_dep = False
+    is_column_reg = False
+    for item in output["schema"]["fields"]:
+        if item["name"] == "code_departement":
+            is_column_dep = True
+            assert item["description"] == "Le code INSEE du département"
+            assert item["type"] == "string"
+            assert item["formatFR"] == "code_departement"
+            assert item["constraints"]["pattern"] == "^(([013-9]\\d|2[AB1-9])$|9\\d{2}$)"
+        if item["name"] == "code_region":
+            is_column_reg = True
+            assert item["description"] == "Le code INSEE de la région"
+            assert item["type"] == "string"
+            assert item["formatFR"] == "code_region"
+            assert item["constraints"]["pattern"] == "^\\d{2}$"
+    assert is_column_dep
+    assert is_column_reg
+
+
+params_csv = [
+    ("csv_file", {"engine": None, "sheet_name": None}),
+    ("file.csv.gz", {"engine": None, "sheet_name": None, "separator": ",", "columns.len": 3}),
+]
+params_others = [
+    ("file.ods", {"engine": "odf"}),
+    # this is a "tricked" xls file that is actually read as odf
+    ("file.xls", {"engine": "odf"}),
+    # this file has an empty first row; check if the sheet we consider is the largest
+    ("file.xlsx", {"engine": "openpyxl", "header_row_idx": 1, "sheet_name": "REI_1987"}),
+    ("xlsx_file", {"engine": "openpyxl"}),
+]
+
+
+@pytest.mark.parametrize("params", params_csv + params_others)
+def test_non_csv_files(params):
+    file_name, checks = params
+    _ = routine(
+        file_path=f"tests/data/{file_name}",
+        num_rows=-1,
+        output_profile=False,
+        save_results=False,
+    )
+    for k, v in checks.items():
+        if v is None:
+            assert not _.get(k)
+        elif "." in k:
+            key, func = k.split(".")
+            assert eval(func)(_[key]) == v
+        else:
+            assert _[k] == v
+
+
+@pytest.fixture
+def mocked_responses():
+    with responses.RequestsMock() as rsps:
+        yield rsps
+
+
+@pytest.mark.parametrize(
+    "params",
+    # ideally we'd like to do the same with params_others but pandas.read_excel uses urllib
+    # which doesn't support the way we mock the response, TBC
+    params_csv + [("a_test_file.csv", {"separator": ";", "header_row_idx": 2, "total_lines": 404})],
+)
+def test_urls(mocked_responses, params):
+    file_name, checks = params
+    url = f"http://example.com/{file_name}"
+    expected_content = open(f"tests/data/{file_name}", "rb").read()
+    mocked_responses.get(
+        url,
+        body=expected_content,
+        status=200,
+    )
+    with patch("urllib.request.urlopen") as mock_urlopen:
+        mock_response = MagicMock()
+        mock_response.read.return_value = expected_content
+        mock_response.__enter__.return_value = mock_response
+        mock_urlopen.return_value = mock_response
+        _ = routine(
+            file_path=url,
+            num_rows=-1,
+            output_profile=False,
+            save_results=False,
+        )
+    for k, v in checks.items():
+        if v is None:
+            assert not _.get(k)
+        elif "." in k:
+            key, func = k.split(".")
+            assert eval(func)(_[key]) == v
+        else:
+            assert _[k] == v
+
+
+@pytest.mark.parametrize(
+    "expected_type",
+    (
+        (True, "int"),
+        (False, "string"),
+    ),
+)
+def test_nan_values(expected_type):
+    # if skipping NaN, the column contains only ints
+    skipna, expected_type = expected_type
+    output = routine(
+        file_path="tests/data/b_test_file.csv",
+        num_rows=-1,
+        save_results=False,
+        skipna=skipna,
+    )
+    assert output["columns"]["partly_empty"]["python_type"] == expected_type
+
+
+def test_output_df():
+    output, df_chunks = routine(
+        file_path="tests/data/b_test_file.csv",
+        num_rows=-1,
+        output_profile=False,
+        save_results=False,
+        output_df=True,
+    )
+    df = pd.concat(df_chunks, ignore_index=True)
+    assert isinstance(output, dict)
+    assert isinstance(df, pd.DataFrame)
+    assert len(df) == 6
+    assert df["partly_empty"].dtype == pd.Int64Dtype()
+
+
+@pytest.mark.parametrize(
+    "cast_json",
+    (
+        (True, dict),
+        (False, str),
+    ),
+)
+def test_cast_json(mocked_responses, cast_json):
+    cast_json, expected_type = cast_json
+    expected_content = 'id,a_simple_dict\n1,{"a": 1}\n2,{"b": 2}\n3,{"c": 3}\n'
+    mocked_responses.get(
+        "http://example.com/test.csv",
+        body=expected_content,
+        status=200,
+    )
+    with patch("urllib.request.urlopen") as mock_urlopen:
+        mock_response = MagicMock()
+        mock_response.read.return_value = expected_content.encode("utf-8")
+        mock_response.__enter__.return_value = mock_response
+        mock_urlopen.return_value = mock_response
+        analysis, df_chunks = routine(
+            file_path="http://example.com/test.csv",
+            num_rows=-1,
+            output_profile=False,
+            save_results=False,
+            output_df=True,
+            cast_json=cast_json,
+        )
+    df = pd.concat(df_chunks, ignore_index=True)
+    assert analysis["columns"]["a_simple_dict"]["python_type"] == "json"
+    assert isinstance(df["a_simple_dict"][0], expected_type)
+
+
+def test_almost_uniform_column(mocked_responses):
+    col_name = "int_not_bool"
+    expected_content = f"{col_name}\n" + "9\n" + "1\n" * int(1e7)
+    mocked_responses.get(
+        "http://example.com/test.csv",
+        body=expected_content,
+        status=200,
+    )
+    with patch("urllib.request.urlopen") as mock_urlopen:
+        mock_response = MagicMock()
+        mock_response.read.return_value = expected_content.encode("utf-8")
+        mock_response.__enter__.return_value = mock_response
+        mock_urlopen.return_value = mock_response
+        analysis = routine(
+            file_path="http://example.com/test.csv",
+            num_rows=-1,
+            output_profile=False,
+            save_results=False,
+        )
+    assert analysis["columns"][col_name]["format"] == "int"
+
+
+def test_full_nan_column(mocked_responses):
+    # we want a file that needs sampling
+    col_name = "only_nan"
+    expected_content = f"{col_name},second_col\n" + ",1\n" * (CHUNK_SIZE + 1)
+    mocked_responses.get(
+        "http://example.com/test.csv",
+        body=expected_content,
+        status=200,
+    )
+    with patch("urllib.request.urlopen") as mock_urlopen:
+        mock_response = MagicMock()
+        mock_response.read.return_value = expected_content.encode("utf-8")
+        mock_response.__enter__.return_value = mock_response
+        mock_urlopen.return_value = mock_response
+        # only NaNs should return "string"
+        analysis = routine(
+            file_path="http://example.com/test.csv",
+            num_rows=-1,
+            output_profile=False,
+            save_results=False,
+        )
+        assert analysis["columns"][col_name]["format"] == "string"
+
+
+@pytest.mark.parametrize(
+    "nb_rows",
+    (100, CHUNK_SIZE + 1),
+)
+def test_count_column(mocked_responses, nb_rows):
+    expected_content = "count,_count\n" + "a,1\n" * nb_rows
+    mocked_responses.get(
+        "http://example.com/test.csv",
+        body=expected_content,
+        status=200,
+    )
+    with patch("urllib.request.urlopen") as mock_urlopen:
+        mock_response = MagicMock()
+        mock_response.read.return_value = expected_content.encode("utf-8")
+        mock_response.__enter__.return_value = mock_response
+        mock_urlopen.return_value = mock_response
+        # only testing it doesn't fail with output_profile=True
+        routine(
+            file_path="http://example.com/test.csv",
+            num_rows=-1,
+            output_profile=True,
+            save_results=False,
+        )
+
+
+def test_multiple_geo_columns(mocked_responses):
+    lat, not_lat = "latitude_obj", "latin"
+    lon, not_lon = "longitude_obj", "longueur"
+    expected_content = f"{lat},{lon},{not_lat},{not_lon}\n" + "1.0,-10.0,1.0,-10.0\n" * 10
+    mocked_responses.get(
+        "http://example.com/test.csv",
+        body=expected_content,
+        status=200,
+    )
+    analysis = routine(
+        file_path="http://example.com/test.csv",
+        num_rows=-1,
+        output_profile=False,
+        save_results=False,
+    )
+    # we want the lat/lon columns to be labelled as such, and either:
+    # - the not lat/lon columns to be labelled as float only
+    # - or the not lat/lon columns to be labelled as lat/lon but with a lower score
+    # both cases are acceptable
+    assert analysis["columns"][lat]["format"] == "latitude_wgs"
+    assert analysis["columns"][lon]["format"] == "longitude_wgs"
+    assert analysis["columns"][not_lat]["format"] == "float" or (
+        analysis["columns"][not_lat]["format"] == "latitude_wgs"
+        and analysis["columns"][not_lat]["score"] < analysis["columns"][lat]["score"]
+    )
+    assert analysis["columns"][not_lon]["format"] == "float" or (
+        analysis["columns"][not_lon]["format"] == "longitude_wgs"
+        and analysis["columns"][not_lon]["score"] < analysis["columns"][lon]["score"]
+    )

tests/test_labels.py

@@ -0,0 +1,26 @@
+import pytest
+
+from csv_detective.format import FormatsManager
+
+fmtm = FormatsManager()
+
+
+# money labels
+def test_money_labels():
+    header = "Montant total"
+    assert fmtm.formats["money"].is_valid_label(header) == 0.5
+
+
+@pytest.mark.parametrize(
+    "params",
+    [
+        ("latitude", 1.0),
+        ("lat", 0.75),
+        ("coord_lat", 0.375),
+        ("y", 0.5),
+        ("nb_cycles", 0.0),
+    ],
+)
+def test_latitude(params):
+    header, expected = params
+    assert expected == fmtm.formats["latitude_wgs"].is_valid_label(header)

tests/test_structure.py

@@ -0,0 +1,45 @@
+import os
+
+import pytest
+
+from csv_detective.format import Format, FormatsManager
+
+fmtm = FormatsManager()
+
+
+def test_all_tests_have_unique_name():
+    formats: list[str] = os.listdir("csv_detective/formats")
+    assert "__init__.py" in formats
+    assert len(formats) == len(set(formats))
+
+
+def test_conformity():
+    for name, format in fmtm.formats.items():
+        assert isinstance(name, str)
+        assert isinstance(format, Format)
+        assert all(
+            getattr(format, attr) is not None
+            for attr in [
+                "name",
+                "func",
+                "_test_values",
+                "labels",
+                "proportion",
+                "tags",
+            ]
+        )
+
+
+@pytest.mark.parametrize(
+    "tags",
+    (
+        ["type"],
+        ["temp", "fr"],
+    ),
+)
+def test_get_from_tags(tags):
+    fmts = fmtm.get_formats_from_tags(tags)
+    assert len(fmts)
+    for fmt in fmts.values():
+        for tag in tags:
+            assert tag in fmt.tags

tests/test_validation.py

@@ -0,0 +1,108 @@
+import json
+
+import pandas as pd
+import pytest
+
+from csv_detective.explore_csv import validate_then_detect
+from csv_detective.validate import validate
+
+
+def set_nested_value(source_dict: dict, key_chain: list[str], value):
+    current_dict = source_dict
+    for key in key_chain[:-1]:
+        if key not in current_dict:
+            current_dict[key] = {}
+        current_dict = current_dict[key]
+    current_dict[key_chain[-1]] = value
+
+
+def get_nested_value(source_dict: dict, key_chain: list[str]):
+    result = source_dict
+    for k in key_chain:
+        result = result[k]
+    return result
+
+
+@pytest.mark.parametrize(
+    "_params",
+    (
+        ((True, pd.DataFrame, dict), {}),
+        ((False, None, None), {"separator": "|"}),
+        ((False, None, None), {"encoding": "unknown"}),
+        ((False, None, None), {"header": ["a", "b"]}),
+        (
+            (False, pd.DataFrame, dict),
+            {
+                "columns.NUMCOM": {
+                    "python_type": "int",
+                    "format": "int",
+                    "score": 1.0,
+                },
+            },
+        ),
+    ),
+)
+def test_validation(_params):
+    (should_be_valid, table_type, analysis_type), modif_previous_analysis = _params
+    with open("tests/data/a_test_file.json", "r") as f:
+        previous_analysis = json.load(f)
+    for dotkey in modif_previous_analysis:
+        keys = dotkey.split(".")
+        set_nested_value(previous_analysis, keys, modif_previous_analysis[dotkey])
+    is_valid, table, analysis, col_values = validate(
+        "tests/data/a_test_file.csv",
+        previous_analysis=previous_analysis,
+    )
+    assert is_valid == should_be_valid
+    if table_type is None:
+        assert table is None
+    else:
+        assert isinstance(table, table_type)
+    if analysis_type is None:
+        assert analysis is None
+    else:
+        assert isinstance(analysis, analysis_type)
+    if should_be_valid:
+        assert isinstance(col_values, dict)
+        assert all(
+            col in table.columns and isinstance(values, pd.Series)
+            for col, values in col_values.items()
+        )
+    else:
+        assert col_values is None
+
+
+@pytest.mark.parametrize(
+    "modif_previous_analysis",
+    (
+        {"separator": "|"},
+        {"encoding": "unknown"},
+        {"header": ["a", "b"]},
+        {"total_lines": 100},
+        {
+            "columns.NUMCOM": {
+                "python_type": "int",
+                "format": "int",
+                "score": 1.0,
+            },
+        },
+    ),
+)
+def test_validate_then_detect(modif_previous_analysis):
+    with open("tests/data/a_test_file.json", "r") as f:
+        previous_analysis = json.load(f)
+    valid_values = {}
+    for dotkey in modif_previous_analysis:
+        keys = dotkey.split(".")
+        valid_values[dotkey] = get_nested_value(previous_analysis, keys)
+        set_nested_value(previous_analysis, keys, modif_previous_analysis[dotkey])
+    analysis = validate_then_detect(
+        "tests/data/a_test_file.csv",
+        previous_analysis=previous_analysis,
+        num_rows=-1,
+        output_profile=True,
+        save_results=False,
+    )
+    # checking that if not valid, the analysis has managed to retrieve the right values
+    for dotkey in modif_previous_analysis:
+        assert get_nested_value(analysis, dotkey.split(".")) == valid_values[dotkey]

csv_detective-0.10.1.dev2669.dist-info/WHEEL

@@ -1,4 +0,0 @@
-Wheel-Version: 1.0
-Generator: uv 0.9.25
-Root-Is-Purelib: true
-Tag: py3-none-any