csv-detective 0.10.4.dev1__py3-none-any.whl → 0.10.12674__py3-none-any.whl

tests/test_file.py

@@ -1,469 +0,0 @@
-from unittest.mock import MagicMock, patch
-
-import pandas as pd
-import pytest
-import responses
-
-from csv_detective import routine
-from csv_detective.output.profile import create_profile
-from csv_detective.parsing.csv import CHUNK_SIZE
-
-
-@pytest.fixture
-def mocked_responses():
-    with responses.RequestsMock() as rsps:
-        yield rsps
-
-
-@pytest.mark.parametrize(
-    "chunk_size",
-    (100, 404, int(1e5)),
-)
-def test_columns_output_on_file(chunk_size):
-    with (
-        # maybe we should refactor later to avoid having to patch everywhere
-        patch("csv_detective.parsing.csv.CHUNK_SIZE", chunk_size),
-        patch("csv_detective.parsing.columns.CHUNK_SIZE", chunk_size),
-    ):
-        output = routine(
-            file_path="tests/data/a_test_file.csv",
-            num_rows=-1,
-            output_profile=False,
-            save_results=False,
-        )
-        assert isinstance(output, dict)
-        assert output["separator"] == ";"
-        assert output["header_row_idx"] == 2
-        assert output["header"] == [
-            "NUMCOM",
-            "NOMCOM",
-            "NUMDEP",
-            "NOMDEP",
-            "NUMEPCI",
-            "NOMEPCI",
-            "TXCOUVGLO_COM_2014",
-            "TXCOUVGLO_DEP_2014",
-            "TXCOUVGLO_EPCI_2014",
-            "STRUCTURED_INFO",
-            "GEO_INFO",
-        ]
-        assert output["total_lines"] == 404
-        assert output["nb_duplicates"] == 7
-        assert output["columns"]["NOMCOM"]["format"] == "commune"
-        assert output["columns"]["NOMDEP"]["format"] == "departement"
-        assert output["columns"]["NUMEPCI"]["format"] == "siren"
-        assert output["columns"]["STRUCTURED_INFO"]["python_type"] == "json"
-        assert output["columns"]["STRUCTURED_INFO"]["format"] == "json"
-        assert output["columns"]["GEO_INFO"]["python_type"] == "json"
-        assert output["columns"]["GEO_INFO"]["format"] == "geojson"
-
-
-def test_profile_output_on_file():
-    output = routine(
-        file_path="tests/data/a_test_file.csv",
-        num_rows=-1,
-        output_profile=True,
-        save_results=False,
-    )
-    assert all(
-        [
-            c in list(output["profile"]["TXCOUVGLO_COM_2014"].keys())
-            for c in [
-                "min",
-                "max",
-                "mean",
-                "std",
-                "tops",
-                "nb_distinct",
-                "nb_missing_values",
-            ]
-        ]
-    )
-    assert not any(
-        [
-            c in list(output["profile"]["NUMCOM"].keys())
-            for c in [
-                "min",
-                "max",
-                "mean",
-                "std",
-            ]
-        ]
-    )
-    assert output["profile"]["TXCOUVGLO_COM_2014"]["min"] == 0.0
-    assert output["profile"]["TXCOUVGLO_COM_2014"]["max"] == 200.2
-    assert round(output["profile"]["TXCOUVGLO_COM_2014"]["mean"]) == 60
-    assert round(output["profile"]["TXCOUVGLO_COM_2014"]["std"]) == 36
-    assert output["profile"]["TXCOUVGLO_COM_2014"]["nb_distinct"] == 290
-    assert output["profile"]["TXCOUVGLO_COM_2014"]["nb_missing_values"] == 3
-    assert output["profile"]["GEO_INFO"]["nb_distinct"] == 1
-
-
-def test_profile_with_num_rows():
-    with pytest.raises(ValueError):
-        routine(
-            file_path="tests/data/a_test_file.csv",
-            num_rows=50,
-            output_profile=True,
-            save_results=False,
-        )
-
-
-@pytest.mark.parametrize(
-    "params",
-    (
-        (
-            True,
-            {
-                "int_with_nan": {"format": "int", "python_type": "int"},
-                "date": {"format": "date", "python_type": "date"},
-            },
-        ),
-        (
-            False,
-            {
-                "int_with_nan": [{"format": "int", "python_type": "int"}],
-                "date": [{"format": "date", "python_type": "date"}],
-            },
-        ),
-    ),
-)
-def test_profile_specific_cases(params):
-    limited_output, columns = params
-    table = pd.DataFrame(
-        {
-            "int_with_nan": ["1", pd.NA, pd.NA],
-            "date": ["1996-01-02", "1996-01-02", "2024-11-12"],
-        }
-    )
-    profile = create_profile(
-        table=table,
-        columns=columns,
-        limited_output=limited_output,
-        num_rows=-1,
-    )
-    assert profile["int_with_nan"] == {
-        "min": 1,
-        "max": 1,
-        "mean": 1,
-        "std": None,
-        "tops": [{"count": 1, "value": "1"}],
-        "nb_distinct": 1,
-        "nb_missing_values": 2,
-    }
-    assert profile["date"] == {
-        "tops": [{"count": 2, "value": "1996-01-02"}, {"count": 1, "value": "2024-11-12"}],
-        "nb_distinct": 2,
-        "nb_missing_values": 0,
-    }
-
-
-def test_exception_different_number_of_columns():
-    """
-    A ValueError should be raised if the number of columns differs between the first rows
-    """
-    with pytest.raises(ValueError):
-        routine(
-            file_path="tests/data/c_test_file.csv",
-            num_rows=-1,
-            output_profile=True,
-            save_results=False,
-        )
-
-
-def test_exception_malformed_columns(mocked_responses):
-    """
-    A ValueError should be raised if any column is Unnamed
-    """
-    url = f"http://example.com/bad_cols.csv"
-    expected_content = b"col1,col2,\n1,2,\n3,4,"
-    mocked_responses.get(
-        url,
-        body=expected_content,
-        status=200,
-    )
-    with patch("urllib.request.urlopen") as mock_urlopen:
-        mock_response = MagicMock()
-        mock_response.read.return_value = expected_content
-        mock_response.__enter__.return_value = mock_response
-        mock_urlopen.return_value = mock_response
-        with pytest.raises(ValueError):
-            routine(file_path=url)
-
-
-def test_code_dep_reg_on_file():
-    output = routine(
-        file_path="tests/data/b_test_file.csv",
-        num_rows=-1,
-        output_profile=False,
-        save_results=False,
-    )
-    assert isinstance(output, dict)
-    assert output["columns"]["code_departement"]["format"] == "code_departement"
-    assert output["columns"]["code_region"]["format"] == "code_region"
-
-
-def test_schema_on_file():
-    output = routine(
-        file_path="tests/data/b_test_file.csv",
-        num_rows=-1,
-        output_schema=True,
-        save_results=False,
-    )
-    assert isinstance(output, dict)
-    is_column_dep = False
-    is_column_reg = False
-    for item in output["schema"]["fields"]:
-        if item["name"] == "code_departement":
-            is_column_dep = True
-            assert item["description"] == "Le code INSEE du département"
-            assert item["type"] == "string"
-            assert item["formatFR"] == "code_departement"
-            assert item["constraints"]["pattern"] == "^(([013-9]\\d|2[AB1-9])$|9\\d{2}$)"
-        if item["name"] == "code_region":
-            is_column_reg = True
-            assert item["description"] == "Le code INSEE de la région"
-            assert item["type"] == "string"
-            assert item["formatFR"] == "code_region"
-            assert item["constraints"]["pattern"] == "^\\d{2}$"
-    assert is_column_dep
-    assert is_column_reg
-
-
-params_csv = [
-    ("csv_file", {"engine": None, "sheet_name": None}),
-    ("file.csv.gz", {"engine": None, "sheet_name": None, "separator": ",", "columns.len": 3}),
-]
-params_others = [
-    ("file.ods", {"engine": "odf"}),
-    # this is a "tricked" xls file that is actually read as odf
-    ("file.xls", {"engine": "odf"}),
-    # this file has an empty first row; check if the sheet we consider is the largest
-    ("file.xlsx", {"engine": "openpyxl", "header_row_idx": 1, "sheet_name": "REI_1987"}),
-    ("xlsx_file", {"engine": "openpyxl"}),
-]
-
-
-@pytest.mark.parametrize("params", params_csv + params_others)
-def test_non_csv_files(params):
-    file_name, checks = params
-    _ = routine(
-        file_path=f"tests/data/{file_name}",
-        num_rows=-1,
-        output_profile=False,
-        save_results=False,
-    )
-    for k, v in checks.items():
-        if v is None:
-            assert not _.get(k)
-        elif "." in k:
-            key, func = k.split(".")
-            assert eval(func)(_[key]) == v
-        else:
-            assert _[k] == v
-
-
-@pytest.mark.parametrize(
-    "params",
-    # ideally we'd like to do the same with params_others but pandas.read_excel uses urllib
-    # which doesn't support the way we mock the response, TBC
-    params_csv + [("a_test_file.csv", {"separator": ";", "header_row_idx": 2, "total_lines": 404})],
-)
-def test_urls(mocked_responses, params):
-    file_name, checks = params
-    url = f"http://example.com/{file_name}"
-    expected_content = open(f"tests/data/{file_name}", "rb").read()
-    mocked_responses.get(
-        url,
-        body=expected_content,
-        status=200,
-    )
-    with patch("urllib.request.urlopen") as mock_urlopen:
-        mock_response = MagicMock()
-        mock_response.read.return_value = expected_content
-        mock_response.__enter__.return_value = mock_response
-        mock_urlopen.return_value = mock_response
-        _ = routine(
-            file_path=url,
-            num_rows=-1,
-            output_profile=False,
-            save_results=False,
-        )
-    for k, v in checks.items():
-        if v is None:
-            assert not _.get(k)
-        elif "." in k:
-            key, func = k.split(".")
-            assert eval(func)(_[key]) == v
-        else:
-            assert _[k] == v
-
-
-@pytest.mark.parametrize(
-    "expected_type",
-    (
-        (True, "int"),
-        (False, "string"),
-    ),
-)
-def test_nan_values(expected_type):
-    # if skipping NaN, the column contains only ints
-    skipna, expected_type = expected_type
-    output = routine(
-        file_path="tests/data/b_test_file.csv",
-        num_rows=-1,
-        save_results=False,
-        skipna=skipna,
-    )
-    assert output["columns"]["partly_empty"]["python_type"] == expected_type
-
-
-def test_output_df():
-    output, df_chunks = routine(
-        file_path="tests/data/b_test_file.csv",
-        num_rows=-1,
-        output_profile=False,
-        save_results=False,
-        output_df=True,
-    )
-    df = pd.concat(df_chunks, ignore_index=True)
-    assert isinstance(output, dict)
-    assert isinstance(df, pd.DataFrame)
-    assert len(df) == 6
-    assert df["partly_empty"].dtype == pd.Int64Dtype()
-
-
-@pytest.mark.parametrize(
-    "cast_json",
-    (
-        (True, dict),
-        (False, str),
-    ),
-)
-def test_cast_json(mocked_responses, cast_json):
-    cast_json, expected_type = cast_json
-    expected_content = 'id,a_simple_dict\n1,{"a": 1}\n2,{"b": 2}\n3,{"c": 3}\n'
-    mocked_responses.get(
-        "http://example.com/test.csv",
-        body=expected_content,
-        status=200,
-    )
-    with patch("urllib.request.urlopen") as mock_urlopen:
-        mock_response = MagicMock()
-        mock_response.read.return_value = expected_content.encode("utf-8")
-        mock_response.__enter__.return_value = mock_response
-        mock_urlopen.return_value = mock_response
-        analysis, df_chunks = routine(
-            file_path="http://example.com/test.csv",
-            num_rows=-1,
-            output_profile=False,
-            save_results=False,
-            output_df=True,
-            cast_json=cast_json,
-        )
-    df = pd.concat(df_chunks, ignore_index=True)
-    assert analysis["columns"]["a_simple_dict"]["python_type"] == "json"
-    assert isinstance(df["a_simple_dict"][0], expected_type)
-
-
-def test_almost_uniform_column(mocked_responses):
-    col_name = "int_not_bool"
-    expected_content = f"{col_name}\n" + "9\n" + "1\n" * int(1e7)
-    mocked_responses.get(
-        "http://example.com/test.csv",
-        body=expected_content,
-        status=200,
-    )
-    with patch("urllib.request.urlopen") as mock_urlopen:
-        mock_response = MagicMock()
-        mock_response.read.return_value = expected_content.encode("utf-8")
-        mock_response.__enter__.return_value = mock_response
-        mock_urlopen.return_value = mock_response
-        analysis = routine(
-            file_path="http://example.com/test.csv",
-            num_rows=-1,
-            output_profile=False,
-            save_results=False,
-        )
-    assert analysis["columns"][col_name]["format"] == "int"
-
-
-@pytest.mark.parametrize("nb_rows", (CHUNK_SIZE // 10, CHUNK_SIZE + 1))
-def test_full_nan_column(mocked_responses, nb_rows):
-    # we want a file that needs sampling
-    col_name = "only_nan"
-    expected_content = f"{col_name},second_col\n" + ",1\n" * nb_rows
-    mocked_responses.get(
-        "http://example.com/test.csv",
-        body=expected_content,
-        status=200,
-    )
-    with patch("urllib.request.urlopen") as mock_urlopen:
-        mock_response = MagicMock()
-        mock_response.read.return_value = expected_content.encode("utf-8")
-        mock_response.__enter__.return_value = mock_response
-        mock_urlopen.return_value = mock_response
-        # only NaNs should return "string"
-        analysis = routine(
-            file_path="http://example.com/test.csv",
-            num_rows=-1,
-            output_profile=False,
-            save_results=False,
-        )
-        assert analysis["columns"][col_name]["format"] == "string"
-
-
-@pytest.mark.parametrize(
-    "nb_rows",
-    (100, CHUNK_SIZE + 1),
-)
-def test_count_column(mocked_responses, nb_rows):
-    expected_content = "count,_count\n" + "a,1\n" * nb_rows
-    mocked_responses.get(
-        "http://example.com/test.csv",
-        body=expected_content,
-        status=200,
-    )
-    with patch("urllib.request.urlopen") as mock_urlopen:
-        mock_response = MagicMock()
-        mock_response.read.return_value = expected_content.encode("utf-8")
-        mock_response.__enter__.return_value = mock_response
-        mock_urlopen.return_value = mock_response
-        # only testing it doesn't fail with output_profile=True
-        routine(
-            file_path="http://example.com/test.csv",
-            num_rows=-1,
-            output_profile=True,
-            save_results=False,
-        )
-
-
-def test_multiple_geo_columns(mocked_responses):
-    lat, not_lat = "latitude_obj", "latin"
-    lon, not_lon = "longitude_obj", "longueur"
-    expected_content = f"{lat},{lon},{not_lat},{not_lon}\n" + "1.0,-10.0,1.0,-10.0\n" * 10
-    mocked_responses.get(
-        "http://example.com/test.csv",
-        body=expected_content,
-        status=200,
-    )
-    analysis = routine(
-        file_path="http://example.com/test.csv",
-        num_rows=-1,
-        output_profile=False,
-        save_results=False,
-    )
-    # we want the lat/lon columns to be labelled as such, and either:
-    # - the not lat/lon columns to be labelled as float only
-    # - or the not lat/lon columns to be labelled as lat/lon but with a lower score
-    # both cases are acceptable
-    assert analysis["columns"][lat]["format"] == "latitude_wgs"
-    assert analysis["columns"][lon]["format"] == "longitude_wgs"
-    assert analysis["columns"][not_lat]["format"] == "float" or (
-        analysis["columns"][not_lat]["format"] == "latitude_wgs"
-        and analysis["columns"][not_lat]["score"] < analysis["columns"][lat]["score"]
-    )
-    assert analysis["columns"][not_lon]["format"] == "float" or (
-        analysis["columns"][not_lon]["format"] == "longitude_wgs"
-        and analysis["columns"][not_lon]["score"] < analysis["columns"][lon]["score"]
-    )

tests/test_labels.py

@@ -1,26 +0,0 @@
-import pytest
-
-from csv_detective.format import FormatsManager
-
-fmtm = FormatsManager()
-
-
-# money labels
-def test_money_labels():
-    header = "Montant total"
-    assert fmtm.formats["money"].is_valid_label(header) == 0.5
-
-
-@pytest.mark.parametrize(
-    "params",
-    [
-        ("latitude", 1.0),
-        ("lat", 0.75),
-        ("coord_lat", 0.375),
-        ("y", 0.5),
-        ("nb_cycles", 0.0),
-    ],
-)
-def test_latitude(params):
-    header, expected = params
-    assert expected == fmtm.formats["latitude_wgs"].is_valid_label(header)

tests/test_structure.py

@@ -1,45 +0,0 @@
-import os
-
-import pytest
-
-from csv_detective.format import Format, FormatsManager
-
-fmtm = FormatsManager()
-
-
-def test_all_tests_have_unique_name():
-    formats: list[str] = os.listdir("csv_detective/formats")
-    assert "__init__.py" in formats
-    assert len(formats) == len(set(formats))
-
-
-def test_conformity():
-    for name, format in fmtm.formats.items():
-        assert isinstance(name, str)
-        assert isinstance(format, Format)
-        assert all(
-            getattr(format, attr) is not None
-            for attr in [
-                "name",
-                "func",
-                "_test_values",
-                "labels",
-                "proportion",
-                "tags",
-            ]
-        )
-
-
-@pytest.mark.parametrize(
-    "tags",
-    (
-        ["type"],
-        ["temp", "fr"],
-    ),
-)
-def test_get_from_tags(tags):
-    fmts = fmtm.get_formats_from_tags(tags)
-    assert len(fmts)
-    for fmt in fmts.values():
-        for tag in tags:
-            assert tag in fmt.tags

tests/test_validation.py

@@ -1,163 +0,0 @@
-import json
-from unittest.mock import MagicMock, patch
-
-import pandas as pd
-import pytest
-
-from csv_detective.explore_csv import validate_then_detect
-from csv_detective.validate import validate
-
-
-def set_nested_value(source_dict: dict, key_chain: list[str], value):
-    current_dict = source_dict
-    for key in key_chain[:-1]:
-        if key not in current_dict:
-            current_dict[key] = {}
-        current_dict = current_dict[key]
-    current_dict[key_chain[-1]] = value
-
-
-def get_nested_value(source_dict: dict, key_chain: list[str]):
-    result = source_dict
-    for k in key_chain:
-        result = result[k]
-    return result
-
-
-@pytest.mark.parametrize(
-    "_params",
-    (
-        ((True, dict), {}),
-        ((False, None), {"separator": "|"}),
-        ((False, None), {"encoding": "unknown"}),
-        ((False, None), {"header": ["a", "b"]}),
-        (
-            (False, None),
-            {
-                "columns.NUMCOM": {
-                    "python_type": "int",
-                    "format": "int",
-                    "score": 1.0,
-                },
-            },
-        ),
-    ),
-)
-def test_validation(_params):
-    (should_be_valid, analysis_type), modif_previous_analysis = _params
-    with open("tests/data/a_test_file.json", "r") as f:
-        previous_analysis = json.load(f)
-    for dotkey in modif_previous_analysis:
-        keys = dotkey.split(".")
-        set_nested_value(previous_analysis, keys, modif_previous_analysis[dotkey])
-    is_valid, analysis, col_values = validate(
-        "tests/data/a_test_file.csv",
-        previous_analysis=previous_analysis,
-    )
-    assert is_valid == should_be_valid
-    if analysis_type is None:
-        assert analysis is None
-    else:
-        assert isinstance(analysis, analysis_type)
-    if should_be_valid:
-        assert isinstance(col_values, dict)
-    else:
-        assert col_values is None
-
-
-@pytest.mark.parametrize(
-    "_params",
-    (
-        # int: proportion = 1, should fail (early)
-        ("12", "1.2", {"python_type": "int", "format": "int", "score": 1.5}, False),
-        # siren: proportion = 0.9, should fail (later)
-        (
-            "130025265",
-            "A13794BC",
-            {"python_type": "string", "format": "siren", "score": 1.5},
-            False,
-        ),
-        # siret: proportion = 0.8, should succeed
-        (
-            "13002526500013",
-            "A13794BC",
-            {"python_type": "string", "format": "siret", "score": 1.5},
-            True,
-        ),
-    ),
-)
-def test_validation_with_proportions(_params):
-    # testing the behaviour for a file that has 15% invalid values, but all in a single chunk
-    valid_value, invalid_value, detected, should_be_valid = _params
-    url = f"http://example.com/test.csv"
-    expected_content = "col\n"
-    for _ in range(60):
-        # 60 rows of valid values
-        expected_content += f"{valid_value}\n"
-    for _ in range(15):
-        # 15 rows of invalid values
-        expected_content += f"{invalid_value}\n"
-    for _ in range(25):
-        # 25 rows of valid values
-        expected_content += f"{valid_value}\n"
-    previous_analysis = {
-        "encoding": "utf-8",
-        "separator": ",",
-        "header_row_idx": 0,
-        "header": ["col"],
-        "columns": {"col": detected},
-        # just setting these keys when validation is successful, they're not used for the validation itself
-        "categorical": [],
-        "columns_fields": {},
-        "columns_labels": {},
-        "formats": {},
-    }
-    with (
-        patch("urllib.request.urlopen") as mock_urlopen,
-        patch("csv_detective.validate.VALIDATION_CHUNK_SIZE", 10),
-    ):
-        mock_response = MagicMock()
-        mock_response.read.return_value = expected_content.encode("utf-8")
-        mock_response.__enter__.return_value = mock_response
-        mock_urlopen.return_value = mock_response
-        is_valid, *_ = validate(
-            file_path=url,
-            previous_analysis=previous_analysis,
-        )
-    assert is_valid == should_be_valid
-
-
-@pytest.mark.parametrize(
-    "modif_previous_analysis",
-    (
-        {"separator": "|"},
-        {"encoding": "unknown"},
-        {"header": ["a", "b"]},
-        {"total_lines": 100},
-        {
-            "columns.NUMCOM": {
-                "python_type": "int",
-                "format": "int",
-                "score": 1.0,
-            },
-        },
-    ),
-)
-def test_validate_then_detect(modif_previous_analysis):
-    with open("tests/data/a_test_file.json", "r") as f:
-        previous_analysis = json.load(f)
-    valid_values = {}
-    for dotkey in modif_previous_analysis:
-        keys = dotkey.split(".")
-        valid_values[dotkey] = get_nested_value(previous_analysis, keys)
-        set_nested_value(previous_analysis, keys, modif_previous_analysis[dotkey])
-    analysis = validate_then_detect(
-        "tests/data/a_test_file.csv",
-        previous_analysis=previous_analysis,
-        num_rows=-1,
-        output_profile=True,
-        save_results=False,
-    )
-    # checking that if not valid, the analysis has managed to retrieve the right values
-    for dotkey in modif_previous_analysis:
-        assert get_nested_value(analysis, dotkey.split(".")) == valid_values[dotkey]