csv-detective 0.9.3.dev1977__py3-none-any.whl → 0.9.3.dev2039__py3-none-any.whl

csv_detective/__init__.py

@@ -1,7 +1,6 @@
-from csv_detective.explore_csv import routine, routine_minio, validate_then_detect
+from csv_detective.explore_csv import routine, validate_then_detect
 
 __all__ = [
     "routine",
-    "routine_minio",
     "validate_then_detect",
 ]

csv_detective/detect_fields/FR/geo/latitude_l93/__init__.py

@@ -3,7 +3,7 @@ from frformat import LatitudeL93
 from csv_detective.detect_fields.other.float import _is as is_float
 from csv_detective.detect_fields.other.float import float_casting
 
-PROPORTION = 0.9
+PROPORTION = 1
 
 _latitudel93 = LatitudeL93()
 

csv_detective/detect_fields/FR/geo/latitude_wgs_fr_metropole/__init__.py

@@ -1,6 +1,6 @@
 from csv_detective.detect_fields.other.float import _is as is_float
 
-PROPORTION = 0.9
+PROPORTION = 1
 
 
 def _is(val):

csv_detective/detect_fields/FR/geo/longitude_l93/__init__.py

@@ -3,7 +3,7 @@ from frformat import LongitudeL93
 from csv_detective.detect_fields.other.float import _is as is_float
 from csv_detective.detect_fields.other.float import float_casting
 
-PROPORTION = 0.9
+PROPORTION = 1
 
 _longitudel93 = LongitudeL93()
 

csv_detective/detect_fields/FR/geo/longitude_wgs_fr_metropole/__init__.py

@@ -1,6 +1,6 @@
 from csv_detective.detect_fields.other.float import _is as is_float
 
-PROPORTION = 0.9
+PROPORTION = 1
 
 
 def _is(val):

csv_detective/detect_fields/geo/latitude_wgs/__init__.py

@@ -1,6 +1,6 @@
 from csv_detective.detect_fields.other.float import _is as is_float
 
-PROPORTION = 0.9
+PROPORTION = 1
 
 
 def _is(val):

csv_detective/detect_fields/geo/longitude_wgs/__init__.py

@@ -1,6 +1,6 @@
 from csv_detective.detect_fields.other.float import _is as is_float
 
-PROPORTION = 0.9
+PROPORTION = 1
 
 
 def _is(val):

csv_detective/explore_csv.py

@@ -1,16 +1,12 @@
-import json
 import logging
-import os
-import tempfile
 from time import time
 from typing import Optional, Union
 
 import pandas as pd
 
 from csv_detective.detection.formats import detect_formats
-from csv_detective.output import generate_output, generate_table_schema
+from csv_detective.output import generate_output
 from csv_detective.parsing.load import load_file
-from csv_detective.s3_utils import download_from_minio, upload_to_minio
 from csv_detective.utils import display_logs_depending_process_time, is_url
 from csv_detective.validate import validate
 
@@ -33,11 +29,11 @@ def routine(
     verbose: bool = False,
     sheet_name: Optional[Union[str, int]] = None,
 ) -> Union[dict, tuple[dict, pd.DataFrame]]:
-    """Returns a dict with information about the csv table and possible
+    """Returns a dict with information about the table and possible
     column contents, and if requested the DataFrame with columns cast according to analysis.
 
     Args:
-        file_path: local path to CSV file if not using Minio
+        file_path: local path or URL to file
         num_rows: number of rows to sample from the file for analysis ; -1 for analysis
         of the whole file
         user_input_tests: tests to run on the file
@@ -173,106 +169,3 @@ def validate_then_detect(
             display_logs_depending_process_time(
                 f"Process completed in {round(time() - start_routine, 3)}s", time() - start_routine
             )
-
-
-def routine_minio(
-    csv_minio_location: dict[str, str],
-    output_minio_location: dict[str, str],
-    tableschema_minio_location: dict[str, str],
-    minio_user: str,
-    minio_pwd: str,
-    **kwargs,
-):
-    """Returns a dict with information about the csv table and possible
-    column contents.
-
-    Args:
-        csv_minio_location: dict with Minio URL, bucket and key of the CSV file
-        output_minio_location: Minio URL, bucket and key to store output file. None if
-        not uploading to Minio.
-        tableschema_minio_location: Minio URL, bucket and key to store tableschema file.
-        None if not uploading the tableschema to Minio.
-        minio_user: user name for the minio instance
-        minio_pwd: password for the minio instance
-        kwargs: arguments for routine
-
-    Returns:
-        dict: a dict with information about the csv and possible types for each column
-    """
-
-    if (
-        (
-            any(
-                [
-                    location_dict is not None
-                    for location_dict in [
-                        csv_minio_location,
-                        output_minio_location,
-                        tableschema_minio_location,
-                    ]
-                ]
-            )
-        )
-        and (minio_user is None)
-        or (minio_pwd is None)
-    ):
-        raise ValueError("Minio credentials are required if using Minio")
-
-    for location_dict in [
-        csv_minio_location,
-        output_minio_location,
-        tableschema_minio_location,
-    ]:
-        if location_dict is not None:
-            if any(
-                [
-                    (location_key not in location_dict) or (location_dict[location_key] is None)
-                    for location_key in ["netloc", "bucket", "key"]
-                ]
-            ):
-                raise ValueError("Minio location dict must contain url, bucket and key")
-
-    file_path = tempfile.NamedTemporaryFile(delete=False).name
-    download_from_minio(
-        netloc=csv_minio_location["netloc"],
-        bucket=csv_minio_location["bucket"],
-        key=csv_minio_location["key"],
-        filepath=file_path,
-        minio_user=minio_user,
-        minio_pwd=minio_pwd,
-    )
-
-    analysis = routine(
-        file_path,
-        save_results=True,
-        **kwargs,
-    )
-
-    # Write report JSON file.
-    output_path_to_store_minio_file = os.path.splitext(file_path)[0] + ".json"
-    with open(output_path_to_store_minio_file, "w", encoding="utf8") as fp:
-        json.dump(analysis, fp, indent=4, separators=(",", ": "))
-
-    upload_to_minio(
-        netloc=output_minio_location["netloc"],
-        bucket=output_minio_location["bucket"],
-        key=output_minio_location["key"],
-        filepath=output_path_to_store_minio_file,
-        minio_user=minio_user,
-        minio_pwd=minio_pwd,
-    )
-
-    os.remove(output_path_to_store_minio_file)
-    os.remove(file_path)
-
-    generate_table_schema(
-        analysis_report=analysis,
-        save_file=True,
-        netloc=tableschema_minio_location["netloc"],
-        bucket=tableschema_minio_location["bucket"],
-        key=tableschema_minio_location["key"],
-        minio_user=minio_user,
-        minio_pwd=minio_pwd,
-    )
-
-    return analysis

csv_detective/output/__init__.py

@@ -51,7 +51,7 @@ def generate_output(
             )
 
     if output_schema:
-        analysis["schema"] = generate_table_schema(analysis, save_file=False, verbose=verbose)
+        analysis["schema"] = generate_table_schema(analysis, save_results=False, verbose=verbose)
 
     if output_df:
         return analysis, cast_df(

csv_detective/output/schema.py

@@ -1,14 +1,9 @@
 import json
 import logging
-import os
-import tempfile
 from datetime import datetime
 from time import time
-from typing import Optional
+from typing import Union
 
-from botocore.exceptions import ClientError
-
-from csv_detective.s3_utils import download_from_minio, get_s3_client, upload_to_minio
 from csv_detective.utils import display_logs_depending_process_time
 
 
@@ -202,25 +197,14 @@ def get_constraints(format: str) -> dict:
 
 def generate_table_schema(
     analysis_report: dict,
-    save_file: bool,
-    netloc: Optional[str] = None,
-    bucket: Optional[str] = None,
-    key: Optional[str] = None,
-    minio_user: Optional[str] = None,
-    minio_pwd: Optional[str] = None,
+    save_results: Union[bool, str] = True,
     verbose: bool = False,
 ) -> dict:
     """Generates a table schema from the analysis report
 
     Args:
         analysis_report (dict): The analysis report from csv_detective
-        save_file (bool): indicate if schema should be saved into minio or just returned
-        netloc (str): The netloc of the minio instance to upload the tableschema
-        bucket (str): The bucket to save the schema in
-        key (str): The key to save the schema in (without extension as we will append
-        version number and extension)
-        minio_user (str): The minio user
-        minio_pwd (str): The minio password
+        save_results (bool or str): whether and where to save the results
 
     Returns:
     """
@@ -277,71 +261,9 @@ def generate_table_schema(
             f"Created schema in {round(time() - start, 3)}s", time() - start
         )
 
-    if not save_file:
-        return schema
-
-    if save_file:
-        if not all([netloc, key, bucket, minio_user, minio_pwd]):
-            raise Exception(
-                "To save schema into minio, parameters : netloc, key, bucket, "
-                "minio_user, minio_pwd should be provided"
-            )
-
-        # Create bucket if does not exist
-        client = get_s3_client(netloc, minio_user, minio_pwd)
-        try:
-            client.head_bucket(Bucket=bucket)
-        except ClientError:
-            client.create_bucket(Bucket=bucket)
-
-        tableschema_objects = client.list_objects(Bucket=bucket, Prefix=key, Delimiter="/")
-        if "Contents" in tableschema_objects:
-            tableschema_keys = [
-                tableschema["Key"]
-                for tableschema in client.list_objects(Bucket=bucket, Prefix=key, Delimiter="/")[
-                    "Contents"
-                ]
-            ]
-            tableschema_versions = [
-                os.path.splitext(tableschema_key)[0].split("_")[-1]
-                for tableschema_key in tableschema_keys
-            ]
-            latest_version = max(tableschema_versions)
+    if save_results:
+        output_path = save_results if isinstance(save_results, str) else "schema.json"
+        with open(output_path, "w", encoding="utf8") as fp:
+            json.dump(schema, fp, indent=4, separators=(",", ": "), ensure_ascii=False, default=str)
 
-            with tempfile.NamedTemporaryFile() as latest_schema_file:
-                with open(latest_schema_file.name, "w") as fp:
-                    download_from_minio(
-                        netloc,
-                        bucket,
-                        f"{key}_{latest_version}.json",
-                        latest_schema_file.name,
-                        minio_user,
-                        minio_pwd,
-                    )
-                    # Check if files are different
-                    with open(latest_schema_file.name, "r") as fp:
-                        latest_schema = json.load(fp)
-                        if latest_schema["fields"] != fields:
-                            latest_version_split = latest_version.split(".")
-                            new_version = (
-                                latest_version_split[0]
-                                + "."
-                                + latest_version_split[1]
-                                + "."
-                                + str(int(latest_version_split[2]) + 1)
-                            )
-                        else:
-                            return None
-
-            schema["version"] = new_version
-
-        tableschema_file = tempfile.NamedTemporaryFile(delete=False)
-        with open(tableschema_file.name, "w") as fp:
-            json.dump(schema, fp, indent=4)
-
-        new_version_key = f"{key}_{new_version}.json"
-        upload_to_minio(
-            netloc, bucket, new_version_key, tableschema_file.name, minio_user, minio_pwd
-        )
-        os.unlink(tableschema_file.name)
-        return {"netloc": netloc, "bucket": bucket, "key": new_version_key}
+    return schema

{csv_detective-0.9.3.dev1977.dist-info → csv_detective-0.9.3.dev2039.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csv-detective
-Version: 0.9.3.dev1977
+Version: 0.9.3.dev2039
 Summary: Detect tabular files column content
 Author-email: Etalab <opendatateam@data.gouv.fr>
 License: MIT
@@ -9,7 +9,6 @@ Keywords: CSV,data processing,encoding,guess,parser,tabular
 Requires-Python: <3.14,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: boto3<2,>=1.34.0
 Requires-Dist: dateparser<2,>=1.2.0
 Requires-Dist: faust-cchardet==2.1.19
 Requires-Dist: pandas<3,>=2.2.0

{csv_detective-0.9.3.dev1977.dist-info → csv_detective-0.9.3.dev2039.dist-info}/RECORD

@@ -1,8 +1,7 @@
-csv_detective/__init__.py,sha256=FsL6q5F-gKLMnWy05-1CJpa4cz9tquheZ2LS1tjkVgI,162
+csv_detective/__init__.py,sha256=qvjDQBcw1ZIpapIrdGg1IUjBJ1q5KPhQda_05fevleg,126
 csv_detective/cli.py,sha256=mu5anmBmaDk52_uZGiA4T37wYZCuV43gZAepjs1Cqzc,1389
-csv_detective/explore_csv.py,sha256=sEMza4Z27ac88fGq7tUiK1zlfvuftztHhHVoa0c2EVU,9191
+csv_detective/explore_csv.py,sha256=1q9ZGGLZWwDwHRancdWwSypk0b_mQwpc2LNvcXMeiKQ,5806
 csv_detective/load_tests.py,sha256=uVKweLq3cf-yB5ZZI-m9tBVs_SWNcOw8sDJa97TOJGo,2266
-csv_detective/s3_utils.py,sha256=z1KTVVkdurMv21o-rZu7_aluMJnSi-d5uxnQbqT2NoI,1407
 csv_detective/utils.py,sha256=xiIO7ZDqkTm9Rnhnq6RaDdnrPIfoG0JV9AsmaOG6plA,1162
 csv_detective/validate.py,sha256=RLHXLrRuynkdcvHUlSEbyglPvdbNYlT1Z4nQI-BdYdA,2898
 csv_detective/detect_fields/__init__.py,sha256=ZZ7u9zsMtCqPC2xxeLp57UTCbqpKFJi6D_LO1ew15BU,1980
@@ -17,10 +16,10 @@ csv_detective/detect_fields/FR/geo/code_region/__init__.py,sha256=9pR2tVS4J2Kryt
 csv_detective/detect_fields/FR/geo/commune/__init__.py,sha256=5vw4zjlmWaR2djxuQOUrmwsNIc9HgAE-zdxwerVR3S0,380
 csv_detective/detect_fields/FR/geo/departement/__init__.py,sha256=UsMEW1EVVgnw-daOc1jBkEaGKvqTONSAGnj1s3QgM8w,400
 csv_detective/detect_fields/FR/geo/insee_canton/__init__.py,sha256=YsAGiblFexBxvu_E3XaXhy_bordc6c1oKPgDzTsDeXw,374
-csv_detective/detect_fields/FR/geo/latitude_l93/__init__.py,sha256=RjkDSZzIbp4nnvDpa5GomDpyIJGvwErX7TgC4dlBJ14,437
-csv_detective/detect_fields/FR/geo/latitude_wgs_fr_metropole/__init__.py,sha256=7xmYpTYoHvFfcuocAhm6dP_j4sMII_hG1PMSrWId4FY,344
-csv_detective/detect_fields/FR/geo/longitude_l93/__init__.py,sha256=JbKuGK5UmUGAQKPFpN4RSLf3axJ5D1aCjzRXYHW-iXU,441
-csv_detective/detect_fields/FR/geo/longitude_wgs_fr_metropole/__init__.py,sha256=5VWDaHZvGhJAJu5XQrj6gLx5CVA9dNOE30eTXQ3pSf0,344
+csv_detective/detect_fields/FR/geo/latitude_l93/__init__.py,sha256=7ONo0MxrJY1gPWRwyPCX4ZDbCINmxnKRV85zscADxT8,435
+csv_detective/detect_fields/FR/geo/latitude_wgs_fr_metropole/__init__.py,sha256=lIgWebNapfrnPt0XeNUMs78Xa_csGNAtTk8VEk9wXXo,342
+csv_detective/detect_fields/FR/geo/longitude_l93/__init__.py,sha256=YXTWSymmcXW9eD2OfiSlmX7N-IUtZkDrNYHd6vTnJTc,439
+csv_detective/detect_fields/FR/geo/longitude_wgs_fr_metropole/__init__.py,sha256=7tZ8sgIkQ9zuSOZ-vGYBkH04Vv1xlPlJDM78xYfD57Y,342
 csv_detective/detect_fields/FR/geo/pays/__init__.py,sha256=85y-5qNRAWJrKqL0wh9iPMUBQjvPwc9lv1cYB2m0daQ,364
 csv_detective/detect_fields/FR/geo/region/__init__.py,sha256=6mJRaGsCPBY5JHHe8EWxEjDpAOIfvBPTaZKJb3_n3gU,1077
 csv_detective/detect_fields/FR/other/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -50,9 +49,9 @@ csv_detective/detect_fields/geo/iso_country_code_alpha3/iso_country_code_alpha3.
 csv_detective/detect_fields/geo/iso_country_code_numeric/__init__.py,sha256=AnAridM4C8hcm4PeNdr8969czgrzM4KemGVZWAJSM1U,436
 csv_detective/detect_fields/geo/iso_country_code_numeric/iso_country_code_numeric.txt,sha256=2GtEhuporsHYV-pU4q9kfXU5iOtfW5C0GYBTTKQtnnA,1004
 csv_detective/detect_fields/geo/json_geojson/__init__.py,sha256=6wlwlxQmsVIZ21g-THvH3nBj-I8FuoF2sBlZAoEMGiQ,393
-csv_detective/detect_fields/geo/latitude_wgs/__init__.py,sha256=sdor-L1WDHv5opg1Le13mru4ImSA-yEbxchlWENuUFE,327
+csv_detective/detect_fields/geo/latitude_wgs/__init__.py,sha256=PI-wlTJmPk6nznzu_Fou_SSCET90wIf78mXwb1W1K70,325
 csv_detective/detect_fields/geo/latlon_wgs/__init__.py,sha256=IXDTqD4YFUJYI1FYZ5ZfkqXY6KvNY7sgBVFRAvgTHtI,454
-csv_detective/detect_fields/geo/longitude_wgs/__init__.py,sha256=gPnNTe-L9xjBVE-30VCJiK6IVZttj6Cy6zu1IL5907Y,330
+csv_detective/detect_fields/geo/longitude_wgs/__init__.py,sha256=B7YFfvEI48DfAn8xbc-vpVERQaKh9_59ERfieo2D6OY,328
 csv_detective/detect_fields/geo/lonlat_wgs/__init__.py,sha256=CnBMYevfGdhBvureF3oc_zqT-RZjG419iAuUlugQFLc,454
 csv_detective/detect_fields/other/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 csv_detective/detect_fields/other/booleen/__init__.py,sha256=37ZUJACrZA9FQBYLDeVJGze7_I9x-ZWv5yWuBcqHcwI,497
@@ -137,11 +136,11 @@ csv_detective/detection/headers.py,sha256=y5iR4jWH5fUtAH_Zg0zxWSVG_INCHlXJFMbhPp
 csv_detective/detection/rows.py,sha256=quf3ZTTFPOo09H-faZ9cRKibb1QGHEKHlpivFRx2Va4,742
 csv_detective/detection/separator.py,sha256=XjeDBqhiBxVfkCPJKem9BAgJqs_hOgQltc_pxrH_-Tg,1547
 csv_detective/detection/variables.py,sha256=wfsA_MOk14TPMOY7gkvpTGpo9-USzMnFaAou3MPHqxc,3536
-csv_detective/output/__init__.py,sha256=bMsLp-XCVf4sNymIof_kdMdqFIY7GocOas-lPNekfQg,1930
+csv_detective/output/__init__.py,sha256=Vo7hK5fq6hfK5019K4fEnv-LcfeRuNPQubQFkZAMszs,1933
 csv_detective/output/dataframe.py,sha256=pjxvpzIWVUW9_xvT3JjoPnOIVUUHnzL7kZo1xQdMDxQ,2139
 csv_detective/output/example.py,sha256=XrnPS_uC0cICn7tgnLWNctpUbnPzl7fIMzNTzJEWGJc,8655
 csv_detective/output/profile.py,sha256=thckCcfy9cES5yYNW6TDGV82gP1OFWJuLhInT1g7JpI,2814
-csv_detective/output/schema.py,sha256=5Duw5qnsJ-LaVC6JgF7p1zZAkehDzsbXA4iTSJUgLNM,13760
+csv_detective/output/schema.py,sha256=YUt9c33mzP2fHoj-NwW7kBcANyrkU3lIBWvXRbugtyU,10485
 csv_detective/output/utils.py,sha256=tbji3dEH7bDc6gLCeVSVquqU3xaHA1CQOMuaJT4Hub8,3297
 csv_detective/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 csv_detective/parsing/columns.py,sha256=HRHJBJ1gftuheegJHzhQmg-u83pVAXXuQ9GKR34mKgk,5696
@@ -150,19 +149,18 @@ csv_detective/parsing/csv.py,sha256=fJkjKvyk7InkNnYKtmivyi48mmcwvrha7gvZ5J4-86A,
 csv_detective/parsing/excel.py,sha256=sKD5PRN1TlzPPOKFnZ3VRb0r1yIjPLlpxVWmZQeLYFk,7027
 csv_detective/parsing/load.py,sha256=C3M8nvgWenOb8aDFi5dpDGCoAw9EBqr4EB63zbz2M14,3699
 csv_detective/parsing/text.py,sha256=uz8wfmNTQnOd_4fjrIZ_5rxmFmgrg343hJh2szB73Hc,1770
-csv_detective-0.9.3.dev1977.dist-info/licenses/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
+csv_detective-0.9.3.dev2039.dist-info/licenses/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_example.py,sha256=uTWswvUzBWEADGXZmMAdZvKhKvIjvT5zWOVVABgCDN4,1987
-tests/test_fields.py,sha256=-6wwuqNmGUIxpNn4u9_OmgqgS95uKWBtahDGy3iw3NI,12566
+tests/test_fields.py,sha256=m5wGyHIl7e8PFmBe5OuWST9J0YVvdqniUEuCvWEj1qU,13558
 tests/test_file.py,sha256=QEBv69P0bAKWBzhQ3KKOR1Z1RQSf5CVEilqBojwP2Yc,10791
 tests/test_labels.py,sha256=Y0XlOpztCyV65pk7iAS_nMMfdysoBujlBmz10vHul9A,469
 tests/test_structure.py,sha256=GRDYKy0UcdqlN4qglzsRC0puFj5cb-SVvONjvcPvtAA,1400
 tests/test_validation.py,sha256=ie-Xf0vk6-M6GQq-x7kY5yse1EmXfxQkbaV7fR3fvYo,3308
 venv/bin/activate_this.py,sha256=NRy3waFmwW1pOaNUp33wNN0vD1Kzkd-zXX-Sgl4EiVI,1286
-venv/bin/jp.py,sha256=7z7dvRg0M7HzpZG4ssQID7nScjvQx7bcYTxJWDOrS6E,1717
 venv/bin/runxlrd.py,sha256=YlZMuycM_V_hzNt2yt3FyXPuwouMCmMhvj1oZaBeeuw,16092
-csv_detective-0.9.3.dev1977.dist-info/METADATA,sha256=ltt9Ve8vQcPHPIaBd-BTAaGuJ_a2KtjHTvr-6d2eIMk,9767
-csv_detective-0.9.3.dev1977.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-csv_detective-0.9.3.dev1977.dist-info/entry_points.txt,sha256=JjweTReFqKJmuvkegzlew2j3D5pZzfxvbEGOtGVGmaY,56
-csv_detective-0.9.3.dev1977.dist-info/top_level.txt,sha256=cYKb4Ok3XgYA7rMDOYtxysjSJp_iUA9lJjynhVzue8g,30
-csv_detective-0.9.3.dev1977.dist-info/RECORD,,
+csv_detective-0.9.3.dev2039.dist-info/METADATA,sha256=tIrO6IwjU0482nXQqYhvwmdPhr4vtuOht6U4CLNCZeQ,9735
+csv_detective-0.9.3.dev2039.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+csv_detective-0.9.3.dev2039.dist-info/entry_points.txt,sha256=JjweTReFqKJmuvkegzlew2j3D5pZzfxvbEGOtGVGmaY,56
+csv_detective-0.9.3.dev2039.dist-info/top_level.txt,sha256=cYKb4Ok3XgYA7rMDOYtxysjSJp_iUA9lJjynhVzue8g,30
+csv_detective-0.9.3.dev2039.dist-info/RECORD,,

tests/test_fields.py

@@ -79,6 +79,7 @@ from csv_detective.detection.variables import (
 from csv_detective.load_tests import return_all_tests
 from csv_detective.output.dataframe import cast
 from csv_detective.output.utils import prepare_output_dict
+from csv_detective.parsing.columns import test_col as col_test  # to prevent pytest from testing it
 
 
 def test_all_tests_return_bool():
@@ -461,3 +462,27 @@ def test_early_detection(args):
         res = module._is(value)
         assert res
         mock_func.assert_not_called()
+
+
+def test_all_proportion_1():
+    all_tests = return_all_tests("ALL", "detect_fields")
+    prop_1 = {
+        t.__name__.split(".")[-1]: eval(
+            t.__name__.split(".")[-1]
+            if t.__name__.split(".")[-1] not in ["int", "float"]
+            else "test_" + t.__name__.split(".")[-1]
+        )
+        for t in all_tests
+        if t.PROPORTION == 1
+    }
+    # building a table that uses only correct values for these formats, except on one row
+    table = pd.DataFrame(
+        {
+            test_name: (fields[test_module][True] * 100)[:100] + ["not_suitable"]
+            for test_name, test_module in prop_1.items()
+        }
+    )
+    # testing columns for all formats
+    returned_table = col_test(table, all_tests, limited_output=True)
+    # the analysis should have found no match on any format
+    assert all(returned_table[col].sum() == 0 for col in table.columns)

csv_detective/s3_utils.py

@@ -1,44 +0,0 @@
-import logging
-
-import boto3
-from botocore.client import Config
-from botocore.exceptions import ClientError
-
-
-def get_minio_url(netloc: str, bucket: str, key: str) -> str:
-    """Returns location of given resource in minio once it is saved"""
-    return netloc + "/" + bucket + "/" + key
-
-
-def get_s3_client(url: str, minio_user: str, minio_pwd: str) -> boto3.client:
-    return boto3.client(
-        "s3",
-        endpoint_url=url,
-        aws_access_key_id=minio_user,
-        aws_secret_access_key=minio_pwd,
-        config=Config(signature_version="s3v4"),
-    )
-
-
-def download_from_minio(
-    netloc: str, bucket: str, key: str, filepath: str, minio_user: str, minio_pwd: str
-) -> None:
-    logging.info("Downloading from minio")
-    s3 = get_s3_client(netloc, minio_user, minio_pwd)
-    try:
-        s3.download_file(bucket, key, filepath)
-        logging.info(f"Resource downloaded from minio at {get_minio_url(netloc, bucket, key)}")
-    except ClientError as e:
-        logging.error(e)
-
-
-def upload_to_minio(
-    netloc: str, bucket: str, key: str, filepath: str, minio_user: str, minio_pwd: str
-) -> None:
-    logging.info("Saving to minio")
-    s3 = get_s3_client(netloc, minio_user, minio_pwd)
-    try:
-        s3.upload_file(filepath, bucket, key)
-        logging.info(f"Resource saved into minio at {get_minio_url(netloc, bucket, key)}")
-    except ClientError as e:
-        logging.error(e)

venv/bin/jp.py

@@ -1,54 +0,0 @@
-#!/home/circleci/project/venv/bin/python
-
-import sys
-import json
-import argparse
-from pprint import pformat
-
-import jmespath
-from jmespath import exceptions
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('expression')
-    parser.add_argument('-f', '--filename',
-                        help=('The filename containing the input data.  '
-                              'If a filename is not given then data is '
-                              'read from stdin.'))
-    parser.add_argument('--ast', action='store_true',
-                        help=('Pretty print the AST, do not search the data.'))
-    args = parser.parse_args()
-    expression = args.expression
-    if args.ast:
-        # Only print the AST
-        expression = jmespath.compile(args.expression)
-        sys.stdout.write(pformat(expression.parsed))
-        sys.stdout.write('\n')
-        return 0
-    if args.filename:
-        with open(args.filename, 'r') as f:
-            data = json.load(f)
-    else:
-        data = sys.stdin.read()
-        data = json.loads(data)
-    try:
-        sys.stdout.write(json.dumps(
-            jmespath.search(expression, data), indent=4, ensure_ascii=False))
-        sys.stdout.write('\n')
-    except exceptions.ArityError as e:
-        sys.stderr.write("invalid-arity: %s\n" % e)
-        return 1
-    except exceptions.JMESPathTypeError as e:
-        sys.stderr.write("invalid-type: %s\n" % e)
-        return 1
-    except exceptions.UnknownFunctionError as e:
-        sys.stderr.write("unknown-function: %s\n" % e)
-        return 1
-    except exceptions.ParseError as e:
-        sys.stderr.write("syntax-error: %s\n" % e)
-        return 1
-
-
-if __name__ == '__main__':
-    sys.exit(main())