csv-detective 0.8.1.dev1362__py3-none-any.whl → 0.8.1.dev1380__py3-none-any.whl

csv_detective/detect_labels/other/email/__init__.py

@@ -1,45 +1,20 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
+def _is(header: str) -> float:
     words_combinations_list = [
-        'email',
-        'mail',
-        'courriel',
-        'contact',
-        'mel',
-        'lieucourriel',
-        'coordinates.emailcontact',
-        'e mail',
-        'mo mail',
-        'adresse mail',
-        'adresse email'
+        "email",
+        "mail",
+        "courriel",
+        "contact",
+        "mel",
+        "lieucourriel",
+        "coordinates.emailcontact",
+        "e mail",
+        "mo mail",
+        "adresse mail",
+        "adresse email",
     ]
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/other/float/__init__.py

@@ -1,33 +1,8 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
-    words_combinations_list = ['part', 'ratio']
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+def _is(header: str) -> float:
+    words_combinations_list = ["part", "ratio", "taux"]
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/other/int/__init__.py

@@ -1,33 +1,8 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
-    words_combinations_list = ['nb', 'nombre', 'nbre']
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+def _is(header: str) -> float:
+    words_combinations_list = ["nb", "nombre", "nbre"]
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/other/money/__init__.py

@@ -1,11 +1,8 @@
-PROPORTION = 0.5
-
+from csv_detective.parsing.text import header_score
 
-def _is(header):
-    '''
-    Returns 1 if at least one of the mentionned words is in the label, else 0
-    '''
+PROPORTION = 0.5
 
-    words_list = ['budget', 'salaire', 'euro', 'euros', 'prêt', 'montant']
 
-    return float(any([word in header.lower() for word in words_list]))
+def _is(header: str) -> float:
+    words_combinations_list = ["budget", "salaire", "euro", "euros", "prêt", "montant"]
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/other/mongo_object_id/__init__.py

@@ -1,33 +1,8 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
+def _is(header: str) -> float:
     words_combinations_list = ['id', 'objectid']
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/other/twitter/__init__.py

@@ -1,33 +1,8 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
-    words_combinations_list = ['twitter', 'twitter account', 'twitter username']
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+def _is(header: str) -> float:
+    words_combinations_list = ["twitter", "twitter account", "twitter username"]
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/other/url/__init__.py

@@ -1,48 +1,23 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
+def _is(header: str) -> float:
     words_combinations_list = [
-        'url',
-        'url source',
-        'site web',
-        'source url',
-        'site internet',
-        'remote url',
-        'web',
-        'site',
-        'lien',
-        'site data',
-        'lien url',
-        'lien vers le fichier',
-        'sitweb',
-        'interneturl'
+        "url",
+        "url source",
+        "site web",
+        "source url",
+        "site internet",
+        "remote url",
+        "web",
+        "site",
+        "lien",
+        "site data",
+        "lien url",
+        "lien vers le fichier",
+        "sitweb",
+        "interneturl",
     ]
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/other/uuid/__init__.py

@@ -1,33 +1,8 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
-    words_combinations_list = ['id', 'uuid', 'guid']
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+def _is(header: str) -> float:
+    words_combinations_list = ["id", "uuid", "guid"]
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/temp/date/__init__.py

@@ -1,53 +1,28 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
+def _is(header: str) -> float:
     words_combinations_list = [
-        'date',
-        'jour',
-        'date de mise a jour',
-        'sns date',
-        'date maj',
-        'rem date',
-        'periode',
-        'date de publication',
-        'dpc',
-        'extract date',
-        'date immatriculation',
-        'date jeu donnees',
-        'datemaj',
-        'dateouv',
-        'date der maj',
-        'dmaj',
-        'jour',
-        'yyyymmdd',
-        'aaaammjj',
+        "date",
+        "jour",
+        "date de mise a jour",
+        "sns date",
+        "date maj",
+        "rem date",
+        "periode",
+        "date de publication",
+        "dpc",
+        "extract date",
+        "date immatriculation",
+        "date jeu donnees",
+        "datemaj",
+        "dateouv",
+        "date der maj",
+        "dmaj",
+        "jour",
+        "yyyymmdd",
+        "aaaammjj",
     ]
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/temp/datetime_iso/__init__.py

@@ -1,45 +1,20 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
+def _is(header: str) -> float:
     words_combinations_list = [
-        'datetime iso',
-        'datetime',
-        'timestamp',
-        'osm_timestamp',
-        'date',
-        'created at',
-        'last update',
-        'date maj',
-        'createdat',
-        'date naissance',
-        'date donnees'
+        "datetime iso",
+        "datetime",
+        "timestamp",
+        "osm_timestamp",
+        "date",
+        "created at",
+        "last update",
+        "date maj",
+        "createdat",
+        "date naissance",
+        "date donnees",
     ]
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/temp/datetime_rfc822/__init__.py

@@ -1,44 +1,19 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
+def _is(header: str) -> float:
     words_combinations_list = [
-        'datetime',
-        'timestamp',
-        'osm_timestamp',
-        'date',
-        'created at',
-        'last update',
-        'date maj',
-        'createdat',
-        'date naissance',
-        'date donnees'
+        "datetime",
+        "timestamp",
+        "osm_timestamp",
+        "date",
+        "created at",
+        "last update",
+        "date maj",
+        "createdat",
+        "date naissance",
+        "date donnees",
     ]  # Almost same as IS0, no example in data
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/temp/year/__init__.py

@@ -1,44 +1,19 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
+def _is(header: str) -> float:
     words_combinations_list = [
-        'year',
-        'annee',
-        'annee depot',
-        'an nais',
-        'exercice',
-        'data year',
-        'annee de publication',
-        'exercice comptable',
-        'annee de naissance',
-        'annee ouverture'
+        "year",
+        "annee",
+        "annee depot",
+        "an nais",
+        "exercice",
+        "data year",
+        "annee de publication",
+        "exercice comptable",
+        "annee de naissance",
+        "annee ouverture",
     ]
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+    return header_score(header, words_combinations_list)

csv_detective/parsing/text.py

@@ -8,6 +8,17 @@ def camel_case_split(identifier: str):
     return " ".join([m.group(0) for m in matches])
 
 
+translate_dict = {
+    " ": ["-", "_", "'", ",", "  "],
+    "a": ["à", "â"],
+    "c": ["ç"],
+    "e": ["é", "è", "ê", "é"],
+    "i": ["î", "ï"],
+    "o": ["ô", "ö"],
+    "u": ["ù", "û", "ü"],
+}
+
+
 # Process text
 def _process_text(val: str):
     """Traitement des chaînes de caractères pour les standardiser.
@@ -15,25 +26,36 @@ def _process_text(val: str):
     des méthodes hybrides, mais aucune ne s'est avérée plus performante."""
     val = camel_case_split(val)
     val = val.lower()
-    val = val.replace("-", " ")
-    val = val.replace("_", " ")
-    val = val.replace("'", " ")
-    val = val.replace(",", " ")
-    val = val.replace("  ", " ")
-    val = val.replace("à", "a")
-    val = val.replace("â", "a")
-    val = val.replace("ç", "c")
-    val = val.replace("é", "e")
-    val = val.replace("é", "e")
-    val = val.replace("è", "e")
-    val = val.replace("ê", "e")
-    val = val.replace("î", "i")
-    val = val.replace("ï", "i")
-    val = val.replace("ô", "o")
-    val = val.replace("ö", "o")
-    val = val.replace("î", "i")
-    val = val.replace("û", "u")
-    val = val.replace("ù", "u")
-    val = val.replace("ü", "u")
+    for target in translate_dict:
+        for source in translate_dict[target]:
+            val = val.replace(source, target)
     val = val.strip()
     return val
+
+
+def is_word_in_string(word: str, string: str):
+    # if the substring is too short, the test can become irrelevant
+    return len(word) > 2 and word in string
+
+
+def header_score(header: str, words_combinations_list: list[str]) -> float:
+    """Returns:
+    - 1 if the header is exactly in the specified list
+    - 0.5 if any of the words is within the header
+    - 0 otherwise"""
+    processed_header = _process_text(header)
+
+    header_matches_words_combination = float(
+        any(
+            words_combination == processed_header for words_combination in words_combinations_list
+        )
+    )
+    words_combination_in_header = 0.5 * (
+        any(
+            is_word_in_string(
+                words_combination, processed_header
+            ) for words_combination in words_combinations_list
+        )
+    )
+
+    return max(header_matches_words_combination, words_combination_in_header)

csv_detective/utils.py

@@ -32,7 +32,3 @@ def prevent_nan(value: float) -> Optional[float]:
     if math.isnan(value):
         return None
     return value
-
-
-def is_word_in_string(word: str, string: str):
-    return word in string