csv-detective 0.8.1.dev1362__py3-none-any.whl → 0.8.1.dev1416__py3-none-any.whl

csv_detective/detect_fields/other/url/__init__.py

@@ -1,13 +1,14 @@
+import re
 
 PROPORTION = 1
+url_pattern = re.compile(
+    r"^((https?|ftp)://|www\.)(([A-Za-z0-9-]+\.)+[A-Za-z]{2,6})"
+    r"(/[A-Za-z0-9._~:/?#[@!$&'()*+,;=%-]*)?$"
+)
 
 
 def _is(val):
-    '''Detects urls'''
+    """Detects urls"""
     if not isinstance(val, str):
         return False
-    a = 'http://' in val
-    b = 'www.' in val
-    c = any([x in val for x in ['.fr', '.com', '.org', '.gouv', '.net']])
-    d = not ('@' in val)
-    return (a or b or c) and d
+    return bool(url_pattern.match(val))

csv_detective/detect_labels/FR/geo/adresse/__init__.py

@@ -1,40 +1,15 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
+def _is(header: str) -> float:
     words_combinations_list = [
-        'adresse',
-        'adresse postale',
-        'adresse geographique',
-        'adr',
-        'adresse complete',
-        'adresse station'
+        "adresse",
+        "adresse postale",
+        "adresse geographique",
+        "adr",
+        "adresse complete",
+        "adresse station",
     ]
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/FR/geo/code_commune_insee/__init__.py

@@ -1,42 +1,17 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
+def _is(header: str) -> float:
     words_combinations_list = [
-        'code commune insee',
-        'code insee',
-        'codes insee',
-        'code commune',
-        'code insee commune',
-        'insee',
-        'code com',
-        'com',
+        "code commune insee",
+        "code insee",
+        "codes insee",
+        "code commune",
+        "code insee commune",
+        "insee",
+        "code com",
+        "com",
     ]
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/FR/geo/code_departement/__init__.py

@@ -1,33 +1,15 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-    # 'dep': Possible confusion with dep name?
-    words_combinations_list = ['code departement', 'code_departement', 'dep', 'departement', 'dept']
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+def _is(header: str) -> float:
+    # "dep": Possible confusion with dep name?
+    words_combinations_list = [
+        "code departement",
+        "code_departement",
+        "dep",
+        "departement",
+        "dept",
+    ]
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/FR/geo/code_fantoir/__init__.py

@@ -1,33 +1,12 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
-    words_combinations_list = ['cadastre1', 'code fantoir', 'fantoir']
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+def _is(header: str) -> float:
+    words_combinations_list = [
+        "cadastre1",
+        "code fantoir",
+        "fantoir",
+    ]
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/FR/geo/code_postal/__init__.py

@@ -1,41 +1,16 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
+def _is(header: str) -> float:
     words_combinations_list = [
-        'code postal',
-        'postal code',
-        'postcode',
-        'post code',
-        'cp',
-        'codes postaux',
-        'location postcode'
+        "code postal",
+        "postal code",
+        "postcode",
+        "post code",
+        "cp",
+        "codes postaux",
+        "location postcode",
     ]
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/FR/geo/code_region/__init__.py

@@ -1,33 +1,14 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-    # 'reg' : possible confusion with region name?
-    words_combinations_list = ['code region', 'reg', 'code insee region', 'region']
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+def _is(header: str) -> float:
+    # "reg" : possible confusion with region name?
+    words_combinations_list = [
+        "code region",
+        "reg",
+        "code insee region",
+        "region",
+    ]
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/FR/geo/commune/__init__.py

@@ -1,33 +1,12 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
-    words_combinations_list = ['commune', 'ville', 'libelle commune']
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+def _is(header: str) -> float:
+    words_combinations_list = [
+        "commune",
+        "ville",
+        "libelle commune",
+    ]
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/FR/geo/departement/__init__.py

@@ -1,47 +1,22 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
+def _is(header: str) -> float:
     words_combinations_list = [
-        'departement',
-        'libelle du departement',
-        'deplib',
-        'nom dept',
-        'dept',
-        'libdepartement',
-        'nom departement',
-        'libelle dep',
-        'libelle departement',
-        'lb departements',
-        'dep libusage',
-        'lb departement',
-        'nom dep'
+        "departement",
+        "libelle du departement",
+        "deplib",
+        "nom dept",
+        "dept",
+        "libdepartement",
+        "nom departement",
+        "libelle dep",
+        "libelle departement",
+        "lb departements",
+        "dep libusage",
+        "lb departement",
+        "nom dep",
     ]
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/FR/geo/insee_canton/__init__.py

@@ -1,33 +1,13 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
-    words_combinations_list = ['insee canton', 'canton', 'cant', 'nom canton']
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+def _is(header: str) -> float:
+    words_combinations_list = [
+        "insee canton",
+        "canton",
+        "cant",
+        "nom canton",
+    ]
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/FR/geo/latitude_l93/__init__.py

@@ -1,54 +1,30 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
+def _is(header: str) -> float:
     # Does not always detect CRS
     words_combinations_list = [
-        'latitude',
-        'lat',
-        'y',
-        'yf',
-        'yd',
-        'y l93',
-        'coordonnee y',
-        'latitude lb93',
-        'coord y',
-        'ycoord',
-        'geocodage y gps',
-        'location latitude',
-        'ylatitude',
-        'ylat',
-        'latitude (y)',
-        'latitudeorg',
-        'coordinates.latitude',
-        'googlemap latitude',
-        'latitudelieu',
-        'latitude googlemap'
-        ]
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+        "latitude",
+        "lat",
+        "y",
+        "yf",
+        "yd",
+        "y l93",
+        "coordonnee y",
+        "latitude lb93",
+        "coord y",
+        "ycoord",
+        "geocodage y gps",
+        "location latitude",
+        "ylatitude",
+        "ylat",
+        "latitude (y)",
+        "latitudeorg",
+        "coordinates.latitude",
+        "googlemap latitude",
+        "latitudelieu",
+        "latitude googlemap",
+    ]
+    return header_score(header, words_combinations_list)

csv_detective/detect_labels/FR/geo/latitude_wgs_fr_metropole/__init__.py

@@ -1,55 +1,30 @@
-from csv_detective.utils import is_word_in_string
-from csv_detective.parsing.text import _process_text
+from csv_detective.parsing.text import header_score
 
 PROPORTION = 0.5
 
 
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
+def _is(header: str) -> float:
     words_combinations_list = [
-        'latitude',
-        'lat',
-        'y',
-        'yf',
-        'yd',
-        'coordonnee y',
-        'coord y',
-        'ycoord',
-        'geocodage y gps',
-        'location latitude',
-        'ylatitude',
-        'ylat',
-        'latitude (y)',
-        'latitudeorg',
-        'coordinates.latitude',
-        'googlemap latitude',
-        'latitudelieu',
-        'latitude googlemap',
-        'latitude wgs84',
-        'y wgs84',
-        'latitude (wgs84)'
+        "latitude",
+        "lat",
+        "y",
+        "yf",
+        "yd",
+        "coordonnee y",
+        "coord y",
+        "ycoord",
+        "geocodage y gps",
+        "location latitude",
+        "ylatitude",
+        "ylat",
+        "latitude (y)",
+        "latitudeorg",
+        "coordinates.latitude",
+        "googlemap latitude",
+        "latitudelieu",
+        "latitude googlemap",
+        "latitude wgs84",
+        "y wgs84",
+        "latitude (wgs84)",
     ]
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                is_word_in_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)
+    return header_score(header, words_combinations_list)