edb-noumea 0.3.6__py3-none-any.whl → 0.3.7__py3-none-any.whl

edb_noumea/details.py

@@ -107,6 +107,32 @@ def get_detailed_results():
         # It seems to start at index 6 in the camelot df.
         df = df.iloc[6:].reset_index(drop=True)
 
+        # Apply ffill to all relevant columns after initial column assignment and before specific cleaning
+        # Replace empty strings with pd.NA first to ensure ffill works
+        for col in df.columns:
+            df[col] = df[col].replace('', pd.NA)
+        
+        # Now, forward-fill all columns. This assumes that empty cells in a data block should inherit from the previous non-empty cell.
+        df = df.ffill()
+
+        # Remove trailing informational rows
+        # Identify the patterns of the rows to be removed
+        unwanted_patterns = [
+            "La qualité des eaux de baignade est évaluée au",
+            "de présence de germes pathogènes",
+            "Suivant l'arrêté N°", # Added to catch more informational rows
+            "Indicateurs microbiologiques" # Added to catch more informational rows
+        ]
+        
+        # Filter out rows that contain these patterns in the 'Nom du site de baignade' or 'Point de prélèvement' column
+        # We check both columns as the content might shift
+        initial_row_count = df.shape[0]
+        df = df[~df['Nom du site de baignade'].fillna('').astype(str).str.contains('|'.join(unwanted_patterns), case=False, na=False)]
+        df = df[~df['Point de prélèvement'].fillna('').astype(str).str.contains('|'.join(unwanted_patterns), case=False, na=False)]
+        
+        if df.shape[0] < initial_row_count:
+            print(f"Removed {initial_row_count - df.shape[0]} informational rows.")
+
     except Exception as e:
         print(f"❌ Une erreur est survenue lors de l'extraction des données du PDF avec Camelot.")
         print(f"   Erreur originale : {e}")
@@ -171,11 +197,11 @@ def get_detailed_results():
 
     # Nettoyer et convertir les colonnes e_coli_npp_100ml et enterocoques_npp_100ml
     if "e_coli_npp_100ml" in cleaned_df.columns:
-        cleaned_df["e_coli_npp_100ml"] = cleaned_df["e_coli_npp_100ml"].astype(str).str.replace(r"<\s*10", "10", regex=True)
+        cleaned_df["e_coli_npp_100ml"] = cleaned_df["e_coli_npp_100ml"].astype(str).str.replace(" ", "", regex=False).str.replace(r"<\s*10", "10", regex=True)
         cleaned_df["e_coli_npp_100ml"] = pd.to_numeric(cleaned_df["e_coli_npp_100ml"], errors="coerce").astype('Int64')
 
     if "enterocoques_npp_100ml" in cleaned_df.columns:
-        cleaned_df["enterocoques_npp_100ml"] = cleaned_df["enterocoques_npp_100ml"].astype(str).str.replace(r"<\s*10", "10", regex=True)
+        cleaned_df["enterocoques_npp_100ml"] = cleaned_df["enterocoques_npp_100ml"].astype(str).str.replace(" ", "", regex=False).str.replace(r"<\s*10", "10", regex=True)
         cleaned_df["enterocoques_npp_100ml"] = pd.to_numeric(cleaned_df["enterocoques_npp_100ml"], errors="coerce").astype('Int64')
 
     # Convertir la colonne 'date' en datetime (format jour/mois/année)

{edb_noumea-0.3.6.dist-info → edb_noumea-0.3.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: edb-noumea
-Version: 0.3.6
+Version: 0.3.7
 Summary: Scraper pour la qualité des eaux de baignade à Nouméa.
 Project-URL: Homepage, https://github.com/adriens/edb-noumea
 Project-URL: Repository, https://github.com/adriens/edb-noumea

edb_noumea-0.3.7.dist-info/RECORD

@@ -0,0 +1,8 @@
+edb_noumea/__init__.py,sha256=G7WKTGLsr2wtW1E2jYpqq4miZLoSGhTifSE36CGNkLo,60
+edb_noumea/details.py,sha256=vdHlaohUBtGtNCi46DrxUxYE34UHjBw4MKz3B_A1o6k,10379
+edb_noumea/main.py,sha256=ekx8XF7b3-w63AuVrZqVPg7IKHRL0cZGYmudsHa6DTk,2137
+edb_noumea-0.3.7.dist-info/licenses/LICENSE,sha256=mNQ0SS064BtPKYHabMRg2yM3m-GDX4MgDQ6ZnDFiueI,1100
+edb_noumea-0.3.7.dist-info/METADATA,sha256=VdFm-Hozr9L5XlGh06DLBqyQlHGzon1nXqgtSjdpEa0,7126
+edb_noumea-0.3.7.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+edb_noumea-0.3.7.dist-info/top_level.txt,sha256=Dj3JusM0b5H9_f9yZeO-IwucCZzI1OHSjLMKtvRjq6k,11
+edb_noumea-0.3.7.dist-info/RECORD,,

edb_noumea-0.3.6.dist-info/RECORD

@@ -1,8 +0,0 @@
-edb_noumea/__init__.py,sha256=G7WKTGLsr2wtW1E2jYpqq4miZLoSGhTifSE36CGNkLo,60
-edb_noumea/details.py,sha256=n9FgkNPzEAZf9HQxTlU7OWjyofi57O00zIqQghyarE8,8821
-edb_noumea/main.py,sha256=ekx8XF7b3-w63AuVrZqVPg7IKHRL0cZGYmudsHa6DTk,2137
-edb_noumea-0.3.6.dist-info/licenses/LICENSE,sha256=mNQ0SS064BtPKYHabMRg2yM3m-GDX4MgDQ6ZnDFiueI,1100
-edb_noumea-0.3.6.dist-info/METADATA,sha256=xAqPGKMwfOvz_HF65QvBcWvyrK8iwGTY5QwwcJmMmYU,7126
-edb_noumea-0.3.6.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-edb_noumea-0.3.6.dist-info/top_level.txt,sha256=Dj3JusM0b5H9_f9yZeO-IwucCZzI1OHSjLMKtvRjq6k,11
-edb_noumea-0.3.6.dist-info/RECORD,,