uk_bin_collection 0.98.0__py3-none-any.whl → 0.98.2__py3-none-any.whl

uk_bin_collection/tests/input.json

@@ -49,7 +49,7 @@
         "wiki_name": "Barnsley Metropolitan Borough Council",
         "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search)."
     },
-"BasildonCouncil": {
+    "BasildonCouncil": {
         "skip_get_url": true,
         "uprn": "10013350430",
         "url": "https://basildonportal.azurewebsites.net/api/getPropertyRefuseInformation",

uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py

@@ -70,10 +70,19 @@ class CouncilClass(AbstractGetBinDataClass):
             headless = kwargs.get("headless")
             web_driver = kwargs.get("web_driver")
             driver = create_webdriver(web_driver, headless, None, __name__)
-            page = "https://account.barnet.gov.uk/Forms/Home/Redirector/Index/?id=6a2ac067-3322-46e5-96e4-16c0c214454a&mod=OA&casetype=BAR&formname=BNTCOLDATE"
+            page = "https://www.barnet.gov.uk/recycling-and-waste/bin-collections/find-your-bin-collection-day"
+
             driver.get(page)
 
-            time.sleep(5)
+            # Wait for the element to be clickable
+            find_your_collection_button = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, '//a[contains(text(), "Find your household collection day")]')
+                )
+            )
+
+            # Click the element
+            find_your_collection_button.click()
 
             try:
                 accept_cookies = WebDriverWait(driver, timeout=10).until(

uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py

@@ -24,7 +24,7 @@ class CouncilClass(AbstractGetBinDataClass):
     def parse_data(self, page: str, **kwargs) -> dict:
         driver = None
         try:
-            page = "https://mybexley.bexley.gov.uk/service/When_is_my_collection_day"
+            page = "https://waste.bexley.gov.uk/waste"
 
             data = {"bins": []}
 
@@ -38,102 +38,102 @@ class CouncilClass(AbstractGetBinDataClass):
             driver = create_webdriver(web_driver, headless, None, __name__)
             driver.get(page)
 
-            # If you bang in the house number (or property name) and postcode in the box it should find your property
-
-            iframe_presense = WebDriverWait(driver, 30).until(
-                EC.presence_of_element_located((By.ID, "fillform-frame-1"))
-            )
-
-            driver.switch_to.frame(iframe_presense)
-            wait = WebDriverWait(driver, 60)
-            start_btn = wait.until(
-                EC.element_to_be_clickable(
-                    (By.XPATH, "//button/span[contains(text(), 'Next')]")
-                )
-            )
-
-            start_btn.click()
+            wait = WebDriverWait(driver, 10)
 
             inputElement_postcodesearch = wait.until(
-                EC.element_to_be_clickable((By.ID, "postcode_search"))
+                EC.element_to_be_clickable((By.ID, "pc"))
             )
             inputElement_postcodesearch.send_keys(user_postcode)
 
+
+
             find_address_btn = wait.until(
-                EC.element_to_be_clickable((By.XPATH, '//*[@id="search"]'))
+                EC.element_to_be_clickable((By.XPATH, '//*[@id="sub"]'))
             )
             find_address_btn.click()
 
             dropdown_options = wait.until(
                 EC.presence_of_element_located(
-                    (By.XPATH, '//*[@id="select2-chosen-1"]')
+                    (By.XPATH, '//*[@id="address"]')
                 )
             )
             time.sleep(2)
             dropdown_options.click()
             time.sleep(1)
-            dropdown_input = wait.until(
+
+            # Wait for the element to be clickable
+            address = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, f'//li[contains(text(), "{user_paon}")]')
+                )
+            )
+
+            # Click the element
+            address.click()
+
+
+            submit_address = wait.until(
                 EC.presence_of_element_located(
-                    (By.XPATH, '//*[@id="s2id_autogen1_search"]')
+                    (By.XPATH, '//*[@id="go"]')
                 )
             )
-            time.sleep(1)
-            dropdown_input.send_keys(user_paon)
-            dropdown_input.send_keys(Keys.ENTER)
+            time.sleep(2)
+            submit_address.click()
 
             results_found = wait.until(
-                EC.presence_of_element_located((By.CLASS_NAME, "found-content"))
-            )
-            finish_btn = wait.until(
                 EC.element_to_be_clickable(
-                    (By.XPATH, "//button/span[contains(text(), 'Next')]")
+                    (By.XPATH, '//h1[contains(text(), "Your bin days")]')
+                )           
                 )
-            )
-            finish_btn.click()
+
             final_page = wait.until(
                 EC.presence_of_element_located(
-                    (By.CLASS_NAME, "waste-header-container")
+                    (By.CLASS_NAME, "waste__collections")
                 )
             )
 
             soup = BeautifulSoup(driver.page_source, features="html.parser")
 
-            bin_fields = soup.find_all("div", class_="waste-panel-container")
-            # Define your XPath
-
-            for bin in bin_fields:
-                # Extract h3 text from the current element
-                h3_text = (
-                    bin.find("h3", class_="container-name").get_text(strip=True)
-                    if bin.find("h3", class_="container-name")
-                    else None
-                )
-
-                date_text = (
-                    bin.find("p", class_="container-status").get_text(strip=True)
-                    if bin.find("p", class_="container-status")
-                    else None
-                )
+            # Find all waste services
 
-                if h3_text and date_text:
-                    # Parse the date using the appropriate format
-                    parsed_date = datetime.strptime(date_text, "%A %d %B")
-
-                    # Assuming the current year is used for the collection date
-                    current_year = datetime.now().year
-
-                    # If the parsed date is in the past, assume it's for the next year
-                    if parsed_date < datetime.now():
-                        current_year += 1
-
-                    data["bins"].append(
-                        {
-                            "type": h3_text,
-                            "collectionDate": parsed_date.replace(
-                                year=current_year
-                            ).strftime("%d/%m/%Y"),
-                        }
-                    )
+            # Initialize the data dictionary
+            data = {"bins": []}
+            bin_sections = soup.find_all("h3", class_="waste-service-name")
+
+            # Loop through each bin field
+            for bin_section in bin_sections:
+                # Extract the bin type (e.g., "Brown Caddy", "Green Wheelie Bin", etc.)
+                bin_type = bin_section.get_text(strip=True).split("\n")[0]  # The first part is the bin type
+
+                # Find the next sibling <dl> tag that contains the next collection information
+                summary_list = bin_section.find_next("dl", class_="govuk-summary-list")
+
+                if summary_list:
+                    # Now, instead of finding by class, we'll search by text within the dt element
+                    next_collection_dt = summary_list.find("dt", string=lambda text: "Next collection" in text)
+
+                    if next_collection_dt:
+                        # Find the sibling <dd> tag for the collection date
+                        next_collection = next_collection_dt.find_next_sibling("dd").get_text(strip=True)
+
+                        if next_collection:
+                            try:
+                                # Parse the next collection date (assuming the format is like "Tuesday 15 October 2024")
+                                parsed_date = datetime.strptime(next_collection, "%A %d %B %Y")
+
+                                # Add the bin information to the data dictionary
+                                data["bins"].append({
+                                    "type": bin_type,
+                                    "collectionDate": parsed_date.strftime(date_format),
+                                })
+                            except ValueError as e:
+                                print(f"Error parsing date for {bin_type}: {e}")
+                        else:
+                            print(f"No next collection date found for {bin_type}")
+                    else:
+                        print(f"No 'Next collection' text found for {bin_type}")
+                else:
+                    print(f"No summary list found for {bin_type}")
 
         except Exception as e:
             # Here you can log the exception if needed

uk_bin_collection/uk_bin_collection/councils/MoleValleyDistrictCouncil.py

@@ -7,28 +7,18 @@ from bs4 import BeautifulSoup
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
-
-# import the wonderful Beautiful Soup and the URL grabber
 class CouncilClass(AbstractGetBinDataClass):
-    """
-    Concrete classes have to implement all abstract operations of the
-    base class. They can also override some operations with a default
-    implementation.
-    """
-
     def parse_data(self, page: str, **kwargs) -> dict:
 
         user_postcode = kwargs.get("postcode")
         check_postcode(user_postcode)
 
-        root_url = "https://myproperty.molevalley.gov.uk/molevalley/api/live_addresses/{}?format=json".format(
-            user_postcode
-        )
-        requests.packages.urllib3.disable_warnings()
+        # Fetch the page content
+        root_url = "https://myproperty.molevalley.gov.uk/molevalley/api/live_addresses/{}?format=json".format(user_postcode)
         response = requests.get(root_url, verify=False)
 
         if not response.ok:
-            raise ValueError("Invalid server response code retreiving data.")
+            raise ValueError("Invalid server response code retrieving data.")
 
         jsonData = response.json()
 
@@ -37,7 +27,6 @@ class CouncilClass(AbstractGetBinDataClass):
 
         properties_found = jsonData["results"]["features"]
 
-        # If UPRN is provided, we can check a specific address.
         html_data = None
         uprn = kwargs.get("uprn")
         if uprn:
@@ -49,53 +38,81 @@ class CouncilClass(AbstractGetBinDataClass):
             if html_data is None:
                 raise ValueError("No collection data found for UPRN provided.")
         else:
-            # If UPRN not provided, just use the first result
             html_data = properties_found[0]["properties"]["three_column_layout_html"]
 
-        soup = BeautifulSoup(html_data, features="html.parser")
-        soup.prettify()
+        # Conditionally replace the commented-out sections (<!-- ... -->)
+        if "<!--" in html_data and "-->" in html_data:
+            print("Commented-out section found, replacing comments...")
+            html_data = html_data.replace("<!--", "").replace("-->", "")
+        else:
+            print("No commented-out section found, processing as is.")
+
+        # Process the updated HTML data with BeautifulSoup
+        soup = BeautifulSoup(html_data, "html.parser")
 
         data = {"bins": []}
         all_collection_dates = []
-        regex_date = re.compile(r".* ([\d]+\/[\d]+\/[\d]+)")
+        regex_date = re.compile(r"(\d{2}/\d{2}/\d{4})")  # Adjusted date regex
         regex_additional_collection = re.compile(r"We also collect (.*) on (.*) -")
 
+        # Debugging to verify the HTML content is parsed correctly
+        print("HTML content parsed successfully.")
+
         # Search for the 'Bins and Recycling' panel
-        for panel in soup.select('div[class*="panel"]'):
-            if panel.h2.text.strip() == "Bins and Recycling":
-
-                # Gather the bin types and dates
-                for collection in panel.select("div > strong"):
-                    bin_type = collection.text.strip()
-                    collection_string = collection.find_next("p").text.strip()
-                    m = regex_date.match(collection_string)
-                    if m:
-                        collection_date = datetime.strptime(
-                            m.group(1), "%d/%m/%Y"
-                        ).date()
-                        data["bins"].append(
-                            {
+        bins_panel = soup.find("h2", string="Bins and Recycling")
+        if bins_panel:
+            panel = bins_panel.find_parent("div", class_="panel")
+            print("Found 'Bins and Recycling' panel.")
+            
+            # Extract bin collection info from the un-commented HTML
+            for strong_tag in panel.find_all("strong"):
+                bin_type = strong_tag.text.strip()
+                collection_string = strong_tag.find_next("p").text.strip()
+                
+                # Debugging output
+                print(f"Processing bin type: {bin_type}")
+                print(f"Collection string: {collection_string}")
+                
+                match = regex_date.search(collection_string)
+                if match:
+                    collection_date = datetime.strptime(match.group(1), "%d/%m/%Y").date()
+                    data["bins"].append({
+                        "type": bin_type,
+                        "collectionDate": collection_date.strftime("%d/%m/%Y"),
+                    })
+                    all_collection_dates.append(collection_date)
+                else:
+                    # Add a debug line to show which collections are missing dates
+                    print(f"No valid date found for bin type: {bin_type}")
+
+            # Search for additional collections like electrical and textiles
+            for p in panel.find_all("p"):
+                additional_match = regex_additional_collection.match(p.text.strip())
+                
+                # Debugging output for additional collections
+                if additional_match:
+                    bin_type = additional_match.group(1)
+                    print(f"Found additional collection: {bin_type}")
+                    if "each collection day" in additional_match.group(2):
+                        if all_collection_dates:
+                            collection_date = min(all_collection_dates)
+                            data["bins"].append({
                                 "type": bin_type,
                                 "collectionDate": collection_date.strftime("%d/%m/%Y"),
-                            }
-                        )
-                        all_collection_dates.append(collection_date)
-
-                # Search for additional collections
-                for p in panel.select("p"):
-                    m2 = regex_additional_collection.match(p.text.strip())
-                    if m2:
-                        bin_type = m2.group(1)
-                        if "each collection day" in m2.group(2):
-                            collection_date = min(all_collection_dates)
-                            data["bins"].append(
-                                {
-                                    "type": bin_type,
-                                    "collectionDate": collection_date.strftime(
-                                        "%d/%m/%Y"
-                                    ),
-                                }
-                            )
-                break
-
-        return data
+                            })
+                        else:
+                            print("No collection dates available for additional collection.")
+                            raise ValueError("No valid bin collection dates found.")
+                else:
+                    print(f"No additional collection found in paragraph: {p.text.strip()}")
+        else:
+            raise ValueError("Unable to find 'Bins and Recycling' panel in the HTML data.")
+
+        # Debugging to check collected data
+        print(f"Collected bin data: {data}")
+        
+        # Handle the case where no collection dates were found
+        if not all_collection_dates:
+            raise ValueError("No valid collection dates were found in the data.")
+        
+        return data

{uk_bin_collection-0.98.0.dist-info → uk_bin_collection-0.98.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.98.0
+Version: 0.98.2
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.98.0.dist-info → uk_bin_collection-0.98.2.dist-info}/RECORD

@@ -2,7 +2,7 @@ uk_bin_collection/README.rst,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 uk_bin_collection/tests/council_feature_input_parity.py,sha256=DO6Mk4ImYgM5ZCZ-cutwz5RoYYWZRLYx2tr6zIs_9Rc,3843
 uk_bin_collection/tests/features/environment.py,sha256=VQZjJdJI_kZn08M0j5cUgvKT4k3iTw8icJge1DGOkoA,127
 uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=SJK-Vc737hrf03tssxxbeg_JIvAH-ddB8f6gU1LTbuQ,251
-uk_bin_collection/tests/input.json,sha256=e7HeP8RqQQuiCBVDTA6GyTEAPzUfs2-Bs-Djg8GEgLs,68070
+uk_bin_collection/tests/input.json,sha256=Lb2LZ6Q1LvG-yaG4KCjPIiKdLyK9IE6N_4vb7wwdExw,68074
 uk_bin_collection/tests/output.schema,sha256=ZwKQBwYyTDEM4G2hJwfLUVM-5v1vKRvRK9W9SS1sd18,1086
 uk_bin_collection/tests/step_defs/step_helpers/file_handler.py,sha256=Ygzi4V0S1MIHqbdstUlIqtRIwnynvhu4UtpweJ6-5N8,1474
 uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=LrOSt_loA1Mw3vTqaO2LpaDMu7rYJy6k5Kr-EOBln7s,3424
@@ -17,7 +17,7 @@ uk_bin_collection/uk_bin_collection/councils/ArmaghBanbridgeCraigavonCouncil.py,
 uk_bin_collection/uk_bin_collection/councils/ArunCouncil.py,sha256=yfhthv9nuogP19VOZ3TYQrq51qqjiCZcSel4sXhiKjs,4012
 uk_bin_collection/uk_bin_collection/councils/AylesburyValeCouncil.py,sha256=LouqjspEMt1TkOGqWHs2zkxwOETIy3n7p64uKIlAgUg,2401
 uk_bin_collection/uk_bin_collection/councils/BCPCouncil.py,sha256=W7QBx6Mgso8RYosuXsaYo3GGNAu-tiyBSmuYxr1JSOU,1707
-uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py,sha256=n6lNsO_ognmaxq_wIz91Dxt5XjBVQe60X5p6w1_mO1A,8174
+uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py,sha256=NccMd0830aiNRQ8SH6mc4r5Hlugfey5-PyUd24QLo4s,8478
 uk_bin_collection/uk_bin_collection/councils/BarnsleyMBCouncil.py,sha256=MgF_7XyIcIoNzFR0OJsjBkLCZKgWxBrV6nTcutMxO1Q,4244
 uk_bin_collection/uk_bin_collection/councils/BasildonCouncil.py,sha256=SBvAa0GZM3V7ygK8ARawbHAPH6R_303U30RH8WYPi5Q,3020
 uk_bin_collection/uk_bin_collection/councils/BasingstokeCouncil.py,sha256=VPWGljnH4C3q8qs5ZmCtqjNjgWQvviALzjk00q3EZeQ,2632
@@ -25,7 +25,7 @@ uk_bin_collection/uk_bin_collection/councils/BathAndNorthEastSomersetCouncil.py,
 uk_bin_collection/uk_bin_collection/councils/BedfordBoroughCouncil.py,sha256=CvGB7w9HMn7XyEtwfd9MWZE_HlZ75pDcaKMsQJz0xhk,1669
 uk_bin_collection/uk_bin_collection/councils/BedfordshireCouncil.py,sha256=U1HOr9YLMAlFoZysfw5n04E0bVuCliO5Yj1FMiiwcHA,2549
 uk_bin_collection/uk_bin_collection/councils/BelfastCityCouncil.py,sha256=mspYVHO8fgoVIwogT6V2Go1tbf3PDbEmr8kZMJug5Gs,4235
-uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py,sha256=9MrbpfR17R6DYjDrItQL3CPF7ie1SJkaoNbwQumPgIQ,5399
+uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py,sha256=nzet8yKKzpiAQXDr0tmqXuJ4R-EdJqtxSNGiUCh79bA,5591
 uk_bin_collection/uk_bin_collection/councils/BirminghamCityCouncil.py,sha256=now2xgpfshYM33UWC18j6xa6BuBydO5Sl7OrDQOo6b0,4687
 uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py,sha256=jHbCK8sL09vdmdP7Xnh8lIrU5AHTnJLEZfOLephPvWg,4090
 uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py,sha256=r7wjrRPT2EinRMnYjGxmsCD6aMFhEOyRNd8_3R9PdQU,4117
@@ -111,7 +111,7 @@ uk_bin_collection/uk_bin_collection/councils/MertonCouncil.py,sha256=3Y2Un4xXo1s
 uk_bin_collection/uk_bin_collection/councils/MidAndEastAntrimBoroughCouncil.py,sha256=oOWwU5FSgGej2Mv7FQ66N-EzS5nZgmGsd0WnfLWUc1I,5238
 uk_bin_collection/uk_bin_collection/councils/MidSussexDistrictCouncil.py,sha256=AZgC9wmDLEjUOtIFvf0ehF5LHturXTH4DkE3ioPSVBA,6254
 uk_bin_collection/uk_bin_collection/councils/MiltonKeynesCityCouncil.py,sha256=3olsWa77L34vz-c7NgeGK9xmNuR4Ws_oAk5D4UpIkPw,2005
-uk_bin_collection/uk_bin_collection/councils/MoleValleyDistrictCouncil.py,sha256=54-autRRGAM4pBxlqmUE6g825rmUF-gRqrcgHL_lkIk,3994
+uk_bin_collection/uk_bin_collection/councils/MoleValleyDistrictCouncil.py,sha256=bvCrC4Qcg0Uzp9zZGcC7-7-oJcMh2cb1VaXfdkB11oc,5257
 uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py,sha256=V9URKAv3dA_deYmStL2Nmn4GbVCM-tU2qnKobivmGew,5583
 uk_bin_collection/uk_bin_collection/councils/NewForestCouncil.py,sha256=ylTn9KmWITtaO9_Z8kJCN2w2ALfhrfGt3SeJ78lgw7M,5391
 uk_bin_collection/uk_bin_collection/councils/NewarkAndSherwoodDC.py,sha256=lAleYfCGUWCKOi7Ye_cjgfpI3pWwTcFctlYmh0hjebM,2140
@@ -210,8 +210,8 @@ uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py,sha256=I2kBYMlsD4bId
 uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=4s9ODGPAwPqwXc8SrTX5Wlfmizs3_58iXUtHc4Ir86o,1162
 uk_bin_collection/uk_bin_collection/create_new_council.py,sha256=m-IhmWmeWQlFsTZC4OxuFvtw5ZtB8EAJHxJTH4O59lQ,1536
 uk_bin_collection/uk_bin_collection/get_bin_data.py,sha256=YvmHfZqanwrJ8ToGch34x-L-7yPe31nB_x77_Mgl_vo,4545
-uk_bin_collection-0.98.0.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
-uk_bin_collection-0.98.0.dist-info/METADATA,sha256=AvCRXeHiq0mKpaa7w0WkFuC78gBQSTyjvvCzB7aHJRM,16843
-uk_bin_collection-0.98.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-uk_bin_collection-0.98.0.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
-uk_bin_collection-0.98.0.dist-info/RECORD,,
+uk_bin_collection-0.98.2.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
+uk_bin_collection-0.98.2.dist-info/METADATA,sha256=lhQtluGdKsFsfZJo7MJXKlWLeGLv-rbTX1aA081Fj3g,16843
+uk_bin_collection-0.98.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+uk_bin_collection-0.98.2.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
+uk_bin_collection-0.98.2.dist-info/RECORD,,