uk_bin_collection 0.151.0__py3-none-any.whl → 0.152.1__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/AngusCouncil.py

@@ -0,0 +1,149 @@
+import time
+import re
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select, WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            user_postcode = kwargs.get("postcode")
+            if not user_postcode:
+                raise ValueError("No postcode provided.")
+            check_postcode(user_postcode)
+
+            user_uprn = kwargs.get("uprn")
+            check_uprn(user_uprn)
+
+            headless = kwargs.get("headless")
+            web_driver = kwargs.get("web_driver")
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            page = "https://www.angus.gov.uk/bins_litter_and_recycling/bin_collection_days"
+
+            driver.get(page)
+
+            wait = WebDriverWait(driver, 10)
+            accept_cookies_button = wait.until(
+                EC.element_to_be_clickable((By.ID, "ccc-recommended-settings"))
+            )
+            accept_cookies_button.click()
+
+            find_your_collection_button = wait.until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, "/html/body/div[2]/div[2]/div/div/section/div[2]/div/article/div/div/p[2]/a")
+                )
+            )
+            find_your_collection_button.click()
+
+            iframe = wait.until(EC.presence_of_element_located((By.ID, "fillform-frame-1")))
+            driver.switch_to.frame(iframe)
+
+            postcode_input = wait.until(EC.presence_of_element_located((By.ID, "searchString")))
+            postcode_input.send_keys(user_postcode + Keys.TAB + Keys.ENTER)
+
+            time.sleep(15)
+
+            select_elem = wait.until(EC.presence_of_element_located((By.ID, "customerAddress")))
+            WebDriverWait(driver, 10).until(
+                lambda d: len(select_elem.find_elements(By.TAG_NAME, "option")) > 1
+            )
+            dropdown = Select(select_elem)
+            dropdown.select_by_value(user_uprn)
+
+            time.sleep(10)
+
+            wait.until(
+                EC.presence_of_element_located(
+                    (By.CSS_SELECTOR, "span.fieldInput.content.html.non-input"))
+            )
+
+            soup = BeautifulSoup(driver.page_source, "html.parser")
+            bin_data = {"bins": []}
+            current_date = datetime.now()
+            current_formatted_date = None
+
+            spans = soup.select("span.fieldInput.content.html.non-input")
+            print(f"Found {len(spans)} bin info spans.")
+
+            for i, span in enumerate(spans):
+                try:
+                    # Look for any non-empty <u> tag recursively
+                    date_tag = next(
+                        (u for u in span.find_all("u") if u and u.text.strip()),
+                        None
+                    )
+                    bin_type_tag = span.find("b")
+
+                    if date_tag:
+                        raw_date = date_tag.text.strip().replace(",", "")
+                        full_date_str = f"{raw_date} {current_date.year}"
+                        full_date_str = re.sub(r"\s+", " ", full_date_str)
+
+                        try:
+                            parsed_date = datetime.strptime(full_date_str, "%A %d %B %Y")
+                            if parsed_date.date() < current_date.date():
+                                parsed_date = parsed_date.replace(year=current_date.year + 1)
+                            current_formatted_date = parsed_date.strftime("%d/%m/%Y")
+                            print(f"[{i}] Parsed date: {current_formatted_date}")
+                        except ValueError as ve:
+                            print(f"[{i}] Could not parse date: '{full_date_str}' - {ve}")
+                            continue
+                    else:
+                        print(f"[{i}] No date tag found, using last valid date: {current_formatted_date}")
+
+                    if not current_formatted_date:
+                        print(f"[{i}] No current date to associate bin type with — skipping.")
+                        continue
+
+                    if not bin_type_tag or not bin_type_tag.text.strip():
+                        print(f"[{i}] No bin type found — skipping.")
+                        continue
+
+                    bin_type = bin_type_tag.text.strip()
+
+                    # Optional seasonal override
+                    try:
+                        overrides_dict = get_seasonal_overrides()
+                        if current_formatted_date in overrides_dict:
+                            current_formatted_date = overrides_dict[current_formatted_date]
+                    except Exception:
+                        pass
+
+                    print(f"[{i}] Found bin: {bin_type} on {current_formatted_date}")
+
+                    bin_data["bins"].append({
+                        "type": bin_type,
+                        "collectionDate": current_formatted_date
+                    })
+
+                except Exception as inner_e:
+                    print(f"[{i}] Skipping span due to error: {inner_e}")
+                    continue
+
+                except Exception as inner_e:
+                    print(f"Skipping span due to error: {inner_e}")
+                    continue
+
+            if not bin_data["bins"]:
+                raise ValueError("No bin data found.")
+
+            print(bin_data)
+            
+            return bin_data
+
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            raise
+
+        finally:
+            if driver:
+                driver.quit()

uk_bin_collection/uk_bin_collection/councils/BarkingDagenham.py

@@ -84,10 +84,19 @@ class CouncilClass(AbstractGetBinDataClass):
                 EC.element_to_be_clickable((By.ID, "address")),
                 message="Address dropdown not found",
             )
+            
             dropdown = Select(address_select)
 
-            dropdown.select_by_visible_text(user_paon)
-            print("Address selected successfully")
+            found = False
+            for option in dropdown.options:
+                if user_paon in option.text:
+                    option.click()
+                    found = True
+                    print("Address selected successfully")
+                    break
+
+            if not found:
+                raise Exception(f"No matching address containing '{user_paon}' found.")
 
             driver.switch_to.active_element.send_keys(Keys.TAB + Keys.ENTER)
             print("Pressed ENTER on Next button")

uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py

@@ -24,74 +24,51 @@ class CouncilClass(AbstractGetBinDataClass):
     def parse_data(self, page: str, **kwargs) -> dict:
         driver = None
         try:
-            page = "https://waste.bexley.gov.uk/waste"
-
-            data = {"bins": []}
-
             user_uprn = kwargs.get("uprn")
-            user_paon = kwargs.get("paon")
-            user_postcode = kwargs.get("postcode")
             web_driver = kwargs.get("web_driver")
             headless = kwargs.get("headless")
 
+            page = f"https://waste.bexley.gov.uk/waste/{user_uprn}"
+
+            print(f"Trying URL: {page}")  # Debug
+
             # Create Selenium webdriver
             driver = create_webdriver(web_driver, headless, None, __name__)
             driver.get(page)
 
-            wait = WebDriverWait(driver, 10)
-
-            inputElement_postcodesearch = wait.until(
-                EC.element_to_be_clickable((By.ID, "pc"))
-            )
-            inputElement_postcodesearch.send_keys(user_postcode)
-
-            find_address_btn = wait.until(
-                EC.element_to_be_clickable((By.XPATH, '//*[@id="sub"]'))
-            )
-            find_address_btn.click()
+            # Wait for the main content container to be present
+            wait = WebDriverWait(driver, 30)  # Increased timeout to 30 seconds
 
-            dropdown_options = wait.until(
-                EC.presence_of_element_located((By.XPATH, '//*[@id="address"]'))
-            )
-            time.sleep(2)
-            dropdown_options.click()
-            time.sleep(1)
-
-            # Wait for the element to be clickable
-            address = WebDriverWait(driver, 10).until(
-                EC.element_to_be_clickable(
-                    (By.XPATH, f'//li[contains(text(), "{user_paon}")]')
+            # First wait for container
+            main_content = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, "/html/body/div[1]/div/div[2]/div")
                 )
             )
 
-            # Click the element
-            address.click()
-
-            submit_address = wait.until(
-                EC.presence_of_element_located((By.XPATH, '//*[@id="go"]'))
-            )
-            time.sleep(2)
-            submit_address.click()
+            # Then wait for loading indicator to disappear
+            wait.until(EC.invisibility_of_element_located((By.ID, "loading-indicator")))
 
-            results_found = wait.until(
-                EC.element_to_be_clickable(
-                    (By.XPATH, '//h1[contains(text(), "Your bin days")]')
-                )
-            )
+            # Add after the loading indicator wait
+            time.sleep(3)  # Give extra time for JavaScript to populate the data
 
-            final_page = wait.until(
-                EC.presence_of_element_located((By.CLASS_NAME, "waste__collections"))
+            # Then wait for at least one bin section to appear
+            wait.until(
+                EC.presence_of_element_located((By.CLASS_NAME, "waste-service-name"))
             )
 
+            # Now parse the page content
             soup = BeautifulSoup(driver.page_source, features="html.parser")
 
-            # Find all waste services
-
-            # Initialize the data dictionary
             data = {"bins": []}
             bin_sections = soup.find_all("h3", class_="waste-service-name")
 
-            # Loop through each bin field
+            if not bin_sections:
+                print("No bin sections found after waiting for content")
+                print(f"Page source: {driver.page_source}")
+                return data
+
+            # Rest of your existing bin processing code
             for bin_section in bin_sections:
                 # Extract the bin type (e.g., "Brown Caddy", "Green Wheelie Bin", etc.)
                 bin_type = bin_section.get_text(strip=True).split("\n")[

uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py

@@ -63,8 +63,16 @@ class CouncilClass(AbstractGetBinDataClass):
 
             # Create a 'Select' for it, then select the first address in the list
             # (Index 0 is "Make a selection from the list")
-            dropdownSelect = Select(parent_element)
-            dropdownSelect.select_by_visible_text(str(user_paon))
+            options = parent_element.find_elements(By.TAG_NAME, "option")
+            found = False
+            for option in options:
+                if user_paon in option.text:
+                    option.click()
+                    found = True
+                    break
+
+            if not found:
+                raise Exception(f"Address containing '{user_paon}' not found in dropdown options")
 
             submit_btn = wait.until(
                 EC.presence_of_element_located(
@@ -125,6 +133,7 @@ class CouncilClass(AbstractGetBinDataClass):
                                 break
                 dict_data = {"type": bin_type, "collectionDate": bin_date}
                 data["bins"].append(dict_data)
+                print(data)
         except Exception as e:
             # Here you can log the exception if needed
             print(f"An error occurred: {e}")

uk_bin_collection/uk_bin_collection/councils/BroadlandDistrictCouncil.py

@@ -83,15 +83,30 @@ class CouncilClass(AbstractGetBinDataClass):
             )
             print("Found address dropdown")
 
-            # Create a Select object for the dropdown
             dropdown_select = Select(address_dropdown)
 
-            # Search for the exact address
-            print(f"Looking for address: {user_paon}")
+            print(f"Looking for address containing: {user_paon}")
 
-            # Select the address by visible text
-            dropdown_select.select_by_visible_text(user_paon)
-            print(f"Selected address: {user_paon}")
+            found = False
+            user_paon_clean = user_paon.lower().strip()
+
+            for option in dropdown_select.options:
+                option_text_clean = option.text.lower().strip()
+
+                if (
+                    option_text_clean == user_paon_clean  # Exact match if full address given
+                    or option_text_clean.startswith(f"{user_paon_clean} ")  # Startswith match if just a number
+                ):
+                    option.click()
+                    found = True
+                    print(f"Selected address: {option.text.strip()}")
+                    break
+
+            if not found:
+                all_options = [opt.text for opt in dropdown_select.options]
+                raise Exception(
+                    f"Could not find a matching address for '{user_paon}'. Available options: {all_options}"
+                )
 
             print("Looking for submit button after address selection...")
             submit_btn = wait.until(

uk_bin_collection/uk_bin_collection/councils/CharnwoodBoroughCouncil.py

@@ -1,10 +1,11 @@
+from datetime import timedelta
+
 from bs4 import BeautifulSoup
+from dateutil.relativedelta import relativedelta
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
-from datetime import timedelta
-from dateutil.relativedelta import relativedelta
-
 
 # import the wonderful Beautiful Soup and the URL grabber
 class CouncilClass(AbstractGetBinDataClass):
@@ -15,7 +16,17 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page: str, **kwargs) -> dict:
+        try:
+            user_uprn = kwargs.get("uprn")
+            check_uprn(user_uprn)
+            url = f"https://my.charnwood.gov.uk/location?put=cbc{user_uprn}&rememberme=0&redirect=%2F"
+            if not user_uprn:
+                url = kwargs.get("url")
+        except Exception as e:
+            raise ValueError(f"Error getting identifier: {str(e)}")
+
         # Make a BS4 object
+        page = requests.get(url)
         soup = BeautifulSoup(page.text, features="html.parser")
         soup.prettify()
 

uk_bin_collection/uk_bin_collection/councils/CheltenhamBoroughCouncil.py

@@ -266,9 +266,9 @@ class CouncilClass(AbstractGetBinDataClass):
                     ).date()
 
         for refuse_date in refuse_dates:
-            collection_date = datetime.strptime(refuse_date, "%d/%m/%Y") + timedelta(
+            collection_date = (datetime.strptime(refuse_date, "%d/%m/%Y") + timedelta(
                 days=refuse_day_offset
-            )
+            )).date()
             if collection_date in bh_dict:
                 collection_date = bh_dict[collection_date]
             collection_date = collection_date.strftime("%d/%m/%Y")
@@ -281,9 +281,9 @@ class CouncilClass(AbstractGetBinDataClass):
 
         for recycling_date in recycling_dates:
 
-            collection_date = datetime.strptime(recycling_date, "%d/%m/%Y") + timedelta(
+            collection_date = (datetime.strptime(recycling_date, "%d/%m/%Y") + timedelta(
                 days=recycling_day_offset
-            )
+            )).date()
             if collection_date in bh_dict:
                 collection_date = bh_dict[collection_date]
             collection_date = collection_date.strftime("%d/%m/%Y")
@@ -296,9 +296,9 @@ class CouncilClass(AbstractGetBinDataClass):
 
         for garden_date in garden_dates:
 
-            collection_date = datetime.strptime(garden_date, "%d/%m/%Y") + timedelta(
+            collection_date = (datetime.strptime(garden_date, "%d/%m/%Y") + timedelta(
                 days=garden_day_offset
-            )
+            )).date()
             if collection_date in bh_dict:
                 collection_date = bh_dict[collection_date]
             collection_date = collection_date.strftime("%d/%m/%Y")
@@ -318,9 +318,9 @@ class CouncilClass(AbstractGetBinDataClass):
 
             for food_date in food_dates:
 
-                collection_date = datetime.strptime(food_date, "%d/%m/%Y") + timedelta(
+                collection_date = (datetime.strptime(food_date, "%d/%m/%Y") + timedelta(
                     days=food_day_offset
-                )
+                )).date()
                 if collection_date in bh_dict:
                     collection_date = bh_dict[collection_date]
                 collection_date = collection_date.strftime("%d/%m/%Y")
@@ -354,9 +354,9 @@ class CouncilClass(AbstractGetBinDataClass):
 
             for food_date in food_dates_first:
 
-                collection_date = datetime.strptime(food_date, "%d/%m/%Y") + timedelta(
+                collection_date = (datetime.strptime(food_date, "%d/%m/%Y") + timedelta(
                     days=food_day_offset
-                )
+                )).date()
                 if collection_date in bh_dict:
                     collection_date = bh_dict[collection_date]
                 collection_date = collection_date.strftime("%d/%m/%Y")
@@ -368,9 +368,9 @@ class CouncilClass(AbstractGetBinDataClass):
                 bindata["bins"].append(dict_data)
             for food_date in food_dates_second:
 
-                collection_date = datetime.strptime(food_date, "%d/%m/%Y") + timedelta(
+                collection_date = (datetime.strptime(food_date, "%d/%m/%Y") + timedelta(
                     days=second_week_offset
-                )
+                )).date()
                 if collection_date in bh_dict:
                     collection_date = bh_dict[collection_date]
                 collection_date = collection_date.strftime("%d/%m/%Y")

uk_bin_collection/uk_bin_collection/councils/CheshireEastCouncil.py

@@ -1,5 +1,8 @@
-from typing import Dict, Any, Optional
-from bs4 import BeautifulSoup, Tag, NavigableString
+from typing import Any, Dict, Optional
+
+from bs4 import BeautifulSoup, NavigableString, Tag
+
+from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 """
@@ -13,6 +16,25 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page: Any, **kwargs: Any) -> Dict[str, Any]:
+
+        try:
+            user_uprn = kwargs.get("uprn")
+            check_uprn(user_uprn)
+            url = f"https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/GetBartecJobList?uprn={user_uprn}"
+            if not user_uprn:
+                # This is a fallback for if the user stored a URL in old system. Ensures backwards compatibility.
+                url = kwargs.get("url")
+        except Exception as e:
+            raise ValueError(f"Error getting identifier: {str(e)}")
+
+        # Add warning suppression for the insecure request
+        import urllib3
+
+        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+        # Make request with SSL verification disabled
+        page = requests.get(url, verify=False)
+
         soup = BeautifulSoup(page.text, features="html.parser")
 
         bin_data_dict: Dict[str, Any] = {"bins": []}