uk_bin_collection 0.153.0__py3-none-any.whl → 0.157.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/ErewashBoroughCouncil.py

@@ -1,4 +1,7 @@
+import json
+
 from bs4 import BeautifulSoup
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
@@ -16,46 +19,41 @@ class CouncilClass(AbstractGetBinDataClass):
         uprn = kwargs.get("uprn")
         check_uprn(uprn)
 
+        label_map = {
+            "domestic-waste-collection-service": "Household Waste",
+            "recycling-collection-service": "Recycling",
+            "garden-waste-collection-service": "Garden Waste",
+        }
+
         requests.packages.urllib3.disable_warnings()
         response = requests.get(
-            f"https://map.erewash.gov.uk/isharelive.web/myerewash.aspx?action=SetAddress&UniqueId={uprn}",
+            f"https://www.erewash.gov.uk/bbd-whitespace/one-year-collection-dates-without-christmas?uprn={uprn}",
             headers={"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64)"},
         )
+        # Parse the JSON response
+        payload = response.json()
+        bin_collection = json.loads(payload) if isinstance(payload, str) else payload
 
-        soup = BeautifulSoup(response.text, features="html.parser")
-        collections = soup.find("div", {"aria-label": "Waste Collection"}).find_all(
-            "div", {"class": "atPanelContent"}
+        cd = next(
+            i["settings"]["collection_dates"]
+            for i in bin_collection
+            if i.get("command") == "settings"
         )
-        for c in collections:
-            bin_type = c.find("h4").get_text(strip=True)
-            if "my next" in bin_type.lower():
-                collection_info = c.find("div", {"class": "atPanelData"}).get_text(
-                    strip=True
-                )
-                results = re.search(
-                    "([A-Za-z]+ \\d+[A-Za-z]+ [A-Za-z]+ \\d*)", collection_info
+
+        for month in cd.values():
+            for e in month:
+                d = e["date"]  # "YYYY-MM-DD"
+                label = label_map.get(
+                    e.get("service-identifier"),
+                    e.get("service") or e.get("service-identifier"),
                 )
-                if results:
-                    collection_date = datetime.strptime(
-                        remove_ordinal_indicator_from_date_string(results[1]).strip(),
-                        "%A %d %B %Y",
-                    ).strftime(date_format)
-                    dict_data = {
-                        "type": bin_type.replace("My Next ", "").replace(
-                            " Collection", ""
-                        ),
-                        "collectionDate": collection_date,
-                    }
-                    data["bins"].append(dict_data)
-                    if "garden waste" in collection_info.lower():
-                        dict_data = {
-                            "type": "Garden Waste",
-                            "collectionDate": collection_date,
-                        }
-                        data["bins"].append(dict_data)
-
-        data["bins"].sort(
-            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
-        )
+
+                dict_data = {
+                    "type": label,
+                    "collectionDate": datetime.strptime(d, "%Y-%m-%d").strftime(
+                        date_format
+                    ),
+                }
+                data["bins"].append(dict_data)
 
         return data

uk_bin_collection/uk_bin_collection/councils/FarehamBoroughCouncil.py

@@ -38,11 +38,14 @@ class CouncilClass(AbstractGetBinDataClass):
         if "rows" in bin_data:
             collection_str = bin_data["rows"][0]["DomesticBinDay"]
 
-            results = re.findall(r"(\d\d?\/\d\d?\/\d{4}) \((\w*)\)", collection_str)
+            results = re.findall(r'(\d{1,2}/\d{1,2}/\d{4}|today)\s*\(([^)]+)\)', collection_str)
 
             if results:
                 for result in results:
-                    collection_date = datetime.strptime(result[0], "%d/%m/%Y")
+                    if (result[0] == "today"):
+                        collection_date = datetime.today()
+                    else:
+                        collection_date = datetime.strptime(result[0], "%d/%m/%Y")
                     dict_data = {
                         "type": result[1],
                         "collectionDate": collection_date.strftime(date_format),

uk_bin_collection/uk_bin_collection/councils/FolkstoneandHytheDistrictCouncil.py

@@ -74,6 +74,28 @@ class CouncilClass(AbstractGetBinDataClass):
                     }
                     bindata["bins"].append(dict_data)
 
+        # Extract the Garden Waste schedule
+        garden_waste_section = soup.find(
+            "span", text=lambda x: x and "Garden Waste" in x
+        )
+        if garden_waste_section:
+            bin_types = garden_waste_section.text.replace("Garden Waste: ", "").split(
+                " / "
+            )
+            garden_waste_dates = garden_waste_section.find_next("ul").find_all("li")
+            for date in garden_waste_dates:
+                for bin_type in bin_types:
+                    dict_data = {
+                        "type": bin_type.strip(),
+                        "collectionDate": datetime.strptime(
+                            remove_ordinal_indicator_from_date_string(
+                                date.text.strip()
+                            ),
+                            "%A %d %B %Y",
+                        ).strftime("%d/%m/%Y"),
+                    }
+                    bindata["bins"].append(dict_data)
+
         bindata["bins"].sort(
             key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
         )

uk_bin_collection/uk_bin_collection/councils/GlasgowCityCouncil.py

@@ -18,7 +18,7 @@ class CouncilClass(AbstractGetBinDataClass):
         try:
             user_uprn = kwargs.get("uprn")
             check_uprn(user_uprn)
-            url = f"https://onlineservices.glasgow.gov.uk/forms/RefuseAndRecyclingWebApplication/CollectionsCalendar.aspx?UPRN={user_uprn}"
+            url = f"https://onlineservices.glasgow.gov.uk/forms/refuseandrecyclingcalendar/CollectionsCalendar.aspx?UPRN={user_uprn}"
             if not user_uprn:
                 # This is a fallback for if the user stored a URL in old system. Ensures backwards compatibility.
                 url = kwargs.get("url")

uk_bin_collection/uk_bin_collection/councils/HartlepoolBoroughCouncil.py

@@ -73,7 +73,9 @@ class CouncilClass(AbstractGetBinDataClass):
         for div in soup.find_all("div"):
             # Extract bin type and date from the span tag
             text = div.find("span").text.strip()
-            bin_type, date = text.split(" ", 1)
+            parts = text.split(" ")
+            date = parts[-1]  # assume the last token is the date
+            bin_type = " ".join(parts[:-1])
             dict_data = {
                 "type": bin_type,
                 "collectionDate": date,

uk_bin_collection/uk_bin_collection/councils/IslingtonCouncil.py

@@ -17,11 +17,14 @@ class CouncilClass(AbstractGetBinDataClass):
 
         data = {"bins": []}
 
-        waste_table = (
-            soup.find(string="Waste and recycling collections")
-            .find_next("div", class_="m-toggle-content")
-            .find("table")
-        )
+        # Find the waste and recycling section with proper null checking
+        waste_section = soup.find(string="Waste and recycling collections")
+        waste_table = None
+        
+        if waste_section:
+            toggle_content = waste_section.find_next("div", class_="m-toggle-content")
+            if toggle_content:
+                waste_table = toggle_content.find("table")
 
         if waste_table:
             rows = waste_table.find_all("tr")

uk_bin_collection/uk_bin_collection/councils/LancasterCityCouncil.py

@@ -57,17 +57,30 @@ class CouncilClass(AbstractGetBinDataClass):
         response = session.get(addr_link)
         new_soup = BeautifulSoup(response.text, features="html.parser")
         services = new_soup.find("section", {"id": "scheduled-collections"})
+        
+        if services is None:
+            raise Exception("Could not find scheduled collections section on the page")
+            
         services_sub = services.find_all("li")
+        if not services_sub:
+            raise Exception("No collection services found")
+            
         for i in range(0, len(services_sub), 3):
-            dt = datetime.strptime(services_sub[i + 1].text.strip(), "%d/%m/%Y").date()
-            bin_type = BeautifulSoup(services_sub[i + 2].text, features="lxml").find(
-                "p"
-            )
-            data["bins"].append(
-                {
-                    "type": bin_type.text.strip().removesuffix(" Collection Service"),
-                    "collectionDate": dt.strftime(date_format),
-                }
-            )
+            if i + 2 < len(services_sub):
+                date_text = services_sub[i + 1].text.strip() if services_sub[i + 1] else None
+                if date_text:
+                    try:
+                        dt = datetime.strptime(date_text, "%d/%m/%Y").date()
+                        bin_type_element = BeautifulSoup(services_sub[i + 2].text, features="lxml").find("p")
+                        if bin_type_element and bin_type_element.text:
+                            data["bins"].append(
+                                {
+                                    "type": bin_type_element.text.strip().removesuffix(" Collection Service"),
+                                    "collectionDate": dt.strftime(date_format),
+                                }
+                            )
+                    except (ValueError, AttributeError) as e:
+                        # Skip invalid date or missing elements
+                        continue
 
         return data

uk_bin_collection/uk_bin_collection/councils/MidSuffolkDistrictCouncil.py

@@ -1,6 +1,7 @@
 import re
 import time
 
+import holidays
 import requests
 from bs4 import BeautifulSoup
 from selenium.webdriver.common.by import By
@@ -50,58 +51,63 @@ class CouncilClass(AbstractGetBinDataClass):
         refuse_dates = get_dates_every_x_days(refusestartDate, 14, 28)
         recycling_dates = get_dates_every_x_days(recyclingstartDate, 14, 28)
 
-        bank_holidays = [
-            ("25/12/2024", 2),
-            ("26/12/2024", 2),
-            ("27/12/2024", 3),
-            ("30/12/2024", 1),
-            ("31/12/2024", 2),
-            ("01/01/2025", 2),
-            ("02/01/2025", 2),
-            ("03/01/2025", 3),
-            ("06/01/2025", 1),
-            ("07/01/2025", 1),
-            ("08/01/2025", 1),
-            ("09/01/2025", 1),
-            ("10/01/2025", 1),
-            ("18/04/2025", 1),
-            ("21/04/2025", 1),
-            ("22/04/2025", 1),
-            ("23/04/2025", 1),
-            ("24/04/2025", 1),
-            ("25/04/2025", 1),
-            ("05/05/2025", 1),
-            ("06/05/2025", 1),
-            ("07/05/2025", 1),
-            ("08/05/2025", 1),
-            ("09/05/2025", 1),
-            ("26/05/2025", 1),
-            ("27/05/2025", 1),
-            ("28/05/2025", 1),
-            ("29/05/2025", 1),
-            ("30/05/2025", 1),
-            ("25/08/2025", 1),
-            ("26/08/2025", 1),
-            ("27/08/2025", 1),
-            ("28/08/2025", 1),
-            ("29/08/2025", 1),
-        ]
+        # Generate bank holidays dynamically using the holidays library
+        def get_bank_holidays_set():
+            """Get set of bank holiday dates for quick lookup."""
+            current_year = datetime.now().year
+            uk_holidays = holidays.UK(years=range(current_year - 1, current_year + 3))
+            return set(uk_holidays.keys())
+
+        def find_next_collection_day(original_date):
+            """Find the next valid collection day, avoiding weekends and bank holidays."""
+            bank_holiday_dates = get_bank_holidays_set()
+            check_date = datetime.strptime(original_date, "%d/%m/%Y")
+
+            # Safety limit to prevent infinite loops
+            max_attempts = 10
+            attempts = 0
+
+            # Keep moving forward until we find a valid collection day
+            while attempts < max_attempts:
+                attempts += 1
+
+                # Check if it's a weekend (Saturday=5, Sunday=6)
+                if check_date.weekday() >= 5:
+                    check_date += timedelta(days=1)
+                    continue
 
-        for refuseDate in refuse_dates:
+                # Check if it's a bank holiday
+                if check_date.date() in bank_holiday_dates:
+                    # Major holidays (Christmas/New Year) get bigger delays
+                    holiday_name = str(holidays.UK().get(check_date.date(), ''))
+                    is_major_holiday = (
+                        'Christmas' in holiday_name or
+                        'Boxing' in holiday_name or
+                        'New Year' in holiday_name
+                    )
+                    delay_days = 2 if is_major_holiday else 1
+                    check_date += timedelta(days=delay_days)
+                    continue
+
+                # Found a valid collection day
+                break
+
+            # If we've exhausted attempts, return the original date as fallback
+            if attempts >= max_attempts:
+                return original_date
+
+            return check_date.strftime("%d/%m/%Y")
+
+        bank_holidays = []  # No longer needed - using smart date calculation
 
-            collection_date = (
+        for refuseDate in refuse_dates:
+            # Calculate initial collection date
+            initial_date = (
                 datetime.strptime(refuseDate, "%d/%m/%Y") + timedelta(days=offset_days)
             ).strftime("%d/%m/%Y")
 
-            holiday_offset = next(
-                (value for date, value in bank_holidays if date == collection_date), 0
-            )
-
-            if holiday_offset > 0:
-                collection_date = (
-                    datetime.strptime(collection_date, "%d/%m/%Y")
-                    + timedelta(days=holiday_offset)
-                ).strftime("%d/%m/%Y")
+            # Find the next valid collection day (handles weekends + cascading holidays)
+            collection_date = find_next_collection_day(initial_date)
 
             dict_data = {
                 "type": "Refuse Bin",
@@ -110,21 +116,14 @@ class CouncilClass(AbstractGetBinDataClass):
             bindata["bins"].append(dict_data)
 
         for recyclingDate in recycling_dates:
-
-            collection_date = (
+            # Calculate initial collection date
+            initial_date = (
                 datetime.strptime(recyclingDate, "%d/%m/%Y")
                 + timedelta(days=offset_days)
             ).strftime("%d/%m/%Y")
 
-            holiday_offset = next(
-                (value for date, value in bank_holidays if date == collection_date), 0
-            )
-
-            if holiday_offset > 0:
-                collection_date = (
-                    datetime.strptime(collection_date, "%d/%m/%Y")
-                    + timedelta(days=holiday_offset)
-                ).strftime("%d/%m/%Y")
+            # Find the next valid collection day (handles weekends + cascading holidays)
+            collection_date = find_next_collection_day(initial_date)
 
             dict_data = {
                 "type": "Recycling Bin",
@@ -140,48 +139,27 @@ class CouncilClass(AbstractGetBinDataClass):
 
             garden_dates = get_dates_every_x_days(gardenstartDate, 14, 28)
 
-            garden_bank_holidays = [
-                ("23/12/2024", 1),
-                ("24/12/2024", 1),
-                ("25/12/2024", 1),
-                ("26/12/2024", 1),
-                ("27/12/2024", 1),
-                ("30/12/2024", 1),
-                ("31/12/2024", 1),
-                ("01/01/2025", 1),
-                ("02/01/2025", 1),
-                ("03/01/2025", 1),
-            ]
+            def is_christmas_period(date_obj):
+                """Check if date is in Christmas/New Year skip period for garden collections."""
+                if date_obj.month == 12 and date_obj.day >= 23:
+                    return True
+                if date_obj.month == 1 and date_obj.day <= 3:
+                    return True
+                return False
 
             for gardenDate in garden_dates:
-
-                collection_date = (
+                # Calculate initial collection date
+                initial_date = (
                     datetime.strptime(gardenDate, "%d/%m/%Y")
                     + timedelta(days=offset_days_garden)
-                ).strftime("%d/%m/%Y")
-
-                garden_holiday = next(
-                    (
-                        value
-                        for date, value in garden_bank_holidays
-                        if date == collection_date
-                    ),
-                    0,
                 )
 
-                if garden_holiday > 0:
+                # Skip garden collections during Christmas/New Year period
+                if is_christmas_period(initial_date):
                     continue
 
-                holiday_offset = next(
-                    (value for date, value in bank_holidays if date == collection_date),
-                    0,
-                )
-
-                if holiday_offset > 0:
-                    collection_date = (
-                        datetime.strptime(collection_date, "%d/%m/%Y")
-                        + timedelta(days=holiday_offset)
-                    ).strftime("%d/%m/%Y")
+                # Find the next valid collection day (handles weekends + holidays)
+                collection_date = find_next_collection_day(initial_date.strftime("%d/%m/%Y"))
 
                 dict_data = {
                     "type": "Garden Bin",

uk_bin_collection/uk_bin_collection/councils/NewForestCouncil.py

@@ -1,16 +1,14 @@
 import logging
-import pickle
 import time
 
-import requests
 from bs4 import BeautifulSoup
 from selenium import webdriver
+from selenium.common.exceptions import NoSuchElementException
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import Select
 from selenium.webdriver.support.wait import WebDriverWait
-from uk_bin_collection.uk_bin_collection.common import *
 
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
@@ -23,17 +21,64 @@ logging.basicConfig(
 
 class CouncilClass(AbstractGetBinDataClass):
 
+    def get_legacy_bins(self, page: str) -> []:
+
+        logging.info("Extracting legacy bin collection data")
+        soup = BeautifulSoup(page, features="html.parser")
+        legacy_bins = []
+
+        # Rubbish and recycling
+        rubbish_recycling = soup.find(
+            "span", class_="CTID-77-_ eb-77-Override-textControl"
+        )
+        if rubbish_recycling:
+            match = re.search(r"collected weekly on (\w+)", rubbish_recycling.text)
+            if match:
+                day_name = match.group(1)
+                next_collection = get_next_day_of_week(day_name)
+                legacy_bins.append(
+                    {
+                        "type": "Rubbish and recycling",
+                        "collectionDate": next_collection,
+                    }
+                )
+                logging.info(f"Rubbish and Recycling: {str(next_collection)}")
+
+        # Glass collection
+        glass_collection = soup.find("span", class_="CTID-78-_ eb-78-textControl")
+        if glass_collection:
+            match = re.search(
+                r"next collection is\s+(\d{2}/\d{2}/\d{4})", glass_collection.text
+            )
+            if match:
+                legacy_bins.append(
+                    {"type": "Glass collection", "collectionDate": match.group(1)}
+                )
+                logging.info(f"Glass: {str(match.group(1))}")
+
+        # Garden waste
+        garden_waste = soup.find("div", class_="eb-2HIpCnWC-Override-EditorInput")
+        if garden_waste:
+            match = re.search(r"(\d{2}/\d{2}/\d{4})", garden_waste.text)
+            if match:
+                legacy_bins.append(
+                    {"type": "Garden waste", "collectionDate": match.group(1)}
+                )
+                logging.info(f"Garden: {str(match.group(1))}")
+
+        # return bins
+        return legacy_bins
+
     def parse_data(self, page: str, **kwargs) -> dict:
         driver = None
         try:
-            data = {"bins": []}
-            collections = []
+            bins = []
             user_uprn = kwargs.get("uprn")
             user_postcode = kwargs.get("postcode")
             web_driver = kwargs.get("web_driver")
             headless = kwargs.get("headless")
             check_postcode(user_postcode)
-            url = "https://forms.newforest.gov.uk/ufs/FIND_MY_COLLECTION.eb"
+            url = "https://forms.newforest.gov.uk/ufs/FIND_MY_BIN_BAR.eb"
 
             # Get session cookies using requests
 
@@ -52,10 +97,20 @@ class CouncilClass(AbstractGetBinDataClass):
 
             logging.info("Entering postcode")
             input_element_postcode = wait.until(
-                EC.presence_of_element_located((By.XPATH, '//input[@id="CTID-1-_-A"]'))
+                EC.element_to_be_clickable(
+                    (By.XPATH, '//input[@id="CTID-JmLqCKl2-_-A"]')
+                )
+            )
+
+            driver.execute_script(
+                "arguments[0].scrollIntoView();", input_element_postcode
             )
 
-            input_element_postcode.send_keys(user_postcode)
+            logging.info(f"Entering postcode '{str(user_postcode)}'")
+            # Force the value through the DOM cos send_keys just don't work for some reason :(
+            driver.execute_script(
+                f"arguments[0].value='{str(user_postcode)}'", input_element_postcode
+            )
 
             logging.info("Searching for postcode")
             input_element_postcode_btn = wait.until(
@@ -66,7 +121,9 @@ class CouncilClass(AbstractGetBinDataClass):
 
             logging.info("Waiting for address dropdown")
             input_element_postcode_dropdown = wait.until(
-                EC.presence_of_element_located((By.XPATH, '//select[@id="CTID-6-_-A"]'))
+                EC.element_to_be_clickable(
+                    (By.XPATH, '//select[@id="CTID-KOeKcmrC-_-A"]')
+                )
             )
 
             logging.info("Selecting address")
@@ -86,51 +143,51 @@ class CouncilClass(AbstractGetBinDataClass):
 
             input_element_address_btn.click()
 
-            logging.info("Waiting for bin collection page")
-            h4_element = wait.until(
-                EC.presence_of_element_located(
-                    (By.XPATH, "//h1[contains(text(), 'Collections days for')]")
+            # Be patient, clicks take time!
+            time.sleep(2)
+            # logging.info(driver.page_source)
+
+            try:
+                link_element = driver.find_element(
+                    By.XPATH,
+                    '//a[contains(text(),"Find your current bin collection day")]',
+                )
+                logging.info(
+                    "Found override panel span, search for link and use old logic"
                 )
-            )
 
-            logging.info("Extracting bin collection data")
-            soup = BeautifulSoup(driver.page_source, features="html.parser")
-            bins = []
+                link_element.click()
 
-            # Rubbish and recycling
-            rubbish_recycling = soup.find(
-                "span", class_="CTID-77-_ eb-77-Override-textControl"
-            )
-            if rubbish_recycling:
-                match = re.search(r"collected weekly on (\w+)", rubbish_recycling.text)
-                if match:
-                    day_name = match.group(1)
-                    next_collection = get_next_day_of_week(day_name)
-                    bins.append(
-                        {
-                            "type": "Rubbish and recycling",
-                            "collectionDate": next_collection,
-                        }
-                    )
+                # Be patient, clicks take time!
+                time.sleep(2)
 
-            # Glass collection
-            glass_collection = soup.find("span", class_="CTID-78-_ eb-78-textControl")
-            if glass_collection:
-                match = re.search(
-                    r"next collection is\s+(\d{2}/\d{2}/\d{4})", glass_collection.text
-                )
-                if match:
-                    bins.append(
-                        {"type": "Glass collection", "collectionDate": match.group(1)}
+                bins = self.get_legacy_bins(driver.page_source)
+
+            except NoSuchElementException:
+                logging.info("Waiting for bin collection table")
+                collections_table = wait.until(
+                    EC.presence_of_element_located(
+                        (
+                            By.XPATH,
+                            '//table[contains(@class,"eb-1j4UaesZ-tableContent")]',
+                        )
                     )
+                )
 
-            # Garden waste
-            garden_waste = soup.find("div", class_="eb-2HIpCnWC-Override-EditorInput")
-            if garden_waste:
-                match = re.search(r"(\d{2}/\d{2}/\d{4})", garden_waste.text)
-                if match:
+                soup = BeautifulSoup(driver.page_source, features="html.parser")
+                rows = soup.find_all(class_="eb-1j4UaesZ-tableRow")
+
+                for row in rows:
+                    cols = row.find_all("td")
+                    date_string = cols[0].findChild("div").findChild("div").get_text()
+                    bin_type = cols[1].findChild("div").findChild("div").get_text()
+
+                    col_date = datetime.strptime(date_string, "%A %B %d, %Y")
                     bins.append(
-                        {"type": "Garden waste", "collectionDate": match.group(1)}
+                        {
+                            "type": bin_type,
+                            "collectionDate": datetime.strftime(col_date, date_format),
+                        }
                     )
 
             return {"bins": bins}