uk_bin_collection 0.154.0__py3-none-any.whl → 0.158.0__py3-none-any.whl

uk_bin_collection/tests/input.json

@@ -760,6 +760,13 @@
         "wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
         "LAD24CD": "E07000040"
     },
+    "EastDunbartonshireCouncil": {
+        "uprn": "132027197",
+        "url": "https://www.eastdunbarton.gov.uk/",
+        "wiki_name": "East Dunbartonshire",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN.",
+        "LAD24CD": "S12000045"
+    },
     "EastHertsCouncil": {
         "LAD24CD": "E07000097",
         "skip_get_url": true,
@@ -1608,7 +1615,7 @@
         "postcode": "SO41 0GJ",
         "skip_get_url": true,
         "uprn": "100060482345",
-        "url": "https://forms.newforest.gov.uk/id/FIND_MY_COLLECTION",
+        "url": "https://forms.newforest.gov.uk/ufs/FIND_MY_BIN_BAR.eb",
         "web_driver": "http://selenium:4444",
         "wiki_name": "New Forest",
         "wiki_note": "Pass the postcode and UPRN. This parser requires a Selenium webdriver.",
@@ -1647,10 +1654,11 @@
     "NewportCityCouncil": {
         "postcode": "NP20 4HE",
         "skip_get_url": true,
-        "uprn": "100100688837",
+        "house_number": "6",
         "url": "https://www.newport.gov.uk/",
+        "web_driver": "http://selenium:4444",
         "wiki_name": "Newport",
-        "wiki_note": "Pass the postcode and UPRN. You can find the UPRN using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
+        "wiki_note": "Pass the postcode and house number in their respective arguments, both wrapped in quotes.",
         "LAD24CD": "W06000022"
     },
     "NorthAyrshireCouncil": {
@@ -2087,10 +2095,11 @@
     "SomersetCouncil": {
         "postcode": "TA6 4AA",
         "skip_get_url": true,
-        "uprn": "10090857775",
+        "house_number": "5",
         "url": "https://www.somerset.gov.uk/",
+        "web_driver": "http://selenium:4444",
         "wiki_name": "Somerset",
-        "wiki_note": "Provide your UPRN and postcode. Find your UPRN using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
+        "wiki_note": "Provide your house number and postcode",
         "LAD24CD": "E06000066"
     },
     "SouthAyrshireCouncil": {
@@ -2122,7 +2131,7 @@
     "SouthGloucestershireCouncil": {
         "skip_get_url": true,
         "uprn": "566419",
-        "url": "https://beta.southglos.gov.uk/waste-and-recycling-collection-date",
+        "url": "https://api.southglos.gov.uk/wastecomp/GetCollectionDetails",
         "wiki_name": "South Gloucestershire",
         "wiki_note": "Provide your UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
         "LAD24CD": "E06000025"
@@ -2415,12 +2424,13 @@
         "LAD24CD": "E07000076"
     },
     "TestValleyBoroughCouncil": {
-        "postcode": "SO51 9ZD",
+        "postcode": "SO51 0BY",
         "skip_get_url": true,
-        "uprn": "200010012019",
-        "url": "https://testvalley.gov.uk/wasteandrecycling/when-are-my-bins-collected",
+        "house_number": "2",
+        "url": "https://testvalley.gov.uk/wasteandrecycling/when-are-my-bins-collected/when-are-my-bins-collected",
+        "web_driver": "http://selenium:4444",
         "wiki_name": "Test Valley",
-        "wiki_note": "Provide your UPRN and postcode. Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN.",
+        "wiki_note": "Provide your house number and postcode",
         "LAD24CD": "E07000093"
     },
     "ThanetDistrictCouncil": {
@@ -2463,6 +2473,7 @@
         "skip_get_url": true,
         "uprn": "10000016984",
         "postcode": "TQ1 1AG",
+        "web_driver": "http://selenium:4444",
         "url": "https://www.torbay.gov.uk/recycling/bin-collections/",
         "wiki_name": "Torbay",
         "wiki_note": "Provide your UPRN. Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find it.",

uk_bin_collection/uk_bin_collection/councils/AberdeenCityCouncil.py

@@ -5,7 +5,6 @@ import requests
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
-
 # import the wonderful Beautiful Soup and the URL grabber
 class CouncilClass(AbstractGetBinDataClass):
     """

uk_bin_collection/uk_bin_collection/councils/DacorumBoroughCouncil.py

@@ -76,19 +76,28 @@ class CouncilClass(AbstractGetBinDataClass):
             )
 
             for Collection in NextCollections:
-                BinType = Collection.find("strong").text.strip()
-                if BinType:
-                    CollectionDate = datetime.strptime(
-                        Collection.find_all("div", {"style": "display:table-cell;"})[1]
-                        .get_text()
-                        .strip(),
-                        "%a, %d %b %Y",
-                    )
-                    dict_data = {
-                        "type": BinType,
-                        "collectionDate": CollectionDate.strftime("%d/%m/%Y"),
-                    }
-                    data["bins"].append(dict_data)
+                strong_element = Collection.find("strong")
+                if strong_element:
+                    BinType = strong_element.text.strip()
+                    # Skip if this is not a bin type (e.g., informational text)
+                    if BinType and not any(skip_text in BinType.lower() for skip_text in 
+                                         ["please note", "we may collect", "bank holiday", "different day"]):
+                        date_cells = Collection.find_all("div", {"style": "display:table-cell;"})
+                        if len(date_cells) > 1:
+                            date_text = date_cells[1].get_text().strip()
+                            if date_text:
+                                try:
+                                    CollectionDate = datetime.strptime(date_text, "%a, %d %b %Y")
+                                    dict_data = {
+                                        "type": BinType,
+                                        "collectionDate": CollectionDate.strftime("%d/%m/%Y"),
+                                    }
+                                    # Check for duplicates before adding
+                                    if dict_data not in data["bins"]:
+                                        data["bins"].append(dict_data)
+                                except ValueError:
+                                    # Skip if date parsing fails
+                                    continue
 
         except Exception as e:
             # Here you can log the exception if needed

uk_bin_collection/uk_bin_collection/councils/EastDunbartonshireCouncil.py

@@ -0,0 +1,52 @@
+import requests
+from bs4 import BeautifulSoup, Tag
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        URI = f"https://www.eastdunbarton.gov.uk/services/a-z-of-services/bins-waste-and-recycling/bins-and-recycling/collections/?uprn={user_uprn}"
+
+        # Make the GET request
+        response = requests.get(URI)
+
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        table = soup.find("table", {"class": "bin-table"})
+
+        tbody = table.find("tbody")
+
+        trs = tbody.find_all("tr")
+
+        for tr in trs:
+            tds = tr.find_all("td")
+            bin_type = tds[0].get_text()
+            collection_date_str = tds[1].find("span").get_text()
+
+            collection_date = datetime.strptime(collection_date_str, "%A, %d %B %Y")
+
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": collection_date.strftime(date_format),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/IslingtonCouncil.py

@@ -17,11 +17,14 @@ class CouncilClass(AbstractGetBinDataClass):
 
         data = {"bins": []}
 
-        waste_table = (
-            soup.find(string="Waste and recycling collections")
-            .find_next("div", class_="m-toggle-content")
-            .find("table")
-        )
+        # Find the waste and recycling section with proper null checking
+        waste_section = soup.find(string="Waste and recycling collections")
+        waste_table = None
+        
+        if waste_section:
+            toggle_content = waste_section.find_next("div", class_="m-toggle-content")
+            if toggle_content:
+                waste_table = toggle_content.find("table")
 
         if waste_table:
             rows = waste_table.find_all("tr")

uk_bin_collection/uk_bin_collection/councils/LancasterCityCouncil.py

@@ -57,17 +57,30 @@ class CouncilClass(AbstractGetBinDataClass):
         response = session.get(addr_link)
         new_soup = BeautifulSoup(response.text, features="html.parser")
         services = new_soup.find("section", {"id": "scheduled-collections"})
+        
+        if services is None:
+            raise Exception("Could not find scheduled collections section on the page")
+            
         services_sub = services.find_all("li")
+        if not services_sub:
+            raise Exception("No collection services found")
+            
         for i in range(0, len(services_sub), 3):
-            dt = datetime.strptime(services_sub[i + 1].text.strip(), "%d/%m/%Y").date()
-            bin_type = BeautifulSoup(services_sub[i + 2].text, features="lxml").find(
-                "p"
-            )
-            data["bins"].append(
-                {
-                    "type": bin_type.text.strip().removesuffix(" Collection Service"),
-                    "collectionDate": dt.strftime(date_format),
-                }
-            )
+            if i + 2 < len(services_sub):
+                date_text = services_sub[i + 1].text.strip() if services_sub[i + 1] else None
+                if date_text:
+                    try:
+                        dt = datetime.strptime(date_text, "%d/%m/%Y").date()
+                        bin_type_element = BeautifulSoup(services_sub[i + 2].text, features="lxml").find("p")
+                        if bin_type_element and bin_type_element.text:
+                            data["bins"].append(
+                                {
+                                    "type": bin_type_element.text.strip().removesuffix(" Collection Service"),
+                                    "collectionDate": dt.strftime(date_format),
+                                }
+                            )
+                    except (ValueError, AttributeError) as e:
+                        # Skip invalid date or missing elements
+                        continue
 
         return data

uk_bin_collection/uk_bin_collection/councils/LondonBoroughSutton.py

@@ -1,30 +1,24 @@
-from time import sleep
-
 import requests
 from bs4 import BeautifulSoup
+from datetime import datetime
+import re
+from time import sleep
 
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
+def remove_ordinal_indicator_from_date_string(date_str):
+    return re.sub(r'(\d+)(st|nd|rd|th)', r'\1', date_str)
 
-# import the wonderful Beautiful Soup and the URL grabber
 class CouncilClass(AbstractGetBinDataClass):
-    """
-    Concrete classes have to implement all abstract operations of the
-    base class. They can also override some operations with a default
-    implementation.
-    """
 
     def parse_data(self, page: str, **kwargs) -> dict:
-
         user_uprn = kwargs.get("uprn")
-        # check_uprn(user_uprn)
         bindata = {"bins": []}
 
         URI = f"https://waste-services.sutton.gov.uk/waste/{user_uprn}"
 
         s = requests.Session()
-
         r = s.get(URI)
         while "Loading your bin days..." in r.text:
             sleep(2)
@@ -32,48 +26,65 @@ class CouncilClass(AbstractGetBinDataClass):
         r.raise_for_status()
 
         soup = BeautifulSoup(r.content, "html.parser")
-
         current_year = datetime.now().year
         next_year = current_year + 1
 
-        services = soup.find_all("h3", class_="govuk-heading-m waste-service-name")
-
+        # Find all h3 headers (bin types)
+        services = soup.find_all("h3")
         for service in services:
-            bin_type = service.get_text(
-                strip=True
-            )  # Bin type name (e.g., 'Food waste', 'Mixed recycling')
-            if bin_type == "Bulky Waste":
+            bin_type = service.get_text(strip=True)
+            if "Bulky Waste" in bin_type:
                 continue
-            service_details = service.find_next("div", class_="govuk-grid-row")
-
-            next_collection = (
-                (
-                    service_details.find("dt", string="Next collection")
-                    .find_next_sibling("dd")
-                    .get_text(strip=True)
-                )
-                .replace("(this collection has been adjusted from its usual time)", "")
-                .strip()
-            )
-
-            next_collection = datetime.strptime(
-                remove_ordinal_indicator_from_date_string(next_collection),
-                "%A, %d %B",
-            )
-
-            if (datetime.now().month == 12) and (next_collection.month == 1):
-                next_collection = next_collection.replace(year=next_year)
-            else:
-                next_collection = next_collection.replace(year=current_year)
-
-            dict_data = {
-                "type": bin_type,
-                "collectionDate": next_collection.strftime("%d/%m/%Y"),
-            }
-            bindata["bins"].append(dict_data)
-
-        bindata["bins"].sort(
-            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
-        )
 
+            # Find the next element (next sibling) which is likely a paragraph with date info
+            next_sib = service.find_next_sibling()
+            while next_sib and getattr(next_sib, 'name', None) not in [None, 'p']:
+                next_sib = next_sib.find_next_sibling()
+
+            next_coll = None
+            if next_sib:
+                text = next_sib.get_text() if hasattr(next_sib, 'get_text') else str(next_sib)
+                match = re.search(r"Next collection\s*([A-Za-z]+,? \d{1,2}(?:st|nd|rd|th)? [A-Za-z]+)", text)
+                if match:
+                    next_coll = match.group(1)
+                else:
+                    # Sometimes the text may be attached without a space after 'Next collection'
+                    match = re.search(r"Next collection([A-Za-z]+,? \d{1,2}(?:st|nd|rd|th)? [A-Za-z]+)", text)
+                    if match:
+                        next_coll = match.group(1)
+
+            # Try several siblings forward if not found
+            if not next_coll:
+                sib_try = service
+                for _ in range(3):
+                    if sib_try:
+                        sib_try = sib_try.find_next_sibling()
+                    else:
+                        break
+                    if sib_try:
+                        text = sib_try.get_text() if hasattr(sib_try, 'get_text') else str(sib_try)
+                        match = re.search(r"Next collection\s*([A-Za-z]+,? \d{1,2}(?:st|nd|rd|th)? [A-Za-z]+)", text)
+                        if match:
+                            next_coll = match.group(1)
+                            break
+
+            if next_coll:
+                next_coll = remove_ordinal_indicator_from_date_string(next_coll)
+                try:
+                    next_collection = datetime.strptime(next_coll, "%A, %d %B")
+                except ValueError:
+                    continue
+
+                if (datetime.now().month == 12 and next_collection.month == 1):
+                    next_collection = next_collection.replace(year=next_year)
+                else:
+                    next_collection = next_collection.replace(year=current_year)
+
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": next_collection.strftime("%d/%m/%Y"),
+                }
+                bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y"))
         return bindata

uk_bin_collection/uk_bin_collection/councils/MidSuffolkDistrictCouncil.py

@@ -1,6 +1,7 @@
 import re
 import time
 
+import holidays
 import requests
 from bs4 import BeautifulSoup
 from selenium.webdriver.common.by import By
@@ -50,58 +51,63 @@ class CouncilClass(AbstractGetBinDataClass):
         refuse_dates = get_dates_every_x_days(refusestartDate, 14, 28)
         recycling_dates = get_dates_every_x_days(recyclingstartDate, 14, 28)
 
-        bank_holidays = [
-            ("25/12/2024", 2),
-            ("26/12/2024", 2),
-            ("27/12/2024", 3),
-            ("30/12/2024", 1),
-            ("31/12/2024", 2),
-            ("01/01/2025", 2),
-            ("02/01/2025", 2),
-            ("03/01/2025", 3),
-            ("06/01/2025", 1),
-            ("07/01/2025", 1),
-            ("08/01/2025", 1),
-            ("09/01/2025", 1),
-            ("10/01/2025", 1),
-            ("18/04/2025", 1),
-            ("21/04/2025", 1),
-            ("22/04/2025", 1),
-            ("23/04/2025", 1),
-            ("24/04/2025", 1),
-            ("25/04/2025", 1),
-            ("05/05/2025", 1),
-            ("06/05/2025", 1),
-            ("07/05/2025", 1),
-            ("08/05/2025", 1),
-            ("09/05/2025", 1),
-            ("26/05/2025", 1),
-            ("27/05/2025", 1),
-            ("28/05/2025", 1),
-            ("29/05/2025", 1),
-            ("30/05/2025", 1),
-            ("25/08/2025", 1),
-            ("26/08/2025", 1),
-            ("27/08/2025", 1),
-            ("28/08/2025", 1),
-            ("29/08/2025", 1),
-        ]
+        # Generate bank holidays dynamically using the holidays library
+        def get_bank_holidays_set():
+            """Get set of bank holiday dates for quick lookup."""
+            current_year = datetime.now().year
+            uk_holidays = holidays.UK(years=range(current_year - 1, current_year + 3))
+            return set(uk_holidays.keys())
+
+        def find_next_collection_day(original_date):
+            """Find the next valid collection day, avoiding weekends and bank holidays."""
+            bank_holiday_dates = get_bank_holidays_set()
+            check_date = datetime.strptime(original_date, "%d/%m/%Y")
+
+            # Safety limit to prevent infinite loops
+            max_attempts = 10
+            attempts = 0
+
+            # Keep moving forward until we find a valid collection day
+            while attempts < max_attempts:
+                attempts += 1
+
+                # Check if it's a weekend (Saturday=5, Sunday=6)
+                if check_date.weekday() >= 5:
+                    check_date += timedelta(days=1)
+                    continue
 
-        for refuseDate in refuse_dates:
+                # Check if it's a bank holiday
+                if check_date.date() in bank_holiday_dates:
+                    # Major holidays (Christmas/New Year) get bigger delays
+                    holiday_name = str(holidays.UK().get(check_date.date(), ''))
+                    is_major_holiday = (
+                        'Christmas' in holiday_name or
+                        'Boxing' in holiday_name or
+                        'New Year' in holiday_name
+                    )
+                    delay_days = 2 if is_major_holiday else 1
+                    check_date += timedelta(days=delay_days)
+                    continue
+
+                # Found a valid collection day
+                break
+
+            # If we've exhausted attempts, return the original date as fallback
+            if attempts >= max_attempts:
+                return original_date
+
+            return check_date.strftime("%d/%m/%Y")
+
+        bank_holidays = []  # No longer needed - using smart date calculation
 
-            collection_date = (
+        for refuseDate in refuse_dates:
+            # Calculate initial collection date
+            initial_date = (
                 datetime.strptime(refuseDate, "%d/%m/%Y") + timedelta(days=offset_days)
             ).strftime("%d/%m/%Y")
 
-            holiday_offset = next(
-                (value for date, value in bank_holidays if date == collection_date), 0
-            )
-
-            if holiday_offset > 0:
-                collection_date = (
-                    datetime.strptime(collection_date, "%d/%m/%Y")
-                    + timedelta(days=holiday_offset)
-                ).strftime("%d/%m/%Y")
+            # Find the next valid collection day (handles weekends + cascading holidays)
+            collection_date = find_next_collection_day(initial_date)
 
             dict_data = {
                 "type": "Refuse Bin",
@@ -110,21 +116,14 @@ class CouncilClass(AbstractGetBinDataClass):
             bindata["bins"].append(dict_data)
 
         for recyclingDate in recycling_dates:
-
-            collection_date = (
+            # Calculate initial collection date
+            initial_date = (
                 datetime.strptime(recyclingDate, "%d/%m/%Y")
                 + timedelta(days=offset_days)
             ).strftime("%d/%m/%Y")
 
-            holiday_offset = next(
-                (value for date, value in bank_holidays if date == collection_date), 0
-            )
-
-            if holiday_offset > 0:
-                collection_date = (
-                    datetime.strptime(collection_date, "%d/%m/%Y")
-                    + timedelta(days=holiday_offset)
-                ).strftime("%d/%m/%Y")
+            # Find the next valid collection day (handles weekends + cascading holidays)
+            collection_date = find_next_collection_day(initial_date)
 
             dict_data = {
                 "type": "Recycling Bin",
@@ -140,48 +139,27 @@ class CouncilClass(AbstractGetBinDataClass):
 
             garden_dates = get_dates_every_x_days(gardenstartDate, 14, 28)
 
-            garden_bank_holidays = [
-                ("23/12/2024", 1),
-                ("24/12/2024", 1),
-                ("25/12/2024", 1),
-                ("26/12/2024", 1),
-                ("27/12/2024", 1),
-                ("30/12/2024", 1),
-                ("31/12/2024", 1),
-                ("01/01/2025", 1),
-                ("02/01/2025", 1),
-                ("03/01/2025", 1),
-            ]
+            def is_christmas_period(date_obj):
+                """Check if date is in Christmas/New Year skip period for garden collections."""
+                if date_obj.month == 12 and date_obj.day >= 23:
+                    return True
+                if date_obj.month == 1 and date_obj.day <= 3:
+                    return True
+                return False
 
             for gardenDate in garden_dates:
-
-                collection_date = (
+                # Calculate initial collection date
+                initial_date = (
                     datetime.strptime(gardenDate, "%d/%m/%Y")
                     + timedelta(days=offset_days_garden)
-                ).strftime("%d/%m/%Y")
-
-                garden_holiday = next(
-                    (
-                        value
-                        for date, value in garden_bank_holidays
-                        if date == collection_date
-                    ),
-                    0,
                 )
 
-                if garden_holiday > 0:
+                # Skip garden collections during Christmas/New Year period
+                if is_christmas_period(initial_date):
                     continue
 
-                holiday_offset = next(
-                    (value for date, value in bank_holidays if date == collection_date),
-                    0,
-                )
-
-                if holiday_offset > 0:
-                    collection_date = (
-                        datetime.strptime(collection_date, "%d/%m/%Y")
-                        + timedelta(days=holiday_offset)
-                    ).strftime("%d/%m/%Y")
+                # Find the next valid collection day (handles weekends + holidays)
+                collection_date = find_next_collection_day(initial_date.strftime("%d/%m/%Y"))
 
                 dict_data = {
                     "type": "Garden Bin",