uk_bin_collection 0.153.0__py3-none-any.whl → 0.154.0__py3-none-any.whl

uk_bin_collection/tests/input.json

@@ -102,12 +102,9 @@
     },
     "BCPCouncil": {
         "LAD24CD": "E06000058",
-        "house_number": "3 HARBOUR VIEW ROAD, POOLE, BH14 0PD",
-        "postcode": "BH14 0PD",
-        "web_driver": "http://selenium:4444",
         "skip_get_url": true,
         "uprn": "100040810214",
-        "url": "https://online.bcpcouncil.gov.uk/bindaylookup/",
+        "url": "https://bcpportal.bcpcouncil.gov.uk/checkyourbincollection",
         "wiki_name": "Bournemouth, Christchurch and Poole",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
@@ -888,7 +885,7 @@
     "ErewashBoroughCouncil": {
         "skip_get_url": true,
         "uprn": "10003582028",
-        "url": "https://map.erewash.gov.uk/isharelive.web/myerewash.aspx",
+        "url": "https://www.erewash.gov.uk",
         "wiki_name": "Erewash",
         "wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
         "LAD24CD": "E07000036"
@@ -995,7 +992,7 @@
     },
     "GlasgowCityCouncil": {
         "uprn": "906700034497",
-        "url": "https://onlineservices.glasgow.gov.uk/forms/RefuseAndRecyclingWebApplication/AddressSearch.aspx",
+        "url": "https://onlineservices.glasgow.gov.uk/forms/refuseandrecyclingcalendar/AddressSearch.aspx",
         "skip_get_url": true,
         "wiki_name": "Glasgow City",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN.",
@@ -1779,21 +1776,22 @@
         "LAD24CD": "E06000065"
     },
     "NorthumberlandCouncil": {
-        "house_number": "22",
-        "postcode": "NE46 1UQ",
+        "uprn": "010096302588",
+        "postcode": "NE65 0ZP",
         "skip_get_url": true,
-        "url": "https://www.northumberland.gov.uk/Waste/Household-waste/Household-bin-collections/Bin-Calendars.aspx",
+        "url": "https://bincollection.northumberland.gov.uk/postcode",
         "web_driver": "http://selenium:4444",
         "wiki_name": "Northumberland",
-        "wiki_note": "Pass the house number and postcode in their respective parameters. This parser requires a Selenium webdriver.",
+        "wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
         "LAD24CD": "E06000057"
     },
     "NorwichCityCouncil": {
-        "uprn": "100090888980",
-        "url": "https://www.norwich.gov.uk",
-        "wiki_command_url_override": "https://www.norwich.gov.uk",
+        "house_number": "2",
+        "postcode": "NR2 3TT",
+        "url": "https://bnr-wrp.whitespacews.com",
+        "wiki_command_url_override": "hhttps://bnr-wrp.whitespacews.com",
         "wiki_name": "Norwich",
-        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN.",
+        "wiki_note": "Pass the house number and postcode in their respective parameters.",
         "LAD24CD": "E07000148"
     },
     "NottinghamCityCouncil": {
@@ -2804,4 +2802,4 @@
         "wiki_note": "Provide your UPRN.",
         "LAD24CD": "E06000014"
     }
-}
+}

uk_bin_collection/uk_bin_collection/councils/BCPCouncil.py

@@ -1,15 +1,13 @@
-import json
 import time
-from datetime import datetime
-from bs4 import BeautifulSoup
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support.ui import WebDriverWait, Select
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.common.keys import Keys
+
+import requests
+from dateutil.relativedelta import relativedelta
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
+# import the wonderful Beautiful Soup and the URL grabber
 class CouncilClass(AbstractGetBinDataClass):
     """
     Concrete classes have to implement all abstract operations of the
@@ -18,116 +16,43 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page: str, **kwargs) -> dict:
-        postcode = kwargs.get("postcode")
-        house_number = kwargs.get("paon")
-        web_driver = kwargs.get("web_driver")
-        headless = kwargs.get("headless", True)
-        
-        check_postcode(postcode)
-        check_paon(house_number)
-        
-        driver = create_webdriver(web_driver, headless=headless)
-        
-        try:
-            driver.get("https://bcpportal.bcpcouncil.gov.uk/checkyourbincollection/")
-            
-            # Handle cookie banner first
-            try:
-                cookie_button = WebDriverWait(driver, 5).until(
-                    EC.element_to_be_clickable((By.XPATH, "//button[contains(text(), 'Okay')]"))
-                )
-                cookie_button.click()
-            except:
-                pass  # Cookie banner might not be present
-            
-            # Wait for and enter postcode
-            postcode_input = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located((By.CSS_SELECTOR, "input[type='text']"))
-            )
-            postcode_input.clear()
-            postcode_input.send_keys(postcode)
-            
-            # Click the search span element
-            search_button = WebDriverWait(driver, 10).until(
-                EC.element_to_be_clickable((By.ID, "searchAddress"))
-            )
-            search_button.click()
-            
-            # Wait for address dropdown
-            select_element = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located((By.TAG_NAME, "select"))
-            )
-            
-            # Find and select the address containing the house number
-            address_option = WebDriverWait(driver, 10).until(
-                EC.element_to_be_clickable((By.XPATH, f"//option[contains(text(), 'HARBOUR VIEW ROAD')]"))
-            )
-            address_option.click()
-            
-            # Wait for bin collection results to load
-            WebDriverWait(driver, 15).until(
-                EC.presence_of_element_located((By.XPATH, "//td[contains(text(), 'collection')] | //th[contains(text(), 'collection')]"))
-            )
-            
-            # Find the table containing collection data by looking for a cell with 'collection' text
-            collection_table = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located((By.XPATH, "//td[contains(text(), 'collection')]/ancestor::table | //th[contains(text(), 'collection')]/ancestor::table"))
-            )
-            
-            # Parse the table data
-            soup = BeautifulSoup(driver.page_source, 'html.parser')
-            data = {"bins": []}
-            
-            # Find the table containing collection information
-            collection_cell = soup.find(['td', 'th'], string=lambda text: text and 'collection' in text.lower())
-            if collection_cell:
-                table = collection_cell.find_parent('table')
-                if table:
-                    rows = table.find_all('tr')
-                    for row in rows[1:]:  # Skip header row
-                        cells = row.find_all(['td', 'th'])
-                        if len(cells) >= 2:  # At least bin type and one collection date
-                            bin_type = cells[0].get_text(strip=True)
-                            next_collection = cells[1].get_text(strip=True) if len(cells) > 1 else ""
-                            following_collection = cells[2].get_text(strip=True) if len(cells) > 2 else ""
-                            
-                            
-                            # Process next collection date
-                            if bin_type and next_collection and "No collection" not in next_collection:
-                                try:
-                                    # Try multiple date formats
-                                    for date_fmt in ["%A, %d %B %Y", "%A %d %B %Y", "%d/%m/%Y", "%d-%m-%Y", "%Y-%m-%d"]:
-                                        try:
-                                            parsed_date = datetime.strptime(next_collection, date_fmt)
-                                            data["bins"].append({
-                                                "type": bin_type,
-                                                "collectionDate": parsed_date.strftime(date_format)
-                                            })
-                                            break
-                                        except ValueError:
-                                            continue
-                                except:
-                                    continue
-                            
-                            # Process following collection date
-                            if bin_type and following_collection and "No collection" not in following_collection and "download PDF" not in following_collection:
-                                try:
-                                    # Clean up the following collection text (remove PDF link text)
-                                    following_collection = following_collection.replace("download PDF", "").strip()
-                                    for date_fmt in ["%A, %d %B %Y", "%A %d %B %Y", "%d/%m/%Y", "%d-%m-%Y", "%Y-%m-%d"]:
-                                        try:
-                                            parsed_date = datetime.strptime(following_collection, date_fmt)
-                                            data["bins"].append({
-                                                "type": bin_type,
-                                                "collectionDate": parsed_date.strftime(date_format)
-                                            })
-                                            break
-                                        except ValueError:
-                                            continue
-                                except:
-                                    continue
-            
-            return data
-            
-        finally:
-            driver.quit()
+        # Make a BS4 object
+        uprn = kwargs.get("uprn")
+        # usrn = kwargs.get("paon")
+        check_uprn(uprn)
+        # check_usrn(usrn)
+        bindata = {"bins": []}
+
+        # uprn = uprn.zfill(12)
+
+        API_URL = "https://prod-17.uksouth.logic.azure.com/workflows/58253d7b7d754447acf9fe5fcf76f493/triggers/manual/paths/invoke?api-version=2016-06-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=TAvYIUFj6dzaP90XQCm2ElY6Cd34ze05I3ba7LKTiBs"
+
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "*/*",
+            "User-Agent": "Mozilla/5.0",
+            "Referer": "https://bcpportal.bcpcouncil.gov.uk/",
+        }
+        s = requests.session()
+        data = {
+            "uprn": uprn,
+        }
+
+        r = s.post(API_URL, json=data, headers=headers)
+        r.raise_for_status()
+
+        data = r.json()
+        rows_data = data["data"]
+        for row in rows_data:
+            bin_type = row["wasteContainerUsageTypeDescription"]
+            collections = row["scheduleDateRange"]
+            for collection in collections:
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": datetime.strptime(
+                        collection, "%Y-%m-%d"
+                    ).strftime(date_format),
+                }
+                bindata["bins"].append(dict_data)
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/BasingstokeCouncil.py

@@ -1,6 +1,8 @@
-from bs4 import BeautifulSoup
 from datetime import datetime
+
 import requests
+from bs4 import BeautifulSoup
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
@@ -10,6 +12,7 @@ COLLECTION_KINDS = {
     "glass": "rteelem_ctl03_pnlCollections_Glass",
     # Garden waste data is only returned if the property is subscribed to the Garden Waste service
     "garden": "rteelem_ctl03_pnlCollections_GardenWaste",
+    "food": "rteelem_ctl03_pnlCollections_Food",
 }
 
 

uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py

@@ -72,7 +72,9 @@ class CouncilClass(AbstractGetBinDataClass):
                     break
 
             if not found:
-                raise Exception(f"Address containing '{user_paon}' not found in dropdown options")
+                raise Exception(
+                    f"Address containing '{user_paon}' not found in dropdown options"
+                )
 
             submit_btn = wait.until(
                 EC.presence_of_element_located(
@@ -84,7 +86,7 @@ class CouncilClass(AbstractGetBinDataClass):
 
             results = wait.until(
                 EC.presence_of_element_located(
-                    (By.XPATH, f'//span[contains(@class,"collection-sub")]')
+                    (By.XPATH, f'//div[contains(@class,"mx-name-listView1")]')
                 )
             )
 
@@ -96,44 +98,21 @@ class CouncilClass(AbstractGetBinDataClass):
             current_date = datetime.now()
 
             # Find all elements with class starting with 'mx-name-index-'
-            bins = soup.find_all(class_=lambda x: x and x.startswith("mx-name-index-"))
+            bin_view = soup.find(class_="mx-name-listView1")
+            bins = bin_view.find_all(
+                class_=lambda x: x and x.startswith("mx-name-index-")
+            )
 
             for bin_item in bins:
-                bin_type = bin_item.find(class_="collection-main").text.strip()
-                day_of_week_elements = bin_item.find_all(class_="collection-header")
-                bin_date = None
-
-                for elem in day_of_week_elements:
-                    if (
-                        elem.text.strip() != bin_type
-                    ):  # Avoid taking the bin type as the date
-                        next_sibling = elem.find_next_sibling()
-                        if next_sibling:
-                            bin_date_str = next_sibling.text.strip()
-                            try:
-                                # Try parsing the date string in the format 'dd Month' (e.g., '30 Dec', '5 January')
-                                bin_date = datetime.strptime(bin_date_str, "%d %b")
-                            except ValueError:
-                                try:
-                                    # If the above format fails, try 'dd MonthName' (e.g., '30 December', '5 January')
-                                    bin_date = datetime.strptime(bin_date_str, "%d %B")
-                                except ValueError:
-                                    pass
-
-                            if bin_date:
-                                # Set the year based on the logic provided
-                                if bin_date.month < current_date.month:
-                                    bin_date = bin_date.replace(
-                                        year=current_date.year + 1
-                                    )
-                                else:
-                                    bin_date = bin_date.replace(year=current_date.year)
-                                # Format the date to the desired format
-                                bin_date = bin_date.strftime("%d/%m/%Y")
-                                break
+                bin_type = bin_item.find(class_="mx-name-text31").text.strip()
+
+                bin_date_str = bin_item.find(class_="mx-name-text29").text.strip()
+
+                bin_date = datetime.strptime(bin_date_str, "%d %B %Y")
+                bin_date = bin_date.strftime(date_format)
+
                 dict_data = {"type": bin_type, "collectionDate": bin_date}
                 data["bins"].append(dict_data)
-                print(data)
         except Exception as e:
             # Here you can log the exception if needed
             print(f"An error occurred: {e}")

uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py

@@ -26,7 +26,9 @@ class CouncilClass(AbstractGetBinDataClass):
         uprn = kwargs.get("uprn")
         check_uprn(uprn)
 
-        post_url = "https://apps.castlepoint.gov.uk/cpapps/index.cfm?fa=myStreet.displayDetails"
+        base_url = "https://apps.castlepoint.gov.uk/cpapps/"
+
+        post_url = f"{base_url}index.cfm?fa=myStreet.displayDetails"
         post_header_str = (
             "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,"
             "image/apng,"
@@ -51,31 +53,60 @@ class CouncilClass(AbstractGetBinDataClass):
         soup = BeautifulSoup(post_response.text, features="html.parser")
         soup.prettify()
 
+        calMonthNext = f"{base_url}{soup.select_one("div.calMonthNext a")["href"]}"
+        nextmonth_response = requests.post(
+            calMonthNext, headers=post_headers, data=form_data, verify=False
+        )
+        soup_nextmonth = BeautifulSoup(nextmonth_response.text, features="html.parser")
+        soup_nextmonth.prettify()
+
         data = {"bins": []}
-        collection_tuple = []
 
-        calendar = soup.find("table", class_="calendar")
-        month = datetime.strptime(
-            soup.find("div", class_="calMonthCurrent").get_text(), "[%b]"
-        ).strftime("%m")
-        year = datetime.strptime(
-            soup.find("h1").get_text(), "About my Street - %B %Y"
-        ).strftime("%Y")
-
-        pink_days = [
-            day.get_text().strip() for day in calendar.find_all("td", class_="pink")
-        ]
-        black_days = [
-            day.get_text().strip() for day in calendar.find_all("td", class_="normal")
-        ]
-
-        for day in pink_days:
-            collection_date = datetime(year=int(year), month=int(month), day=int(day))
-            collection_tuple.append(("Pink collection", collection_date))
-
-        for day in black_days:
-            collection_date = datetime(year=int(year), month=int(month), day=int(day))
-            collection_tuple.append(("Normal collection", collection_date))
+        def parse_calendar_month(soup_one_month):
+            out = []
+
+            calendar = soup_one_month.find("table", class_="calendar")
+            if not calendar:
+                return out  # be robust
+
+            # e.g. "[Aug]"
+            month_txt = soup_one_month.find("div", class_="calMonthCurrent").get_text(
+                strip=True
+            )
+            month = datetime.strptime(month_txt, "[%b]").strftime("%m")
+
+            # e.g. "About my Street - August 2025"
+            year_txt = soup_one_month.find("h1").get_text(strip=True)
+            year = datetime.strptime(year_txt, "About my Street - %B %Y").strftime("%Y")
+
+            pink_days = [
+                td.get_text(strip=True) for td in calendar.find_all("td", class_="pink")
+            ]
+            black_days = [
+                td.get_text(strip=True)
+                for td in calendar.find_all("td", class_="normal")
+            ]
+
+            for day in pink_days:
+                out.append(
+                    (
+                        "Pink collection",
+                        datetime(year=int(year), month=int(month), day=int(day)),
+                    )
+                )
+            for day in black_days:
+                out.append(
+                    (
+                        "Normal collection",
+                        datetime(year=int(year), month=int(month), day=int(day)),
+                    )
+                )
+
+            return out
+
+        collection_tuple = []
+        for s in (soup, soup_nextmonth):
+            collection_tuple.extend(parse_calendar_month(s))
 
         ordered_data = sorted(collection_tuple, key=lambda x: x[1])
 

uk_bin_collection/uk_bin_collection/councils/ErewashBoroughCouncil.py

@@ -1,4 +1,7 @@
+import json
+
 from bs4 import BeautifulSoup
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
@@ -16,46 +19,41 @@ class CouncilClass(AbstractGetBinDataClass):
         uprn = kwargs.get("uprn")
         check_uprn(uprn)
 
+        label_map = {
+            "domestic-waste-collection-service": "Household Waste",
+            "recycling-collection-service": "Recycling",
+            "garden-waste-collection-service": "Garden Waste",
+        }
+
         requests.packages.urllib3.disable_warnings()
         response = requests.get(
-            f"https://map.erewash.gov.uk/isharelive.web/myerewash.aspx?action=SetAddress&UniqueId={uprn}",
+            f"https://www.erewash.gov.uk/bbd-whitespace/one-year-collection-dates-without-christmas?uprn={uprn}",
             headers={"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64)"},
         )
+        # Parse the JSON response
+        payload = response.json()
+        bin_collection = json.loads(payload) if isinstance(payload, str) else payload
 
-        soup = BeautifulSoup(response.text, features="html.parser")
-        collections = soup.find("div", {"aria-label": "Waste Collection"}).find_all(
-            "div", {"class": "atPanelContent"}
+        cd = next(
+            i["settings"]["collection_dates"]
+            for i in bin_collection
+            if i.get("command") == "settings"
         )
-        for c in collections:
-            bin_type = c.find("h4").get_text(strip=True)
-            if "my next" in bin_type.lower():
-                collection_info = c.find("div", {"class": "atPanelData"}).get_text(
-                    strip=True
-                )
-                results = re.search(
-                    "([A-Za-z]+ \\d+[A-Za-z]+ [A-Za-z]+ \\d*)", collection_info
+
+        for month in cd.values():
+            for e in month:
+                d = e["date"]  # "YYYY-MM-DD"
+                label = label_map.get(
+                    e.get("service-identifier"),
+                    e.get("service") or e.get("service-identifier"),
                 )
-                if results:
-                    collection_date = datetime.strptime(
-                        remove_ordinal_indicator_from_date_string(results[1]).strip(),
-                        "%A %d %B %Y",
-                    ).strftime(date_format)
-                    dict_data = {
-                        "type": bin_type.replace("My Next ", "").replace(
-                            " Collection", ""
-                        ),
-                        "collectionDate": collection_date,
-                    }
-                    data["bins"].append(dict_data)
-                    if "garden waste" in collection_info.lower():
-                        dict_data = {
-                            "type": "Garden Waste",
-                            "collectionDate": collection_date,
-                        }
-                        data["bins"].append(dict_data)
-
-        data["bins"].sort(
-            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
-        )
+
+                dict_data = {
+                    "type": label,
+                    "collectionDate": datetime.strptime(d, "%Y-%m-%d").strftime(
+                        date_format
+                    ),
+                }
+                data["bins"].append(dict_data)
 
         return data

uk_bin_collection/uk_bin_collection/councils/FarehamBoroughCouncil.py

@@ -38,11 +38,14 @@ class CouncilClass(AbstractGetBinDataClass):
         if "rows" in bin_data:
             collection_str = bin_data["rows"][0]["DomesticBinDay"]
 
-            results = re.findall(r"(\d\d?\/\d\d?\/\d{4}) \((\w*)\)", collection_str)
+            results = re.findall(r'(\d{1,2}/\d{1,2}/\d{4}|today)\s*\(([^)]+)\)', collection_str)
 
             if results:
                 for result in results:
-                    collection_date = datetime.strptime(result[0], "%d/%m/%Y")
+                    if (result[0] == "today"):
+                        collection_date = datetime.today()
+                    else:
+                        collection_date = datetime.strptime(result[0], "%d/%m/%Y")
                     dict_data = {
                         "type": result[1],
                         "collectionDate": collection_date.strftime(date_format),

uk_bin_collection/uk_bin_collection/councils/FolkstoneandHytheDistrictCouncil.py

@@ -74,6 +74,28 @@ class CouncilClass(AbstractGetBinDataClass):
                     }
                     bindata["bins"].append(dict_data)
 
+        # Extract the Garden Waste schedule
+        garden_waste_section = soup.find(
+            "span", text=lambda x: x and "Garden Waste" in x
+        )
+        if garden_waste_section:
+            bin_types = garden_waste_section.text.replace("Garden Waste: ", "").split(
+                " / "
+            )
+            garden_waste_dates = garden_waste_section.find_next("ul").find_all("li")
+            for date in garden_waste_dates:
+                for bin_type in bin_types:
+                    dict_data = {
+                        "type": bin_type.strip(),
+                        "collectionDate": datetime.strptime(
+                            remove_ordinal_indicator_from_date_string(
+                                date.text.strip()
+                            ),
+                            "%A %d %B %Y",
+                        ).strftime("%d/%m/%Y"),
+                    }
+                    bindata["bins"].append(dict_data)
+
         bindata["bins"].sort(
             key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
         )

uk_bin_collection/uk_bin_collection/councils/GlasgowCityCouncil.py

@@ -18,7 +18,7 @@ class CouncilClass(AbstractGetBinDataClass):
         try:
             user_uprn = kwargs.get("uprn")
             check_uprn(user_uprn)
-            url = f"https://onlineservices.glasgow.gov.uk/forms/RefuseAndRecyclingWebApplication/CollectionsCalendar.aspx?UPRN={user_uprn}"
+            url = f"https://onlineservices.glasgow.gov.uk/forms/refuseandrecyclingcalendar/CollectionsCalendar.aspx?UPRN={user_uprn}"
             if not user_uprn:
                 # This is a fallback for if the user stored a URL in old system. Ensures backwards compatibility.
                 url = kwargs.get("url")

uk_bin_collection/uk_bin_collection/councils/HartlepoolBoroughCouncil.py

@@ -73,7 +73,9 @@ class CouncilClass(AbstractGetBinDataClass):
         for div in soup.find_all("div"):
             # Extract bin type and date from the span tag
             text = div.find("span").text.strip()
-            bin_type, date = text.split(" ", 1)
+            parts = text.split(" ")
+            date = parts[-1]  # assume the last token is the date
+            bin_type = " ".join(parts[:-1])
             dict_data = {
                 "type": bin_type,
                 "collectionDate": date,