uk_bin_collection 0.134.3__py3-none-any.whl → 0.135.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/CherwellDistrictCouncil.py

@@ -0,0 +1,75 @@
+from datetime import datetime, timedelta
+
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        URI = f"https://www.cherwell.gov.uk/homepage/129/bin-collection-search?uprn={user_uprn}"
+
+        # Make the GET request
+        response = requests.get(URI)
+
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        def get_full_date(date_str):
+            # Get the current year
+            current_year = datetime.today().year
+
+            date_str = remove_ordinal_indicator_from_date_string(date_str)
+
+            # Convert the input string to a datetime object (assuming the current year first)
+            date_obj = datetime.strptime(f"{date_str} {current_year}", "%d %B %Y")
+
+            # If the date has already passed this year, use next year
+            if date_obj < datetime.today():
+                date_obj = datetime.strptime(
+                    f"{date_str} {current_year + 1}", "%d %B %Y"
+                )
+
+            return date_obj.strftime(date_format)  # Return in YYYY-MM-DD format
+
+        # print(soup)
+
+        div = soup.find("div", class_="bin-collection-results__tasks")
+
+        for item in div.find_all("li", class_="list__item"):
+            # Extract bin type
+            bin_type_tag = item.find("h3", class_="bin-collection-tasks__heading")
+            bin_type = (
+                "".join(bin_type_tag.find_all(text=True, recursive=False)).strip()
+                if bin_type_tag
+                else "Unknown Bin"
+            )
+
+            # Extract collection date
+            date_tag = item.find("p", class_="bin-collection-tasks__date")
+            collection_date = date_tag.text.strip() if date_tag else "Unknown Date"
+
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": get_full_date(collection_date),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/ConwyCountyBorough.py

@@ -1,12 +1,20 @@
+from datetime import datetime
+
+import requests
 from bs4 import BeautifulSoup
-from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
 from uk_bin_collection.uk_bin_collection.common import *
-from datetime import datetime
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
 class CouncilClass(AbstractGetBinDataClass):
     def parse_data(self, page: str, **kwargs) -> dict:
-        soup = BeautifulSoup(page.text, features="html.parser")
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        uri = f"https://www.conwy.gov.uk/Contensis-Forms/erf/collection-result-soap-xmas2025.asp?ilangid=1&uprn={user_uprn}"
+
+        response = requests.get(uri)
+        soup = BeautifulSoup(response.content, features="html.parser")
         data = {"bins": []}
 
         for bin_section in soup.select('div[class*="containererf"]'):

uk_bin_collection/uk_bin_collection/councils/CotswoldDistrictCouncil.py

@@ -41,15 +41,13 @@ class CouncilClass(AbstractGetBinDataClass):
             # If you bang in the house number (or property name) and postcode in the box it should find your property
             wait = WebDriverWait(driver, 60)
             address_entry_field = wait.until(
-                EC.presence_of_element_located(
-                    (By.XPATH, '//*[@id="combobox-input-20"]')
-                )
+                EC.element_to_be_clickable((By.XPATH, '//*[@id="combobox-input-22"]'))
             )
 
             address_entry_field.send_keys(str(full_address))
 
             address_entry_field = wait.until(
-                EC.element_to_be_clickable((By.XPATH, '//*[@id="combobox-input-20"]'))
+                EC.element_to_be_clickable((By.XPATH, '//*[@id="combobox-input-22"]'))
             )
             address_entry_field.click()
             address_entry_field.send_keys(Keys.BACKSPACE)
@@ -57,7 +55,7 @@ class CouncilClass(AbstractGetBinDataClass):
 
             first_found_address = wait.until(
                 EC.element_to_be_clickable(
-                    (By.XPATH, '//*[@id="dropdown-element-20"]/ul')
+                    (By.XPATH, '//*[@id="dropdown-element-22"]/ul')
                 )
             )
 

uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py

@@ -1,8 +1,5 @@
+import requests
 from bs4 import BeautifulSoup
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.support.ui import Select
-from selenium.webdriver.support.wait import WebDriverWait
 
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
@@ -19,63 +16,70 @@ class CouncilClass(AbstractGetBinDataClass):
     def parse_data(self, page: str, **kwargs) -> dict:
         driver = None
         try:
-            page = "https://selfserve.derbyshiredales.gov.uk/renderform.aspx?t=103&k=9644C066D2168A4C21BCDA351DA2642526359DFF"
+            uri = "https://selfserve.derbyshiredales.gov.uk/renderform.aspx?t=103&k=9644C066D2168A4C21BCDA351DA2642526359DFF"
 
-            data = {"bins": []}
+            bindata = {"bins": []}
 
             user_uprn = kwargs.get("uprn")
             user_postcode = kwargs.get("postcode")
-            web_driver = kwargs.get("web_driver")
-            headless = kwargs.get("headless")
             check_uprn(user_uprn)
             check_postcode(user_postcode)
 
-            # Create Selenium webdriver
-            driver = create_webdriver(web_driver, headless, None, __name__)
-            driver.get(page)
+            # Start a session
+            session = requests.Session()
 
-            # Populate postcode field
-            inputElement_postcode = driver.find_element(
-                By.ID,
-                "ctl00_ContentPlaceHolder1_FF2924TB",
-            )
-            inputElement_postcode.send_keys(user_postcode)
-
-            # Click search button
-            driver.find_element(
-                By.ID,
-                "ctl00_ContentPlaceHolder1_FF2924BTN",
-            ).click()
-
-            # Wait for the 'Select address' dropdown to appear and select option matching UPRN
-            dropdown = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located(
-                    (By.ID, "ctl00_ContentPlaceHolder1_FF2924DDL")
-                )
-            )
-            # Create a 'Select' for it, then select the matching URPN option
-            dropdownSelect = Select(dropdown)
-            dropdownSelect.select_by_value("U" + user_uprn)
-
-            # Wait for the submit button to appear, then click it to get the collection dates
-            submit = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located(
-                    (By.ID, "ctl00_ContentPlaceHolder1_btnSubmit")
-                )
-            )
-            submit.click()
+            response = session.get(uri)
 
-            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            soup = BeautifulSoup(response.content, features="html.parser")
 
-            bin_rows = (
-                soup.find("div", id="ctl00_ContentPlaceHolder1_pnlConfirmation")
-                .find("div", {"class": "row"})
-                .find_all("div", {"class": "row"})
-            )
+            # Function to extract hidden input values
+            def get_hidden_value(soup, name):
+                element = soup.find("input", {"name": name})
+                return element["value"] if element else None
+
+            # Extract the required values
+            data = {
+                "__RequestVerificationToken": get_hidden_value(
+                    soup, "__RequestVerificationToken"
+                ),
+                "FormGuid": get_hidden_value(soup, "FormGuid"),
+                "ObjectTemplateID": get_hidden_value(soup, "ObjectTemplateID"),
+                "Trigger": "submit",
+                "CurrentSectionID": get_hidden_value(soup, "CurrentSectionID"),
+                "TriggerCtl": "",
+                "FF2924": "U" + user_uprn,
+                "FF2924lbltxt": "Collection address",
+                "FF2924-text": user_postcode,
+            }
+
+            # Print extracted data
+            # print("Extracted Data:", data)
+
+            # Step 2: Submit the extracted data via a POST request
+            headers = {
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+                "Referer": uri,
+                "Content-Type": "application/x-www-form-urlencoded",
+            }
+
+            URI = "https://selfserve.derbyshiredales.gov.uk/renderform/Form"
+
+            # Make the POST request
+            post_response = session.post(URI, data=data, headers=headers)
+
+            soup = BeautifulSoup(post_response.content, features="html.parser")
+
+            # print(soup)
+
+            bin_rows = soup.find("div", {"class": "ss_confPanel"})
+
+            bin_rows = bin_rows.find_all("div", {"class": "row"})
             if bin_rows:
                 for bin_row in bin_rows:
                     bin_data = bin_row.find_all("div")
                     if bin_data and bin_data[0] and bin_data[1]:
+                        if bin_data[0].get_text(strip=True) == "Your Collections":
+                            continue
                         collection_date = datetime.strptime(
                             bin_data[0].get_text(strip=True), "%A%d %B, %Y"
                         )
@@ -83,9 +87,9 @@ class CouncilClass(AbstractGetBinDataClass):
                             "type": bin_data[1].get_text(strip=True),
                             "collectionDate": collection_date.strftime(date_format),
                         }
-                        data["bins"].append(dict_data)
+                        bindata["bins"].append(dict_data)
 
-            data["bins"].sort(
+            bindata["bins"].sort(
                 key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
             )
         except Exception as e:
@@ -97,4 +101,4 @@ class CouncilClass(AbstractGetBinDataClass):
             # This block ensures that the driver is closed regardless of an exception
             if driver:
                 driver.quit()
-        return data
+        return bindata

uk_bin_collection/uk_bin_collection/councils/EpsomandEwellBoroughCouncil.py

@@ -0,0 +1,86 @@
+from datetime import datetime, timedelta
+
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        URI = f"https://maps.epsom-ewell.gov.uk/myeebc.aspx?action=SetAddress&UniqueId={user_uprn}"
+
+        # Make the GET request
+        response = requests.get(URI)
+
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        # print(soup)
+
+        div = soup.find_all("div", class_="atPanelContent atAlt1 atLast")
+
+        # print(div[1])
+
+        panels = div[1].find_all("div", class_="atPanelData")
+
+        # print(panels)
+
+        def get_full_date(date_str):
+            # Get the current year
+            current_year = datetime.today().year
+
+            # Convert the input string to a datetime object (assuming the current year first)
+            date_obj = datetime.strptime(f"{date_str} {current_year}", "%A %d %B %Y")
+
+            # If the date has already passed this year, use next year
+            if date_obj < datetime.today():
+                date_obj = datetime.strptime(
+                    f"{date_str} {current_year + 1}", "%A %d %B %Y"
+                )
+
+            return date_obj.strftime(date_format)  # Return in YYYY-MM-DD format
+
+        for panel in panels:
+            bin_type_tag = panel.find("h4")  # Extracts bin type
+            date_text = panel.find_all("td")  # Extracts collection date
+
+            date_text = date_text[1]
+
+            if bin_type_tag and date_text:
+                bin_type = bin_type_tag.text.strip()
+                try:
+                    collection_date = date_text.text.strip().split(":")[1]
+                except IndexError:
+                    continue
+
+                bin_type = (
+                    (" ".join(bin_type.splitlines())).replace("  ", " ")
+                ).lstrip()
+                collection_date = (
+                    (" ".join(collection_date.splitlines())).replace("  ", " ")
+                ).lstrip()
+
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": get_full_date(collection_date),
+                }
+                bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/GloucesterCityCouncil.py

@@ -113,7 +113,7 @@ class CouncilClass(AbstractGetBinDataClass):
                 bin_data["bins"].append(dict_data)
 
             bin_data["bins"].sort(
-                key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+                key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
             )
 
         except Exception as e:

uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py

@@ -104,7 +104,8 @@ class CouncilClass(AbstractGetBinDataClass):
             bin_types = soup.find_all("ul", class_="binCollectionTimesList")
 
             for bin_collection_dates in bin_types:
-                bin_collection_list = bin_collection_dates.find_all("li", class_="")
+
+                bin_collection_list = bin_collection_dates.find_all("li")
 
                 if bin_collection_list:
                     collection_dates = [

uk_bin_collection/uk_bin_collection/councils/MiddlesbroughCouncil.py

@@ -0,0 +1,100 @@
+import time
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select, WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+import re
+
+class CouncilClass(AbstractGetBinDataClass):
+    def parse_data(self, page: str, **kwargs) -> dict:
+        try:
+            data = {"bins": []}
+
+            user_paon = kwargs.get("paon")
+            headless = kwargs.get("headless")
+            web_driver = kwargs.get("web_driver")
+            driver = create_webdriver(web_driver, headless, None, __name__)
+
+            page = "https://www.middlesbrough.gov.uk/recycling-and-rubbish/bin-collection-dates/"
+            driver.get(page)
+
+            address_box = WebDriverWait(driver, timeout=15).until(
+                EC.presence_of_element_located((By.ID, "row-input-0"))
+            )
+            address_box.click()
+            address_box.send_keys(user_paon)
+
+            search_button = WebDriverWait(driver, timeout=15).until(
+                EC.presence_of_element_located((By.ID, "rCbtn-search"))
+            )
+            search_button.click()
+
+            iframe_presense = WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located((By.ID, "recollect-frame"))
+            )
+            driver.switch_to.frame(iframe_presense)
+
+            results = WebDriverWait(driver, timeout=15).until(
+                EC.presence_of_element_located((By.ID, "rCpage-place_calendar"))
+            )
+
+            html_content = driver.page_source
+            soup = BeautifulSoup(html_content, "html.parser")
+
+            calendar_section = soup.find("section", {"id": "alt-calendar-list"})
+            if not calendar_section:
+                raise ValueError("Calendar section not found in the HTML.")
+
+            date_headers = calendar_section.find_all("h3")
+            collection_lists = calendar_section.find_all("ul")
+
+            current_month = datetime.now().month
+            current_year = datetime.now().year
+
+            for date_header, collection_list in zip(date_headers, collection_lists):
+                raw_date = date_header.text.strip()
+
+                # **Regex to match "Wednesday, February 19" format**
+                match = re.match(r"([A-Za-z]+), ([A-Za-z]+) (\d{1,2})", raw_date)
+                
+                if match:
+                    day_name, month_name, day_number = match.groups()  # Extract components
+                    extracted_month = datetime.strptime(month_name, "%B").month
+                    extracted_day = int(day_number)
+
+                    # Handle Dec-Jan rollover: If month is before the current month, assume next year
+                    inferred_year = current_year + 1 if extracted_month < current_month else current_year
+
+                    # **Correct the raw_date format before parsing**
+                    raw_date = f"{day_name}, {month_name} {day_number}, {inferred_year}"
+
+                print(f"DEBUG: Final raw_date before parsing -> {raw_date}")  # Debugging output
+
+                # Convert to required format (%d/%m/%Y)
+                try:
+                    parsed_date = datetime.strptime(raw_date, "%A, %B %d, %Y")
+                    formatted_date = parsed_date.strftime(date_format)
+                except ValueError:
+                    raise ValueError(f"Date format error after inference: {raw_date}")
+
+                for li in collection_list.find_all("li"):
+                    bin_type = li.get_text(strip=True).split(".")[0]
+                    data["bins"].append(
+                        {"type": bin_type, "collectionDate": formatted_date}
+                    )
+
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            raise
+        finally:
+            if driver:
+                driver.quit()
+
+        return data

uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py

@@ -100,6 +100,8 @@ class CouncilClass(AbstractGetBinDataClass):
                 {"id": "contentInner"},
             )
 
+            soup = soup.find("div", class_="umb-block-grid__layout-item")
+
             # Get the dates
             for date in soup.find_all("h2"):
                 if date.get_text(strip=True) != "Bank Holidays":

uk_bin_collection/uk_bin_collection/councils/RedcarandClevelandCouncil.py

@@ -0,0 +1,108 @@
+import time
+
+import requests
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_postcode = kwargs.get("postcode")
+        user_paon = kwargs.get("paon")
+        check_postcode(user_postcode)
+        check_paon(user_paon)
+        bindata = {"bins": []}
+
+        URI = "https://api.eu.recollect.net/api/areas/RedcarandClevelandUK/services/50006/address-suggest"
+
+        params = {
+            "q": user_postcode,
+            "locale": "en-GB",
+            "_": str(int(time.time() * 1000)),
+        }
+
+        # print(params)
+
+        # Send GET request
+        response = requests.get(URI, params=params)
+
+        addresses = response.json()
+
+        place_id = next(
+            (
+                item["place_id"]
+                for item in addresses
+                if item.get("name", "").startswith(user_paon)
+            ),
+            None,
+        )
+
+        # print(addresses)
+        # print(place_id)
+
+        URI = (
+            f"https://api.eu.recollect.net/api/places/{place_id}/services/50006/events"
+        )
+
+        after = datetime.today()
+        before = after + timedelta(days=30)
+
+        after = after.strftime("%Y-%m-%d")
+        before = before.strftime("%Y-%m-%d")
+
+        # print(after)
+        # print(before)
+
+        params = {
+            "nomerge": 1,
+            "hide": "reminder_only",
+            "after": after,
+            "before": before,
+            "locale": "en-GB",
+            "include_message": "email",
+            "_": str(int(time.time() * 1000)),
+        }
+
+        # print(params)
+
+        # Send GET request
+        response = requests.get(URI, params=params)
+
+        response = response.json()
+
+        bin_collection = response["events"]
+
+        # print(bin_collection)
+
+        # Extract "end_day" and "name"
+        events = [
+            (event["end_day"], flag["name"])
+            for event in bin_collection
+            for flag in event.get("flags", [])
+        ]
+
+        # Print results
+        for end_day, bin_type in events:
+
+            date = datetime.strptime(end_day, "%Y-%m-%d")
+
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": date.strftime(date_format),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/RunnymedeBoroughCouncil.py

@@ -0,0 +1,54 @@
+from datetime import datetime, timedelta
+
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        URI = f"https://www.runnymede.gov.uk/homepage/150/check-your-bin-collection-day?address={user_uprn}"
+
+        # Make the GET request
+        response = requests.get(URI)
+
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        div = soup.find("div", class_="widget-bin-collection")
+
+        table = div.find("table")
+
+        tbody = table.find("tbody")
+
+        for tr in tbody.find_all("tr"):
+            tds = tr.find_all("td")
+            bin_type = tds[0].text.strip()
+            date_text = tds[1].text.strip()
+
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": (
+                    datetime.strptime(date_text, "%A, %d %B %Y")
+                ).strftime(date_format),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        return bindata