uk_bin_collection 0.78.0__py3-none-any.whl → 0.79.1__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/HounslowCouncil.py

@@ -0,0 +1,122 @@
+import time
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.keys import Keys
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_date(self, date_str):
+        date_formats = [
+            "This %A - %d %b %Y",  # Changed %B to %b to accommodate abbreviated month names
+            "Next %A - %d %b %Y",  # Same change here
+            "%A %d %b %Y",  # And here
+        ]
+        for format in date_formats:
+            try:
+                return datetime.strptime(date_str, format).strftime("%d/%m/%Y")
+            except ValueError:
+                continue
+        raise ValueError(f"Date format not recognized: {date_str}")
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            # Make a BS4 object
+
+            page = "https://www.hounslow.gov.uk/info/20272/recycling_and_waste_collection_day_finder"
+
+            user_postcode = kwargs.get("postcode")
+            user_uprn = kwargs.get("uprn")
+            user_paon = kwargs.get("paon")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(page)
+
+            wait = WebDriverWait(driver, 60)
+
+            inputElement_postcodesearch = wait.until(
+                EC.element_to_be_clickable((By.ID, "Postcode"))
+            )
+
+            inputElement_postcodesearch.send_keys(user_postcode)
+
+            inputElement_postcodesearch_btn = wait.until(
+                EC.element_to_be_clickable((By.ID, "findAddress"))
+            )
+            inputElement_postcodesearch_btn.click()
+
+            inputElement_select_address = wait.until(
+                EC.element_to_be_clickable((By.ID, "UPRN"))
+            )
+
+            select_element = wait.until(
+                EC.visibility_of_element_located((By.ID, "UPRN"))
+            )  # Adjust this ID to your element's ID
+
+            # Create a Select object
+            select = Select(select_element)
+
+            # Fetch all options
+            options = select.options
+
+            # Loop through options to find the one that starts with the UPRN
+            for option in options:
+                if option.get_attribute("value").startswith(f"{user_uprn}|"):
+                    option.click()  # Select the matching option
+                    break
+
+            results = wait.until(
+                EC.element_to_be_clickable((By.CLASS_NAME, "bin_day_main_wrapper"))
+            )
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            soup.prettify()
+
+            # Find all headers which include collection dates
+            collection_headers = soup.find_all("h4")
+            bins_data = []
+
+            # Process each collection date and corresponding bins
+            for header in collection_headers:
+                date_text = header.get_text(strip=True)
+                collection_date = self.parse_date(date_text)
+
+                # Get next sibling which should be the list of bins
+                bin_list = header.find_next_sibling("ul")
+                if bin_list:
+                    for item in bin_list.find_all("li", class_="list-group-item"):
+                        bin_type = item.get_text(strip=True)
+                        bins_data.append(
+                            {"type": bin_type, "collectionDate": collection_date}
+                        )
+
+            # Construct the final JSON object
+            json_data = {"bins": bins_data}
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return json_data

uk_bin_collection/uk_bin_collection/councils/KirkleesCouncil.py

@@ -61,7 +61,9 @@ class CouncilClass(AbstractGetBinDataClass):
         user_postcode = kwargs["postcode"]
 
         self._driver = driver = create_webdriver(
-            web_driver=kwargs["web_driver"], headless=kwargs.get("headless", True), session_name=__name__
+            web_driver=kwargs["web_driver"],
+            headless=kwargs.get("headless", True),
+            session_name=__name__,
         )
         driver.implicitly_wait(1)
 

uk_bin_collection/uk_bin_collection/councils/MoleValleyDistrictCouncil.py

@@ -1,7 +1,9 @@
-from bs4 import BeautifulSoup
-from datetime import datetime
 import re
+from datetime import datetime
+
 import requests
+from bs4 import BeautifulSoup
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
@@ -19,35 +21,36 @@ class CouncilClass(AbstractGetBinDataClass):
         user_postcode = kwargs.get("postcode")
         check_postcode(user_postcode)
 
-        root_url = "https://molevalley.cloudmappin.com/my-mv-address-search/search/{}/0".format(
+        root_url = "https://myproperty.molevalley.gov.uk/molevalley/api/live_addresses/{}?format=json".format(
             user_postcode
         )
-        response = requests.get(root_url)
+        requests.packages.urllib3.disable_warnings()
+        response = requests.get(root_url, verify=False)
 
         if not response.ok:
             raise ValueError("Invalid server response code retreiving data.")
 
         jsonData = response.json()
 
-        if len(jsonData["results"]) == 0:
+        if len(jsonData["results"]["features"]) == 0:
             raise ValueError("No collection data found for postcode provided.")
 
-        properties_found = jsonData["results"][0]["items"]
+        properties_found = jsonData["results"]["features"]
 
         # If UPRN is provided, we can check a specific address.
         html_data = None
         uprn = kwargs.get("uprn")
         if uprn:
             check_uprn(uprn)
-            for n, item in enumerate(properties_found):
-                if uprn == str(int(item["info"][0][1]["value"])):
-                    html_data = properties_found[n]["info"][2][1]["value"]
+            for item in properties_found:
+                if uprn == str(int(item["properties"]["blpu_uprn"])):
+                    html_data = item["properties"]["three_column_layout_html"]
                     break
             if html_data is None:
                 raise ValueError("No collection data found for UPRN provided.")
         else:
             # If UPRN not provided, just use the first result
-            html_data = properties_found[0]["info"][2][1]["value"]
+            html_data = properties_found[0]["properties"]["three_column_layout_html"]
 
         soup = BeautifulSoup(html_data, features="html.parser")
         soup.prettify()

uk_bin_collection/uk_bin_collection/councils/SouthKestevenDistrictCouncil.py

@@ -0,0 +1,151 @@
+import time
+from datetime import datetime
+
+from selenium.webdriver.support.ui import Select
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.keys import Keys
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    # Extract data from the table
+    def format_date(self, date_str):
+        # Convert date format from "Fri 31 May 2024" to "31/05/2024"
+        date_match = re.search(r"\d{1,2} \w+ \d{4}", date_str)
+        if date_match:
+            date_obj = re.search(r"(\d{1,2}) (\w+) (\d{4})", date_match.group(0))
+            day = date_obj.group(1).zfill(2)
+            month_name = date_obj.group(2)
+            month = {
+                "January": "01",
+                "February": "02",
+                "March": "03",
+                "April": "04",
+                "May": "05",
+                "June": "06",
+                "July": "07",
+                "August": "08",
+                "September": "09",
+                "October": "10",
+                "November": "11",
+                "December": "12",
+            }[month_name]
+            year = date_obj.group(3)
+            formatted_date = f"{day}/{month}/{year}"
+        else:
+            formatted_date = "Unknown Date"
+        return formatted_date
+
+    def extract_bin_data(self, article):
+        date = article.find("div", class_="binday__cell--day").text.strip()
+        bin_type_class = article.get("class")[
+            1
+        ]  # Assuming the second class indicates the bin type
+        bin_type = "black" if "black" in bin_type_class else "silver"
+        formatted_date = self.format_date(date)
+        return {"type": bin_type, "collectionDate": formatted_date}
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            # Make a BS4 object
+
+            page = "https://pre.southkesteven.gov.uk/BinSearch.aspx"
+
+            user_postcode = kwargs.get("postcode")
+            user_uprn = kwargs.get("uprn")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            house_number = kwargs.get("paon")
+
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(page)
+
+            wait = WebDriverWait(driver, 60)
+
+            inputElement_postcodesearch = wait.until(
+                EC.visibility_of_element_located((By.ID, "title"))
+            )
+            inputElement_postcodesearch.clear()
+
+            inputElement_postcodesearch.send_keys(user_postcode)
+
+            inputElement_postcodesearch_btn = wait.until(
+                EC.element_to_be_clickable((By.XPATH, "//button/span[text()='Search']"))
+            )
+            inputElement_postcodesearch_btn.click()
+
+            inputElement_select_address = wait.until(
+                EC.element_to_be_clickable((By.ID, "address"))
+            )
+
+            # Now create a Select object based on the found element
+            dropdown = Select(inputElement_select_address)
+
+            # Select the option by visible text
+            dropdown.select_by_visible_text(house_number)
+
+            inputElement_results_btn = wait.until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, "//button[text()='View your bin days']")
+                )
+            )
+            inputElement_results_btn.click()
+
+            p_element = wait.until(
+                EC.presence_of_element_located(
+                    (
+                        By.XPATH,
+                        "//p[contains(text(), 'Your next bin collection date is ')]",
+                    )
+                )
+            )
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            soup.prettify()
+
+            bin_data = []
+
+            # Extract data from the first aside element
+            first_aside = soup.find("aside", class_="alert")
+            if first_aside:
+                next_collection_date = first_aside.find(
+                    "span", class_="alert__heading alpha"
+                ).text.strip()
+                bin_info = {
+                    "type": "purple",  # Based on the provided information in the HTML, assuming it's a purple bin day.
+                    "collectionDate": self.format_date(next_collection_date),
+                }
+                bin_data.append(bin_info)
+
+            # Extract data from articles
+            articles = soup.find_all("article", class_="binday")
+            for article in articles:
+                bin_info = self.extract_bin_data(article)
+                bin_data.append(bin_info)
+
+            result = {"bins": bin_data}
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return result

uk_bin_collection/uk_bin_collection/councils/StroudDistrictCouncil.py

@@ -0,0 +1,94 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+from datetime import datetime, timedelta
+from typing import Dict, Any
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def get_next_weekday(self, day_name: str) -> str:
+        days_of_week = [
+            "Monday",
+            "Tuesday",
+            "Wednesday",
+            "Thursday",
+            "Friday",
+            "Saturday",
+            "Sunday",
+        ]
+        today = datetime.today()
+        target_day = days_of_week.index(day_name)
+        days_until_target = (target_day - today.weekday() + 7) % 7
+        if days_until_target == 0:
+            days_until_target = 7  # Next occurrence should be next week
+        next_weekday = today + timedelta(days=days_until_target)
+        return next_weekday.strftime("%d/%m/%Y")
+
+    def parse_data(self, page: Any, **kwargs: Any) -> Dict[str, Any]:
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+        # Find the section with the title "Bins, rubbish & recycling"
+        h2_header = soup.find("h2", id="rubbish-header")
+
+        # Mapping original titles to new titles
+        title_mapping = {
+            "Next rubbish collection date": "Rubbish",
+            "Next recycling collection date": "Recycling",
+            "Food waste collection": "Food Waste",
+            "Garden waste collection": "Garden Waste",
+        }
+
+        # Extract the list items following this section
+        if h2_header:
+            list_items = h2_header.find_next("ul", class_="list-group").find_all("li")
+
+            extracted_data = {}
+            for item in list_items:
+                header = item.find("h3")
+                if header:
+                    key = header.text.strip()
+                    date = item.find("p").strong.text.strip()
+                    extracted_data[key] = date
+                else:
+                    # Special handling for garden waste collection
+                    if "Garden waste collection" in item.text:
+                        key = "Garden waste collection"
+                        date = item.find_all("strong")[1].text.strip()
+                        extracted_data[key] = date
+
+            print("Extracted data:", extracted_data)
+
+            # Transform the data to the required schema
+            bin_data = {"bins": []}
+
+            for key, value in extracted_data.items():
+                if value.startswith("Every"):
+                    # Extract the day name
+                    day_name = value.split()[1]
+                    # Convert to the next occurrence of that day
+                    formatted_date = self.get_next_weekday(day_name)
+                else:
+                    # Convert date format from "Tuesday 28 May 2024" to "28/05/2024"
+                    date_obj = datetime.strptime(value, "%A %d %B %Y")
+                    formatted_date = date_obj.strftime("%d/%m/%Y")
+
+                bin_entry = {
+                    "type": title_mapping.get(key, key),
+                    "collectionDate": formatted_date,
+                }
+
+                bin_data["bins"].append(bin_entry)
+
+            return bin_data
+        else:
+            print("Section not found")
+            return data

uk_bin_collection/uk_bin_collection/councils/TendringDistrictCouncil.py

@@ -0,0 +1,110 @@
+import time
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+# import the wonderful Beautiful Soup and the URL grabber
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            page = "https://tendring-self.achieveservice.com/en/service/Rubbish_and_recycling_collection_days"
+
+            bin_data = {"bins": []}
+
+            user_uprn = kwargs.get("uprn")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_uprn(user_uprn)
+            check_postcode(user_postcode)
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(page)
+
+            cookies_button = WebDriverWait(driver, timeout=15).until(
+                EC.presence_of_element_located((By.ID, "close-cookie-message"))
+            )
+            cookies_button.click()
+
+            without_login_button = WebDriverWait(driver, timeout=15).until(
+                EC.presence_of_element_located(
+                    (By.LINK_TEXT, "or, continue without an account")
+                )
+            )
+            without_login_button.click()
+
+            iframe_presense = WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located((By.ID, "fillform-frame-1"))
+            )
+
+            driver.switch_to.frame(iframe_presense)
+            wait = WebDriverWait(driver, 60)
+            inputElement_postcodesearch = wait.until(
+                EC.element_to_be_clickable((By.NAME, "postcode_search"))
+            )
+
+            inputElement_postcodesearch.send_keys(user_postcode)
+
+            # Wait for the 'Select address' dropdown to be updated
+            time.sleep(1)
+
+            dropdown = wait.until(
+                EC.element_to_be_clickable((By.NAME, "selectAddress"))
+            )
+            # Create a 'Select' for it, then select the first address in the list
+            # (Index 0 is "Select...")
+            dropdownSelect = Select(dropdown)
+            dropdownSelect.select_by_value(str(user_uprn))
+
+            # Wait for 'wasteTable' to be shown
+            wait.until(EC.presence_of_element_located((By.CLASS_NAME, "wasteTable")))
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            bins = (
+                soup.find("table", {"class": "wasteTable"}).find("tbody").find_all("tr")
+            )
+            for bin_row in bins:
+                bin = bin_row.find_all("td")
+                if bin:
+                    if bin[1].get_text(strip=True) != "":
+                        bin_date = datetime.strptime(
+                            bin[1].get_text(strip=True), "%d/%m/%Y"
+                        )
+                        dict_data = {
+                            "type": re.sub(
+                                r"\([^)]*\)", "", bin[0].get_text(strip=True)
+                            ),
+                            "collectionDate": bin_date.strftime(date_format),
+                        }
+                        bin_data["bins"].append(dict_data)
+
+            bin_data["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+            )
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return bin_data

uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py

@@ -17,7 +17,9 @@ class CouncilClass(AbstractGetBinDataClass):
         try:
             # Create Selenium webdriver
             headless = kwargs.get("headless")
-            driver = create_webdriver(kwargs.get("web_driver"), headless, None, __name__)
+            driver = create_webdriver(
+                kwargs.get("web_driver"), headless, None, __name__
+            )
             driver.get(kwargs.get("url"))
 
             # Make a BS4 object

uk_bin_collection/uk_bin_collection/councils/WalthamForest.py

@@ -0,0 +1,127 @@
+import time
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.keys import Keys
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            page = "https://portal.walthamforest.gov.uk/AchieveForms/?mode=fill&consentMessage=yes&form_uri=sandbox-publish://AF-Process-d62ccdd2-3de9-48eb-a229-8e20cbdd6393/AF-Stage-8bf39bf9-5391-4c24-857f-0dc2025c67f4/definition.json&process=1&process_uri=sandbox-processes://AF-Process-d62ccdd2-3de9-48eb-a229-8e20cbdd6393&process_id=AF-Process-d62ccdd2-3de9-48eb-a229-8e20cbdd6393"
+
+            user_postcode = kwargs.get("postcode")
+            user_uprn = kwargs.get("uprn")
+            user_paon = kwargs.get("paon")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(page)
+
+            iframe_presense = WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located((By.ID, "fillform-frame-1"))
+            )
+
+            driver.switch_to.frame(iframe_presense)
+            wait = WebDriverWait(driver, 60)
+            inputElement_postcodesearch = wait.until(
+                EC.element_to_be_clickable((By.NAME, "postcode_search"))
+            )
+
+            inputElement_postcodesearch.send_keys(user_postcode)
+            find_address_button = wait.until(
+                EC.element_to_be_clickable((By.ID, "lookupPostcode"))
+            )
+
+            find_address_button.send_keys(Keys.RETURN)
+
+            dropdown = wait.until(
+                EC.element_to_be_clickable((By.CLASS_NAME, "select2-choice"))
+            )
+
+            time.sleep(1)
+            dropdown.click()
+
+            dropdown_search = wait.until(
+                EC.element_to_be_clickable((By.CLASS_NAME, "select2-input"))
+            )
+            dropdown_search.click()
+
+            dropdown_search.send_keys(user_paon)
+            dropdown_search.send_keys(Keys.RETURN)
+
+            find_ac_button = wait.until(
+                EC.element_to_be_clickable((By.ID, "confirmSearchUPRN"))
+            )
+
+            find_ac_button.send_keys(Keys.RETURN)
+            h4_element = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, "//h4[contains(text(), 'Your Collections')]")
+                )
+            )
+
+            data_table = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (
+                        By.XPATH,
+                        '//div[contains(@class, "fieldContent")]',
+                    )
+                )
+            )
+            # Make a BS4 object
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            data = {"bins": []}
+
+            collection_divs = soup.find_all("div", {"style": "text-align: center;"})
+
+            for collection_div in collection_divs:
+                h5_tag = collection_div.find("h5")
+                p_tag = collection_div.find("p")
+
+                if h5_tag and p_tag:
+                    bin_type = h5_tag.get_text(strip=True)
+                    collection_date_text = p_tag.find("b").get_text(strip=True)
+
+                    # Extract and format the date
+                    date_match = re.search(r"(\d+ \w+)", collection_date_text)
+                    if date_match:
+                        date_str = date_match.group(1)
+                        date_obj = datetime.strptime(
+                            date_str + " " + str(datetime.today().year), "%d %B %Y"
+                        )
+                        collection_date = get_next_occurrence_from_day_month(
+                            date_obj
+                        ).strftime(date_format)
+
+                        data["bins"].append(
+                            {"type": bin_type, "collectionDate": collection_date}
+                        )
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data