uk_bin_collection 0.74.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/HuntingdonDistrictCouncil.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+# This script pulls (in one hit) the data from
+# Huntingdon District Council District Council Bins Data
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+from uk_bin_collection.uk_bin_collection.common import date_format
+from datetime import datetime
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page, **kwargs) -> None:
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+
+        no_garden_message = "Your property does not receive a garden waste collection"
+        results = soup.find("ul", class_="d-print-none").find_all("li")
+
+        for result in results:
+            if no_garden_message in result.get_text(strip=True):
+                continue
+            else:
+                data["bins"].append(
+                    {
+                        "type": " ".join(
+                            result.get_text(strip=True).split(" ")[5:7]
+                        ).capitalize(),
+                        "collectionDate": datetime.strptime(
+                            result.find("strong").get_text(strip=True), "%A %d %B %Y"
+                        ).strftime(date_format),
+                    }
+                )
+
+        return data

uk_bin_collection/uk_bin_collection/councils/KingstonUponThamesCouncil.py

@@ -0,0 +1,84 @@
+# alternative implementation for retrieving bin data from Kingston Upon Thames Council
+# principal URL is https://waste-services.kingston.gov.uk/waste/[uprn]
+# https://www.kingston.gov.uk/info/200287/bins_and_recycling/1113/check_your_bin_collection_day
+
+# switched to using Selenium as the htmx elements are not rendered reliably with requests
+
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+import re
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        driver = None
+        try:
+
+            headless = kwargs.get("headless")
+            web_driver = kwargs.get("web_driver")
+            driver = create_webdriver(web_driver, headless)
+            driver.get(kwargs.get("url"))
+            wait = WebDriverWait(driver, 15, 2)
+
+            wait.until(
+                EC.presence_of_element_located((By.CLASS_NAME, "waste-service-name"))
+            )
+
+            data = {"bins": []}
+
+            soup = BeautifulSoup(driver.page_source, "html.parser")
+            collections = soup.find_all("h3", {"class": "waste-service-name"})
+            for c in collections:
+                rows = c.find_next_sibling("div", {"class": "govuk-grid-row"}).find_all(
+                    "div", {"class": "govuk-summary-list__row"}
+                )
+                for row in rows:
+                    if row.find("dt").get_text().strip().lower() == "next collection":
+                        collection_date = remove_ordinal_indicator_from_date_string(
+                            row.find("dd").get_text()
+                        ).strip()
+                        # strip out any text inside of the date string
+                        collection_date = re.sub(
+                            r"\n\s*\(this.*?\)", "", collection_date
+                        )
+                        dict_data = {
+                            "type": c.get_text().strip().capitalize(),
+                            "collectionDate": get_next_occurrence_from_day_month(
+                                datetime.strptime(
+                                    collection_date
+                                    + " "
+                                    + datetime.now().strftime("%Y"),
+                                    "%A, %d %B %Y",
+                                )
+                            ).strftime(date_format),
+                        }
+                        data["bins"].append(dict_data)
+
+            data["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+            )
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+
+        return data

uk_bin_collection/uk_bin_collection/councils/KirkleesCouncil.py

@@ -0,0 +1,130 @@
+from datetime import datetime
+from typing import Optional
+
+from selenium.common import TimeoutException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.remote.webdriver import WebDriver
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import create_webdriver
+from uk_bin_collection.uk_bin_collection.common import date_format
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+def wait_for_element(driver, element_type, element: str, timeout: int = 5):
+    element_present = EC.presence_of_element_located((element_type, element))
+    wait_for_element_conditions(driver, element_present, timeout=timeout)
+
+
+def wait_for_element_conditions(driver, conditions, timeout: int = 5):
+    try:
+        WebDriverWait(driver, timeout).until(conditions)
+    except TimeoutException:
+        print("Timed out waiting for page to load")
+        raise
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def __init__(self):
+        self._driver: Optional[WebDriver] = None
+
+    def parse_data(self, *args, **kwargs) -> dict:
+        try:
+            return self._parse_data(*args, **kwargs)
+        finally:
+            if self._driver:
+                self._driver.quit()
+
+    def _parse_data(self, page: str, **kwargs) -> dict:
+        """
+        Process:
+
+        - Use a house number and postcode that is known to be domestic and resolves to a
+          single unique address. When the address search form is submitted with
+          those details, a session is created
+
+        - Now a session exists, navigate to the calendar URL, specifying the UPRN
+
+        - Extract info from the 'alt' attribute of the images on that page
+        """
+        bins = []
+
+        user_paon = kwargs["paon"]
+        user_postcode = kwargs["postcode"]
+
+        self._driver = driver = create_webdriver(
+            web_driver=kwargs["web_driver"], headless=kwargs.get("headless", True)
+        )
+        driver.implicitly_wait(1)
+
+        driver.get(
+            "https://www.kirklees.gov.uk/beta/your-property-bins-recycling/your-bins/default.aspx"
+        )
+
+        wait_for_element(
+            driver, By.ID, "cphPageBody_cphContent_thisGeoSearch_txtGeoPremises"
+        )
+
+        house_input = driver.find_element(
+            By.ID, "cphPageBody_cphContent_thisGeoSearch_txtGeoPremises"
+        )
+        house_input.send_keys(user_paon)
+
+        postcode_input = driver.find_element(
+            By.ID, "cphPageBody_cphContent_thisGeoSearch_txtGeoSearch"
+        )
+        postcode_input.send_keys(user_postcode)
+
+        # submit address search
+        driver.find_element(By.ID, "butGeoSearch").send_keys(Keys.RETURN)
+
+        wait_for_element(
+            driver,
+            By.ID,
+            "cphPageBody_cphContent_wtcDomestic240__lnkAccordionAnchor",
+            # submitting can be slow
+            timeout=30,
+        )
+
+        # Open the panel
+        driver.find_element(
+            By.ID, "cphPageBody_cphContent_wtcDomestic240__lnkAccordionAnchor"
+        ).click()
+
+        # Domestic waste calendar
+        wait_for_element(
+            driver, By.ID, "cphPageBody_cphContent_wtcDomestic240__LnkCalendar"
+        )
+        calendar_link = driver.find_element(
+            By.ID, "cphPageBody_cphContent_wtcDomestic240__LnkCalendar"
+        )
+        driver.execute_script("arguments[0].click();", calendar_link)
+
+        # <img alt="Recycling                      collection date 14 March 2024"
+        # <img alt="Domestic                       collection date 21 March 2024
+        date_strings = driver.find_elements(
+            By.CSS_SELECTOR, 'img[alt*="collection date"]'
+        )
+
+        for date in date_strings:
+            bin_type, _, _, day, month, year = date.get_attribute("alt").split()
+            collection_date = datetime.strptime(
+                f"{day} {month} {year}", "%d %B %Y"
+            ).strftime(date_format)
+
+            bins.append(
+                {
+                    "type": bin_type,
+                    "collectionDate": collection_date,
+                }
+            )
+
+        return {"bins": bins}

uk_bin_collection/uk_bin_collection/councils/KnowsleyMBCouncil.py

@@ -0,0 +1,139 @@
+import time
+
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            data = {"bins": []}
+            collections = []
+            user_paon = kwargs.get("paon")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_paon(user_paon)
+            check_postcode(user_postcode)
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless)
+            driver.get(
+                "https://knowsleytransaction.mendixcloud.com/link/youarebeingredirected?target=bincollectioninformation"
+            )
+
+            # Wait for the postcode field to appear then populate it
+            inputElement_postcode = WebDriverWait(driver, 30).until(
+                EC.visibility_of_element_located(
+                    (
+                        By.XPATH,
+                        "/html/body/div[1]/div/div/div/div/div/div[2]/div/div/div/div/div/div[3]/div/div[1]/div/div[1]/div/div/input",
+                    )
+                )
+            )
+            inputElement_postcode.send_keys(user_postcode)
+
+            # Wait for address search button, then click it
+            addressSearch_button = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (
+                        By.XPATH,
+                        "/html/body/div[1]/div/div/div/div/div/div[2]/div/div/div/div/div/div[3]/div/div[1]/div/div[2]/div/button",
+                    )
+                )
+            )
+            addressSearch_button.click()
+
+            # Wait until the address list has loaded
+            WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located(
+                    (
+                        By.XPATH,
+                        "/html/body/div[1]/div/div/div/div/div/div[2]/div/div/div/div/div/div[3]/div/div[1]/div/div[3]/div/div",
+                    )
+                )
+            )
+
+            # Select the correct address from the list
+            addressList_rows = driver.find_elements(By.CLASS_NAME, "row")
+            for row in addressList_rows:
+                option_name = row.text[0 : len(user_paon)]
+                if option_name == user_paon:
+                    break
+            address_to_select = row.find_element(By.LINK_TEXT, "Choose this address")
+            address_to_select.click()
+
+            # Wait for bin dates to load
+            WebDriverWait(driver, 20).until(
+                EC.presence_of_element_located(
+                    (
+                        By.XPATH,
+                        "/html/body/div[1]/div/div/div/div/div/div[2]/div/div/div/div/div/div[3]/div/div[1]/div/div[4]/div/div",
+                    )
+                )
+            )
+
+            # Parse the HTML from the WebDriver
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            soup.prettify()
+
+            z = soup.find(
+                "div", {"class": "mx-name-textBox5 mx-textbox form-group"}
+            ).find_next("div", {"class": "form-control-static"})
+
+            maroon_bin_date = datetime.strptime(
+                soup.find("div", {"class": "mx-name-textBox3 mx-textbox form-group"})
+                .find_next("div", {"class": "form-control-static"})
+                .get_text(strip=True),
+                "%A %d/%m/%Y",
+            )
+            collections.append(("Maroon bin", maroon_bin_date))
+
+            grey_bin_date = datetime.strptime(
+                soup.find("div", {"class": "mx-name-textBox4 mx-textbox form-group"})
+                .find_next("div", {"class": "form-control-static"})
+                .get_text(strip=True),
+                "%A %d/%m/%Y",
+            )
+            collections.append(("Grey bin", grey_bin_date))
+
+            blue_bin_date = datetime.strptime(
+                soup.find("div", {"class": "mx-name-textBox5 mx-textbox form-group"})
+                .find_next("div", {"class": "form-control-static"})
+                .get_text(strip=True),
+                "%A %d/%m/%Y",
+            )
+            collections.append(("Blue bin", blue_bin_date))
+
+            ordered_data = sorted(collections, key=lambda x: x[1])
+            for item in ordered_data:
+                dict_data = {
+                    "type": item[0].capitalize(),
+                    "collectionDate": item[1].strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+
+        return data

uk_bin_collection/uk_bin_collection/councils/LancasterCityCouncil.py

@@ -0,0 +1,71 @@
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+from uk_bin_collection.uk_bin_collection.common import *
+from bs4 import BeautifulSoup
+from datetime import datetime
+import requests
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # data to return
+        data = {"bins": []}
+
+        # start session
+        # note: this ignores the given url
+        base_url = "https://lcc-wrp.whitespacews.com"
+        session = requests.session()
+        response = session.get(base_url + "/#!")
+        links = [
+            a["href"]
+            for a in BeautifulSoup(response.text, features="html.parser").select("a")
+        ]
+        portal_link = ""
+        for l in links:
+            if "seq=1" in l:
+                portal_link = l
+
+        # fill address form
+        response = session.get(portal_link)
+        form = BeautifulSoup(response.text, features="html.parser").find("form")
+        form_url = dict(form.attrs).get("action")
+        payload = {
+            "address_name_number": kwargs.get("number"),
+            "address_street": "",
+            "address_postcode": kwargs.get("postcode"),
+        }
+
+        # get (first) found address
+        response = session.post(form_url, data=payload)
+        links = [
+            a["href"]
+            for a in BeautifulSoup(response.text, features="html.parser").select("a")
+        ]
+        addr_link = ""
+        for l in links:
+            if "seq=3" in l:
+                addr_link = base_url + "/" + l
+
+        # get json formatted bin data for addr
+        response = session.get(addr_link)
+        new_soup = BeautifulSoup(response.text, features="html.parser")
+        services = new_soup.find("section", {"id": "scheduled-collections"})
+        services_sub = services.find_all("li")
+        for i in range(0, len(services_sub), 3):
+            dt = datetime.strptime(services_sub[i + 1].text.strip(), "%d/%m/%Y").date()
+            bin_type = BeautifulSoup(services_sub[i + 2].text, features="lxml").find(
+                "p"
+            )
+            data["bins"].append(
+                {
+                    "type": bin_type.text.strip().removesuffix(" Collection Service"),
+                    "collectionDate": dt.strftime(date_format),
+                }
+            )
+
+        return data

uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py

@@ -0,0 +1,137 @@
+from datetime import datetime
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+from bs4 import BeautifulSoup
+from datetime import datetime
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.keys import Keys
+
+import pandas as pd
+import urllib.request
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the base
+    class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            """
+            Parse council provided CSVs to get the latest bin collections for address
+            """
+
+            user_uprn = kwargs.get("uprn")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_uprn(user_uprn)
+            check_postcode(user_postcode)
+            # Create Selenium webdriver
+            page = f"https://www.leeds.gov.uk/residents/bins-and-recycling/check-your-bin-day"
+
+            driver = create_webdriver(web_driver, headless)
+            driver.get(page)
+
+            wait = WebDriverWait(driver, 60)
+            postcode_box = wait.until(
+                EC.element_to_be_clickable(
+                    (
+                        By.ID,
+                        "ctl00_ctl48_g_eea1a8ba_4306_488e_96f2_97f22038e29f_ctl00_txtPostCode",
+                    )
+                )
+            )
+            postcode_box.send_keys(user_postcode)
+            postcode_btn_present = wait.until(
+                EC.presence_of_element_located(
+                    (
+                        By.ID,
+                        "ctl00_ctl48_g_eea1a8ba_4306_488e_96f2_97f22038e29f_ctl00_btnSearchAddress",
+                    )
+                )
+            )
+            postcode_btn = wait.until(
+                EC.element_to_be_clickable(
+                    (
+                        By.XPATH,
+                        '//*[@id="ctl00_ctl48_g_eea1a8ba_4306_488e_96f2_97f22038e29f_ctl00_btnSearchAddress"]',
+                    )
+                )
+            )
+
+            postcode_btn.send_keys(Keys.ENTER)
+
+            dropdown_present = wait.until(
+                EC.presence_of_element_located(
+                    (
+                        By.XPATH,
+                        '//*[@id="ctl00_ctl48_g_eea1a8ba_4306_488e_96f2_97f22038e29f_ctl00_ddlAddressList"]/option',
+                    )
+                )
+            )
+            address_dropdown = wait.until(
+                EC.element_to_be_clickable(
+                    (
+                        By.ID,
+                        "ctl00_ctl48_g_eea1a8ba_4306_488e_96f2_97f22038e29f_ctl00_ddlAddressList",
+                    )
+                )
+            )
+
+            dropdown_present.click()
+
+            dropdownSelect = Select(address_dropdown)
+            dropdownSelect.select_by_value(str(user_uprn))
+            results = wait.until(
+                EC.presence_of_element_located(
+                    (
+                        By.ID,
+                        "ctl00_ctl48_g_eea1a8ba_4306_488e_96f2_97f22038e29f_ctl00_BinResultsDetails",
+                    )
+                )
+            )
+
+            data = {"bins": []}  # dictionary for data
+            soup = BeautifulSoup(driver.page_source, "html.parser")
+
+            bin_types = soup.find_all("ul", class_="binCollectionTimesList")
+
+            for bin_collection_dates in bin_types:
+                bin_collection_list = bin_collection_dates.find_all("li", class_="")
+
+                if bin_collection_list:
+                    collection_dates = [
+                        date.text.strip() for date in bin_collection_list
+                    ]
+
+                    # Convert the collection dates to the desired format
+                    formatted_dates = [
+                        datetime.strptime(date, "%A %d %b %Y").strftime(date_format)
+                        for date in collection_dates
+                    ]
+
+                    # Extract the type of bin from the header
+                    bin_type = bin_collection_dates.find_previous("h3").text.split()[0]
+
+                    # Adding data to the 'bins' dictionary for each date
+                    for date in formatted_dates:
+                        dict_data = {"type": bin_type, "collectionDate": date}
+                        data["bins"].append(dict_data)
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data