uk_bin_collection 0.74.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/LisburnCastlereaghCityCouncil.py

@@ -0,0 +1,101 @@
+import difflib
+from datetime import date, datetime
+
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    base_url = "https://lisburn.isl-fusion.com"
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        """
+        This function will make a request to the search endpoint with the postcode, extract the
+        house numbers from the responses, then retrieve the ID of the entry with the house number that matches,
+        to then retrieve the bin schedule.
+
+        The API here is a weird combination of HTML in json responses.
+        """
+        postcode = kwargs.get("postcode")
+        paon = kwargs.get("paon")
+
+        if not postcode:
+            raise ValueError("Must provide a postcode")
+
+        if not paon:
+            raise ValueError("Must provide a house number")
+
+        search_url = f"{self.base_url}/address/{postcode}"
+
+        requests.packages.urllib3.disable_warnings()
+        s = requests.Session()
+        response = s.get(search_url)
+        response.raise_for_status()
+
+        address_data = response.json()
+
+        address_list = address_data["html"]
+
+        soup = BeautifulSoup(address_list, features="html.parser")
+
+        address_by_id = {}
+
+        for li in soup.find_all("li"):
+            link = li.find_all("a")[0]
+            address_id = link.attrs["href"]
+            address = link.text
+
+            address_by_id[address_id] = address
+
+        addresses = list(address_by_id.values())
+
+        common = difflib.SequenceMatcher(
+            a=addresses[0], b=addresses[1]
+        ).find_longest_match()
+        extra_bit = addresses[0][common.a : common.a + common.size]
+
+        ids_by_paon = {
+            a.replace(extra_bit, ""): a_id.replace("/view/", "").replace("/", "")
+            for a_id, a in address_by_id.items()
+        }
+
+        property_id = ids_by_paon.get(paon)
+        if not property_id:
+            raise ValueError(
+                f"Invalid house number, valid values are {', '.join(ids_by_paon.keys())}"
+            )
+
+        today = date.today()
+        calendar_url = (
+            f"{self.base_url}/calendar/{property_id}/{today.strftime('%Y-%m-%d')}"
+        )
+        response = s.get(calendar_url)
+        response.raise_for_status()
+        calendar_data = response.json()
+        next_collections = calendar_data["nextCollections"]
+
+        collections = list(next_collections["collections"].values())
+
+        data = {"bins": []}
+
+        for collection in collections:
+            collection_date = datetime.strptime(collection["date"], "%Y-%m-%d")
+            bins = [c["name"] for c in collection["collections"].values()]
+
+            for bin in bins:
+                data["bins"].append(
+                    {
+                        "type": bin,
+                        "collectionDate": collection_date.strftime(date_format),
+                    }
+                )
+        return data

uk_bin_collection/uk_bin_collection/councils/LiverpoolCityCouncil.py

@@ -0,0 +1,65 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+from dateutil.relativedelta import relativedelta
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Add in some variables we need
+        data = {"bins": []}
+        collections = []
+        curr_date = datetime.today()
+
+        # Parse the page
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        # Get all table rows on the page - enumerate gives us an index, which is handy for to keep a row count.
+        # In this case, the first (0th) row is headings, so we can skip it, then parse the other data.
+        for idx, row in enumerate(soup.find_all("tr")):
+            if idx == 0:
+                continue
+
+            row_type = row.find("th").text.strip()
+            row_data = row.find_all("td")
+
+            # When we get the row data, we can loop through it all and parse it to datetime. Because there are no
+            # years, we must add it in, then check if we need to overflow it to the following year.
+            for item in row_data:
+                item_text = item.text.strip()
+
+                if item_text == "Today":
+                    collections.append((row_type, curr_date))
+                elif item_text == "Tomorrow":
+                    collections.append((row_type, curr_date + relativedelta(days=1)))
+                else:
+                    bin_date = datetime.strptime(
+                        remove_ordinal_indicator_from_date_string(item_text),
+                        "%A, %d %B",
+                    ).replace(year=curr_date.year)
+
+                    if curr_date.month == 12 and bin_date.month == 1:
+                        bin_date = bin_date + relativedelta(years=1)
+
+                    collections.append((row_type, bin_date))
+
+        # Sort the text and list elements by date
+        ordered_data = sorted(collections, key=lambda x: x[1])
+
+        # Put the elements into the dictionary
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0],
+                "collectionDate": item[1].strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/LondonBoroughHounslow.py

@@ -0,0 +1,82 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        api_url = "https://www.hounslow.gov.uk/homepage/86/recycling_and_waste_collection_day_finder"
+        user_uprn = kwargs.get("uprn")
+
+        # Check the UPRN is valid
+        check_uprn(user_uprn)
+
+        # Create the form data
+        form_data = {
+            "UPRN": user_uprn,
+        }
+
+        # Make a request to the API
+        requests.packages.urllib3.disable_warnings()
+        response = requests.post(api_url, data=form_data)
+
+        # Make a BS4 object
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+
+        # Get the div element
+        div_element = soup.find("div", {"class": "bin_day_main_wrapper"})
+
+        # Get all bins with their corresponding dates using list comprehension
+        # This creates a list of tuples, where each tuple contains the bin type and collection date
+        bins_with_dates = [
+            (
+                bin.get_text().strip(),
+                h4.get_text().replace("This ", "").replace("Next ", ""),
+            )
+            # This first for loop iterates over each h4 element
+            for h4 in div_element.find_all("h4")
+            # This nested for loop iterates over each li element within the corresponding ul element
+            for bin in h4.find_next_sibling("ul").find_all("li")
+        ]
+
+        for bin_type, collection_date in bins_with_dates:
+            if "-" in collection_date:
+                date_part = collection_date.split(" - ")[1]
+                data["bins"].append(
+                    {
+                        "type": bin_type,
+                        "collectionDate": datetime.strptime(
+                            date_part, "%d %b %Y"
+                        ).strftime(date_format),
+                    }
+                )
+            elif len(collection_date.split(" ")) == 4:
+                data["bins"].append(
+                    {
+                        "type": bin_type,
+                        "collectionDate": datetime.strptime(
+                            collection_date, "%A %d %b %Y"
+                        ).strftime(date_format),
+                    }
+                )
+            else:
+                data["bins"].append(
+                    {
+                        "type": bin_type,
+                        "collectionDate": datetime.strptime(
+                            collection_date, "%d %b %Y"
+                        ).strftime(date_format),
+                    }
+                )
+
+        return data

uk_bin_collection/uk_bin_collection/councils/LondonBoroughRedbridge.py

@@ -0,0 +1,161 @@
+import re
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+# This script pulls (in one hit) the data from Bromley Council Bins Data
+import datetime
+from datetime import datetime
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.keys import Keys
+import time
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            data = {"bins": []}
+            headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64)"}
+
+            uprn = kwargs.get("uprn")
+            postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            driver = create_webdriver(web_driver, headless)
+            driver.get(kwargs.get("url"))
+
+            wait = WebDriverWait(driver, 60)
+            post_code_search = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, f"//input[contains(@class, 'searchAddress')]")
+                )
+            )
+            post_code_search.send_keys(postcode)
+
+            submit_btn = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, f"//button[contains(@class, 'searchAddressButton')]")
+                )
+            )
+
+            submit_btn.send_keys(Keys.ENTER)
+
+            address_link = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, f'//a[contains(@data-uprn,"{uprn}")]')
+                )
+            )
+
+            address_link.send_keys(Keys.ENTER)
+
+            address_results = wait.until(
+                EC.presence_of_element_located(
+                    (By.CLASS_NAME, "your-collection-schedule-container")
+                )
+            )
+
+            # Make a BS4 object
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            data = {"bins": []}
+
+            # Get the current month and year
+            current_month = datetime.now().month
+            current_year = datetime.now().year
+
+            # Function to extract collection data
+            def extract_collection_data(collection_div, collection_type):
+                if collection_div:
+                    date_element = (
+                        collection_div.find(class_="recycling-collection-day-numeric")
+                        or collection_div.find(class_="refuse-collection-day-numeric")
+                        or collection_div.find(class_="garden-collection-day-numeric")
+                    )
+                    month_element = (
+                        collection_div.find(class_="recycling-collection-month")
+                        or collection_div.find(class_="refuse-collection-month")
+                        or collection_div.find(class_="garden-collection-month")
+                    )
+
+                    if date_element and month_element:
+                        collection_date = date_element.get_text(strip=True)
+                        collection_month = month_element.get_text(strip=True)
+
+                        # Combine month, date, and year into a string
+                        date_string = (
+                            f"{collection_date} {collection_month} {current_year}"
+                        )
+
+                        try:
+                            # Convert the date string to a datetime object
+                            collection_date_obj = datetime.strptime(
+                                date_string, "%d %B %Y"
+                            )
+
+                            # Check if the month is ahead of the current month
+                            if collection_date_obj.month >= current_month:
+                                # If the month is ahead, use the current year
+                                formatted_date = collection_date_obj.strftime(
+                                    date_format
+                                )
+                            else:
+                                # If the month is before the current month, use the next year
+                                formatted_date = collection_date_obj.replace(
+                                    year=current_year + 1
+                                ).strftime(date_format)
+                            # Create a dictionary for each collection entry
+                            dict_data = {
+                                "type": collection_type,
+                                "collectionDate": formatted_date,
+                            }
+
+                            # Append dictionary data to the 'bins' list in the 'data' dictionary
+                            data["bins"].append(dict_data)
+
+                        except ValueError as e:
+                            # Handle the case where the date format is invalid
+                            formatted_date = "Invalid Date Format"
+
+            # Extract Recycling collection data
+            extract_collection_data(
+                soup.find(
+                    class_="container-fluid RegularCollectionDay"
+                ).find_next_sibling("div"),
+                "Recycling",
+            )
+
+            # Extract Refuse collection data
+            for refuse_div in soup.find_all(
+                class_="container-fluid RegularCollectionDay"
+            ):
+                extract_collection_data(refuse_div, "Refuse")
+
+            # Extract Garden Waste collection data
+            extract_collection_data(
+                soup.find(class_="container-fluid gardenwasteCollectionDay"),
+                "Garden Waste",
+            )
+
+            # Print the extracted data
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/MaldonDistrictCouncil.py

@@ -0,0 +1,52 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        data = {"bins": []}
+        uprn = kwargs.get("uprn")
+        check_uprn(uprn)
+
+        requests.packages.urllib3.disable_warnings()
+        response = requests.get(
+            f"https://maldon.suez.co.uk/maldon/ServiceSummary?uprn={uprn}",
+            headers={"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64)"},
+        )
+        if response.status_code != 200:
+            raise ValueError("No bin data found for provided UPRN.")
+
+        soup = BeautifulSoup(response.text, features="html.parser")
+        collections = soup.find_all("div", {"class": "panel"})
+        for c in collections:
+            binType = c.find("div", {"class": "panel-heading"}).get_text(strip=True)
+            collectionDate = ""
+            rows = c.find("div", {"class": "panel-body"}).find_all(
+                "div", {"class": "row"}
+            )
+            for row in rows:
+                if row.find("strong").get_text(strip=True).lower() == "next collection":
+                    collectionDate = row.find("div", {"class": "col-sm-9"}).get_text(
+                        strip=True
+                    )
+
+            if collectionDate != "":
+                collection_data = {
+                    "type": binType,
+                    "collectionDate": collectionDate,
+                }
+                data["bins"].append(collection_data)
+
+        data["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        return data

uk_bin_collection/uk_bin_collection/councils/MalvernHillsDC.py

@@ -0,0 +1,57 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        api_url = "https://swict.malvernhills.gov.uk/mhdcroundlookup/HandleSearchScreen"
+
+        user_uprn = kwargs.get("uprn")
+        # Check the UPRN is valid
+        check_uprn(user_uprn)
+
+        # Create the form data
+        form_data = {"nmalAddrtxt": "", "alAddrsel": user_uprn}
+        # expects postcode to be looked up and then uprn used.
+        # we can just provide uprn
+
+        # Make a request to the API
+        requests.packages.urllib3.disable_warnings()
+        response = requests.post(api_url, data=form_data, verify=False)
+
+        # Make a BS4 object
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+
+        # Find results table
+        table_element = soup.find("table")
+        table_body = table_element.find("tbody")
+        rows = table_body.find_all("tr")
+
+        data = {"bins": []}
+
+        for row in rows:
+            columns = row.find_all("td")
+            columns = [ele.text.strip() for ele in columns]
+
+            thisCollection = [ele for ele in columns if ele]  # Get rid of empty values
+
+            # if not signed up for garden waste, this appears as Not applicable
+            if "Not applicable" not in thisCollection[1]:
+                bin_type = thisCollection[0].replace("collection", "").strip()
+                date = datetime.strptime(thisCollection[1], "%A %d/%m/%Y")
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": date.strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/ManchesterCityCouncil.py

@@ -0,0 +1,106 @@
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Get and check UPRN
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        # Start a new session to walk through the form
+        requests.packages.urllib3.disable_warnings()
+        s = requests.session()
+
+        # There's a cookie that makes the whole thing valid when you search for a postcode,
+        # but postcode and UPRN is a hassle imo, so this makes a request for the session to get a cookie
+        # using a Manchester City Council postcode I hardcoded in the data payload
+        postcode_request_header = {
+            "authority": "www.manchester.gov.uk",
+            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,"
+            "image/webp,image/apng,*/*;q=0.8",
+            "accept-language": "en-GB,en;q=0.6",
+            "cache-control": "max-age=0",
+            # Requests sorts cookies= alphabetically
+            "origin": "https://www.manchester.gov.uk",
+            "referer": "https://www.manchester.gov.uk/bincollections",
+            "sec-fetch-dest": "document",
+            "sec-fetch-mode": "navigate",
+            "sec-fetch-site": "same-origin",
+            "sec-fetch-user": "?1",
+            "sec-gpc": "1",
+            "upgrade-insecure-requests": "1",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, "
+            "like Gecko) Chrome/104.0.5112.102 Safari/537.36",
+        }
+        postcode_request_data = {
+            "mcc_bin_dates_search_term": "M2 5DB",
+            "mcc_bin_dates_submit": "Go",
+        }
+        response = s.post(
+            "https://www.manchester.gov.uk/bincollections",
+            headers=postcode_request_header,
+            data=postcode_request_data,
+        )
+
+        # Make a POST with the same cookie-fied session using the user's UPRN data
+        uprn_request_headers = {
+            "authority": "www.manchester.gov.uk",
+            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+            "accept-language": "en-GB,en;q=0.6",
+            "cache-control": "max-age=0",
+            # Requests sorts cookies= alphabetically
+            # 'cookie': 'TestCookie=Test; CookieConsent={stamp:%27D8rypjMDBJhpfMWybSMdGXP1hCZWGJYtGETiMTu1UuXTdRIKl8SU5g==%27%2Cnecessary:true%2Cpreferences:true%2Cstatistics:true%2Cmarketing:true%2Cver:6%2Cutc:1661783732090%2Cregion:%27gb%27}; PHPSESSID=kElJxYAt%2Cf-4ZWoskt0s5tn32BUQRXDYUVp3G-NsqOAOaeIcKlm2T4r7ATSgqfz6',
+            "origin": "https://www.manchester.gov.uk",
+            "referer": "https://www.manchester.gov.uk/bincollections",
+            "sec-fetch-dest": "document",
+            "sec-fetch-mode": "navigate",
+            "sec-fetch-site": "same-origin",
+            "sec-fetch-user": "?1",
+            "sec-gpc": "1",
+            "upgrade-insecure-requests": "1",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36",
+        }
+        uprn_request_data = {
+            "mcc_bin_dates_uprn": user_uprn,
+            "mcc_bin_dates_submit": "Go",
+        }
+        response = s.post(
+            "https://www.manchester.gov.uk/bincollections",
+            headers=uprn_request_headers,
+            data=uprn_request_data,
+        )
+
+        # Make that BS4 object and use it to prettify the response
+        soup = BeautifulSoup(response.content, features="html.parser")
+        soup.prettify()
+
+        # Get the collection items on the page and strip the bits of text that we don't care for
+        collections = []
+        for bin in soup.find_all("div", {"class": "collection"}):
+            bin_type = bin.find_next("h3").text.replace("  DUE TODAY", "").strip()
+            next_collection = bin.find_next("p").text.replace("Next collection ", "")
+            next_collection = datetime.strptime(next_collection, "%A %d %b %Y")
+            collections.append((bin_type, next_collection))
+
+        # Sort the collections by date order rather than bin type, then return as a dictionary (with str date)
+        ordered_data = sorted(collections, key=lambda x: x[1])
+        data = {"bins": []}
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0],
+                "collectionDate": item[1].strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/MansfieldDistrictCouncil.py

@@ -0,0 +1,38 @@
+import requests
+
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        data = {"bins": []}
+        api_url = f'https://portal.mansfield.gov.uk/MDCWhiteSpaceWebService/WhiteSpaceWS.asmx/GetCollectionByUPRNAndDate?apiKey=mDc-wN3-B0f-f4P&UPRN={user_uprn}&coldate={datetime.now().strftime("%d/%m/%Y")}'
+
+        response = requests.get(api_url)
+        if response.status_code != 200:
+            raise ConnectionError("Could not get latest data!")
+
+        json_data = response.json()["Collections"]
+        for item in json_data:
+
+            dict_data = {
+                "type": item.get("Service").split(" ")[0] + " bin",
+                "collectionDate": datetime.strptime(
+                    item.get("Date"), "%d/%m/%Y %H:%M:%S"
+                ).strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data