uk_bin_collection 0.74.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/ValeofWhiteHorseCouncil.py

@@ -0,0 +1,103 @@
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        # UPRN is passed in via a cookie. Set cookies/params and GET the page
+        cookies = {
+            "SVBINZONE": f"VALE%3AUPRN%40{user_uprn}",
+        }
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+            "Accept-Language": "en-GB,en;q=0.7",
+            "Cache-Control": "max-age=0",
+            "Connection": "keep-alive",
+            "Referer": "https://eform.whitehorsedc.gov.uk/ebase/BINZONE_DESKTOP.eb?SOVA_TAG=VALE&ebd=0&ebz=1_1704201201813",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "same-origin",
+            "Sec-Fetch-User": "?1",
+            "Sec-GPC": "1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36",
+        }
+        params = {
+            "SOVA_TAG": "VALE",
+            "ebd": "0",
+        }
+        requests.packages.urllib3.disable_warnings()
+        response = requests.get(
+            "https://eform.whitehorsedc.gov.uk/ebase/BINZONE_DESKTOP.eb",
+            params=params,
+            headers=headers,
+            cookies=cookies,
+        )
+
+        # Parse response text for super speedy finding
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+
+        # Page has slider info side by side, which are two instances of this class
+        for bin in soup.find_all("div", {"class": "bintxt"}):
+            try:
+                # Check bin type heading and make that bin type and colour
+                bin_type_info = list(bin.stripped_strings)
+                if "rubbish" in bin_type_info[0]:
+                    bin_type = "Rubbish"
+                    bin_colour = "Black"
+                elif "recycling" in bin_type_info[0]:
+                    bin_type = "Recycling"
+                    bin_colour = "Green"
+                else:
+                    raise ValueError(f"No bin info found in {bin_type_info[0]}")
+
+                bin_date_info = list(
+                    bin.find_next("div", {"class": "binextra"}).stripped_strings
+                )
+                # On standard collection schedule, date will be contained in the first string
+                if contains_date(bin_date_info[0]):
+                    bin_date = get_next_occurrence_from_day_month(
+                        datetime.strptime(
+                            bin_date_info[0] + " " + datetime.today().strftime("%Y"),
+                            "%A %d %B - %Y",
+                        )
+                    ).strftime(date_format)
+                # On exceptional collection schedule (e.g. around English Bank Holidays), date will be contained in the second stripped string
+                else:
+                    bin_date = get_next_occurrence_from_day_month(
+                        datetime.strptime(
+                            bin_date_info[1] + " " + datetime.today().strftime("%Y"),
+                            "%A %d %B - %Y",
+                        )
+                    ).strftime(date_format)
+            except Exception as ex:
+                raise ValueError(f"Error parsing bin data: {ex}")
+
+            # Build data dict for each entry
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": bin_date,
+                "colour": bin_colour,
+            }
+            data["bins"].append(dict_data)
+
+        data["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        return data

uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py

@@ -0,0 +1,89 @@
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the base
+    class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            # Create Selenium webdriver
+            headless = kwargs.get("headless")
+            driver = create_webdriver(kwargs.get("web_driver"), headless)
+            driver.get(kwargs.get("url"))
+
+            # Make a BS4 object
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            soup.prettify()
+
+            data = {"bins": []}
+            sections = soup.find_all("div", {"class": "wil_c-content-section_heading"})
+            for s in sections:
+                if s.get_text(strip=True).lower() == "bin collections":
+                    rows = s.find_next_sibling(
+                        "div", {"class": "c-content-section_body"}
+                    ).find_all("div", {"class": "u-mb-8"})
+                    for row in rows:
+                        title = row.find("div", {"class": "u-mb-4"})
+                        collections = row.find_all("div", {"class": "u-mb-2"})
+                        if title and collections:
+                            for c in collections:
+                                if (
+                                    c.get_text(strip=True)
+                                    .lower()
+                                    .startswith("next collection")
+                                ):
+                                    # add next collection
+                                    next_collection_date = datetime.strptime(
+                                        c.get_text(strip=True).replace(
+                                            "Next collection - ", ""
+                                        ),
+                                        "%A, %d %B %Y",
+                                    ).strftime(date_format)
+                                    dict_data = {
+                                        "type": title.get_text(strip=True).capitalize(),
+                                        "collectionDate": next_collection_date,
+                                    }
+                                    data["bins"].append(dict_data)
+                                    # add future collections without duplicating next collection
+                                    future_collections = row.find(
+                                        "ul", {"class": "u-mt-4"}
+                                    ).find_all("li")
+                                    for c in future_collections:
+                                        future_collection_date = datetime.strptime(
+                                            c.get_text(strip=True),
+                                            "%A, %d %B %Y",
+                                        ).strftime(date_format)
+                                        if (
+                                            future_collection_date
+                                            != next_collection_date
+                                        ):
+                                            dict_data = {
+                                                "type": title.get_text(
+                                                    strip=True
+                                                ).capitalize(),
+                                                "collectionDate": future_collection_date,
+                                            }
+                                            data["bins"].append(dict_data)
+
+            data["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+            )
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/WarwickDistrictCouncil.py

@@ -0,0 +1,34 @@
+# This script pulls (in one hit) the data
+# from Warick District Council Bins Data
+
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    baseclass. They can also override some
+    operations with a default implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+
+        for element in soup.find_all("strong"):
+            bin_type = element.next_element
+            bin_type = bin_type.lstrip()
+            collectionDateElement = element.next_sibling.next_element.next_element
+            collectionDate = collectionDateElement.getText()
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": collectionDate,
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/WaverleyBoroughCouncil.py

@@ -0,0 +1,119 @@
+from datetime import date, datetime
+
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # pindex isn't actually paon, it's a url parameter that I'm guessing the council use as a property id
+        data = {"bins": []}
+        pindex = kwargs.get("paon")
+        user_postcode = kwargs.get("postcode")
+        check_postcode(user_postcode)
+
+        # WBC use a url parameter called "Track" that's generated when you start a form session.
+        # So first off, open the page, find the page link and copy it with the Track
+        start_url = "https://wav-wrp.whitespacews.com/"
+        s = requests.session()
+        response = s.get(start_url)
+        soup = BeautifulSoup(response.content, features="html.parser")
+        soup.prettify()
+        collection_page_link = soup.find_all(
+            "p", {"class": "govuk-body govuk-!-margin-bottom-0 colorblue lineheight15"}
+        )[0].find("a")["href"]
+        track_id = collection_page_link[33:60]
+
+        # Next we need to search using the postcode, but this is actually an important POST request
+        pc_headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,"
+            "image/webp,image/apng,*/*;q=0.8",
+            "Accept-Language": "en-GB,en;q=0.9",
+            "Cache-Control": "max-age=0",
+            "Connection": "keep-alive",
+            "Origin": "https://wav-wrp.whitespacews.com",
+            "Referer": "https://wav-wrp.whitespacews.com/"
+            + track_id
+            + "&serviceID=A&seq=2",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "same-origin",
+            "Sec-Fetch-User": "?1",
+            "Sec-GPC": "1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, "
+            "like Gecko) Chrome/106.0.0.0 Safari/537.36",
+        }
+        form_data = {
+            "address_name_number": "",
+            "address_street": "",
+            "street_town": "",
+            "address_postcode": user_postcode,
+        }
+        response = s.post(
+            "https://wav-wrp.whitespacews.com/mop.php?serviceID=A&"
+            + track_id
+            + "&seq=2",
+            headers=pc_headers,
+            data=form_data,
+        )
+
+        # Finally, we can use pindex to find the address and get some data
+        request_url = (
+            "https://wav-wrp.whitespacews.com/mop.php?"
+            + track_id
+            + "&serviceID=A&seq=3&pIndex="
+            + pindex
+        )
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,"
+            "image/webp,image/apng,*/*;q=0.8",
+            "Accept-Language": "en-GB,en;q=0.9",
+            "Connection": "keep-alive",
+            "Referer": "https://wav-wrp.whitespacews.com/mop.php?serviceID=A&"
+            + track_id
+            + "&seq=2",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "same-origin",
+            "Sec-Fetch-User": "?1",
+            "Sec-GPC": "1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, "
+            "like Gecko) Chrome/106.0.0.0 Safari/537.36",
+        }
+
+        response = s.get(request_url, headers=headers)
+        soup = BeautifulSoup(response.content, features="html.parser")
+        soup.prettify()
+
+        # Find the list elements
+        u1_block = soup.find_all(
+            "u1",
+            {
+                "class": "displayinlineblock justifycontentleft alignitemscenter margin0 padding0"
+            },
+        )
+
+        for element in u1_block:
+            x = element.find_all_next(
+                "li", {"class": "displayinlineblock padding0px20px5px0px"}
+            )
+            dict_data = {
+                "type": x[2].text.strip(),
+                "collectionDate": datetime.strptime(
+                    x[1].text.strip(), date_format
+                ).strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/WealdenDistrictCouncil.py

@@ -0,0 +1,86 @@
+import json
+
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        headers = {
+            "authority": "www.wealden.gov.uk",
+            "accept": "*/*",
+            "accept-language": "en-GB,en;q=0.7",
+            "content-type": "application/x-www-form-urlencoded; charset=UTF-8",
+            # Requests sorts cookies= alphabetically
+            # 'cookie': 'ARRAffinity=e45c20b343b490e3866d5d35c3dbda687e4a1357c2163c32922209862abb5872; ARRAffinitySameSite=e45c20b343b490e3866d5d35c3dbda687e4a1357c2163c32922209862abb5872',
+            "origin": "https://www.wealden.gov.uk",
+            "referer": "https://www.wealden.gov.uk/recycling-and-waste/bin-search/?uprn=10033413624",
+            "sec-fetch-dest": "empty",
+            "sec-fetch-mode": "cors",
+            "sec-fetch-site": "same-origin",
+            "sec-gpc": "1",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36",
+            "x-requested-with": "XMLHttpRequest",
+        }
+
+        data = {
+            "action": "wealden_get_collections_for_uprn",
+            "uprn": user_uprn,
+        }
+
+        requests.packages.urllib3.disable_warnings()
+        response = requests.post(
+            "https://www.wealden.gov.uk/wp-admin/admin-ajax.php",
+            headers=headers,
+            data=data,
+        )
+        json_data = json.loads(response.text)
+
+        if json_data["status"] != "success":
+            raise ValueError("Error parsing data. Please open an issue on GitHub.")
+
+        property_data = json_data["collection"]
+        data = {"bins": []}
+        collections = []
+
+        if len(property_data["refuseCollectionDate"]) > 0:
+            bin_type = "Rubbish Bin"
+            bin_date = datetime.strptime(
+                property_data["refuseCollectionDate"], "%Y-%m-%dT%H:%M:%S"
+            )
+            collections.append((bin_type, bin_date))
+        if len(property_data["recyclingCollectionDate"]) > 0:
+            bin_type = "Recycling Bin"
+            bin_date = datetime.strptime(
+                property_data["recyclingCollectionDate"], "%Y-%m-%dT%H:%M:%S"
+            )
+            collections.append((bin_type, bin_date))
+        if len(property_data["gardenCollectionDate"]) > 0:
+            bin_type = "Garden Bin"
+            bin_date = datetime.strptime(
+                property_data["gardenCollectionDate"], "%Y-%m-%dT%H:%M:%S"
+            )
+            collections.append((bin_type, bin_date))
+
+        ordered_data = sorted(collections, key=lambda x: x[1])
+        data = {"bins": []}
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0],
+                "collectionDate": item[1].strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/WelhatCouncil.py

@@ -0,0 +1,73 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+def get_token(page) -> str:
+    """
+    Get a __token to include in the form data
+        :param page: Page html
+        :return: Form __token
+    """
+    soup = BeautifulSoup(page.text, features="html.parser")
+    soup.prettify()
+    token = soup.find("input", {"name": "__token"}).get("value")
+    return token
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        uprn = kwargs.get("uprn")
+        postcode = kwargs.get("postcode")
+        check_uprn(uprn)
+        check_postcode(postcode)
+
+        values = {
+            "__token": get_token(page),
+            "page": "492",
+            "locale": "en_GB",
+            "q9f451fe0ca70775687eeedd1e54b359e55f7c10c_0_0": postcode,
+            "q9f451fe0ca70775687eeedd1e54b359e55f7c10c_1_0": uprn,
+            "next": "Next",
+        }
+        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64)"}
+        requests.packages.urllib3.disable_warnings()
+        response = requests.request(
+            "POST",
+            "https://www.welhat.gov.uk/xfp/form/214",
+            headers=headers,
+            data=values,
+        )
+
+        soup = BeautifulSoup(response.text, features="html.parser")
+
+        rows = soup.find("table").find_all("tr")
+
+        # Form a JSON wrapper
+        data = {"bins": []}
+
+        # Loops the Rows
+        for row in rows:
+            cells = row.find_all("td")
+            if cells:
+                binType = cells[0].get_text(strip=True)
+                collectionDate = datetime.strptime(
+                    cells[1].get_text(strip=True), "%A %d %B  %Y"
+                ).strftime(date_format)
+
+                # Make each Bin element in the JSON
+                dict_data = {
+                    "type": binType,
+                    "collectionDate": collectionDate,
+                }
+
+                # Add data to the main JSON Wrapper
+                data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/WestBerkshireCouncil.py

@@ -0,0 +1,134 @@
+import time
+
+from bs4 import BeautifulSoup
+from dateutil.relativedelta import relativedelta
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            data = {"bins": []}
+            collections = []
+            user_paon = kwargs.get("paon")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_paon(user_paon)
+            check_postcode(user_postcode)
+
+            # Create Selenium webdriver
+            # driver = create_webdriver(web_driver, headless)
+            driver = webdriver.Chrome(
+                service=ChromeService(ChromeDriverManager().install())
+            )
+            driver.get("https://www.westberks.gov.uk/binday")
+
+            # Wait for the postcode field to appear then populate it
+            inputElement_postcode = WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located(
+                    (By.ID, "FINDYOURBINDAYS_ADDRESSLOOKUPPOSTCODE")
+                )
+            )
+            inputElement_postcode.send_keys(user_postcode)
+
+            # Click search button
+            findAddress = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.ID, "FINDYOURBINDAYS_ADDRESSLOOKUPSEARCH")
+                )
+            )
+            findAddress.click()
+
+            WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (
+                        By.XPATH,
+                        ""
+                        "//*[@id='FINDYOURBINDAYS_ADDRESSLOOKUPADDRESS']//option[contains(., '"
+                        + user_paon
+                        + "')]",
+                    )
+                )
+            ).click()
+
+            # Wait for the submit button to appear, then click it to get the collection dates
+            WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located(
+                    (By.XPATH, '//*[@id="FINDYOURBINDAYS_RUBBISHDATE"]/div')
+                )
+            )
+            time.sleep(2)
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            soup.prettify()
+
+            rubbish_date = datetime.strptime(
+                " ".join(
+                    soup.find("div", {"id": "FINDYOURBINDAYS_RUBBISHDATE_OUTERDIV"})
+                    .get_text(strip=True)
+                    .split()[6:8]
+                ),
+                "%d %B",
+            ).replace(year=datetime.now().year)
+            recycling_date = datetime.strptime(
+                " ".join(
+                    soup.find("div", {"id": "FINDYOURBINDAYS_RECYCLINGDATE_OUTERDIV"})
+                    .get_text(strip=True)
+                    .split()[6:8]
+                ),
+                "%d %B",
+            ).replace(year=datetime.now().year)
+            food_date = datetime.strptime(
+                " ".join(
+                    soup.find("div", {"id": "FINDYOURBINDAYS_FOODWASTEDATE_OUTERDIV"})
+                    .get_text(strip=True)
+                    .split()[8:10]
+                ),
+                "%d %B",
+            ).replace(year=datetime.now().year)
+
+            if datetime.now().month == 12 and rubbish_date.month == 1:
+                rubbish_date = rubbish_date + relativedelta(years=1)
+            if datetime.now().month == 12 and recycling_date.month == 1:
+                recycling_date = recycling_date + relativedelta(years=1)
+            if datetime.now().month == 12 and food_date.month == 1:
+                food_date = food_date + relativedelta(years=1)
+
+            collections.append(("Rubbish bin", rubbish_date))
+            collections.append(("Recycling bin", recycling_date))
+            collections.append(("Food waste bin", food_date))
+
+            ordered_data = sorted(collections, key=lambda x: x[1])
+            for item in ordered_data:
+                dict_data = {
+                    "type": item[0].capitalize(),
+                    "collectionDate": item[1].strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+            print()
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data