uk_bin_collection 0.74.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py

@@ -0,0 +1,180 @@
+import time
+
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.ui import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+def get_seasonal_overrides():
+    url = "https://www.barnet.gov.uk/recycling-and-waste/bin-collections/find-your-bin-collection-day"
+    response = requests.get(url)
+    if response.status_code == 200:
+        soup = BeautifulSoup(response.text, "html.parser")
+        body_div = soup.find("div", class_="field--name-body")
+        ul_element = body_div.find("ul")
+        if ul_element:
+            li_elements = ul_element.find_all("li")
+            overrides_dict = {}
+            for li_element in li_elements:
+                li_text = li_element.text.strip()
+                li_text = re.sub(r"\([^)]*\)", "", li_text).strip()
+                if "Collections for" in li_text and "will be revised to" in li_text:
+                    parts = li_text.split("will be revised to")
+                    original_date = (
+                        parts[0]
+                        .replace("Collections for", "")
+                        .replace("\xa0", " ")
+                        .strip()
+                    )
+                    revised_date = parts[1].strip()
+
+                    # Extract day and month
+                    date_parts = original_date.split()[1:]
+                    if len(date_parts) == 2:
+                        day, month = date_parts
+                        # Ensure original_date has leading zeros for single-digit days
+                        day = day.zfill(2)
+                        original_date = f"{original_date.split()[0]} {day} {month}"
+
+                    # Store the information in the dictionary
+                    overrides_dict[original_date] = revised_date
+            return overrides_dict
+        else:
+            print("UL element not found within the specified div.")
+    else:
+        print(f"Failed to retrieve the page. Status code: {response.status_code}")
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            user_postcode = kwargs.get("postcode")
+            if not user_postcode:
+                raise ValueError("No postcode provided.")
+            check_postcode(user_postcode)
+
+            user_paon = kwargs.get("paon")
+            check_paon(user_paon)
+            headless = kwargs.get("headless")
+            web_driver = kwargs.get("web_driver")
+            driver = create_webdriver(web_driver, headless)
+            page = "https://account.barnet.gov.uk/Forms/Home/Redirector/Index/?id=6a2ac067-3322-46e5-96e4-16c0c214454a&mod=OA&casetype=BAR&formname=BNTCOLDATE"
+            driver.get(page)
+
+            time.sleep(5)
+
+            postcode_input = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.CSS_SELECTOR, '[aria-label="Postcode"]')
+                )
+            )
+
+            postcode_input.send_keys(user_postcode)
+
+            find_address_button = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.CSS_SELECTOR, '[value="Find address"]')
+                )
+            )
+            find_address_button.click()
+
+            time.sleep(15)
+            # Wait for address box to be visible
+            select_address_input = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (
+                        By.ID,
+                        "MainContent_CUSTOM_FIELD_808562d4b07f437ea751317cabd19d9eeaf8742f49cb4f7fa9bef99405b859f2",
+                    )
+                )
+            )
+
+            # Select address based
+            select = Select(select_address_input)
+            addr_label = f"{user_postcode}, {user_paon},"
+            for addr_option in select.options:
+                option_name = addr_option.accessible_name[0 : len(addr_label)]
+                if option_name == addr_label:
+                    break
+            select.select_by_value(addr_option.text)
+
+            time.sleep(10)
+            # Wait for the specified div to be present
+            target_div_id = "MainContent_CUSTOM_FIELD_808562d4b07f437ea751317cabd19d9ed93a174c32b14f839b65f6abc42d8108_div"
+            target_div = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.ID, target_div_id))
+            )
+
+            time.sleep(5)
+            soup = BeautifulSoup(driver.page_source, "html.parser")
+
+            # Find the div with the specified id
+            target_div = soup.find("div", {"id": target_div_id})
+
+            # Handle the additional table of info for xmas
+            try:
+                overrides_dict = get_seasonal_overrides()
+            except Exception as e:
+                overrides_dict = {}
+
+            # Check if the div is found
+            if target_div:
+                bin_data = {"bins": []}
+
+                for bin_div in target_div.find_all(
+                    "div",
+                    {"style": re.compile("background-color:.*; padding-left: 4px;")},
+                ):
+                    bin_type = bin_div.find("strong").text.strip()
+                    collection_date_string = (
+                        re.search(r"Next collection date:\s+(.*)", bin_div.text)
+                        .group(1)
+                        .strip()
+                        .replace(",", "")
+                    )
+                    if collection_date_string in overrides_dict:
+                        # Replace with the revised date from overrides_dict
+                        collection_date_string = overrides_dict[collection_date_string]
+
+                    current_date = datetime.now()
+                    parsed_date = datetime.strptime(
+                        collection_date_string + f" {current_date.year}", "%A %d %B %Y"
+                    )
+                    # Check if the parsed date is in the past and not today
+                    if parsed_date.date() < current_date.date():
+                        # If so, set the year to the next year
+                        parsed_date = parsed_date.replace(year=current_date.year + 1)
+                    else:
+                        # If not, set the year to the current year
+                        parsed_date = parsed_date.replace(year=current_date.year)
+                    formatted_date = parsed_date.strftime("%d/%m/%Y")
+
+                    contains_date(formatted_date)
+                    bin_info = {"type": bin_type, "collectionDate": formatted_date}
+                    bin_data["bins"].append(bin_info)
+            else:
+                raise ValueError("Collection data not found.")
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return bin_data

uk_bin_collection/uk_bin_collection/councils/BarnsleyMBCouncil.py

@@ -0,0 +1,109 @@
+from typing import Dict, Any
+from bs4 import BeautifulSoup
+from dateutil.relativedelta import relativedelta
+import requests
+from datetime import datetime
+from uk_bin_collection.uk_bin_collection.common import (
+    check_postcode,
+    check_uprn,
+    date_format,
+)
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs: Any) -> Dict[str, Any]:
+        data: Dict[str, Any] = {"bins": []}
+
+        # Get UPRN and postcode from kwargs
+        user_uprn = str(kwargs.get("uprn"))
+        user_postcode = str(kwargs.get("postcode"))
+        check_postcode(user_postcode)
+        check_uprn(user_uprn)
+
+        # Pass in form data and make the POST request
+        headers = {
+            "authority": "waste.barnsley.gov.uk",
+            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "accept-language": "en-GB,en;q=0.9",
+            "cache-control": "no-cache",
+            "content-type": "application/x-www-form-urlencoded",
+            "origin": "https://waste.barnsley.gov.uk",
+            "pragma": "no-cache",
+            "referer": "https://waste.barnsley.gov.uk/ViewCollection/SelectAddress",
+            "sec-ch-ua": '"Chromium";v="118", "Opera GX";v="104", "Not=A?Brand";v="99"',
+            "sec-ch-ua-mobile": "?0",
+            "sec-ch-ua-platform": '"Windows"',
+            "sec-fetch-dest": "document",
+            "sec-fetch-mode": "navigate",
+            "sec-fetch-site": "same-origin",
+            "sec-fetch-user": "?1",
+            "upgrade-insecure-requests": "1",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.5993.118 Safari/537.36",
+        }
+        form_data = {
+            "personInfo.person1.HouseNumberOrName": "",
+            "personInfo.person1.Postcode": f"{user_postcode}",
+            "personInfo.person1.UPRN": f"{user_uprn}",
+            "person1_SelectAddress": "Select address",
+        }
+        response = requests.post(
+            "https://waste.barnsley.gov.uk/ViewCollection/SelectAddress",
+            headers=headers,
+            data=form_data,
+        )
+
+        if response.status_code != 200:
+            raise ConnectionRefusedError(
+                "Error getting results from website! Please open an issue on GitHub!"
+            )
+
+        soup = BeautifulSoup(response.text, features="html.parser")
+
+        results = soup.find_all("fieldset")
+
+        # Next collection details
+        highlight_content = results[0].find("div", {"class": "highlight-content"})
+        bin_date_str = highlight_content.find(
+            "em", {"class": "ui-bin-next-date"}
+        ).text.strip()
+        bin_type = (
+            highlight_content.find("p", {"class": "ui-bin-next-type"}).text.strip()
+            + " bin"
+        )
+
+        if bin_date_str == "Today":
+            bin_date = datetime.today()
+        elif bin_date_str == "Tomorrow":
+            bin_date = datetime.today() + relativedelta(days=1)
+        else:
+            bin_date = datetime.strptime(bin_date_str, "%A, %B %d, %Y")
+
+        dict_data = {
+            "type": bin_type,
+            "collectionDate": bin_date.strftime(date_format),
+        }
+        data["bins"].append(dict_data)
+
+        # Upcoming collections
+        upcoming_collections = results[1].find("tbody").find_all("tr")
+        for row in upcoming_collections:
+            columns = row.find_all("td")
+            bin_date_str = columns[0].text.strip()
+            bin_types = columns[1].text.strip().split(", ")
+
+            for bin_type in bin_types:
+                bin_date = datetime.strptime(bin_date_str, "%A, %B %d, %Y")
+                dict_data = {
+                    "type": bin_type.strip() + " bin",
+                    "collectionDate": bin_date.strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/BasingstokeCouncil.py

@@ -0,0 +1,72 @@
+from bs4 import BeautifulSoup
+from datetime import datetime
+import requests
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+COLLECTION_KINDS = {
+    "waste": "rteelem_ctl03_pnlCollections_Refuse",
+    "recycling": "rteelem_ctl03_pnlCollections_Recycling",
+    "glass": "rteelem_ctl03_pnlCollections_Glass",
+    # Garden waste data is only returned if the property is subscribed to the Garden Waste service
+    "garden": "rteelem_ctl03_pnlCollections_GardenWaste",
+}
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    def parse_data(self, page: str, **kwargs) -> dict:
+        requests.packages.urllib3.disable_warnings()
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        cookies = {
+            "cookie_control_popup": "A",
+            "WhenAreMyBinsCollected": f"{user_uprn}",
+        }
+
+        headers = {
+            "Accept": "*/*",
+            "Accept-Language": "en-GB,en;q=0.9",
+            "Referer": "https://www.basingstoke.gov.uk/",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "cross-site",
+            "Sec-Fetch-User": "?1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.188 Safari/537.36",
+        }
+
+        response = requests.get(
+            "https://www.basingstoke.gov.uk/bincollections",
+            cookies=cookies,
+            headers=headers,
+            verify=False,
+        )
+
+        if response.status_code != 200 or response.text == "0|error|500||":
+            raise SystemError(
+                "Error retrieving data! Please try again or raise an issue on GitHub!"
+            )
+
+        # Make a BS4 object
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+
+        bins = []
+
+        for collection_type, collection_class in COLLECTION_KINDS.items():
+            for date in soup.select(f"div#{collection_class} li"):
+                date_pattern = r"\d{1,2}\s\w+\s\d{4}"  # Regex pattern to extract date
+                match = re.search(date_pattern, date.get_text(strip=True))
+
+                if match:
+                    extracted_date = match.group()
+                    formatted_date = datetime.strptime(
+                        extracted_date, "%d %B %Y"
+                    ).strftime(date_format)
+
+                    bins.append(
+                        {"type": collection_type, "collectionDate": formatted_date}
+                    )
+
+        return {"bins": bins}

uk_bin_collection/uk_bin_collection/councils/BathAndNorthEastSomersetCouncil.py

@@ -0,0 +1,100 @@
+import json
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+import ssl
+import urllib3
+
+
+class CustomHttpAdapter(requests.adapters.HTTPAdapter):
+    """Transport adapter" that allows us to use custom ssl_context."""
+
+    def __init__(self, ssl_context=None, **kwargs):
+        self.ssl_context = ssl_context
+        super().__init__(**kwargs)
+
+    def init_poolmanager(self, connections, maxsize, block=False):
+        self.poolmanager = urllib3.poolmanager.PoolManager(
+            num_pools=connections,
+            maxsize=maxsize,
+            block=block,
+            ssl_context=self.ssl_context,
+        )
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "en-GB,en;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json; charset=utf-8",
+            "Pragma": "no-cache",
+            "Referer": "https://www.bathnes.gov.uk/webforms/waste/collectionday/",
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-origin",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.188 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+        session = requests.Session()
+        ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
+        ctx.options |= 0x4
+        session.mount("https://", CustomHttpAdapter(ctx))
+
+        requests.packages.urllib3.disable_warnings()
+        response = session.get(
+            f"https://www.bathnes.gov.uk/webapi/api/BinsAPI/v2/getbartecroute/{user_uprn}/true",
+            headers=headers,
+        )
+        if response.text == "":
+            raise ValueError(
+                "Error parsing data. Please check the provided UPRN. "
+                "If this error continues please open an issue on GitHub."
+            )
+        json_data = json.loads(response.text)
+
+        data = {"bins": []}
+
+        if len(json_data["residualNextDate"]) > 0:
+            dict_data = {
+                "type": "Black Rubbish Bin",
+                "collectionDate": datetime.strptime(
+                    json_data["residualNextDate"], "%Y-%m-%dT%H:%M:%S"
+                ).strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+        if len(json_data["recyclingNextDate"]) > 0:
+            dict_data = {
+                "type": "Recycling Containers",
+                "collectionDate": datetime.strptime(
+                    json_data["recyclingNextDate"], "%Y-%m-%dT%H:%M:%S"
+                ).strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+        if len(json_data["organicNextDate"]) > 0:
+            dict_data = {
+                "type": "Garden Waste",
+                "collectionDate": datetime.strptime(
+                    json_data["organicNextDate"], "%Y-%m-%dT%H:%M:%S"
+                ).strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        data["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        return data

uk_bin_collection/uk_bin_collection/councils/BedfordBoroughCouncil.py

@@ -0,0 +1,49 @@
+import json
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        api_url = f"https://bbaz-as-prod-bartecapi.azurewebsites.net/api/bincollections/residential/getbyuprn/{user_uprn}/35"
+
+        requests.packages.urllib3.disable_warnings()
+        response = requests.get(api_url)
+
+        if response.status_code != 200:
+            raise ConnectionError("Could not get latest data!")
+
+        json_data = json.loads(response.text)["BinCollections"]
+        data = {"bins": []}
+        collections = []
+
+        for day in json_data:
+            for bin in day:
+                bin_type = bin["BinType"]
+                next_date = datetime.strptime(
+                    bin["JobScheduledStart"], "%Y-%m-%dT%H:%M:%S"
+                )
+                collections.append((bin_type, next_date))
+
+        ordered_data = sorted(collections, key=lambda x: x[1])
+        data = {"bins": []}
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0],
+                "collectionDate": item[1].strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/BedfordshireCouncil.py

@@ -0,0 +1,70 @@
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        user_postcode = kwargs.get("postcode")
+
+        check_uprn(user_uprn)
+        check_postcode(user_postcode)
+
+        # Start a new session to walk through the form
+        requests.packages.urllib3.disable_warnings()
+        s = requests.session()
+
+        headers = {
+            "Origin": "https://www.centralbedfordshire.gov.uk",
+            "Referer": "https://www.centralbedfordshire.gov.uk/info/163/bins_and_waste_collections_-_check_bin_collection_day",
+        }
+
+        files = {
+            "postcode": (None, user_postcode),
+            "address": (None, user_uprn),
+        }
+
+        response = requests.post(
+            "https://www.centralbedfordshire.gov.uk/info/163/bins_and_waste_collections_-_check_bin_collection_day#my_bin_collections",
+            headers=headers,
+            files=files,
+        )
+
+        # Make that BS4 object and use it to prettify the response
+        soup = BeautifulSoup(response.content, features="html.parser")
+        soup.prettify()
+
+        collections_div = soup.find(id="collections")
+
+        # Get the collection items on the page and strip the bits of text that we don't care for
+        collections = []
+        for bin in collections_div.find_all("h3"):
+            next_bin = bin.next_sibling
+
+            while next_bin.name != "h3" and next_bin.name != "p":
+                if next_bin.name != "br":
+                    collection_date = datetime.strptime(bin.text, "%A, %d %B %Y")
+                    collections.append((next_bin, collection_date))
+                next_bin = next_bin.next_sibling
+
+        # Sort the collections by date order rather than bin type, then return as a dictionary (with str date)
+        ordered_data = sorted(collections, key=lambda x: x[1])
+        data = {"bins": []}
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0],
+                "collectionDate": item[1].strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data