uk_bin_collection 0.74.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py

@@ -0,0 +1,147 @@
+from bs4 import BeautifulSoup
+from datetime import datetime
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.keys import Keys
+
+import time
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            page = "https://mybexley.bexley.gov.uk/service/When_is_my_collection_day"
+
+            data = {"bins": []}
+
+            user_uprn = kwargs.get("uprn")
+            user_paon = kwargs.get("paon")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless)
+            driver.get(page)
+
+            # If you bang in the house number (or property name) and postcode in the box it should find your property
+
+            iframe_presense = WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located((By.ID, "fillform-frame-1"))
+            )
+
+            driver.switch_to.frame(iframe_presense)
+            wait = WebDriverWait(driver, 60)
+            start_btn = wait.until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, "//button/span[contains(text(), 'Next')]")
+                )
+            )
+
+            start_btn.click()
+
+            inputElement_postcodesearch = wait.until(
+                EC.element_to_be_clickable((By.ID, "postcode_search"))
+            )
+            inputElement_postcodesearch.send_keys(user_postcode)
+
+            find_address_btn = wait.until(
+                EC.element_to_be_clickable((By.XPATH, '//*[@id="search"]'))
+            )
+            find_address_btn.click()
+
+            dropdown_options = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, '//*[@id="select2-chosen-1"]')
+                )
+            )
+            time.sleep(2)
+            dropdown_options.click()
+            time.sleep(1)
+            dropdown_input = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, '//*[@id="s2id_autogen1_search"]')
+                )
+            )
+            time.sleep(1)
+            dropdown_input.send_keys(user_paon)
+            dropdown_input.send_keys(Keys.ENTER)
+
+            results_found = wait.until(
+                EC.presence_of_element_located((By.CLASS_NAME, "found-content"))
+            )
+            finish_btn = wait.until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, "//button/span[contains(text(), 'Next')]")
+                )
+            )
+            finish_btn.click()
+            final_page = wait.until(
+                EC.presence_of_element_located(
+                    (By.CLASS_NAME, "waste-header-container")
+                )
+            )
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            bin_fields = soup.find_all("div", class_="waste-panel-container")
+            # Define your XPath
+
+            for bin in bin_fields:
+                # Extract h3 text from the current element
+                h3_text = (
+                    bin.find("h3", class_="container-name").get_text(strip=True)
+                    if bin.find("h3", class_="container-name")
+                    else None
+                )
+
+                date_text = (
+                    bin.find("p", class_="container-status").get_text(strip=True)
+                    if bin.find("p", class_="container-status")
+                    else None
+                )
+
+                if h3_text and date_text:
+                    # Parse the date using the appropriate format
+                    parsed_date = datetime.strptime(date_text, "%A %d %B")
+
+                    # Assuming the current year is used for the collection date
+                    current_year = datetime.now().year
+
+                    # If the parsed date is in the past, assume it's for the next year
+                    if parsed_date < datetime.now():
+                        current_year += 1
+
+                    data["bins"].append(
+                        {
+                            "type": h3_text,
+                            "collectionDate": parsed_date.replace(
+                                year=current_year
+                            ).strftime("%d/%m/%Y"),
+                        }
+                    )
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/BirminghamCityCouncil.py

@@ -0,0 +1,119 @@
+from dateutil.relativedelta import relativedelta
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+def get_token(page) -> str:
+    """
+    Get a __token to include in the form data
+        :param page: Page html
+        :return: Form __token
+    """
+    soup = BeautifulSoup(page.text, features="html.parser")
+    soup.prettify()
+    token = soup.find("input", {"name": "__token"}).get("value")
+    return token
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def get_data(self, url) -> str:
+        """This method makes the request to the council
+
+        Keyword arguments:
+        url -- the url to get the data from
+        """
+        # Set a user agent so we look like a browser ;-)
+        user_agent = (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/108.0.0.0 Safari/537.36"
+        )
+        headers = {"User-Agent": user_agent}
+        requests.packages.urllib3.disable_warnings()
+
+        # Make the Request - change the URL - find out your property number
+        try:
+            session = requests.Session()
+            session.headers.update(headers)
+            full_page = session.get(url)
+            return full_page
+        except requests.exceptions.HTTPError as errh:
+            _LOGGER.error(f"Http Error: {errh}")
+            raise
+        except requests.exceptions.ConnectionError as errc:
+            _LOGGER.error(f"Error Connecting: {errc}")
+            raise
+        except requests.exceptions.Timeout as errt:
+            _LOGGER.error(f"Timeout Error: {errt}")
+            raise
+        except requests.exceptions.RequestException as err:
+            _LOGGER.error(f"Oops: Something Else {err}")
+            raise
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        uprn = kwargs.get("uprn")
+        postcode = kwargs.get("postcode")
+        check_uprn(uprn)
+        check_postcode(postcode)
+
+        values = {
+            "__token": get_token(page),
+            "page": "491",
+            "locale": "en_GB",
+            "q1f8ccce1d1e2f58649b4069712be6879a839233f_0_0": postcode,
+            "q1f8ccce1d1e2f58649b4069712be6879a839233f_1_0": uprn,
+            "next": "Next",
+        }
+        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64)"}
+        requests.packages.urllib3.disable_warnings()
+        response = requests.request(
+            "POST",
+            "https://www.birmingham.gov.uk/xfp/form/619",
+            headers=headers,
+            data=values,
+        )
+
+        soup = BeautifulSoup(response.text, features="html.parser")
+
+        rows = soup.find("table").find_all("tr")
+
+        # Form a JSON wrapper
+        data = {"bins": []}
+
+        # Loops the Rows
+        for row in rows:
+            cells = row.find_all("td")
+            if cells:
+                bin_type = cells[0].get_text(strip=True)
+                collection_next = cells[1].get_text(strip=True)
+
+                collection_date = re.findall("\(.*?\)", collection_next)
+
+                if len(collection_date) != 1:
+                    continue
+
+                collection_date_obj = parse(
+                    re.sub("[()]", "", collection_date[0])
+                ).date()
+
+                # since we only have the next collection day, if the parsed date is in the past,
+                # assume the day is instead next month
+                if collection_date_obj < datetime.now().date():
+                    collection_date_obj += relativedelta(months=1)
+
+                # Make each Bin element in the JSON
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": collection_date_obj.strftime(date_format),
+                }
+
+                # Add data to the main JSON Wrapper
+                data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py

@@ -0,0 +1,105 @@
+import json
+from collections import OrderedDict
+from datetime import datetime
+from bs4 import BeautifulSoup
+import requests
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+import ssl
+import urllib3
+import logging
+
+
+class CustomHttpAdapter(requests.adapters.HTTPAdapter):
+    """Transport adapter" that allows us to use custom ssl_context."""
+
+    def __init__(self, ssl_context=None, **kwargs):
+        self.ssl_context = ssl_context
+        super().__init__(**kwargs)
+
+    def init_poolmanager(self, connections, maxsize, block=False):
+        self.poolmanager = urllib3.poolmanager.PoolManager(
+            num_pools=connections,
+            maxsize=maxsize,
+            block=block,
+            ssl_context=self.ssl_context,
+        )
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+
+        driver = None
+        try:
+            data = {"bins": []}
+            uprn = kwargs.get("uprn")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            current_month = datetime.today().strftime("%m")
+            current_year = datetime.today().strftime("%Y")
+            url = (
+                f"https://mybins.blackburn.gov.uk/api/mybins/getbincollectiondays?uprn={uprn}&month={current_month}"
+                f"&year={current_year}"
+            )
+            driver = create_webdriver(web_driver, headless)
+            driver.get(url)
+
+            soup = BeautifulSoup(driver.page_source, "html.parser")
+
+            # Find the <pre> tag that contains the JSON data
+            pre_tag = soup.find("pre")
+
+            if pre_tag:
+                # Extract the text content within the <pre> tag
+
+                # Return JSON from response and loop through collections
+                json_result = json.loads(pre_tag.contents[0])
+                bin_collections = json_result["BinCollectionDays"]
+                for collection in bin_collections:
+                    if collection is not None:
+                        bin_type = collection[0].get("BinType")
+                        current_collection_date = datetime.strptime(
+                            collection[0].get("CollectionDate"), "%Y-%m-%d"
+                        )
+                        next_collection_date = datetime.strptime(
+                            collection[0].get("NextScheduledCollectionDate"), "%Y-%m-%d"
+                        )
+
+                        # Work out the most recent collection date to display
+                        if (
+                            datetime.today().date()
+                            <= current_collection_date.date()
+                            < next_collection_date.date()
+                        ):
+                            collection_date = current_collection_date
+                        else:
+                            collection_date = next_collection_date
+
+                        dict_data = {
+                            "type": bin_type,
+                            "collectionDate": collection_date.strftime(date_format),
+                        }
+                        data["bins"].append(dict_data)
+
+                        data["bins"].sort(
+                            key=lambda x: datetime.strptime(
+                                x.get("collectionDate"), date_format
+                            )
+                        )
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py

@@ -0,0 +1,104 @@
+from bs4 import BeautifulSoup
+from datetime import datetime
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.keys import Keys
+
+import time
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            user_uprn = kwargs.get("uprn")
+            check_uprn(user_uprn)
+
+            user_postcode = kwargs.get("postcode")
+            check_postcode(user_postcode)
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+
+            data = {"bins": []}
+
+            # Get our initial session running
+            page = "https://carehomes.bolton.gov.uk/bins.aspx"
+
+            driver = create_webdriver(web_driver, headless)
+            driver.get(page)
+
+            # If you bang in the house number (or property name) and postcode in the box it should find your property
+            wait = WebDriverWait(driver, 30)
+
+            pc_search_box = wait.until(
+                EC.presence_of_element_located((By.ID, "txtPostcode"))
+            )
+
+            pc_search_box.send_keys(user_postcode)
+
+            pcsearch_btn = wait.until(EC.element_to_be_clickable((By.ID, "btnSubmit")))
+
+            pcsearch_btn.click()
+
+            # Wait for the 'Select your property' dropdown to appear and select the first result
+            dropdown = wait.until(EC.element_to_be_clickable((By.ID, "ddlAddresses")))
+
+            dropdown_options = wait.until(
+                EC.presence_of_element_located((By.XPATH, "//select/option[1]"))
+            )
+            time.sleep(1)
+            # Create a 'Select' for it, then select the first address in the list
+            # (Index 0 is "Make a selection from the list")
+            dropdownSelect = Select(dropdown)
+            dropdownSelect.select_by_value(str(user_uprn))
+            dropdown_options = wait.until(
+                EC.presence_of_element_located((By.ID, "pnlStep3"))
+            )
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            soup.prettify()
+
+            collections = []
+
+            # Find section with bins in
+            sections = soup.find_all("div", {"class": "bin-info"})
+
+            # For each bin section, get the text and the list elements
+            for item in sections:
+                words = item.find_next("strong").text.split()[2:4]
+                bin_type = " ".join(words).capitalize()
+                date_list = item.find_all("p")
+                for d in date_list:
+                    next_collection = datetime.strptime(d.text.strip(), "%A %d %B %Y")
+                    collections.append((bin_type, next_collection))
+
+            # Sort the text and list elements by date
+            ordered_data = sorted(collections, key=lambda x: x[1])
+
+            # Put the elements into the dictionary
+            for item in ordered_data:
+                dict_data = {
+                    "type": item[0],
+                    "collectionDate": item[1].strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/BradfordMDC.py

@@ -0,0 +1,103 @@
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        # UPRN is passed in via a cookie. Set cookies/params and GET the page
+        cookies = {
+            "COLLECTIONDATES": f"{user_uprn}",
+        }
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+            "Accept-Language": "en-GB,en;q=0.7",
+            "Cache-Control": "max-age=0",
+            "Connection": "keep-alive",
+            "Referer": "https://onlineforms.bradford.gov.uk/ufs/collectiondates.eb",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "same-origin",
+            "Sec-Fetch-User": "?1",
+            "Sec-GPC": "1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36",
+        }
+        params = {
+            "ebp": "30",
+            "ebd": "0",
+            "ebz": "1_1713270660323",
+        }
+        requests.packages.urllib3.disable_warnings()
+        response = requests.get(
+            "https://onlineforms.bradford.gov.uk/ufs/collectiondates.eb",
+            params=params,
+            headers=headers,
+            cookies=cookies,
+        )
+
+        # Parse response text for super speedy finding
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+
+        # BradfordMDC site has lots of embedded tables, find the table titled 'Your next general/recycling collections are:'
+        for bin in soup.find_all(attrs={"class": "CTID-FHGh1Q77-_"}):
+            if bin.find_all(attrs={"class": "CTID-62bNngCB-_"}):
+                bin_type = "General Waste"
+                bin_colour = "Green"
+                bin_date_text = bin.find(attrs={"class": "CTID-62bNngCB-_"}).get_text()
+            elif bin.find_all(attrs={"class": "CTID-LHo9iO0y-_"}):
+                bin_type = "Recycling Waste"
+                bin_colour = "Grey"
+                bin_date_text = bin.find(attrs={"class": "CTID-LHo9iO0y-_"}).get_text()
+            else:
+                raise ValueError(f"No bin info found in {bin_type_info[0]}")
+
+            # Collection Date info is alongside the bin type, we got the whole line in the if/elif above
+            # below strips the text off at the beginning, to get a date, though recycling is a character shorter hence the lstrip
+            bin_date_info = bin_date_text[29:50].lstrip(" ")
+
+            if contains_date(bin_date_info):
+                bin_date = get_next_occurrence_from_day_month(
+                    datetime.strptime(
+                        bin_date_info,  # + " " + datetime.today().strftime("%Y"),
+                        "%a %b %d %Y",
+                    )
+                ).strftime(date_format)
+                # print(bin_date_info)
+                # print(bin_date)
+            # On exceptional collection schedule (e.g. around English Bank Holidays), date will be contained in the second stripped string
+            else:
+                bin_date = get_next_occurrence_from_day_month(
+                    datetime.strptime(
+                        bin_date_info[1] + " " + datetime.today().strftime("%Y"),
+                        "%a %b %d %Y",
+                    )
+                ).strftime(date_format)
+
+        # Build data dict for each entry
+        dict_data = {
+            "type": bin_type,
+            "collectionDate": bin_date,
+            "colour": bin_colour,
+        }
+        data["bins"].append(dict_data)
+
+        data["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        return data