uk_bin_collection 0.74.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py

@@ -0,0 +1,137 @@
+import re
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+# This script pulls (in one hit) the data from Bromley Council Bins Data
+import datetime
+from datetime import datetime
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.keys import Keys
+import time
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            data = {"bins": []}
+            headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64)"}
+
+            uprn = kwargs.get("uprn")
+            user_paon = kwargs.get("paon")
+            postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            driver = create_webdriver(web_driver, headless)
+            driver.get(kwargs.get("url"))
+
+            wait = WebDriverWait(driver, 60)
+            post_code_search = wait.until(
+                EC.presence_of_element_located((By.CLASS_NAME, "form-control"))
+            )
+            post_code_search.send_keys(postcode)
+
+            submit_btn = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, f"//button[contains(@class, 'mx-name-actionButton3')]")
+                )
+            )
+
+            submit_btn.send_keys(Keys.ENTER)
+
+            dropdown_options = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, f'//option[contains(text(), "{user_paon}")]')
+                )
+            )
+            parent_element = dropdown_options.find_element(
+                By.XPATH, ".."
+            )  # Using ".." to move up to the parent element
+
+            # Create a 'Select' for it, then select the first address in the list
+            # (Index 0 is "Make a selection from the list")
+            dropdownSelect = Select(parent_element)
+            dropdownSelect.select_by_visible_text(str(user_paon))
+
+            submit_btn = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, f"//button[contains(@class, 'mx-name-actionButton5')]")
+                )
+            )
+
+            submit_btn.send_keys(Keys.ENTER)
+
+            results = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, f'//span[contains(@class,"collection-sub")]')
+                )
+            )
+
+            # Make a BS4 object
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            # Initialize current month and year (you can modify these values based on your requirement)
+            data = {"bins": []}
+
+            current_date = datetime.now()
+
+            # Find all elements with class starting with 'mx-name-index-'
+            bins = soup.find_all(class_=lambda x: x and x.startswith("mx-name-index-"))
+
+            for bin_item in bins:
+                bin_type = bin_item.find(class_="collection-main").text.strip()
+                day_of_week_elements = bin_item.find_all(class_="collection-header")
+                bin_date = None
+
+                for elem in day_of_week_elements:
+                    if (
+                        elem.text.strip() != bin_type
+                    ):  # Avoid taking the bin type as the date
+                        next_sibling = elem.find_next_sibling()
+                        if next_sibling:
+                            bin_date_str = next_sibling.text.strip()
+                            try:
+                                # Try parsing the date string in the format 'dd Month' (e.g., '30 Dec', '5 January')
+                                bin_date = datetime.strptime(bin_date_str, "%d %b")
+                            except ValueError:
+                                try:
+                                    # If the above format fails, try 'dd MonthName' (e.g., '30 December', '5 January')
+                                    bin_date = datetime.strptime(bin_date_str, "%d %B")
+                                except ValueError:
+                                    pass
+
+                            if bin_date:
+                                # Set the year based on the logic provided
+                                if bin_date.month < current_date.month:
+                                    bin_date = bin_date.replace(
+                                        year=current_date.year + 1
+                                    )
+                                else:
+                                    bin_date = bin_date.replace(year=current_date.year)
+                                # Format the date to the desired format
+                                bin_date = bin_date.strftime("%d/%m/%Y")
+                                break
+                dict_data = {"type": bin_type, "collectionDate": bin_date}
+                data["bins"].append(dict_data)
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/BristolCityCouncil.py

@@ -0,0 +1,141 @@
+import ast
+
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        requests.packages.urllib3.disable_warnings()
+        s = requests.session()
+
+        service_type_headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,"
+            "image/webp,image/apng,*/*;q=0.8",
+            "Accept-Language": "en-GB,en;q=0.9",
+            "Cache-Control": "max-age=0",
+            "Connection": "keep-alive",
+            "Referer": "https://www.bristol.gov.uk/",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "cross-site",
+            "Sec-Fetch-User": "?1",
+            "Sec-GPC": "1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, "
+            "like Gecko) Chrome/105.0.0.0 Safari/537.36",
+        }
+        service_type_params = {
+            "servicetypeid": "7dce896c-b3ba-ea11-a812-000d3a7f1cdc",
+        }
+        response = s.get(
+            "https://bristolcouncil.powerappsportals.com/completedynamicformunauth/",
+            params=service_type_params,
+            headers=service_type_headers,
+        )
+
+        llpg_headers = {
+            "Accept": "*/*",
+            "Accept-Language": "en-GB,en;q=0.9",
+            "Connection": "keep-alive",
+            "Ocp-Apim-Subscription-Key": "47ffd667d69c4a858f92fc38dc24b150",
+            "Ocp-Apim-Trace": "true",
+            "Origin": "https://bristolcouncil.powerappsportals.com",
+            "Referer": "https://bristolcouncil.powerappsportals.com/",
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "cross-site",
+            "Sec-GPC": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, "
+            "like Gecko) Chrome/105.0.0.0 Safari/537.36",
+        }
+        llpg_uprn = "UPRN" + user_uprn
+        llpg_json_data = {
+            "Uprn": llpg_uprn,
+        }
+        response = s.post(
+            "https://bcprdapidyna002.azure-api.net/bcprdfundyna001-llpg/DetailedLLPG",
+            headers=llpg_headers,
+            json=llpg_json_data,
+        )
+
+        headers = {
+            "Accept": "*/*",
+            "Accept-Language": "en-GB,en;q=0.9",
+            "Connection": "keep-alive",
+            # Already added when you pass json=
+            # 'Content-Type': 'application/json',
+            "Ocp-Apim-Subscription-Key": "47ffd667d69c4a858f92fc38dc24b150",
+            "Ocp-Apim-Trace": "true",
+            "Origin": "https://bristolcouncil.powerappsportals.com",
+            "Referer": "https://bristolcouncil.powerappsportals.com/",
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "cross-site",
+            "Sec-GPC": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36",
+        }
+        json_data = {
+            "uprn": user_uprn,
+        }
+        response = s.post(
+            "https://bcprdapidyna002.azure-api.net/bcprdfundyna001-alloy/NextCollectionDates",
+            headers=headers,
+            json=json_data,
+        )
+
+        # Make a BS4 object
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+
+        # Soup returns API response rather than HTML, so parse those strings
+        string_data = soup.text.split("data")[1]
+        collection_data = string_data.split("]}")
+
+        # Remove the spare ] and , characters at the of each list element
+        fixed_data = [i[1:] for i in collection_data]
+
+        # Remove the last list element since it's garbage (funny since this is a bin project)
+        fixed_data.pop()
+        collection_data.clear()
+
+        # Make some more changes:
+        idx = 0
+        for i in fixed_data:
+            if idx == 0:
+                # Remove two extra characters if it's the first element
+                i = i[2:]
+            # Append some characters to the end of each line to make to dict
+            i = i + "]}"
+            idx += 1
+            # Reuse the collection_data list to make a list of dictionaries - one for each bin
+            collection_data.append(ast.literal_eval(i))
+
+        collections = []
+        for bin in collection_data:
+            bin_type = bin["containerName"]
+            next_collection = datetime.strptime(
+                bin["collection"][0]["nextCollectionDate"], "%Y-%m-%dT%H:%M:%S"
+            ).strftime(date_format)
+            # Could work out next date using the roundDescription and the is_holiday function in common.py
+            collections.append((bin_type, next_collection))
+
+        ordered_data = sorted(collections, key=lambda x: x[1])
+        data = {"bins": []}
+        for item in ordered_data:
+            dict_data = {"type": item[0], "collectionDate": item[1]}
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py

@@ -0,0 +1,115 @@
+# This script pulls (in one hit) the data from Bromley Council Bins Data
+import datetime
+from bs4 import BeautifulSoup
+from datetime import datetime
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.keys import Keys
+import time
+
+from dateutil.relativedelta import relativedelta
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+        driver = None
+        try:
+            bin_data_dict = {"bins": []}
+            collections = []
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+
+            data = {"bins": []}
+
+            # Get our initial session running
+            driver = create_webdriver(web_driver, headless)
+            driver.get(kwargs.get("url"))
+
+            wait = WebDriverWait(driver, 30)
+            results = wait.until(
+                EC.presence_of_element_located((By.CLASS_NAME, "waste-service-image"))
+            )
+            # Search for the specific bins in the table using BS
+            # Parse the HTML content
+            # Find all elements with the class 'container-name' to extract bin types
+            # Parse the HTML content
+            soup = BeautifulSoup(driver.page_source, "html.parser")
+            soup.prettify
+
+            # Find all elements with class 'govuk-summary-list'
+            bin_info = []
+            waste_services = soup.find_all(
+                "h3", class_="govuk-heading-m waste-service-name"
+            )
+
+            for service in waste_services:
+                service_title = service.get_text(strip=True)
+                next_collection = service.find_next_sibling().find(
+                    "dt", text="Next collection"
+                )
+
+                if next_collection:
+                    next_collection_date = next_collection.find_next_sibling().get_text(
+                        strip=True
+                    )
+                    # Extract date part and remove the suffix
+                    next_collection_date_parse = next_collection_date.split(",")[
+                        1
+                    ].strip()
+                    day = next_collection_date_parse.split()[0]
+                    month = next_collection_date_parse.split()[1]
+
+                    # Remove the suffix (e.g., 'th', 'nd', 'rd', 'st') from the day
+                    if day.endswith(("th", "nd", "rd", "st")):
+                        day = day[:-2]  # Remove the last two characters
+
+                    # Reconstruct the date string without the suffix
+                    date_without_suffix = f"{day} {month}"
+
+                    # Parse the date string to a datetime object
+                    date_object = datetime.strptime(date_without_suffix, "%d %B")
+
+                    # Get the current year
+                    current_year = datetime.now().year
+
+                    # Check if the parsed date is in the past compared to the current date
+                    if date_object < datetime.now():
+                        # If the parsed date is in the past, assume it's for the next year
+                        current_year += 1
+                    # Append the year to the date
+                    date_with_year = date_object.replace(year=current_year)
+
+                    # Format the date with the year
+                    date_with_year_formatted = date_with_year.strftime(
+                        "%d/%m/%Y"
+                    )  # Format the date as '%d/%m/%Y'
+
+                    # Create the dictionary with the formatted data
+                    dict_data = {
+                        "type": service_title,
+                        "collectionDate": date_with_year_formatted,
+                    }
+                    data["bins"].append(dict_data)
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py

@@ -0,0 +1,107 @@
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            page = "https://selfservice.broxtowe.gov.uk/renderform.aspx?t=217&k=9D2EF214E144EE796430597FB475C3892C43C528"
+
+            data = {"bins": []}
+
+            user_uprn = kwargs.get("uprn")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_uprn(user_uprn)
+            check_postcode(user_postcode)
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless)
+            driver.get(page)
+
+            # Populate postcode field
+            inputElement_postcode = driver.find_element(
+                By.ID,
+                "ctl00_ContentPlaceHolder1_FF5683TB",
+            )
+            inputElement_postcode.send_keys(user_postcode)
+
+            # Click search button
+            driver.find_element(
+                By.ID,
+                "ctl00_ContentPlaceHolder1_FF5683BTN",
+            ).click()
+
+            # Wait for the 'Select address' dropdown to appear and select option matching UPRN
+            dropdown = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.ID, "ctl00_ContentPlaceHolder1_FF5683DDL")
+                )
+            )
+            # Create a 'Select' for it, then select the matching URPN option
+            dropdownSelect = Select(dropdown)
+            dropdownSelect.select_by_value("U" + user_uprn)
+
+            # Wait for the submit button to appear, then click it to get the collection dates
+            submit = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.ID, "ctl00_ContentPlaceHolder1_btnSubmit")
+                )
+            )
+            submit.click()
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            bins_div = soup.find("div", id="ctl00_ContentPlaceHolder1_FF5686FormGroup")
+            if bins_div:
+                bins_table = bins_div.find("table")
+                if bins_table:
+                    # Get table rows
+                    for row in bins_table.find_all("tr"):
+                        # Get the rows cells
+                        cells = row.find_all("td")
+                        bin_type = cells[0].get_text(strip=True)
+                        # Skip header row
+                        if bin_type and cells[3] and bin_type != "Bin Type":
+                            if len(cells[3].get_text(strip=True)) > 0:
+                                collection_date = datetime.strptime(
+                                    cells[3].get_text(strip=True), "%A, %d %B %Y"
+                                )
+                                dict_data = {
+                                    "type": bin_type,
+                                    "collectionDate": collection_date.strftime(
+                                        date_format
+                                    ),
+                                }
+                                data["bins"].append(dict_data)
+
+                            data["bins"].sort(
+                                key=lambda x: datetime.strptime(
+                                    x.get("collectionDate"), "%d/%m/%Y"
+                                )
+                            )
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py

@@ -0,0 +1,95 @@
+import pandas as pd
+import time
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support.ui import Select
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def get_data(self, df) -> dict:
+        # Create dictionary of data to be returned
+        data = {"bins": []}
+
+        # Output collection data into dictionary
+        for i, row in df.iterrows():
+            dict_data = {
+                "type": row["Collection Name"],
+                "collectionDate": row["Next Collection Due"],
+            }
+
+            data["bins"].append(dict_data)
+
+        return data
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            page = "https://chiltern.gov.uk/collection-dates"
+
+            # Assign user info
+            user_postcode = kwargs.get("postcode")
+            user_paon = kwargs.get("paon")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless)
+            driver.get(page)
+
+            # Enter postcode in text box and wait
+            inputElement_pc = driver.find_element(
+                By.ID,
+                "COPYOFECHOCOLLECTIONDATES_ADDRESSSELECTION_ADDRESSSELECTIONPOSTCODE",
+            )
+            inputElement_pc.send_keys(user_postcode)
+            inputElement_pc.send_keys(Keys.ENTER)
+
+            time.sleep(4)
+
+            # Select address from dropdown and wait
+            inputElement_ad = Select(
+                driver.find_element(
+                    By.ID,
+                    "COPYOFECHOCOLLECTIONDATES_ADDRESSSELECTION_ADDRESSSELECTIONADDRESS",
+                )
+            )
+
+            inputElement_ad.select_by_visible_text(user_paon)
+
+            time.sleep(4)
+
+            # Submit address information and wait
+            inputElement_bn = driver.find_element(
+                By.ID, "COPYOFECHOCOLLECTIONDATES_ADDRESSSELECTION_NAV1_NEXT"
+            ).click()
+
+            time.sleep(4)
+
+            # Read next collection information into Pandas
+            table = driver.find_element(
+                By.ID, "COPYOFECHOCOLLECTIONDATES_PAGE1_DATES2"
+            ).get_attribute("outerHTML")
+            df = pd.read_html(table, header=[1])
+            df = df[0]
+
+            # Parse data into dict
+            data = self.get_data(df)
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/BuryCouncil.py

@@ -0,0 +1,65 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+        collections = []
+        data = {"bins": []}
+
+        # Get and check postcode and PAON
+        postcode = kwargs.get("postcode")
+        paon = kwargs.get("paon")
+        check_postcode(postcode)
+        check_paon(paon)
+
+        # Make API call to get property info using postcode
+        addr_response = requests.get(
+            f'https://www.bury.gov.uk/app-services/getProperties?postcode={postcode.replace(" ", "")}'
+        )
+        if addr_response.status_code != 200:
+            raise ConnectionAbortedError("Issue encountered getting addresses.")
+        address_json = json.loads(addr_response.text)["response"]
+
+        # This makes addr the next item that has the house number. Since these are ordered by house number, a single
+        # number like 3 wouldn't return 33
+        addr = next(item for item in address_json if paon in item["addressLine1"])
+
+        # Make API call to get bin data using property ID
+        response = requests.get(
+            f'https://www.bury.gov.uk/app-services/getPropertyById?id={addr.get("id")}'
+        )
+        if response.status_code != 200:
+            raise ConnectionAbortedError("Issue encountered getting bin data.")
+        bin_list = json.loads(response.text)["response"]["bins"]
+
+        # The JSON actually returns the next collections and a large calendar. But I opted just for the next dates.
+        for bin_colour, collection_data in bin_list.items():
+            bin_type = bin_colour
+            bin_date = datetime.strptime(
+                remove_ordinal_indicator_from_date_string(
+                    collection_data.get("nextCollection")
+                ),
+                "%A %d %B %Y",
+            )
+            collections.append((bin_type, bin_date))
+
+        # Dates are ordered correctly - soonest first
+        ordered_data = sorted(collections, key=lambda x: x[1])
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0],
+                "collectionDate": item[1].strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data