uk_bin_collection 0.74.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py

@@ -0,0 +1,100 @@
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            page = "https://selfserve.derbyshiredales.gov.uk/renderform.aspx?t=103&k=9644C066D2168A4C21BCDA351DA2642526359DFF"
+
+            data = {"bins": []}
+
+            user_uprn = kwargs.get("uprn")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_uprn(user_uprn)
+            check_postcode(user_postcode)
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless)
+            driver.get(page)
+
+            # Populate postcode field
+            inputElement_postcode = driver.find_element(
+                By.ID,
+                "ctl00_ContentPlaceHolder1_FF2924TB",
+            )
+            inputElement_postcode.send_keys(user_postcode)
+
+            # Click search button
+            driver.find_element(
+                By.ID,
+                "ctl00_ContentPlaceHolder1_FF2924BTN",
+            ).click()
+
+            # Wait for the 'Select address' dropdown to appear and select option matching UPRN
+            dropdown = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.ID, "ctl00_ContentPlaceHolder1_FF2924DDL")
+                )
+            )
+            # Create a 'Select' for it, then select the matching URPN option
+            dropdownSelect = Select(dropdown)
+            dropdownSelect.select_by_value("U" + user_uprn)
+
+            # Wait for the submit button to appear, then click it to get the collection dates
+            submit = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.ID, "ctl00_ContentPlaceHolder1_btnSubmit")
+                )
+            )
+            submit.click()
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            bin_rows = (
+                soup.find("div", id="ctl00_ContentPlaceHolder1_pnlConfirmation")
+                .find("div", {"class": "row"})
+                .find_all("div", {"class": "row"})
+            )
+            if bin_rows:
+                for bin_row in bin_rows:
+                    bin_data = bin_row.find_all("div")
+                    if bin_data and bin_data[0] and bin_data[1]:
+                        collection_date = datetime.strptime(
+                            bin_data[0].get_text(strip=True), "%A%d %B, %Y"
+                        )
+                        dict_data = {
+                            "type": bin_data[1].get_text(strip=True),
+                            "collectionDate": collection_date.strftime(date_format),
+                        }
+                        data["bins"].append(dict_data)
+
+            data["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+            )
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/DoncasterCouncil.py

@@ -0,0 +1,77 @@
+import json
+import math
+from datetime import timedelta
+
+import requests
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # I need to point out that this one gave me a good head scratch. Mainly because I wrote lots
+        # of code to parse the form and all that, then realised this url returns json data... oops.
+        base_url = "https://www.doncaster.gov.uk/Compass/PremiseDetail/GetCollectionsForCalendar"
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        # Working with epoch times, otherwise known as posix/unix timestamps. The number of weeks
+        # to return can actually be customised in the below timedelta
+        today = math.floor(datetime.today().timestamp())
+        four_weeks = math.floor((datetime.today() + timedelta(days=4 * 7)).timestamp())
+
+        # For some reason, the actual web form uses a property id that's completely different
+        # from the uprn - luckily this one is easy to find!
+        params = {
+            "UPRN": user_uprn,
+            "Start": str(today),
+            "End": str(four_weeks),
+        }
+
+        requests.packages.urllib3.disable_warnings()
+        response = requests.get(base_url, params=params)
+
+        # 200 = ok. I got a 500 in testing, so assumed no data for that address
+        if response.status_code != 200:
+            raise ValueError("No bins found for provided UPRN.")
+
+        # Load the json results
+        json_results = json.loads(response.text)["slots"]
+
+        data = {"bins": []}
+        collections = []
+
+        # Each item is a dictionary, so accessing is easy
+        for item in json_results:
+            bin_type = item["title"]
+
+            # item["start"] actually returns a string, so we want to only take digits or +s.
+            # OK, we don't actually want the +s... or anything on the end of them, that's why
+            # we split the string then cast the remaining epoch to a float
+            epoch = "".join([i for i in item["start"] if i.isdigit() or i == "+"])
+            epoch = epoch.split("+")[0]
+            epoch = float(epoch)
+            bin_date = datetime.strptime(
+                str(datetime.fromtimestamp(epoch / 1000)), "%Y-%m-%d %H:%M:%S"
+            )
+            collections.append((bin_type, bin_date))
+
+            # This orders the data we just parsed to date order
+            ordered_data = sorted(collections, key=lambda x: x[1])
+            data = {"bins": []}
+            for bin in ordered_data:
+                dict_data = {
+                    "type": bin[0],
+                    "collectionDate": bin[1].strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/DorsetCouncil.py

@@ -0,0 +1,58 @@
+from bs4 import BeautifulSoup, element
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        data = {"bins": []}
+        collections = []
+
+        # Parse the page and find all the result boxes
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+        results = soup.find_all("li", {"class": "resultListItem"})
+
+        # If the result box has a wanted string in, we can use it. Check the contents of each box and find the
+        # desired text and dates
+        for r in results:
+            if "Your next" in r.text:
+                if type(r.contents[10]) is element.NavigableString:
+                    bin_text = r.contents[10].text.split(" ")[2].title() + " bin"
+                    bin_date = datetime.strptime(
+                        remove_ordinal_indicator_from_date_string(
+                            r.contents[11].text.strip()
+                        ),
+                        "%A %d %B %Y",
+                    )
+                else:
+                    bin_text = r.contents[11].text.split(" ")[2].title() + " bin"
+                    bin_date = datetime.strptime(
+                        remove_ordinal_indicator_from_date_string(
+                            r.contents[12].text.strip()
+                        ),
+                        "%A %d %B %Y",
+                    )
+
+                if bin_date.date() >= datetime.now().date():
+                    collections.append((bin_text, bin_date))
+
+                # Sort the text and date elements by date
+                ordered_data = sorted(collections, key=lambda x: x[1])
+
+        # Put the elements into the dictionary
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0],
+                "collectionDate": item[1].strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/DoverDistrictCouncil.py

@@ -0,0 +1,41 @@
+from bs4 import BeautifulSoup
+from datetime import datetime
+import re
+from uk_bin_collection.uk_bin_collection.common import *  # Consider specific imports
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+class CouncilClass(AbstractGetBinDataClass):
+    def parse_data(self, page: str, **kwargs) -> dict:
+        soup = BeautifulSoup(page.text, 'html.parser')
+
+        bins_data = {"bins": []}
+        bin_collections = []
+
+        results_wrapper = soup.find("div", {"class": "results-table-wrapper"})
+        if not results_wrapper:
+            return bins_data  # Return empty if the results wrapper is not found
+
+        bins = results_wrapper.find_all("div", {"class": "service-wrapper"})
+        for bin_item in bins:
+            service_name = bin_item.find("h3", {"class": "service-name"})
+            next_service = bin_item.find("td", {"class": "next-service"})
+
+            if service_name and next_service:
+                bin_type = service_name.get_text().replace("Collection", "bin").strip()
+                date_span = next_service.find("span", {"class": "table-label"})
+                date_text = date_span.next_sibling.get_text().strip() if date_span else None
+
+                if date_text and re.match(r"\d{2}/\d{2}/\d{4}", date_text):
+                    try:
+                        bin_date = datetime.strptime(date_text, "%d/%m/%Y")
+                        bin_collections.append((bin_type, bin_date))
+                    except ValueError:
+                        continue
+
+        for bin_type, bin_date in sorted(bin_collections, key=lambda x: x[1]):
+            bins_data["bins"].append({
+                "type": bin_type.capitalize(),
+                "collectionDate": bin_date.strftime("%d/%m/%Y"),
+            })
+
+        return bins_data

uk_bin_collection/uk_bin_collection/councils/DurhamCouncil.py

@@ -0,0 +1,49 @@
+import re
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        url = "https://www.durham.gov.uk/bincollections?uprn="
+        uprn = kwargs.get("uprn")
+        check_uprn(uprn)
+        url += uprn
+        requests.packages.urllib3.disable_warnings()
+        page = requests.get(url)
+
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+
+        data = {"bins": []}
+
+        for bin_type in ["rubbish", "recycling", "gardenwaste"]:
+            bin_info = soup.find(class_=f"bins{bin_type}")
+
+            if bin_info:
+                collection_text = bin_info.get_text(strip=True)
+
+                if collection_text:
+                    results = re.search("\\d\\d? [A-Za-z]+ \\d{4}", collection_text)
+                    if results:
+                        date = datetime.strptime(results[0], "%d %B %Y")
+                        if date:
+                            data["bins"].append(
+                                {
+                                    "type": bin_type,
+                                    "collectionDate": date.strftime(date_format),
+                                }
+                            )
+
+        return data

uk_bin_collection/uk_bin_collection/councils/EastCambridgeshireCouncil.py

@@ -0,0 +1,44 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        uprn = kwargs.get("uprn")
+        # Check the UPRN is valid
+        check_uprn(uprn)
+
+        # Request URL
+        url = f"https://eastcambs-self.achieveservice.com/appshost/firmstep/self/apps/custompage/bincollections?language=en&uprn={uprn}"
+
+        # Make Request
+        requests.packages.urllib3.disable_warnings()
+        s = requests.session()
+        page = s.get(url)
+
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        # Form a JSON wrapper
+        data = {"bins": []}
+
+        for bins in soup.find_all("div", {"class": "row collectionsrow"}):
+            # Find the collection dates
+            _, bin_type, date = bins.find_all("div")
+            bin_type = bin_type.text
+            date = datetime.strptime(date.text, "%a - %d %b %Y").date()
+
+            data["bins"].append(
+                {"type": bin_type, "collectionDate": date.strftime(date_format)}
+            )
+
+        return data

uk_bin_collection/uk_bin_collection/councils/EastDevonDC.py

@@ -0,0 +1,74 @@
+import re
+from datetime import datetime
+
+import pandas as pd
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import date_format
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    baseclass. They can also override some
+    operations with a default implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+        month_class_name = 'class="eventmonth"'
+        regular_collection_class_name = "collectiondate regular-collection"
+        holiday_collection_class_name = "collectiondate bankholiday-change"
+        regex_string = "[^0-9]"
+
+        calendar_collection = soup.find("ol", {"class": "nonumbers news collections"})
+        calendar_list = calendar_collection.find_all("li")
+        current_month = ""
+        current_year = ""
+
+        for element in calendar_list:
+            element_tag = str(element)
+            if month_class_name in element_tag:
+                current_month = datetime.strptime(element.text, "%B %Y").strftime("%m")
+                current_year = datetime.strptime(element.text, "%B %Y").strftime("%Y")
+            elif regular_collection_class_name in element_tag:
+                week_value = element.find_next(
+                    "span", {"class": f"{regular_collection_class_name}"}
+                )
+                day_of_week = re.sub(regex_string, "", week_value.text).strip()
+                collection_date = datetime(
+                    int(current_year), int(current_month), int(day_of_week)
+                ).strftime(date_format)
+                collections = week_value.find_next_siblings("span")
+                for item in collections:
+                    x = item.text
+                    bin_type = item.text.strip()
+                    if len(bin_type) > 1:
+                        dict_data = {
+                            "type": bin_type,
+                            "collectionDate": collection_date,
+                        }
+                        data["bins"].append(dict_data)
+            elif holiday_collection_class_name in element_tag:
+                week_value = element.find_next(
+                    "span", {"class": f"{holiday_collection_class_name}"}
+                )
+                day_of_week = re.sub(regex_string, "", week_value.text).strip()
+                collection_date = datetime(
+                    int(current_year), int(current_month), int(day_of_week)
+                ).strftime(date_format)
+                collections = week_value.find_next_siblings("span")
+                for item in collections:
+                    x = item.text
+                    bin_type = item.text.strip()
+                    if len(bin_type) > 1:
+                        dict_data = {
+                            "type": bin_type + " (bank holiday replacement)",
+                            "collectionDate": collection_date,
+                        }
+                        data["bins"].append(dict_data)
+        return data

uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py

@@ -0,0 +1,108 @@
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            data = {"bins": []}
+            user_paon = kwargs.get("paon")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_paon(user_paon)
+            check_postcode(user_postcode)
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless)
+            driver.get(
+                "https://www.e-lindsey.gov.uk/article/6714/Your-Waste-Collection-Days"
+            )
+
+            # Wait for the postcode field to appear then populate it
+            inputElement_postcode = WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located(
+                    (By.ID, "WASTECOLLECTIONDAYS202324_LOOKUP_ADDRESSLOOKUPPOSTCODE")
+                )
+            )
+            inputElement_postcode.send_keys(user_postcode)
+
+            # Click search button
+            findAddress = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.ID, "WASTECOLLECTIONDAYS202324_LOOKUP_ADDRESSLOOKUPSEARCH")
+                )
+            )
+            findAddress.click()
+
+            # Wait for the 'Select address' dropdown to appear and select option matching the house name/number
+            WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (
+                        By.XPATH,
+                        "//select[@id='WASTECOLLECTIONDAYS202324_LOOKUP_ADDRESSLOOKUPADDRESS']//option[contains(., '"
+                        + user_paon
+                        + "')]",
+                    )
+                )
+            ).click()
+
+            # Wait for the submit button to appear, then click it to get the collection dates
+            submit = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.ID, "WASTECOLLECTIONDAYS202324_LOOKUP_FIELD2_NEXT")
+                )
+            )
+            submit.click()
+
+            # Wait for the collections table to appear
+            WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.CSS_SELECTOR, ".waste-results"))
+            )
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            # Get collections
+            for collection in soup.find_all("div", {"class": "waste-result"}):
+                ptags = collection.find_all("p")
+                dict_data = {
+                    "type": collection.find("h3").get_text(strip=True),
+                    "collectionDate": datetime.strptime(
+                        remove_ordinal_indicator_from_date_string(
+                            ptags[1]
+                            .get_text()
+                            .replace("The date of your next collection is", "")
+                            .replace(".", "")
+                            .strip()
+                        ),
+                        "%A %d %B %Y",
+                    ).strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+            data["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+            )
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data