uk_bin_collection 0.74.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/MertonCouncil.py

@@ -0,0 +1,58 @@
+# This script pulls (in one hit) the data from Merton Council Bins Data
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+        collections = []
+
+        # Search for the specific bin in the table using BS4
+        rows = soup.find("table", class_=("collectiondays")).find_all(
+            "tr",
+            class_=(
+                "food-caddy",
+                "papercard-wheelie",
+                "plastics-boxes",
+                "rubbish-wheelie",
+                "textiles",
+                "batteries",
+            ),
+        )
+
+        # Loops the Rows
+        for row in rows:
+            # Get all the cells
+            cells = row.find_all("td")
+            # First cell is the bin_type
+            bin_type = cells[0].get_text().strip()
+            # Date is on the second cell, second paragraph, wrapped in p
+            collectionDate = datetime.strptime(
+                cells[1].select("p > b")[2].get_text(strip=True), "%d %B %Y"
+            )
+
+            # Add each collection to the list as a tuple
+            collections.append((bin_type, collectionDate))
+
+        ordered_data = sorted(collections, key=lambda x: x[1])
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0].capitalize(),
+                "collectionDate": item[1].strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/MidAndEastAntrimBoroughCouncil.py

@@ -0,0 +1,128 @@
+from bs4 import BeautifulSoup
+import time
+from dateutil.relativedelta import relativedelta
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support.ui import Select
+from selenium.common.exceptions import NoSuchElementException
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            page = "https://www.midandeastantrim.gov.uk/resident/waste-recycling/collection-dates/"
+
+            # Assign user info
+            user_postcode = kwargs.get("postcode")
+            # not used: user_paon = kwargs.get("paon")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+
+            # Create Selenium webdriver
+            options = webdriver.ChromeOptions()
+            options.add_experimental_option("excludeSwitches", ["enable-logging"])
+            driver = create_webdriver(web_driver, headless)
+
+            driver.get(page)
+
+            time.sleep(5)
+            number = 0
+            driver.switch_to.frame(number)
+            # Enter postcode in text box and wait
+            inputElement_pc = driver.find_element(By.ID, "txtAjaxSearch")
+            inputElement_pc.send_keys(user_postcode)
+
+            time.sleep(5)
+
+            # Submit address information and wait - selecting the top one only
+            # if it is an exact match then it will go straight to the results
+            try:
+                button = driver.find_element(By.XPATH, '//*[@id="show-button-0"]')
+                driver.execute_script("arguments[0].click();", button)
+            except NoSuchElementException:
+                pass
+
+            time.sleep(4)
+
+            # Read next collection information
+            page = driver.find_element(By.ID, "divCalendarGraphics").get_attribute(
+                "outerHTML"
+            )
+
+            # Make a BS4 object - remove bold tags and add @ so we can split the lines later
+            soup = BeautifulSoup(
+                page.strip()
+                .replace("<b>", "")
+                .replace("</b>", "")
+                .replace("<br>", "@"),
+                features="html.parser",
+            )
+            soup.prettify()
+
+            # Data to return
+            data = {"bins": []}
+
+            # Valid bin types
+            binTypes = ["Refuse", "Garden"]
+
+            # Value to create dict for bin values
+            keys, values = [], []
+
+            # Loop though html for text containing bins
+            # example of html (bold tags removed above)
+            # <div id="divCalendarGraphics">
+            # <br>  <b>Refuse</b>: Tue 14 Nov then every alternate  Tue<br><b>Recycling</b>: No Recycling waste collection for this address<br><b>Garden</b>: Tue 21 Nov then every alternate  Tue<br><img src="img/Gif-Spacer.gif" alt="spacer" height="1" width="30">
+            # split by br tag and take first 4 splits
+            lines = soup.text.split("@", 4)
+            for line in lines[1:4]:
+                keys.append(line.split(":")[0].strip())
+                # strip out the day and month from the text
+                values.append(line.split(":")[1].strip().split(" ")[:3])
+
+            # Create dict for bin name and string dates
+            binDict = dict(zip(keys, values))
+
+            # Process dict for valid bin types
+            for bin in list(binDict):
+                if bin in binTypes:
+                    # Convert date - no year value so take it from todays date
+                    if binDict[bin][0] == "Tomorrow":
+                        date = datetime.today() + relativedelta(days=1)
+                    elif binDict[bin][0] == "Today":
+                        date = datetime.today()
+                    else:
+                        date = datetime.strptime(
+                            " ".join(binDict[bin][1:]), "%d %b"
+                        ).replace(year=datetime.today().year)
+                        # if the date is in the past then it means the collection is next year so add a year
+                        if date < datetime.today():
+                            date = date + relativedelta(years=1)
+
+                    # Set bin data
+                    dict_data = {
+                        "type": bin,
+                        "collectionDate": date.strftime(date_format),
+                    }
+                    data["bins"].append(dict_data)
+
+        # Quit Selenium webdriver to release session
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/MidSussexDistrictCouncil.py

@@ -0,0 +1,80 @@
+import re
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+def get_token(res) -> str:
+    """
+    Get a UFPRT code for the form data to be processed
+        :param res:
+        :return:
+    """
+    soup = BeautifulSoup(res, features="html.parser")
+    soup.prettify()
+    token = soup.find("input", {"name": "ufprt"}).get("value")
+    return token
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        api_url = "https://www.midsussex.gov.uk/waste-recycling/bin-collection/"
+        user_postcode = kwargs.get("postcode")
+        user_paon = kwargs.get("paon")
+        postcode_re = "^([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})$"
+        user_full_addr = f"{user_paon} {user_postcode}"
+
+        check_postcode(user_postcode)
+        check_paon(user_paon)
+
+        form_data = {
+            "PostCodeStep.strAddressSearch": user_postcode,
+            "AddressStep.strAddressSelect": user_full_addr,
+            "Next": "true",
+            "StepIndex": "1",
+        }
+
+        # Get a ufprt by posting here (I have no idea how ufprt works, so may as well grab one from the server)
+        requests.packages.urllib3.disable_warnings()
+        init = requests.post(api_url, data=form_data)
+        ufprt = get_token(init.text)
+        form_data.update({"ufprt": ufprt})
+
+        response = requests.post(api_url, data=form_data)
+
+        # Make a BS4 object
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+
+        table_element = soup.find("table", {"class": "collDates"})
+        table_rows = table_element.find_all_next("tr")
+
+        row_index = 0
+        for row in table_rows:
+            if row_index < 1:
+                row_index += 1
+                continue
+            else:
+                details = row.find_all_next("td")
+                dict_data = {
+                    "type": details[1].get_text().replace("collection", "").strip(),
+                    "collectionDate": datetime.strptime(
+                        details[2].get_text(), "%A %d %B %Y"
+                    ).strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+                row_index += 1
+
+        return data

uk_bin_collection/uk_bin_collection/councils/MiltonKeynesCityCouncil.py

@@ -0,0 +1,54 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        data = {"bins": []}
+
+        # Get the estate from the UPRN field
+        estate = kwargs.get("uprn")
+
+        # Parse the council's website
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        # Get a list of lists of estates and their collection days, then check for a match on estate name
+        collection_days = [
+            item.text.strip().replace("\xa0", " ").split(" - ")
+            for item in soup.find(
+                "div",
+                {
+                    "class": "field field--name-localgov-paragraphs field--type-entity-reference-revisions field--label-hidden field__items"
+                },
+            ).find_all("li")
+        ]
+        result = [
+            result for result in collection_days if result[0].lower() == estate.lower()
+        ]
+
+        # If there is a match, we can process it by getting the next eight dates for that day. Else, raise an exception.
+        if result is not None:
+            day_number = days_of_week.get(result[0][1].split()[0])
+            collection_dates = get_weekday_dates_in_period(
+                datetime.now(), day_number, 8
+            )
+
+            for date in collection_dates:
+                dict_data = {
+                    "type": f"Weekly collection",
+                    "collectionDate": date,
+                }
+                data["bins"].append(dict_data)
+        else:
+            raise ValueError("Estate not found on website.")
+
+        return data

uk_bin_collection/uk_bin_collection/councils/MoleValleyDistrictCouncil.py

@@ -0,0 +1,98 @@
+from bs4 import BeautifulSoup
+from datetime import datetime
+import re
+import requests
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_postcode = kwargs.get("postcode")
+        check_postcode(user_postcode)
+
+        root_url = "https://molevalley.cloudmappin.com/my-mv-address-search/search/{}/0".format(
+            user_postcode
+        )
+        response = requests.get(root_url)
+
+        if not response.ok:
+            raise ValueError("Invalid server response code retreiving data.")
+
+        jsonData = response.json()
+
+        if len(jsonData["results"]) == 0:
+            raise ValueError("No collection data found for postcode provided.")
+
+        properties_found = jsonData["results"][0]["items"]
+
+        # If UPRN is provided, we can check a specific address.
+        html_data = None
+        uprn = kwargs.get("uprn")
+        if uprn:
+            check_uprn(uprn)
+            for n, item in enumerate(properties_found):
+                if uprn == str(int(item["info"][0][1]["value"])):
+                    html_data = properties_found[n]["info"][2][1]["value"]
+                    break
+            if html_data is None:
+                raise ValueError("No collection data found for UPRN provided.")
+        else:
+            # If UPRN not provided, just use the first result
+            html_data = properties_found[0]["info"][2][1]["value"]
+
+        soup = BeautifulSoup(html_data, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+        all_collection_dates = []
+        regex_date = re.compile(r".* ([\d]+\/[\d]+\/[\d]+)")
+        regex_additional_collection = re.compile(r"We also collect (.*) on (.*) -")
+
+        # Search for the 'Bins and Recycling' panel
+        for panel in soup.select('div[class*="panel"]'):
+            if panel.h2.text.strip() == "Bins and Recycling":
+
+                # Gather the bin types and dates
+                for collection in panel.select("div > strong"):
+                    bin_type = collection.text.strip()
+                    collection_string = collection.find_next("p").text.strip()
+                    m = regex_date.match(collection_string)
+                    if m:
+                        collection_date = datetime.strptime(
+                            m.group(1), "%d/%m/%Y"
+                        ).date()
+                        data["bins"].append(
+                            {
+                                "type": bin_type,
+                                "collectionDate": collection_date.strftime("%d/%m/%Y"),
+                            }
+                        )
+                        all_collection_dates.append(collection_date)
+
+                # Search for additional collections
+                for p in panel.select("p"):
+                    m2 = regex_additional_collection.match(p.text.strip())
+                    if m2:
+                        bin_type = m2.group(1)
+                        if "each collection day" in m2.group(2):
+                            collection_date = min(all_collection_dates)
+                            data["bins"].append(
+                                {
+                                    "type": bin_type,
+                                    "collectionDate": collection_date.strftime(
+                                        "%d/%m/%Y"
+                                    ),
+                                }
+                            )
+                break
+
+        return data

uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py

@@ -0,0 +1,139 @@
+import time
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            data = {"bins": []}
+            user_uprn = kwargs.get("uprn")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_uprn(user_uprn)
+            check_postcode(user_postcode)
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless)
+            driver.get("https://www.npt.gov.uk/2195")
+
+            # Accept cookies banner
+            cookieAccept = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.ID, "ccc-notify-accept"))
+            )
+            cookieAccept.click()
+
+            # Populate postcode field
+            inputElement_postcode = driver.find_element(
+                By.ID,
+                "ContentPlaceHolderDefault_ctl13_nptLLPG2_25_addresslookup_txtTmpPostcode",
+            )
+            inputElement_postcode.send_keys(user_postcode)
+
+            # Click search button
+            findAddress = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (
+                        By.ID,
+                        "ContentPlaceHolderDefault_ctl13_nptLLPG2_25_addresslookup_btnFindAddress",
+                    )
+                )
+            )
+            findAddress.click()
+
+            time.sleep(1)
+
+            # Wait for the 'Select address' dropdown to appear and select option matching UPRN
+            dropdown = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (
+                        By.ID,
+                        "ContentPlaceHolderDefault_ctl13_nptLLPG2_25_addresslookup_ddlAddressLookup",
+                    )
+                )
+            )
+            # Create a 'Select' for it, then select the matching URPN option
+            dropdownSelect = Select(dropdown)
+            dropdownSelect.select_by_value(user_uprn)
+
+            # Remove back to top button if exists
+            driver.execute_script(
+                """
+            if (document.contains(document.querySelector(".backtotop"))) {
+                document.querySelector(".backtotop").remove();
+            }
+            """
+            )
+
+            # Wait for the submit button to appear, then click it to get the collection dates
+            submit = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.ID, "ContentPlaceHolderDefault_ctl13_nptLLPG2_25_btnDisplay")
+                )
+            )
+            submit.click()
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            # Get the property details
+            property_details = soup.find(
+                "div",
+                {
+                    "id": "ContentPlaceHolderDefault_ctl13_nptLLPG2_25_divPropertyDetails"
+                },
+            )
+
+            # Get the dates
+            for date in property_details.find_all("h2"):
+                if date.get_text(strip=True) != "Bank Holidays":
+                    bin_date = datetime.strptime(
+                        date.get_text(strip=True).replace("&nbsp", " ")
+                        + " "
+                        + datetime.now().strftime("%Y"),
+                        "%A, %d %B %Y",
+                    )
+                    bin_types_wrapper = date.find_next_sibling("div")
+                    for bin_type_wrapper in bin_types_wrapper.find_all(
+                        "div", {"class": "card"}
+                    ):
+                        if bin_date and bin_type_wrapper:
+                            bin_type = bin_type_wrapper.find("a").get_text(strip=True)
+                            bin_type += (
+                                " ("
+                                + bin_type_wrapper.find("span").get_text(strip=True)
+                                + ")"
+                            )
+                            dict_data = {
+                                "type": bin_type,
+                                "collectionDate": bin_date.strftime(date_format),
+                            }
+                            data["bins"].append(dict_data)
+
+            data["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+            )
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/NewarkAndSherwoodDC.py

@@ -0,0 +1,52 @@
+from datetime import timedelta
+
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Get page with BS4
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        # Work out some date bounds
+        today = datetime.today()
+        eight_weeks = datetime.today() + timedelta(days=8 * 7)
+        data = {"bins": []}
+
+        # Each month calendar is a table, so get the object then find all rows in that object.
+        # Month and year is also a row and not included in the date, so save it then remove the row
+        for month in soup.select('table[class*="table table-condensed"]'):
+            info = month.find_all("tr")
+            month_year = info[0].text.strip()
+            info.pop(0)
+            # Each remaining item is a bin collection, so get the type and tidy up the date.
+            for item in info:
+                bin_type = item.text.split(",")[0].strip()
+                bin_date = datetime.strptime(
+                    remove_ordinal_indicator_from_date_string(
+                        item.text.split(",")[1].strip() + " " + month_year
+                    ),
+                    "%A %d %B %Y",
+                )
+                # Only include dates on or after today, but also only within eight weeks
+                if (
+                    today.date() <= bin_date.date() <= eight_weeks.date()
+                    and "cancelled" not in bin_type
+                ):
+                    dict_data = {
+                        "type": bin_type,
+                        "collectionDate": bin_date.strftime(date_format),
+                    }
+                    data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/NewcastleCityCouncil.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+# This script pulls (in one hit) the data from
+# Newcastle City Council Bins Data
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import date_format
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page, **kwargs) -> None:
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+
+        for element in soup.find_all("strong"):
+            collectionInfo = ""
+            # Domestic Waste is formatted differently to other bins
+            if "Green Bin (Domestic Waste) details:" in str(element):
+                if element.next_sibling.find("br"):
+                    collectionInfo = element.next_sibling.find("br").next_element
+            elif "Next collection" in str(
+                element.next_sibling.next_sibling.next_sibling.next_sibling
+            ):
+                collectionInfo = (
+                    element.next_sibling.next_sibling.next_sibling.next_sibling
+                )
+
+            if collectionInfo != "" and collectionInfo != "Next collection : n/a":
+                bin_type = str(element)[
+                    str(element).find("(") + 1 : str(element).find(")")
+                ]
+                collectionDate = str(
+                    datetime.strptime(
+                        str(collectionInfo).replace("Next collection : ", ""),
+                        "%d-%b-%Y",
+                    )
+                    .date()
+                    .strftime(date_format)
+                )
+
+                dict_data = {"type": bin_type, "collectionDate": collectionDate}
+
+                data["bins"].append(dict_data)
+
+        return data