uk_bin_collection 0.74.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/NewhamCouncil.py

@@ -0,0 +1,58 @@
+import urllib3
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # get the page data
+        request = urllib3.request(method="get", url=kwargs["url"])
+        page_data = request.data
+
+        # Make a BS4 object
+        soup = BeautifulSoup(page_data, features="html.parser")
+        soup.prettify()
+
+        # Form a JSON wrapper
+        data = {"bins": []}
+
+        # Find section with bins in
+        sections = soup.find_all("div", {"class": "card h-100"})
+
+        # there may also be a recycling one too
+        sections_recycling = soup.find_all(
+            "div", {"class": "card h-100 card-recycling"}
+        )
+        if len(sections_recycling) > 0:
+            sections.append(sections_recycling[0])
+
+        # For each bin section, get the text and the list elements
+        for item in sections:
+            header = item.find("div", {"class": "card-header"})
+            bin_type_element = header.find_next("b")
+            if bin_type_element is not None:
+                bin_type = bin_type_element.text
+                array_expected_types = ["Domestic", "Recycling"]
+                if bin_type in array_expected_types:
+                    date = (
+                        item.find_next("p", {"class": "card-text"})
+                        .find_next("mark")
+                        .next_sibling.strip()
+                    )
+                    next_collection = datetime.strptime(date, "%d/%m/%Y")
+
+                    dict_data = {
+                        "type": bin_type,
+                        "collectionDate": next_collection.strftime(date_format),
+                    }
+                    data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/NewportCityCouncil.py

@@ -0,0 +1,203 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_postcode = kwargs.get("postcode")
+        check_postcode(user_postcode)
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/87.0.4280.141 Safari/537.36"
+        }
+
+        requests.packages.urllib3.disable_warnings()
+        with requests.Session() as s:
+            # Set Headers
+            s.headers = headers
+
+            # Get the first page - This is the Search for property by Post Code page
+            resource = s.get(
+                "https://iweb.itouchvision.com/portal/f?p=customer:BIN_DAYS:::NO:RP:UID:6CDD2A34C912312074D8E2410531401A8C00EFF7"
+            )
+            # Create a BeautifulSoup object from the page's HTML
+            soup = BeautifulSoup(resource.text, "html.parser")
+
+            # The page contains a number of values that must be passed into subsequent requests - extract them here
+            payload = {
+                i["name"]: i.get("value", "") for i in soup.select("input[name]")
+            }
+            payload2 = {
+                i["data-for"]: i.get("value", "")
+                for i in soup.select("input[data-for]")
+            }
+            payload_salt = soup.select_one('input[id="pSalt"]').get("value")
+            payload_protected = soup.select_one('input[id="pPageItemsProtected"]').get(
+                "value"
+            )
+
+            # Add the PostCode and 'SEARCH' to the payload
+            payload["p_request"] = "SEARCH"
+            payload["P153_POST_CODE"] = user_postcode
+
+            # Manipulate the lists and build the JSON that must be submitted in further requests - some data is nested
+            merged_list = {**payload, **payload2}
+            new_list = []
+            other_list = {}
+            for key in merged_list.keys():
+                temp_list = {}
+                val = merged_list[key]
+                if key in [
+                    "P153_UPRN",
+                    "P153_TEMP",
+                    "P153_SYSDATE",
+                    "P0_LANGUAGE",
+                    "P153_POST_CODE",
+                ]:
+                    temp_list = {"n": key, "v": val}
+                    new_list.append(temp_list)
+                elif key in [
+                    "p_flow_id",
+                    "p_flow_step_id",
+                    "p_instance",
+                    "p_page_submission_id",
+                    "p_request",
+                    "p_reload_on_submit",
+                ]:
+                    other_list[key] = val
+                else:
+                    temp_list = {"n": key, "v": "", "ck": val}
+                    new_list.append(temp_list)
+
+            json_builder = {
+                "pageItems": {
+                    "itemsToSubmit": new_list,
+                    "protected": payload_protected,
+                    "rowVersion": "",
+                    "formRegionChecksums": [],
+                },
+                "salt": payload_salt,
+            }
+            json_object = json.dumps(json_builder, separators=(",", ":"))
+            other_list["p_json"] = json_object
+
+            # Set Referrer header
+            s.headers.update(
+                {
+                    "referer": "https://iweb.itouchvision.com/portal/f?p=customer:BIN_DAYS:::NO:RP:UID:6CDD2A34C912312074D8E2410531401A8C00EFF7"
+                }
+            )
+
+            # Generate POST including all the JSON we just built
+            s.post(
+                "https://iweb.itouchvision.com/portal/wwv_flow.accept", data=other_list
+            )
+
+            # The second page on the portal would normally allow you to select your property from a dropdown list of
+            # those that are at the postcode entered on the previous page
+            # The required cookies are stored within the session so re-use the session to keep them
+            resource = s.get(
+                "https://iweb.itouchvision.com/portal/itouchvision/r/customer/bin_days"
+            )
+
+            # Create a BeautifulSoup object from the page's HTML
+            soup = BeautifulSoup(resource.text, "html.parser")
+
+            # The page contains a number of values that must be passed into subsequent requests - extract them here
+            payload = {
+                i["name"]: i.get("value", "") for i in soup.select("input[name]")
+            }
+            payload2 = {
+                i["data-for"]: i.get("value", "")
+                for i in soup.select("input[data-for]")
+            }
+            payload_salt = soup.select_one('input[id="pSalt"]').get("value")
+            payload_protected = soup.select_one('input[id="pPageItemsProtected"]').get(
+                "value"
+            )
+
+            # Add the UPRN and 'SUBMIT' to the payload
+            payload["p_request"] = "SUBMIT"
+            payload["P153_UPRN"] = user_uprn
+
+            # Manipulate the lists and build the JSON that must be submitted in further requests - some data is nested
+            merged_list = {**payload, **payload2}
+            new_list = []
+            other_list = {}
+            for key in merged_list.keys():
+                temp_list = {}
+                val = merged_list[key]
+                if key in ["P153_UPRN", "P153_TEMP", "P153_SYSDATE", "P0_LANGUAGE"]:
+                    temp_list = {"n": key, "v": val}
+                    new_list.append(temp_list)
+                elif key in ["P153_ZABY"]:
+                    temp_list = {"n": key, "v": "1", "ck": val}
+                    new_list.append(temp_list)
+                elif key in ["P153_POST_CODE"]:
+                    temp_list = {"n": key, "v": user_postcode, "ck": val}
+                    new_list.append(temp_list)
+                elif key in [
+                    "p_flow_id",
+                    "p_flow_step_id",
+                    "p_instance",
+                    "p_page_submission_id",
+                    "p_request",
+                    "p_reload_on_submit",
+                ]:
+                    other_list[key] = val
+                else:
+                    temp_list = {"n": key, "v": "", "ck": val}
+                    new_list.append(temp_list)
+
+            json_builder = {
+                "pageItems": {
+                    "itemsToSubmit": new_list,
+                    "protected": payload_protected,
+                    "rowVersion": "",
+                    "formRegionChecksums": [],
+                },
+                "salt": payload_salt,
+            }
+
+            json_object = json.dumps(json_builder, separators=(",", ":"))
+            other_list["p_json"] = json_object
+
+            # Generate POST including all the JSON we just built
+            s.post(
+                "https://iweb.itouchvision.com/portal/wwv_flow.accept", data=other_list
+            )
+
+            # The third and final page on the portal shows the detail of the waste collection services
+            # The required cookies are stored within the session so re-use the session to keep them
+            resource = s.get(
+                "https://iweb.itouchvision.com/portal/itouchvision/r/customer/bin_days"
+            )
+
+            # Create a BeautifulSoup object from the page's HTML
+            soup = BeautifulSoup(resource.text, "html.parser")
+            data = {"bins": []}
+
+            # Loop through the items on the page and build a JSON object for ingestion
+            for item in soup.select(".t-MediaList-item"):
+                for value in item.select(".t-MediaList-body"):
+                    dict_data = {
+                        "type": value.select("span")[1].get_text(strip=True).title(),
+                        "collectionDate": datetime.strptime(
+                            value.select(".t-MediaList-desc")[0].get_text(strip=True),
+                            "%A, %d %B, %Y",
+                        ).strftime(date_format),
+                    }
+                    data["bins"].append(dict_data)
+
+            return data

uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py

@@ -0,0 +1,115 @@
+from bs4 import BeautifulSoup
+from datetime import datetime
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            page = (
+                "https://myselfservice.ne-derbyshire.gov.uk/service/Check_your_Bin_Day"
+            )
+
+            data = {"bins": []}
+
+            user_uprn = kwargs.get("uprn")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_uprn(user_uprn)
+            check_postcode(user_postcode)
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless)
+            driver.get(page)
+
+            # If you bang in the house number (or property name) and postcode in the box it should find your property
+
+            iframe_presense = WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located((By.ID, "fillform-frame-1"))
+            )
+
+            driver.switch_to.frame(iframe_presense)
+            wait = WebDriverWait(driver, 60)
+            inputElement_postcodesearch = wait.until(
+                EC.element_to_be_clickable((By.NAME, "postcode_search"))
+            )
+
+            inputElement_postcodesearch.send_keys(str(user_postcode))
+
+            # Wait for the 'Select your property' dropdown to appear and select the first result
+            dropdown = wait.until(EC.element_to_be_clickable((By.NAME, "selAddress")))
+
+            dropdown_options = wait.until(
+                EC.presence_of_element_located((By.CLASS_NAME, "lookup-option"))
+            )
+
+            # Create a 'Select' for it, then select the first address in the list
+            # (Index 0 is "Make a selection from the list")
+            drop_down_values = Select(dropdown)
+            option_element = wait.until(
+                EC.presence_of_element_located(
+                    (By.CSS_SELECTOR, f'option.lookup-option[value="{str(user_uprn)}"]')
+                )
+            )
+
+            drop_down_values.select_by_value(str(user_uprn))
+
+            # Wait for the 'View more' link to appear, then click it to get the full set of dates
+            h3_element = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, "//th[contains(text(), 'Waste Collection')]")
+                )
+            )
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            target_h3 = soup.find("h3", string="Collection Details")
+            tables_after_h3 = target_h3.parent.parent.find_next("table")
+
+            table_rows = tables_after_h3.find_all("tr")
+            for row in table_rows:
+                rowdata = row.find_all("td")
+                if len(rowdata) == 3:
+                    labels = rowdata[0].find_all("label")
+                    # Strip the day (i.e., Monday) out of the collection date string for parsing
+                    if len(labels) >= 2:
+                        date_label = labels[1]
+                        datestring = date_label.text.strip()
+
+                    # Add the bin type and collection date to the 'data' dictionary
+                    data["bins"].append(
+                        {
+                            "type": rowdata[2].text.strip(),
+                            "collectionDate": datetime.strptime(
+                                datestring, "%d/%m/%Y"
+                            ).strftime(
+                                date_format
+                            ),  # Format the date as needed
+                        }
+                    )
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/NorthEastLincs.py

@@ -0,0 +1,53 @@
+import pandas as pd
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import date_format
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    baseclass. They can also override some
+    operations with a default implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+
+        # Get list items that can be seen on page
+        for element in soup.find_all(
+            "li", {"class": "list-group-item p-0 p-3 bin-collection-item"}
+        ):
+            element_text = element.text.strip().split("\n\n")
+            element_text = [x.strip() for x in element_text]
+
+            bin_type = element_text[1]
+            collection_date = pd.Timestamp(element_text[0]).strftime(date_format)
+
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": collection_date,
+            }
+            data["bins"].append(dict_data)
+
+        # Get hidden list items too
+        for element in soup.find_all(
+            "li", {"class": "list-group-item p-0 p-3 bin-collection-item d-none"}
+        ):
+            element_text = element.text.strip().split("\n\n")
+            element_text = [x.strip() for x in element_text]
+
+            bin_type = element_text[1]
+            collection_date = pd.Timestamp(element_text[0]).strftime(date_format)
+
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": collection_date,
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/NorthKestevenDistrictCouncil.py

@@ -0,0 +1,45 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+
+        for bins in soup.find_all(
+            "div", {"class": lambda L: L and L.startswith("bg-")}
+        ):
+            for bin in bins.find_all("p"):
+                if bin.find("strong"):
+                    results = re.search(
+                        "Next Collection:(.*?) on ([A-Za-z]+, \\d\\d? [A-Za-z]+ \\d{4})",
+                        bin.find("strong").get_text(strip=True),
+                    )
+                    if results:
+                        collection_date = datetime.strptime(
+                            results.groups()[1], "%A, %d %B %Y"
+                        )
+                        data["bins"].append(
+                            {
+                                "type": bins.h3.get_text(strip=True),
+                                "collectionDate": collection_date.strftime(date_format),
+                            }
+                        )
+
+        data["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        return data

uk_bin_collection/uk_bin_collection/councils/NorthLanarkshireCouncil.py

@@ -0,0 +1,46 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+        for bins in soup.select("div[class^=waste-type-container]"):
+            bin_type = bins.div.h3.text.strip()
+            collection_date = bins.select("div > p")[0].get_text(strip=True)
+            next_collection_date = bins.select("div > p")[1].get_text(strip=True)
+            if collection_date:
+                # Add collection date
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": datetime.strptime(
+                        collection_date, "%d %B %Y"
+                    ).strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+                # Add next collection date
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": datetime.strptime(
+                        next_collection_date, "%d %B %Y"
+                    ).strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+        data["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        return data

uk_bin_collection/uk_bin_collection/councils/NorthLincolnshireCouncil.py

@@ -0,0 +1,58 @@
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        uprn = kwargs.get("uprn")
+        check_uprn(uprn)
+
+        headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
+            "Connection": "keep-alive",
+            "Host": "m.northlincs.gov.uk",
+            "Origin": "https://www.northlincs.gov.uk",
+            "Referer": "https://www.northlincs.gov.uk/",
+            "sec-ch-ua": '"Chromium";v="110", "Not A(Brand";v="24", "Google Chrome";v="110"',
+            "sec-ch-ua-mobile": "?0",
+            "sec-ch-ua-platform": '"Windows"',
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-site",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36",
+        }
+        requests.packages.urllib3.disable_warnings()
+        response = requests.get(
+            f"https://m.northlincs.gov.uk/bin_collections?no_collections=20&uprn={uprn}",
+            headers=headers,
+        )
+        if response.status_code != 200:
+            raise ValueError("No bin data found for provided UPRN.")
+        json_data = json.loads(response.text.encode().decode("utf-8-sig"))
+
+        data = {"bins": []}
+        for c in json_data["Collections"]:
+            bin_type = c["BinCodeDescription"].strip()
+            if bin_type.lower() != "textiles bag":
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": get_next_occurrence_from_day_month(
+                        datetime.strptime(
+                            c["BinCollectionDate"].replace(" (*)", "").strip()
+                            + " "
+                            + datetime.now().strftime("%Y"),
+                            "%A %d %B %Y",
+                        )
+                    ).strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+        return data