uk_bin_collection 0.74.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/SalfordCityCouncil.py

@@ -0,0 +1,70 @@
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        api_url = "https://www.salford.gov.uk/bins-and-recycling/bin-collection-days/your-bin-collections"
+        user_uprn = kwargs.get("uprn")
+
+        # Check the UPRN is valid
+        check_uprn(user_uprn)
+
+        # Create the form data
+        params = {
+            "UPRN": user_uprn,
+        }
+
+        # Make a request to the API
+        requests.packages.urllib3.disable_warnings()
+        response = requests.get(api_url, params=params)
+
+        # Make a BS4 object
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+
+        # Get the div element
+        div_element = soup.find("div", {"class": "wastefurther"})
+
+        # Get the bins
+        bin_lists = div_element.find_all("ul")
+
+        # Loop through each <ul> tag to extract the bin information
+        for i, bin_list in enumerate(bin_lists):
+            # Find the <p> tag containing the bin type string
+            bin_type = bin_list.find_previous_sibling("p").find("strong").text.strip()
+
+            # Loop through each <li> tag in the <ul> tag to extract the collection date
+            for li in bin_list.find_all("li"):
+                # Convert the collection time to a datetime object
+                collection_date = datetime.strptime(li.text, "%A %d %B %Y")
+
+                # Add the bin to the data dict
+                data["bins"].append(
+                    {
+                        # remove the ":" from the end of the bin type
+                        "type": bin_type[:-1],
+                        "collectionDate": collection_date,
+                    }
+                )
+
+        # Sort the bins by collection time
+        data["bins"] = sorted(data["bins"], key=lambda x: x["collectionDate"])
+
+        # Convert the datetime objects to strings in the desired format
+        for bin in data["bins"]:
+            bin["collectionDate"] = bin["collectionDate"].strftime(date_format)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/SevenoaksDistrictCouncil.py

@@ -0,0 +1,106 @@
+import time
+from typing import Any
+
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
+from selenium.webdriver.support.ui import Select, WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from dateutil.parser import parse
+
+from uk_bin_collection.uk_bin_collection.common import create_webdriver, date_format
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    def wait_for_element_conditions(self, driver, conditions, timeout: int = 5):
+        try:
+            WebDriverWait(driver, timeout).until(conditions)
+        except TimeoutException:
+            print("Timed out waiting for page to load")
+            raise
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            page = "https://sevenoaks-dc-host01.oncreate.app/w/webpage/waste-collection-day"
+
+            # Assign user info
+            user_postcode = kwargs.get("postcode")
+            user_paon = kwargs.get("paon")
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless)
+            driver.get(page)
+
+            # Enter postcode
+            postcode_css_selector = "#address_search_postcode"
+            self.wait_for_element_conditions(
+                driver,
+                EC.presence_of_element_located(
+                    (By.CSS_SELECTOR, postcode_css_selector)
+                ),
+            )
+            postcode_input_box = driver.find_element(
+                By.CSS_SELECTOR, postcode_css_selector
+            )
+            postcode_input_box.send_keys(user_postcode)
+            postcode_input_box.send_keys(Keys.ENTER)
+
+            # Select the dropdown
+            self.wait_for_element_conditions(
+                driver, EC.presence_of_element_located((By.XPATH, "//select/option[2]"))
+            )
+            select_address_dropdown = Select(driver.find_element(By.XPATH, "//select"))
+
+            if user_paon is not None:
+                for option in select_address_dropdown.options:
+                    if user_paon in option.text:
+                        select_address_dropdown.select_by_visible_text(option.text)
+                        break
+            else:
+                # If we've not been supplied an address, pick the second entry
+                select_address_dropdown.select_by_index(1)
+
+            # Grab the response blocks
+            response_xpath_selector = "//div[@data-class_name]//h4/../../../.."
+            self.wait_for_element_conditions(
+                driver,
+                EC.presence_of_element_located((By.XPATH, response_xpath_selector)),
+            )
+            elements = driver.find_elements(By.XPATH, response_xpath_selector)
+
+            # Iterate through them
+            data = {"bins": []}
+            for element in elements:
+                try:
+                    raw_bin_name = element.find_element(By.TAG_NAME, "h4").text
+                    raw_next_collection_date = element.find_elements(
+                        By.XPATH, ".//div[input]"
+                    )[1].text
+
+                    parsed_bin_date = parse(
+                        raw_next_collection_date, fuzzy_with_tokens=True
+                    )[0]
+
+                    dict_data = {
+                        "type": raw_bin_name,
+                        "collectionDate": parsed_bin_date.strftime(date_format),
+                    }
+
+                    data["bins"].append(dict_data)
+
+                except (IndexError, NoSuchElementException):
+                    print("Error finding element for bin")
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/SheffieldCityCouncil.py

@@ -0,0 +1,54 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        # Form a JSON wrapper
+        data = {"bins": []}
+
+        # Search for the specific table using BS4
+        rows = soup.find("table", {"class": re.compile("table")}).find_all("tr")
+
+        # Loops the Rows
+        for row in rows:
+            cells = row.find_all(
+                "td", {"class": lambda L: L and L.startswith("service-name")}
+            )
+
+            if len(cells) > 0:
+                collectionDatesRawData = row.find_all(
+                    "td", {"class": lambda L: L and L.startswith("next-service")}
+                )[0].get_text(strip=True)
+                collectionDate = collectionDatesRawData[
+                    16 : len(collectionDatesRawData)
+                ].split(",")
+                bin_type = row.find_all(
+                    "td", {"class": lambda L: L and L.startswith("service-name")}
+                )[0].h4.get_text(strip=True)
+
+                for collectDate in collectionDate:
+                    # Make each Bin element in the JSON
+                    dict_data = {
+                        "type": bin_type,
+                        "collectionDate": datetime.strptime(
+                            collectDate.strip(), "%d %b %Y"
+                        ).strftime(date_format),
+                    }
+
+                    # Add data to the main JSON Wrapper
+                    data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/ShropshireCouncil.py

@@ -0,0 +1,45 @@
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        # Form a JSON wrapper
+        data = {"bins": []}
+
+        # Find section with bins in
+        sections = (
+            soup.find("div", {"class": "container results-table-wrapper"})
+            .find("tbody")
+            .find_all("tr")
+        )
+
+        # For each bin section, get the text and the list elements
+        for item in sections:
+            words = item.find_next("a").text.split()[1:2]
+            bin_type = " ".join(words).capitalize()
+            date = (
+                item.find("td", {"class": "next-service"})
+                .find_next("span")
+                .next_sibling.strip()
+            )
+            next_collection = datetime.strptime(date, "%d/%m/%Y")
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": next_collection.strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/SolihullCouncil.py

@@ -0,0 +1,48 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+        collections = []
+
+        for bin in soup.find_all("div", class_="mb-4 card"):
+            bin_type = (
+                bin.find("div", class_="card-title")
+                .find("h4")
+                .get_text()
+                .strip()
+                .replace("Wheelie ", "")
+            )
+            bin_date_text = (
+                bin.find_all("div", class_="mt-1")[1]
+                .find("strong")
+                .get_text()
+                .strip()
+                .replace(",", "")
+            )
+            bin_date = datetime.strptime(bin_date_text, "%A %d %B %Y")
+            collections.append((bin_type, bin_date))
+
+        ordered_data = sorted(collections, key=lambda x: x[1])
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0].capitalize(),
+                "collectionDate": item[1].strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/SomersetCouncil.py

@@ -0,0 +1,203 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_postcode = kwargs.get("postcode")
+        check_postcode(user_postcode)
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/87.0.4280.141 Safari/537.36"
+        }
+
+        requests.packages.urllib3.disable_warnings()
+        with requests.Session() as s:
+            # Set Headers
+            s.headers = headers
+
+            # Get the first page - This is the Search for property by Post Code page
+            resource = s.get(
+                "https://iweb.itouchvision.com/portal/f?p=customer:BIN_DAYS:::NO:RP:UID:625C791B4D9301137723E9095361401AE8C03934"
+            )
+            # Create a BeautifulSoup object from the page's HTML
+            soup = BeautifulSoup(resource.text, "html.parser")
+
+            # The page contains a number of values that must be passed into subsequent requests - extract them here
+            payload = {
+                i["name"]: i.get("value", "") for i in soup.select("input[name]")
+            }
+            payload2 = {
+                i["data-for"]: i.get("value", "")
+                for i in soup.select("input[data-for]")
+            }
+            payload_salt = soup.select_one('input[id="pSalt"]').get("value")
+            payload_protected = soup.select_one('input[id="pPageItemsProtected"]').get(
+                "value"
+            )
+
+            # Add the PostCode and 'SEARCH' to the payload
+            payload["p_request"] = "SEARCH"
+            payload["P153_POST_CODE"] = user_postcode
+
+            # Manipulate the lists and build the JSON that must be submitted in further requests - some data is nested
+            merged_list = {**payload, **payload2}
+            new_list = []
+            other_list = {}
+            for key in merged_list.keys():
+                temp_list = {}
+                val = merged_list[key]
+                if key in [
+                    "P153_UPRN",
+                    "P153_TEMP",
+                    "P153_SYSDATE",
+                    "P0_LANGUAGE",
+                    "P153_POST_CODE",
+                ]:
+                    temp_list = {"n": key, "v": val}
+                    new_list.append(temp_list)
+                elif key in [
+                    "p_flow_id",
+                    "p_flow_step_id",
+                    "p_instance",
+                    "p_page_submission_id",
+                    "p_request",
+                    "p_reload_on_submit",
+                ]:
+                    other_list[key] = val
+                else:
+                    temp_list = {"n": key, "v": "", "ck": val}
+                    new_list.append(temp_list)
+
+            json_builder = {
+                "pageItems": {
+                    "itemsToSubmit": new_list,
+                    "protected": payload_protected,
+                    "rowVersion": "",
+                    "formRegionChecksums": [],
+                },
+                "salt": payload_salt,
+            }
+            json_object = json.dumps(json_builder, separators=(",", ":"))
+            other_list["p_json"] = json_object
+
+            # Set Referrer header
+            s.headers.update(
+                {
+                    "referer": "https://iweb.itouchvision.com/portal/f?p=customer:BIN_DAYS:::NO:RP:UID:625C791B4D9301137723E9095361401AE8C03934"
+                }
+            )
+
+            # Generate POST including all the JSON we just built
+            s.post(
+                "https://iweb.itouchvision.com/portal/wwv_flow.accept", data=other_list
+            )
+
+            # The second page on the portal would normally allow you to select your property from a dropdown list of
+            # those that are at the postcode entered on the previous page
+            # The required cookies are stored within the session so re-use the session to keep them
+            resource = s.get(
+                "https://iweb.itouchvision.com/portal/itouchvision/r/customer/bin_days"
+            )
+
+            # Create a BeautifulSoup object from the page's HTML
+            soup = BeautifulSoup(resource.text, "html.parser")
+
+            # The page contains a number of values that must be passed into subsequent requests - extract them here
+            payload = {
+                i["name"]: i.get("value", "") for i in soup.select("input[name]")
+            }
+            payload2 = {
+                i["data-for"]: i.get("value", "")
+                for i in soup.select("input[data-for]")
+            }
+            payload_salt = soup.select_one('input[id="pSalt"]').get("value")
+            payload_protected = soup.select_one('input[id="pPageItemsProtected"]').get(
+                "value"
+            )
+
+            # Add the UPRN and 'SUBMIT' to the payload
+            payload["p_request"] = "SUBMIT"
+            payload["P153_UPRN"] = user_uprn
+
+            # Manipulate the lists and build the JSON that must be submitted in further requests - some data is nested
+            merged_list = {**payload, **payload2}
+            new_list = []
+            other_list = {}
+            for key in merged_list.keys():
+                temp_list = {}
+                val = merged_list[key]
+                if key in ["P153_UPRN", "P153_TEMP", "P153_SYSDATE", "P0_LANGUAGE"]:
+                    temp_list = {"n": key, "v": val}
+                    new_list.append(temp_list)
+                elif key in ["P153_ZABY"]:
+                    temp_list = {"n": key, "v": "1", "ck": val}
+                    new_list.append(temp_list)
+                elif key in ["P153_POST_CODE"]:
+                    temp_list = {"n": key, "v": user_postcode, "ck": val}
+                    new_list.append(temp_list)
+                elif key in [
+                    "p_flow_id",
+                    "p_flow_step_id",
+                    "p_instance",
+                    "p_page_submission_id",
+                    "p_request",
+                    "p_reload_on_submit",
+                ]:
+                    other_list[key] = val
+                else:
+                    temp_list = {"n": key, "v": "", "ck": val}
+                    new_list.append(temp_list)
+
+            json_builder = {
+                "pageItems": {
+                    "itemsToSubmit": new_list,
+                    "protected": payload_protected,
+                    "rowVersion": "",
+                    "formRegionChecksums": [],
+                },
+                "salt": payload_salt,
+            }
+
+            json_object = json.dumps(json_builder, separators=(",", ":"))
+            other_list["p_json"] = json_object
+
+            # Generate POST including all the JSON we just built
+            s.post(
+                "https://iweb.itouchvision.com/portal/wwv_flow.accept", data=other_list
+            )
+
+            # The third and final page on the portal shows the detail of the waste collection services
+            # The required cookies are stored within the session so re-use the session to keep them
+            resource = s.get(
+                "https://iweb.itouchvision.com/portal/itouchvision/r/customer/bin_days"
+            )
+
+            # Create a BeautifulSoup object from the page's HTML
+            soup = BeautifulSoup(resource.text, "html.parser")
+            data = {"bins": []}
+
+            # Loop through the items on the page and build a JSON object for ingestion
+            for item in soup.select(".t-MediaList-item"):
+                for value in item.select(".t-MediaList-body"):
+                    dict_data = {
+                        "type": value.select("span")[1].get_text(strip=True).title(),
+                        "collectionDate": datetime.strptime(
+                            value.select(".t-MediaList-desc")[0].get_text(strip=True),
+                            "%A, %d %B, %Y",
+                        ).strftime(date_format),
+                    }
+                    data["bins"].append(dict_data)
+
+            return data

uk_bin_collection/uk_bin_collection/councils/SouthAyrshireCouncil.py

@@ -0,0 +1,73 @@
+import json
+from datetime import timedelta
+
+import requests
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Get and check both the passed UPRN and postcode
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        user_postcode = kwargs.get("postcode")
+        check_postcode(user_postcode)
+
+        # Build the headers, specify the parameters and then make a GET for the calendar
+        headers = {
+            "Connection": "Keep-Alive",
+            "User-Agent": "okhttp/3.14.9",
+        }
+        params = {
+            "end_date": "2024-01-01",
+            "rn": user_uprn,
+            "device": "undefined",
+            "postcode": user_postcode,
+            "OS": "android",
+            "OS_ver": "31",
+            "app_ver": "35",
+        }
+        requests.packages.urllib3.disable_warnings()
+        response = requests.get(
+            "http://www.sac-bins.co.uk/get_calendar.php", params=params, headers=headers
+        )
+
+        # Load the response as JSON
+        json_data = json.loads(response.text)
+
+        # The response loads well over a year's worth of data, so figure out some dates to limit output
+        today = datetime.today()
+        eight_weeks = datetime.today() + timedelta(days=8 * 7)
+        data = {"bins": []}
+
+        # The bin titles are pretty weird and colours are too basic, so make the names match the app
+        bin_friendly_names = {
+            "blue": "Blue Bin",
+            "red": "Food Caddy",
+            "green": "Green Bin",
+            "grey": "Grey Bin",
+            "purple": "Purple Bin",
+            "brown": "Brown Bin",
+        }
+
+        # Loop through the results. When a date is found that's on or greater than today's date AND less than
+        # eight weeks away, we want it in the output. So look up the friendly name and add it in.
+        for item in json_data:
+            bin_date = datetime.strptime(item["start"], "%Y-%m-%d").date()
+            if today.date() <= bin_date <= eight_weeks.date():
+                bin_type = bin_friendly_names.get(item["className"])
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": bin_date.strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/SouthCambridgeshireCouncil.py

@@ -0,0 +1,65 @@
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        API_URLS = {
+            "address_search": "https://servicelayer3c.azure-api.net/wastecalendar/address/search/",
+            "collection": "https://servicelayer3c.azure-api.net/wastecalendar/collection/search/{}/",
+        }
+        headers = {
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36",
+        }
+
+        user_postcode = kwargs.get("postcode")
+        user_paon = kwargs.get("paon")
+
+        # Establish session
+        requests.packages.urllib3.disable_warnings()
+        s = requests.Session()
+        r = s.get(
+            "https://www.scambs.gov.uk/recycling-and-bins/find-your-household-bin-collection-day/",
+            headers=headers,
+        )
+
+        # Select address
+        r = s.get(
+            API_URLS["address_search"],
+            headers=headers,
+            params={"postCode": user_postcode},
+        )
+        addresses = r.json()
+        address_ids = [
+            x["id"] for x in addresses if x["houseNumber"].capitalize() == user_paon
+        ]
+        if len(address_ids) == 0:
+            raise Exception(f"Could not match address {user_paon}, {user_postcode}")
+
+        # Get the schedule
+        r = s.get(
+            API_URLS["collection"].format(address_ids[0]),
+            headers=headers,
+        )
+        schedule = r.json()["collections"]
+
+        data = {"bins": []}
+
+        for collection in schedule:
+            dt = datetime.strptime(collection["date"], "%Y-%m-%dT%H:%M:%SZ").strftime(
+                date_format
+            )
+            for round in collection["roundTypes"]:
+                dict_data = {"type": round.title(), "collectionDate": dt}
+                data["bins"].append(dict_data)
+
+        return data