uk_bin_collection 0.74.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/SouthGloucestershireCouncil.py

@@ -0,0 +1,74 @@
+from datetime import timedelta
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+def format_bin_data(key: str, date: datetime):
+    formatted_date = date.strftime(date_format)
+
+    if re.match(r"^R\d+$", key) is not None:
+        # RX matches both general waste and recycling
+        return [
+            ("General Waste (Black Bin)", formatted_date),
+            ("Recycling & Food Waste", formatted_date),
+        ]
+    elif re.match(r"^G\d+$", key) is not None:
+        return [("Garden Waste (Green Bin)", formatted_date)]
+    elif re.match(r"^C\d+$", key) is not None:
+        return [("Recycling & Food Waste", formatted_date)]
+    else:
+        return None
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    def parse_data(self, page: str, **kwargs) -> dict:
+        uprn = kwargs.get("uprn")
+        check_uprn(uprn)
+
+        api_url = (
+            f"https://webapps.southglos.gov.uk/Webservices/SGC.RefuseCollectionService/RefuseCollectionService"
+            f".svc/getCollections/{uprn}"
+        )
+
+        headers = {"content-type": "application/json"}
+
+        response = requests.get(api_url, headers=headers)
+
+        json_response = json.loads(response.content)
+        if not json_response:
+            raise ValueError("No collection data found for provided UPRN.")
+
+        collection_data = json_response[0]
+
+        today = datetime.today()
+        eight_weeks = datetime.today() + timedelta(days=8 * 7)
+        data = {"bins": []}
+        collection_tuple = []
+
+        for key in collection_data:
+            if key == "CalendarName":
+                continue
+
+            item = collection_data[key]
+
+            if item == "":
+                continue
+
+            collection_date = datetime.strptime(item, date_format)
+            if today.date() <= collection_date.date() <= eight_weeks.date():
+                bin_data = format_bin_data(key, collection_date)
+                if bin_data is not None:
+                    for bin_date in bin_data:
+                        collection_tuple.append(bin_date)
+
+        ordered_data = sorted(collection_tuple, key=lambda x: x[1])
+
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0],
+                "collectionDate": item[1],
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/SouthLanarkshireCouncil.py

@@ -0,0 +1,78 @@
+import time
+from datetime import timedelta
+
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        data = {"bins": []}
+        collection_types = [
+            "non recyclable waste",
+            "food and garden",
+            "paper and card",
+            "glass, cans and plastics",
+        ]
+
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        week_details = soup.find("div", {"class": "bin-dir-snip"})
+        week_dates = week_details.find("div", {"class": "clearfix"}).find("p")
+        week_collections = week_details.find_all_next("h4")
+
+        results = re.search(
+            "([A-Za-z0-9 ]+) to ([A-Za-z0-9 ]+)", week_dates.get_text().strip()
+        )
+        if results:
+            week_start = datetime.strptime(results.groups()[0], "%A %d %B %Y")
+            week_end = datetime.strptime(results.groups()[1], "%A %d %B %Y")
+            week_days = (
+                week_start + timedelta(days=i)
+                for i in range((week_end - week_start).days + 1)
+            )
+
+            week_collection_types = []
+            for week_collection in week_collections:
+                week_collection = (
+                    week_collection.get_text().strip().lower().replace("-", " ")
+                )
+                for collection_type in collection_types:
+                    if collection_type in week_collection:
+                        week_collection_types.append(collection_type)
+
+            collection_schedule = (
+                soup.find("div", {"class": "serviceDetails"})
+                .find("table")
+                .find_all_next("tr")
+            )
+
+            for day in week_days:
+                for row in collection_schedule:
+                    schedule_type = row.find("th").get_text().strip()
+                    results2 = re.search("([^(]+)", row.find("td").get_text().strip())
+                    if results2:
+                        schedule_day = results2[1].strip()
+                        for collection_type in week_collection_types:
+                            if collection_type in schedule_type.lower():
+                                if (
+                                    day.weekday()
+                                    == time.strptime(schedule_day, "%A").tm_wday
+                                ):
+                                    dict_data = {
+                                        "type": schedule_type,
+                                        "collectionDate": day.strftime(date_format),
+                                    }
+                                    data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/SouthNorfolkCouncil.py

@@ -0,0 +1,91 @@
+from xml.etree import ElementTree
+
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    baseclass. They can also override some
+    operations with a default implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        uprn = kwargs.get("uprn")
+        check_uprn(uprn)
+        council = "SNO"
+
+        # Make SOAP request
+        headers = {
+            "Accept": "*/*",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
+            "Connection": "keep-alive",
+            "Content-Type": "text/xml; charset=UTF-8",
+            "Host": "collections-southnorfolk.azurewebsites.net",
+            "Origin": "https://collections-southnorfolk.azurewebsites.net",
+            "Referer": "https://collections-southnorfolk.azurewebsites.net/calendar.html",
+            "sec-ch-ua": '"Chromium";v="110", "Not A(Brand";v="24", "Google Chrome";v="110"',
+            "sec-ch-ua-mobile": "?0",
+            "sec-ch-ua-platform": '"Windows"',
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-origin",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36",
+        }
+        requests.packages.urllib3.disable_warnings()
+        post_data = (
+            '<?xml version="1.0" encoding="utf-8"?>'
+            '<soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">'
+            '<soap:Body><getRoundCalendarForUPRN xmlns="http://webaspx-collections.azurewebsites.net/">'
+            "<council>" + council + "</council><UPRN>" + uprn + "</UPRN>"
+            "<from>Chtml</from></getRoundCalendarForUPRN></soap:Body></soap:Envelope>"
+        )
+        response = requests.post(
+            "https://collections-southnorfolk.azurewebsites.net/WSCollExternal.asmx",
+            headers=headers,
+            data=post_data,
+        )
+        if response.status_code != 200:
+            raise ValueError("No bin data found for provided UPRN.")
+
+        # Get HTML from SOAP response
+        xmltree = ElementTree.fromstring(response.text)
+        html = xmltree.find(
+            ".//{http://webaspx-collections.azurewebsites.net/}getRoundCalendarForUPRNResult"
+        ).text
+        # Parse with BS4
+        soup = BeautifulSoup(html, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+        for bin_type in ["RefuseBin", "RecycleBin", "GardenBin"]:
+            bin_el = soup.find("b", string=bin_type)
+            if bin_el:
+                bin_info = bin_el.next_sibling.split(": ")[1]
+                collection_date = ""
+                results = re.search("([A-Za-z]+ \\d\\d? [A-Za-z]+) then", bin_info)
+                if results:
+                    date = get_next_occurrence_from_day_month(
+                        datetime.strptime(
+                            results[1] + " " + datetime.now().strftime("%Y"),
+                            "%a %d %b %Y",
+                        )
+                    )
+                    if date:
+                        collection_date = date.strftime(date_format)
+                else:
+                    results2 = re.search("([A-Za-z]+) then", bin_info)
+                    if results2:
+                        collection_date = results2[1]
+
+                if collection_date != "":
+                    dict_data = {
+                        "type": bin_type,
+                        "collectionDate": collection_date,
+                    }
+                    data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/SouthOxfordshireCouncil.py

@@ -0,0 +1,93 @@
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        # UPRN is passed in via a cookie. Set cookies/params and GET the page
+        cookies = {
+            # 'JSESSIONID': '96F2A15C14569B2ED2BBEB140FE86532',
+            "SVBINZONE": f"SOUTH%3AUPRN%40{user_uprn}",
+        }
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+            "Accept-Language": "en-GB,en;q=0.7",
+            "Cache-Control": "max-age=0",
+            "Connection": "keep-alive",
+            "Referer": "https://eform.southoxon.gov.uk/ebase/BINZONE_DESKTOP.eb?SOVA_TAG=SOUTH&ebd=0&ebz=1_1668467255368",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "same-origin",
+            "Sec-Fetch-User": "?1",
+            "Sec-GPC": "1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36",
+        }
+        params = {
+            "SOVA_TAG": "SOUTH",
+            "ebd": "0",
+            # 'ebz':      '1_1668467255368',
+        }
+        requests.packages.urllib3.disable_warnings()
+        response = requests.get(
+            "https://eform.southoxon.gov.uk/ebase/BINZONE_DESKTOP.eb",
+            params=params,
+            headers=headers,
+            cookies=cookies,
+        )
+
+        # Parse response text for super speedy finding
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+
+        # Page has slider info side by side, which are two instances of this class
+        for bin in soup.find_all("div", {"class": "binextra"}):
+            bin_info = list(bin.stripped_strings)
+            try:
+                # On standard collection schedule, date will be contained in the first stripped string
+                if contains_date(bin_info[0]):
+                    bin_date = get_next_occurrence_from_day_month(
+                        datetime.strptime(
+                            bin_info[0] + " " + datetime.today().strftime("%Y"),
+                            "%A %d %B - %Y",
+                        )
+                    ).strftime(date_format)
+                    bin_type = str.capitalize(bin_info[1])
+                # On exceptional collection schedule (e.g. around English Bank Holidays), date will be contained in the second stripped string
+                else:
+                    bin_date = get_next_occurrence_from_day_month(
+                        datetime.strptime(
+                            bin_info[1] + " " + datetime.today().strftime("%Y"),
+                            "%A %d %B - %Y",
+                        )
+                    ).strftime(date_format)
+                    bin_type = str.capitalize(bin_info[2])
+            except Exception as ex:
+                raise ValueError(f"Error parsing bin data: {ex}")
+
+            # Build data dict for each entry
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": bin_date,
+            }
+            data["bins"].append(dict_data)
+
+        data["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        return data

uk_bin_collection/uk_bin_collection/councils/SouthTynesideCouncil.py

@@ -0,0 +1,98 @@
+import json
+from datetime import datetime
+
+import requests
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+def get_address_uprn(postcode: str, paon: str, api_url: str) -> str:
+    """
+    Gets the UPRN and address in desired format
+        :rtype: str
+        :param postcode: Postcode to use
+        :param paon: House number to use
+        :param api_url: API to POST
+        :return: UPRN and postcode in str format
+    """
+    addr = ""
+    payload = json.dumps(
+        {
+            "jsonrpc": "2.0",
+            "id": "1689431267990",
+            "method": "stc.common.snippets.getAddressList",
+            "params": {"postcode": f"{postcode.replace(' ', '')}", "localonly": "true"},
+        }
+    )
+    headers = {"Content-Type": "application/json"}
+    response = requests.post(api_url, data=payload, headers=headers)
+
+    json_response = json.loads(response.content)
+    results = json_response["result"]["ReturnedList"]
+
+    for item in results:
+        if item["Address"].split()[0] == paon.strip():
+            addr = item["UPRN"] + "|" + item["Address"]
+            break
+
+    return addr
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        requests.packages.urllib3.disable_warnings()
+        api_url = "https://www.southtyneside.gov.uk/apiserver/ajaxlibrary/"
+        user_postcode = kwargs.get("postcode")
+        user_paon = kwargs.get("paon")
+        data = {"bins": []}
+
+        check_postcode(user_postcode)
+        check_paon(user_paon)
+
+        try:
+            if user_paon is None:
+                raise ValueError("Invalid house number")
+        except Exception as ex:
+            print(f"Exception encountered: {ex}")
+            print(
+                "Please check the provided house number. If this error continues, please first trying setting the "
+                "house number manually on line 25 before raising an issue."
+            )
+            exit(1)
+
+        # Get the "UPRN" (actually the UPRN + address)
+        uprn = get_address_uprn(user_postcode, user_paon, api_url)
+
+        # Set up payload and headers, then post to API to get schedule
+        payload = json.dumps(
+            {
+                "jsonrpc": "2.0",
+                "id": "1689431609779",
+                "method": "stc.waste.collections.getDates",
+                "params": {"addresscode": uprn},
+            }
+        )
+        headers = {"Content-Type": "application/json"}
+        response = requests.request("POST", api_url, headers=headers, data=payload)
+
+        # Break down the resulting JSON and load into dictionary
+        json_result = json.loads(response.text)["result"]
+        months = json_result["SortedCollections"]
+        for month in months:
+            collections_in_month = month["Collections"]
+            for item in collections_in_month:
+                dict_data = {
+                    "type": item["Type"],
+                    "collectionDate": datetime.strptime(
+                        item["DateString"], "%d %B %Y"
+                    ).strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/StAlbansCityAndDistrictCouncil.py

@@ -0,0 +1,43 @@
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+from dateutil import parser
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        data = {"bins": []}
+
+        headers = {
+            "Content-Type": "application/json; charset=UTF-8",
+        }
+
+        req_data = {"uprn": user_uprn, "noticeBoard": "default"}
+
+        url = "https://gis.stalbans.gov.uk/NoticeBoard9/VeoliaProxy.NoticeBoard.asmx/GetServicesByUprnAndNoticeBoard"
+
+        response = requests.post(url, json=req_data, headers=headers)
+
+        collections_response = response.json()
+
+        collections = []
+
+        for collection in collections_response["d"]:
+            collection_data = collection["ServiceHeaders"][0]
+            bin_type = collection_data["TaskType"]
+            collection_date = collection_data["Next"]
+            next_collection = parser.isoparser().isoparse(collection_date)
+            collections.append((bin_type, next_collection))
+
+        ordered_data = sorted(collections, key=lambda x: x[1])
+
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0],
+                "collectionDate": item[1].strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/StHelensBC.py

@@ -0,0 +1,56 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    baseclass. They can also override some
+    operations with a default implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        uprn = kwargs.get("uprn")
+        # Check the UPRN is valid
+        check_uprn(uprn)
+
+        # Request URL
+        url = f"https://secure.sthelens.net/website/CollectionDates.nsf/servlet.xsp/NextCollections?source=1&refid={uprn}"
+
+        # Make Request
+        requests.packages.urllib3.disable_warnings()
+        s = requests.session()
+        page = s.get(url)
+
+        # Make a BS4 object
+        soup = BeautifulSoup(
+            re.sub("<div([^>]+)>", "", page.text).replace("</div>", ""),
+            features="html.parser",
+        )
+        soup.prettify()
+
+        data = {"bins": []}
+        collection_rows = (
+            soup.find("table", {"class": "multitable"}).find("tbody").find_all("tr")
+        )
+
+        for collection_row in collection_rows:
+            # Get bin collection type
+            bin_type = collection_row.find("th")
+            if bin_type:
+                bin_type = bin_type.get_text(strip=True)
+                # Get bin collection dates
+                for bin_date in collection_row.find_all("td"):
+                    if bin_date.get_text(strip=True) != "Dates not allocated":
+                        collection_date = datetime.strptime(
+                            bin_date.get_text(strip=True), "%a %d %b %Y"
+                        )
+                        dict_data = {
+                            "type": bin_type,
+                            "collectionDate": collection_date.strftime(date_format),
+                        }
+                        data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py

@@ -0,0 +1,112 @@
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        data = {"bins": []}
+        user_uprn = kwargs.get("uprn")
+        user_postcode = kwargs.get("postcode")
+        web_driver = kwargs.get("web_driver")
+        headless = kwargs.get("headless")
+        check_uprn(user_uprn)
+        check_postcode(user_postcode)
+
+        # Create Selenium webdriver
+        driver = create_webdriver(web_driver, headless)
+        driver.get("https://www.staffsmoorlands.gov.uk/findyourbinday")
+
+        # Close cookies banner
+        cookieAccept = WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located(
+                (By.CSS_SELECTOR, ".cookiemessage__link--close")
+            )
+        )
+        cookieAccept.click()
+
+        # Wait for the postcode field to appear then populate it
+        inputElement_postcode = WebDriverWait(driver, 30).until(
+            EC.presence_of_element_located(
+                (By.ID, "FINDBINDAYSSTAFFORDSHIREMOORLANDS_POSTCODESELECT_POSTCODE")
+            )
+        )
+        inputElement_postcode.send_keys(user_postcode)
+
+        # Click search button
+        findAddress = WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located(
+                (
+                    By.ID,
+                    "FINDBINDAYSSTAFFORDSHIREMOORLANDS_POSTCODESELECT_PAGE1NEXT_NEXT",
+                )
+            )
+        )
+        findAddress.click()
+
+        # Wait for the 'Select address' dropdown to appear and select option matching UPRN
+        dropdown = WebDriverWait(driver, 30).until(
+            EC.presence_of_element_located(
+                (By.ID, "FINDBINDAYSSTAFFORDSHIREMOORLANDS_ADDRESSSELECT_ADDRESS")
+            )
+        )
+        # Create a 'Select' for it, then select the matching URPN option
+        dropdownSelect = Select(dropdown)
+        dropdownSelect.select_by_value(user_uprn)
+
+        # Wait for the submit button to appear, then click it to get the collection dates
+        submit = WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located(
+                (
+                    By.ID,
+                    "FINDBINDAYSSTAFFORDSHIREMOORLANDS_ADDRESSSELECT_ADDRESSSELECTNEXTBTN_NEXT",
+                )
+            )
+        )
+        submit.click()
+
+        soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+        # Quit Selenium webdriver to release session
+        driver.quit()
+
+        # Get months
+        for month_wrapper in soup.find_all("div", {"class": "bin-collection__month"}):
+            if month_wrapper:
+                month_year = month_wrapper.find(
+                    "h3", {"class": "bin-collection__title"}
+                ).get_text(strip=True)
+                # Get collections
+                for collection in month_wrapper.find_all(
+                    "li", {"class": "bin-collection__item"}
+                ):
+                    day = collection.find(
+                        "span", {"class": "bin-collection__number"}
+                    ).get_text(strip=True)
+                    if month_year and day:
+                        bin_date = datetime.strptime(day + " " + month_year, "%d %B %Y")
+                        dict_data = {
+                            "type": collection.find(
+                                "span", {"class": "bin-collection__type"}
+                            ).get_text(strip=True),
+                            "collectionDate": bin_date.strftime(date_format),
+                        }
+                        data["bins"].append(dict_data)
+
+        data["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return data