uk_bin_collection 0.74.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/WestLindseyDistrictCouncil.py

@@ -0,0 +1,118 @@
+import urllib.parse
+
+from bs4 import BeautifulSoup
+from dateutil.relativedelta import relativedelta
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        data = {"bins": []}
+
+        user_postcode = kwargs.get("postcode")
+        user_number = kwargs.get("paon")
+
+        user_address = "{} {}".format(user_number, user_postcode)
+        user_address = urllib.parse.quote(user_address)
+
+        # This first URL checks against a string representing the users address and returns values used for a second lookup.
+        stage1_url = "https://wlnk.statmap.co.uk/map/Cluster.svc/findLocation?callback=getAddressesCallback1702938375023&script=%5CCluster%5CCluster.AuroraScript%24&address={}".format(
+            user_address
+        )
+
+        address_data = requests.get(stage1_url).text
+
+        # Strip data and parse the JSON
+        address_data = json.loads(
+            re.sub("getAddressesCallback\d+\(", "", address_data)[:-2]
+        )
+
+        if address_data["TotalHits"] == 0:
+            raise Exception(
+                "No address found for string {}. See Wiki".format(user_address)
+            )
+        elif address_data["TotalHits"] != 1:
+            # Multiple hits returned. Let's pick the first one. We could raise an exception here if this causes problems.
+            pass
+
+        # Pull out the address data needed for the next step
+        address_id = address_data["Locations"][0]["Id"]
+        address_x = address_data["Locations"][0]["X"]
+        address_y = address_data["Locations"][0]["Y"]
+
+        stage2_url = "https://wlnk.statmap.co.uk/map/Cluster.svc/getpage?script=\Cluster\Cluster.AuroraScript$&taskId=bins&format=js&updateOnly=true&query=x%3D{}%3By%3D{}%3Bid%3D{}".format(
+            address_x, address_y, address_id
+        )
+
+        bin_query = requests.get(stage2_url).text
+
+        # Test that what we got is good
+        if "injectCss" not in bin_query:
+            raise Exception(
+                "Error. Data has not been returned correctly. Please raise an issue on the GitHub page"
+            )
+
+        # Return only the HTML contained within the Javascript function payload.
+        pattern = 'document\.getElementById\("DR1"\)\.innerHTML="(.+)";'
+
+        bin_html = re.findall(pattern, bin_query)
+
+        if len(bin_html) != 1:
+            # This exception is raised if the regular expression above finds anything other than one expected match.
+            raise Exception(
+                "Incorrect number of matches found during phase 2 search. Please raise an issue on the Github page"
+            )
+
+        # Some silly python foo required here to unescape the unicode contained.
+        bin_html = bin_html[0].encode().decode("unicode-escape")
+
+        soup = BeautifulSoup(bin_html, "html.parser")
+
+        collection_rows = soup.find("li", {"class": "auroraListItem"}).find_all("li")
+
+        collections = []
+
+        for row in collection_rows:
+            # Get bin type
+            bin_type = row.find("span").text
+
+            # Get bin date
+            bin_date_text = row.text
+            pattern = "\d+\/\d+"
+            bin_dates = re.findall(pattern, bin_date_text)
+
+            for bin_date in bin_dates:
+                # Split the bin date into day and month and build a full date with the current year
+                split_date = bin_date.split("/")
+                full_date = datetime(
+                    datetime.now().year, int(split_date[1]), int(split_date[0])
+                )
+
+                # If the current month is December and one of the next collections is in January, increment the year
+                if datetime.now().month == 12 and int(split_date[1]) == 1:
+                    full_date = bin_date + relativedelta(years=1)
+
+                # Since data in unordered, add to a tuple
+                collections.append((bin_type.title(), full_date))
+
+        # Sort the tuple by date
+        ordered_data = sorted(collections, key=lambda x: x[1])
+
+        # Add everything into the dictionary
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0],
+                "collectionDate": item[1].strftime(date_format),
+            }
+
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py

@@ -0,0 +1,103 @@
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            data = {"bins": []}
+            user_paon = kwargs.get("paon")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_paon(user_paon)
+            check_postcode(user_postcode)
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless)
+            driver.get(
+                "https://www.westlothian.gov.uk/article/31528/Bin-Collection-Calendar-Dates"
+            )
+
+            # Close feedback banner
+            feedbackBanner = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.CSS_SELECTOR, ".feedback__link--no"))
+            )
+            feedbackBanner.click()
+
+            # Wait for the postcode field to appear then populate it
+            inputElement_postcode = WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located(
+                    (By.ID, "WLBINCOLLECTION_PAGE1_ADDRESSLOOKUPPOSTCODE")
+                )
+            )
+            inputElement_postcode.send_keys(user_postcode)
+
+            # Click search button
+            findAddress = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.ID, "WLBINCOLLECTION_PAGE1_ADDRESSLOOKUPSEARCH")
+                )
+            )
+            findAddress.click()
+
+            # Wait for the 'Select address' dropdown to appear and select option matching the house name/number
+            WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (
+                        By.XPATH,
+                        "//select[@id='WLBINCOLLECTION_PAGE1_ADDRESSLOOKUPADDRESS']//option[contains(., '"
+                        + user_paon
+                        + "')]",
+                    )
+                )
+            ).click()
+
+            # Wait for the collections table to appear
+            WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.CSS_SELECTOR, ".bin-collections"))
+            )
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            # Get collections
+            for collection in soup.find_all("div", {"class": "bin-collect"}):
+                dict_data = {
+                    "type": collection.find("h3").get_text(strip=True),
+                    "collectionDate": datetime.strptime(
+                        remove_ordinal_indicator_from_date_string(
+                            collection.find(
+                                "span", {"class": "bin-collect__date"}
+                            ).get_text(strip=True)
+                        ),
+                        "%A, %B %d %Y",
+                    ).strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+            data["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+            )
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection/uk_bin_collection/councils/WestNorthamptonshireCouncil.py

@@ -0,0 +1,34 @@
+import urllib.parse
+
+import requests
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_postcode = kwargs.get("postcode")
+        check_postcode(user_postcode)
+
+        api_url = f"https://mycouncil.northampton.digital/BinRoundFinder?postcode={urllib.parse.quote(user_postcode)}"
+        json_data = requests.get(api_url).json()
+
+        data = {"bins": []}
+
+        dict_data = {
+            "type": json_data["type"].capitalize(),
+            "collectionDate": datetime.strptime(
+                json_data["date"], "%Y%m%d%H%M"
+            ).strftime(date_format),
+        }
+        data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/WestSuffolkCouncil.py

@@ -0,0 +1,64 @@
+import itertools
+
+from bs4 import BeautifulSoup, Tag
+from dateutil.parser import parse as date_parse
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        data = {"bins": []}
+        user_uprn = kwargs.get("uprn")
+
+        api_url = (
+            f"https://maps.westsuffolk.gov.uk/MyWestSuffolk.aspx?action=SetAddress&UniqueId={user_uprn}"
+        )
+
+        response = requests.get(api_url)
+
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+
+        def panel_search(cur_tag: Tag):
+            """
+            Helper function to find the correct tag
+            """
+            if cur_tag.name != "div":
+                return False
+
+            tag_class = cur_tag.attrs.get("class", None)
+            if tag_class is None:
+                return False
+
+            parent_has_header = cur_tag.parent.find_all("h4", string="Bin collection days")
+            if len(parent_has_header) < 1:
+                return False
+
+            return "atPanelData" in tag_class
+
+        collection_tag = soup.body.find_all(panel_search)
+
+        # Parse the resultant div
+        for tag in collection_tag:
+            text_list = list(tag.stripped_strings)
+            # Create and parse the list as tuples of name:date
+            for bin_name, collection_date in itertools.batched(text_list, 2):
+                # Clean-up the bin_name
+                bin_name_clean = bin_name.strip().replace("\r", "").replace("\n", "")
+                bin_name_clean = re.sub(' +', ' ', bin_name_clean)
+
+                # Parse the date
+                next_collection = date_parse(collection_date)
+                next_collection = next_collection.replace(year=datetime.now().year)
+
+                dict_data = {
+                    "type": bin_name_clean,
+                    "collectionDate": next_collection.strftime(date_format),
+                }
+
+                data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/WiganBoroughCouncil.py

@@ -0,0 +1,97 @@
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Get and check UPRN
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        user_uprn = user_uprn.zfill(
+            12
+        )  # Wigan is expecting 12 character UPRN or else it falls over, expects 0 padded UPRNS at the start for any that aren't 12 chars
+
+        user_postcode = kwargs.get("postcode")
+        check_postcode(user_postcode)
+
+        # Start a new session to walk through the form
+        requests.packages.urllib3.disable_warnings()
+        s = requests.session()
+
+        # Get our initial session running
+        response = s.get("https://apps.wigan.gov.uk/MyNeighbourhood/")
+
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+
+        # Grab the ASP variables needed to continue
+        payload = {
+            "__VIEWSTATE": (soup.find("input", {"id": "__VIEWSTATE"}).get("value")),
+            "__VIEWSTATEGENERATOR": (
+                soup.find("input", {"id": "__VIEWSTATEGENERATOR"}).get("value")
+            ),
+            "__EVENTVALIDATION": (
+                soup.find("input", {"id": "__EVENTVALIDATION"}).get("value")
+            ),
+            "ctl00$ContentPlaceHolder1$txtPostcode": (user_postcode),
+            "ctl00$ContentPlaceHolder1$btnPostcodeSearch": ("Search"),
+        }
+
+        # Use the above to get to the next page with address selection
+        response = s.post("https://apps.wigan.gov.uk/MyNeighbourhood/", payload)
+
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+
+        # Load the new variables that are constant and can't be gotten from the page
+        payload = {
+            "__EVENTTARGET": ("ctl00$ContentPlaceHolder1$lstAddresses"),
+            "__EVENTARGUMENT": (""),
+            "__LASTFOCUS": (""),
+            "__VIEWSTATE": (soup.find("input", {"id": "__VIEWSTATE"}).get("value")),
+            "__VIEWSTATEGENERATOR": (
+                soup.find("input", {"id": "__VIEWSTATEGENERATOR"}).get("value")
+            ),
+            "__EVENTVALIDATION": (
+                soup.find("input", {"id": "__EVENTVALIDATION"}).get("value")
+            ),
+            "ctl00$ContentPlaceHolder1$txtPostcode": (user_postcode),
+            "ctl00$ContentPlaceHolder1$lstAddresses": ("UPRN" + user_uprn),
+        }
+
+        # Get the final page with the actual dates
+        response = s.post("https://apps.wigan.gov.uk/MyNeighbourhood/", payload)
+
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+
+        # Get the dates.
+        for bins in soup.find_all("div", {"class": "BinsRecycling"}):
+            bin_type = bins.find("h2").text
+            binCollection = bins.find("div", {"class": "dateWrapper-next"}).get_text(
+                strip=True
+            )
+            binData = datetime.strptime(
+                re.sub(r"(\d)(st|nd|rd|th)", r"\1", binCollection), "%A%d%b%Y"
+            )
+            if binData:
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": binData.strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/WiltshireCouncil.py

@@ -0,0 +1,135 @@
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        requests.packages.urllib3.disable_warnings()
+        # Define some months to get from the calendar
+        this_month = datetime.now().month
+        this_year = datetime.now().year
+        one_month = this_month + 1
+        two_month = this_month + 2
+        months = [this_month, one_month, two_month]
+
+        # Get and check the postcode and UPRN values
+        user_postcode = kwargs.get("postcode")
+        check_postcode(user_postcode)
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        # Some data for the request
+        cookies = {
+            "ARRAffinity": "c5a9db7fe43cef907f06528c3d34a997365656f757206fbdf34193e2c3b6f737",
+            "ARRAffinitySameSite": "c5a9db7fe43cef907f06528c3d34a997365656f757206fbdf34193e2c3b6f737",
+        }
+        headers = {
+            "Accept": "*/*",
+            "Accept-Language": "en-GB,en;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+            # 'Cookie': 'ARRAffinity=c5a9db7fe43cef907f06528c3d34a997365656f757206fbdf34193e2c3b6f737; ARRAffinitySameSite=c5a9db7fe43cef907f06528c3d34a997365656f757206fbdf34193e2c3b6f737',
+            "Origin": "https://ilambassadorformsprod.azurewebsites.net",
+            "Pragma": "no-cache",
+            "Referer": "https://ilambassadorformsprod.azurewebsites.net/wastecollectiondays/index",
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-origin",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 OPR/98.0.0.0",
+            "X-Requested-With": "XMLHttpRequest",
+            "sec-ch-ua": '"Chromium";v="112", "Not_A Brand";v="24", "Opera GX";v="98"',
+            "sec-ch-ua-mobile": "?0",
+            "sec-ch-ua-platform": '"Windows"',
+        }
+
+        collections = []
+
+        # For each of the months we defined
+        for cal_month in months:
+            # If we're in Nov/Dec, the calculations won't work since its just adding one, so roll it
+            # to next year correctly
+            if cal_month == 13:
+                cal_month = 1
+                cal_year = this_year + 1
+            elif cal_month == 14:
+                cal_month = 2
+                cal_year = this_year + 1
+            else:
+                cal_year = this_year
+
+            # Data for the calendar
+            data = {
+                "Month": cal_month,
+                "Year": cal_year,
+                "Postcode": user_postcode,
+                "Uprn": user_uprn,
+            }
+
+            # Send it all as a POST
+            response = requests.post(
+                "https://ilambassadorformsprod.azurewebsites.net/wastecollectiondays/collectionlist",
+                cookies=cookies,
+                headers=headers,
+                data=data,
+            )
+
+            # If we don't get a HTTP200, throw an error
+            if response.status_code != 200:
+                raise SystemError(
+                    "Error retrieving data! Please try again or raise an issue on GitHub!"
+                )
+
+            soup = BeautifulSoup(response.text, features="html.parser")
+            soup.prettify()
+
+            # Find all the bits of the current calendar that contain an event
+            events = soup.find_all("div", {"class": "rc-event-container"})
+
+            for event in events:
+                # Get the date and type of each bin collection
+                bin_date = datetime.strptime(
+                    event.find_next("a").attrs.get("data-original-datetext"),
+                    "%A %d %B, %Y",
+                )
+                bin_type = event.find_next("a").attrs.get("data-original-title")
+                # Only process it if it's today or in the future
+                if bin_date.date() >= datetime.now().date():
+                    # Split the really long type up into two separate bins
+                    if (
+                        bin_type
+                        == "Mixed dry recycling (blue lidded bin) and glass (black box or basket)"
+                    ):
+                        collections.append(
+                            (
+                                "Mixed dry recycling (blue lidded bin)",
+                                datetime.strftime(bin_date, date_format),
+                            )
+                        )
+                        collections.append(
+                            (
+                                "Glass (black box or basket)",
+                                datetime.strftime(bin_date, date_format),
+                            )
+                        )
+                    else:
+                        collections.append(
+                            (bin_type, datetime.strftime(bin_date, date_format))
+                        )
+
+        data = {"bins": []}
+
+        # Now there's a list of collections, yeet them into the dictionary for nice JSON
+        for item in collections:
+            dict_data = {"type": item[0], "collectionDate": item[1]}
+            data["bins"].append(dict_data)
+
+        return data