uk_bin_collection 0.119.0__py3-none-any.whl → 0.123.1__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/LondonBoroughLewisham.py

@@ -0,0 +1,140 @@
+import re
+import time
+
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select, WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        user_postcode = kwargs.get("postcode")
+        web_driver = kwargs.get("web_driver")
+        headless = kwargs.get("headless")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        # Initialize the WebDriver (Chrome in this case)
+        with create_webdriver(
+            web_driver,
+            headless,
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
+            __name__,
+        ) as driver:
+
+            # Step 1: Navigate to the form page
+            driver.get(
+                "https://lewisham.gov.uk/myservices/recycling-and-rubbish/your-bins/collection"
+            )
+
+            try:
+                cookie_accept_button = WebDriverWait(driver, 5).until(
+                    EC.element_to_be_clickable(
+                        (By.ID, "CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll")
+                    )
+                )
+                cookie_accept_button.click()
+            except Exception:
+                print("No cookie consent banner found or already dismissed.")
+
+            # Wait for the form to load
+            WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.CLASS_NAME, "address-finder"))
+            )
+
+            # Step 2: Locate the input field for the postcode
+            postcode_input = driver.find_element(
+                By.CLASS_NAME, "js-address-finder-input"
+            )
+
+            # Enter the postcode
+            postcode_input.send_keys(
+                user_postcode
+            )  # Replace with your desired postcode
+            time.sleep(1)  # Optional: Wait for the UI to react
+
+            # Step 4: Click the "Find address" button with retry logic
+            find_button = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (By.CLASS_NAME, "js-address-finder-step-address")
+                )
+            )
+            find_button.click()
+
+            # Wait for the address selector to appear and options to load
+            WebDriverWait(driver, 10).until(
+                lambda d: len(
+                    d.find_element(By.ID, "address-selector").find_elements(
+                        By.TAG_NAME, "option"
+                    )
+                )
+                > 1
+            )
+
+            # Select the dropdown and print available options
+            address_selector = driver.find_element(By.ID, "address-selector")
+
+            # Use Select class to interact with the dropdown
+            select = Select(address_selector)
+            if len(select.options) > 1:
+                select.select_by_value(user_uprn)
+            else:
+                print("No additional addresses available to select")
+
+            # Wait until the URL contains the expected substring
+            WebDriverWait(driver, 10).until(
+                EC.url_contains("/find-your-collection-day-result")
+            )
+
+            # Parse the HTML
+            soup = BeautifulSoup(driver.page_source, "html.parser")
+
+            # Extract the main container
+            collection_result = soup.find("div", class_="js-find-collection-result")
+
+            # Extract each collection type and its frequency/day
+            for strong_tag in collection_result.find_all("strong"):
+                bin_type = strong_tag.text.strip()  # e.g., "Food waste"
+                # Extract the sibling text
+                schedule_text = (
+                    strong_tag.next_sibling.next_sibling.next_sibling.text.strip()
+                    .replace("\n", " ")
+                    .replace("\t", " ")
+                )
+
+                # Extract the day using regex
+                print(schedule_text)
+                day_match = re.search(r"on\s*(\w+day)", schedule_text)
+                print(day_match)
+                day = day_match.group(1) if day_match else None
+
+                # Extract the next collection date using regex
+                date_match = re.search(
+                    r"Your next collection date is\s*(\d{2}/\d{2}/\d{4})(.?)",
+                    schedule_text,
+                )
+                if date_match:
+                    next_collection_date = date_match.group(1)
+                else:
+                    next_collection_date = get_next_day_of_week(day, date_format)
+
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": next_collection_date,
+                }
+                bindata["bins"].append(dict_data)
+
+            return bindata

uk_bin_collection/uk_bin_collection/councils/MidSuffolkDistrictCouncil.py

@@ -24,6 +24,7 @@ class CouncilClass(AbstractGetBinDataClass):
 
         collection_day = kwargs.get("paon")
         garden_collection_week = kwargs.get("postcode")
+        garden_collection_day = kwargs.get("uprn")
         bindata = {"bins": []}
 
         days_of_week = [
@@ -42,6 +43,7 @@ class CouncilClass(AbstractGetBinDataClass):
         recyclingstartDate = datetime(2024, 11, 4)
 
         offset_days = days_of_week.index(collection_day)
+        offset_days_garden = days_of_week.index(garden_collection_day)
         if garden_collection_week:
             garden_collection = garden_week.index(garden_collection_week)
 
@@ -155,7 +157,7 @@ class CouncilClass(AbstractGetBinDataClass):
 
                 collection_date = (
                     datetime.strptime(gardenDate, "%d/%m/%Y")
-                    + timedelta(days=offset_days)
+                    + timedelta(days=offset_days_garden)
                 ).strftime("%d/%m/%Y")
 
                 garden_holiday = next(

uk_bin_collection/uk_bin_collection/councils/MonmouthshireCountyCouncil.py

@@ -0,0 +1,70 @@
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        URI = (
+            f"https://maps.monmouthshire.gov.uk/?action=SetAddress&UniqueId={user_uprn}"
+        )
+
+        # Make the GET request
+        response = requests.get(URI)
+
+        # Parse the HTML
+        soup = BeautifulSoup(response.content, "html.parser")
+
+        waste_collections_div = soup.find("div", {"aria-label": "Waste Collections"})
+
+        # Find all bin collection panels
+        bin_panels = waste_collections_div.find_all("div", class_="atPanelContent")
+
+        current_year = datetime.now().year
+        current_month = datetime.now().month
+
+        for panel in bin_panels:
+            # Extract bin name (e.g., "Household rubbish bag")
+            bin_name = panel.find("h4").text.strip().replace("\r", "").replace("\n", "")
+
+            # Extract collection date (e.g., "Monday 9th December")
+            date_tag = panel.find("p")
+            if date_tag and "Your next collection date is" in date_tag.text:
+                collection_date = date_tag.find("strong").text.strip()
+            else:
+                continue
+
+            collection_date = datetime.strptime(
+                remove_ordinal_indicator_from_date_string(collection_date), "%A %d %B"
+            )
+
+            if (current_month > 9) and (collection_date.month < 4):
+                collection_date = collection_date.replace(year=(current_year + 1))
+            else:
+                collection_date = collection_date.replace(year=current_year)
+
+            dict_data = {
+                "type": bin_name,
+                "collectionDate": collection_date.strftime("%d/%m/%Y"),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/MorayCouncil.py

@@ -0,0 +1,65 @@
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        bindata = {"bins": []}
+
+        user_uprn = user_uprn.zfill(8)
+
+        year = datetime.today().year
+        response = requests.get(
+            f"https://bindayfinder.moray.gov.uk/cal_{year}_view.php",
+            params={"id": user_uprn},
+        )
+        if response.status_code != 200:
+            # fall back to known good calendar URL
+            response = requests.get(
+                "https://bindayfinder.moray.gov.uk/cal_2024_view.php",
+                params={"id": user_uprn},
+            )
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        bin_types = {
+            "G": "Green",
+            "B": "Brown",
+            "P": "Purple",
+            "C": "Blue",
+            "O": "Orange",
+        }
+
+        for month_container in soup.findAll("div", class_="month-container"):
+            for div in month_container.findAll("div"):
+                if "month-header" in div["class"]:
+                    month = div.text
+                elif div["class"] and div["class"][0] in ["B", "GPOC", "GBPOC"]:
+                    bins = div["class"][0]
+                    dom = int(div.text)
+                    for i in bins:
+                        dict_data = {
+                            "type": bin_types.get(i),
+                            "collectionDate": datetime.strptime(
+                                f"{dom} {month} {year}",
+                                "%d %B %Y",
+                            ).strftime("%d/%m/%Y"),
+                        }
+                        bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/NewcastleUnderLymeCouncil.py

@@ -0,0 +1,66 @@
+import requests
+from bs4 import BeautifulSoup
+from dateutil.relativedelta import relativedelta
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        URI = f"https://www.newcastle-staffs.gov.uk/homepage/97/check-your-bin-day?uprn={user_uprn}"
+
+        # Make the GET request
+        response = requests.get(URI)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+
+        # Find the table
+        table = soup.find("table", {"class": "data-table"})
+
+        if table:
+            rows = table.find("tbody").find_all("tr")
+            for row in rows:
+                date = datetime.strptime(
+                    (
+                        row.find_all("td")[0]
+                        .get_text(strip=True)
+                        .replace("Date:", "")
+                        .strip()
+                    ),
+                    "%A %d %B",
+                ).replace(year=datetime.now().year)
+                if datetime.now().month > 10 and date.month < 3:
+                    date = date + relativedelta(years=1)
+                bin_types = (
+                    row.find_all("td")[1]
+                    .text.replace("Collection Type:", "")
+                    .splitlines()
+                )
+                for bin_type in bin_types:
+                    bin_type = bin_type.strip()
+                    if bin_type:
+                        dict_data = {
+                            "type": bin_type.strip(),
+                            "collectionDate": date.strftime("%d/%m/%Y"),
+                        }
+                        bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/NorthHertfordshireDistrictCouncil.py

@@ -0,0 +1,93 @@
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_postcode = kwargs.get("postcode")
+        user_paon = kwargs.get("paon")
+        check_postcode(user_postcode)
+        check_paon(user_paon)
+        bindata = {"bins": []}
+
+        URI = "https://uhtn-wrp.whitespacews.com/"
+
+        session = requests.Session()
+
+        # get link from first page as has some kind of unique hash
+        r = session.get(
+            URI,
+        )
+        r.raise_for_status()
+        soup = BeautifulSoup(r.text, features="html.parser")
+
+        alink = soup.find("a", text="Find my bin collection day")
+
+        if alink is None:
+            raise Exception("Initial page did not load correctly")
+
+        # greplace 'seq' query string to skip next step
+        nextpageurl = alink["href"].replace("seq=1", "seq=2")
+
+        data = {
+            "address_name_number": user_paon,
+            "address_postcode": user_postcode,
+        }
+
+        # get list of addresses
+        r = session.post(nextpageurl, data)
+        r.raise_for_status()
+
+        soup = BeautifulSoup(r.text, features="html.parser")
+
+        # get first address (if you don't enter enough argument values this won't find the right address)
+        alink = soup.find("div", id="property_list").find("a")
+
+        if alink is None:
+            raise Exception("Address not found")
+
+        nextpageurl = URI + alink["href"]
+
+        # get collection page
+        r = session.get(
+            nextpageurl,
+        )
+        r.raise_for_status()
+        soup = BeautifulSoup(r.text, features="html.parser")
+
+        if soup.find("span", id="waste-hint"):
+            raise Exception("No scheduled services at this address")
+
+        u1s = soup.find("section", id="scheduled-collections").find_all("u1")
+
+        for u1 in u1s:
+            lis = u1.find_all("li", recursive=False)
+
+            date = lis[1].text.replace("\n", "")
+            bin_type = lis[2].text.replace("\n", "")
+
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": datetime.strptime(
+                    date,
+                    "%d/%m/%Y",
+                ).strftime(date_format),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/RoyalBoroughofGreenwich.py

@@ -0,0 +1,113 @@
+import time
+
+import requests
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_postcode = kwargs.get("postcode")
+        user_paon = kwargs.get("paon")
+        check_postcode(user_postcode)
+        check_paon(user_paon)
+        bindata = {"bins": []}
+
+        user_postcode = user_postcode.replace(" ", "+")
+
+        URI = f"https://www.royalgreenwich.gov.uk/site/custom_scripts/apps/waste-collection/new2023/source.php?term={user_postcode}"
+
+        # Make the GET request
+        response = requests.get(URI)
+
+        for address in response.json():
+            if user_paon in address:
+                collection_address = address
+                break
+
+        URI = "https://www.royalgreenwich.gov.uk/site/custom_scripts/repo/apps/waste-collection/new2023/ajax-response-uprn.php"
+
+        data = {"address": collection_address}
+
+        response = requests.post(URI, data=data)
+
+        response = response.json()
+
+        collection_day = response["Day"]
+        week = response["Frequency"]
+
+        days_of_week = [
+            "Monday",
+            "Tuesday",
+            "Wednesday",
+            "Thursday",
+            "Friday",
+            "Saturday",
+            "Sunday",
+        ]
+        collectionweek = ["Week A", "Week B"]
+
+        offset_days = days_of_week.index(collection_day)
+        week = collectionweek.index(week)
+
+        greenstartDate = datetime(2024, 11, 25)
+        bluestartDate = datetime(2024, 11, 25)
+        if week == 0:
+            blackstartDate = datetime(2024, 11, 18)
+        elif week == 1:
+            blackstartDate = datetime(2024, 11, 25)
+
+        green_dates = get_dates_every_x_days(greenstartDate, 7, 100)
+        blue_dates = get_dates_every_x_days(bluestartDate, 7, 100)
+        black_dates = get_dates_every_x_days(blackstartDate, 14, 50)
+
+        for greenDate in green_dates:
+
+            collection_date = (
+                datetime.strptime(greenDate, "%d/%m/%Y") + timedelta(days=offset_days)
+            ).strftime("%d/%m/%Y")
+
+            dict_data = {
+                "type": "Green Bin",
+                "collectionDate": collection_date,
+            }
+            bindata["bins"].append(dict_data)
+
+        for blueDate in blue_dates:
+
+            collection_date = (
+                datetime.strptime(blueDate, "%d/%m/%Y") + timedelta(days=offset_days)
+            ).strftime("%d/%m/%Y")
+
+            dict_data = {
+                "type": "Blue Bin",
+                "collectionDate": collection_date,
+            }
+            bindata["bins"].append(dict_data)
+
+        for blackDate in black_dates:
+
+            collection_date = (
+                datetime.strptime(blackDate, "%d/%m/%Y") + timedelta(days=offset_days)
+            ).strftime("%d/%m/%Y")
+
+            dict_data = {
+                "type": "Black Bin",
+                "collectionDate": collection_date,
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/SandwellBoroughCouncil.py

@@ -0,0 +1,87 @@
+import time
+
+import requests
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        SESSION_URL = "https://my.sandwell.gov.uk/authapi/isauthenticated?uri=https%253A%252F%252Fmy.sandwell.gov.uk%252Fen%252FAchieveForms%252F%253Fform_uri%253Dsandbox-publish%253A%252F%252FAF-Process-ebaa26a2-393c-4a3c-84f5-e61564192a8a%252FAF-Stage-e4c2cb32-db55-4ff5-845c-8b27f87346c4%252Fdefinition.json%2526redirectlink%253D%25252Fen%2526cancelRedirectLink%253D%25252Fen%2526consentMessage%253Dyes&hostname=my.sandwell.gov.uk&withCredentials=true"
+
+        API_URL = "https://my.sandwell.gov.uk/apibroker/runLookup"
+
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+            "User-Agent": "Mozilla/5.0",
+            "X-Requested-With": "XMLHttpRequest",
+            "Referer": "https://my.sandwell.gov.uk/fillform/?iframe_id=fillform-frame-1&db_id=",
+        }
+        s = requests.session()
+        r = s.get(SESSION_URL)
+        r.raise_for_status()
+        session_data = r.json()
+        sid = session_data["auth-session"]
+
+        data = {
+            "formValues": {
+                "Property details": {
+                    "Uprn": {
+                        "value": user_uprn,
+                    },
+                    "NextCollectionFromDate": {
+                        "value": datetime.now().strftime("%Y-%m-%d"),
+                    },
+                },
+            },
+        }
+
+        params = {
+            "id": "58a1a71694992",
+            "repeat_against": "",
+            "noRetry": "false",
+            "getOnlyTokens": "undefined",
+            "log_id": "",
+            "app_name": "AF-Renderer::Self",
+            # unix_timestamp
+            "_": str(int(time.time() * 1000)),
+            "sid": sid,
+        }
+
+        r = s.post(API_URL, json=data, headers=headers, params=params)
+        r.raise_for_status()
+
+        data = r.json()
+        rows_data = data["integration"]["transformed"]["rows_data"]
+        if not isinstance(rows_data, dict):
+            raise ValueError("Invalid data returned from API")
+        bin_types = {
+            "Recycling (Blue)",
+            "Household Waste (Grey)",
+            "Food Waste (Brown)",
+            "Garden Waste (Green)",
+        }
+        for row in rows_data.items():
+            date = row[1]["DWDate"]
+            for bin_type in bin_types:
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": date,
+                }
+                bindata["bins"].append(dict_data)
+
+        return bindata