uk_bin_collection 0.114.6__py3-none-any.whl → 0.116.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/BroxbourneCouncil.py

@@ -0,0 +1,71 @@
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        user_postcode = kwargs.get("postcode")
+        check_uprn(user_uprn)
+        check_postcode(user_postcode)
+        bindata = {"bins": []}
+
+        API_URL = "https://www.broxbourne.gov.uk/xfp/form/205"
+
+        post_data = {
+            "page": "490",
+            "locale": "en_GB",
+            "qacf7e570cf99fae4cb3a2e14d5a75fd0d6561058_0_0": user_postcode,
+            "qacf7e570cf99fae4cb3a2e14d5a75fd0d6561058_1_0": user_uprn,
+            "next": "Next",
+        }
+
+        r = requests.post(API_URL, data=post_data)
+        r.raise_for_status()
+
+        soup = BeautifulSoup(r.content, features="html.parser")
+        soup.prettify()
+
+        form__instructions = soup.find(attrs={"class": "form__instructions"})
+        table = form__instructions.find("table")
+
+        rows = table.find_all("tr")
+
+        current_year = datetime.now().year
+
+        # Process each row into a list of dictionaries
+        for row in rows[1:]:  # Skip the header row
+            columns = row.find_all("td")
+            collection_date = (
+                columns[0].get_text(separator=" ").replace("\xa0", " ").strip()
+            )
+            service = columns[1].get_text(separator=" ").replace("\xa0", " ").strip()
+
+            collection_date = datetime.strptime(collection_date, "%a %d %b")
+
+            if collection_date.month == 1:
+                collection_date = collection_date.replace(year=current_year + 1)
+            else:
+                collection_date = collection_date.replace(year=current_year)
+
+            dict_data = {
+                "type": service,
+                "collectionDate": (collection_date).strftime(date_format),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+        return bindata

uk_bin_collection/uk_bin_collection/councils/CheshireWestAndChesterCouncil.py

@@ -1,123 +1,105 @@
-import logging
 import time
 
-from bs4 import BeautifulSoup
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.support.ui import Select
-from selenium.webdriver.support.wait import WebDriverWait
+import requests
 
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
-# Set up logging
-logging.basicConfig(
-    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
-)
 
+# import the wonderful Beautiful Soup and the URL grabber
 class CouncilClass(AbstractGetBinDataClass):
-    def parse_data(self, page: str, **kwargs) -> dict:
-        driver = None
-        try:
-            data = {"bins": []}
-            collections = []
-            user_uprn = kwargs.get("uprn")
-            user_paon = kwargs.get("paon")
-            user_postcode = kwargs.get("postcode")
-            web_driver = kwargs.get("web_driver")
-            headless = kwargs.get("headless")
-            check_paon(user_paon)
-            check_postcode(user_postcode)
-
-            # Create Selenium webdriver
-            driver = create_webdriver(web_driver, headless, None, __name__)
-            if headless:
-                driver.set_window_size(1920, 1080)
-
-            driver.get(
-                "https://www.cheshirewestandchester.gov.uk/residents/waste-and-recycling/your-bin-collection/collection-day"
-            )
-            wait = WebDriverWait(driver, 60)
-
-            def click_element(by, value):
-                element = wait.until(EC.element_to_be_clickable((by, value)))
-                driver.execute_script("arguments[0].scrollIntoView();", element)
-                element.click()
-
-            logging.info("Accepting cookies")
-            click_element(By.ID, "ccc-close")
-
-            logging.info("Finding collection day")
-            click_element(By.LINK_TEXT, "Find your collection day")
-
-            logging.info("Switching to iframe")
-            iframe_presence = wait.until(
-                EC.presence_of_element_located((By.ID, "fillform-frame-1"))
-            )
-            driver.switch_to.frame(iframe_presence)
-
-            logging.info("Entering postcode")
-            input_element_postcode = wait.until(
-                EC.presence_of_element_located(
-                    (By.XPATH, '//input[@id="postcode_search"]')
-                )
-            )
-            input_element_postcode.send_keys(user_postcode)
-
-            pcsearch_btn = wait.until(
-                EC.element_to_be_clickable((By.XPATH, "//input[@id='postcode_search']"))
-            )
-            click_element(By.XPATH, "//input[@id='postcode_search']")
-
-            logging.info("Selecting address")
-            dropdown = wait.until(EC.element_to_be_clickable((By.ID, "Choose_Address")))
-            dropdown_options = wait.until(
-                EC.presence_of_element_located((By.CLASS_NAME, "lookup-option"))
-            )
-            drop_down_values = Select(dropdown)
-            option_element = wait.until(
-                EC.presence_of_element_located(
-                    (By.CSS_SELECTOR, f'option.lookup-option[value="{str(user_uprn)}"]')
-                )
-            )
-            driver.execute_script("arguments[0].scrollIntoView();", option_element)
-            drop_down_values.select_by_value(str(user_uprn))
-
-            logging.info("Waiting for bin schedule")
-            wait.until(
-                EC.presence_of_element_located(
-                    (By.CLASS_NAME, "bin-schedule-content-bin-card")
-                )
-            )
-
-            logging.info("Extracting bin collection data")
-            soup = BeautifulSoup(driver.page_source, features="html.parser")
-            bin_cards = soup.find_all("div", {"class": "bin-schedule-content-bin-card"})
-            collections = []
-
-            for card in bin_cards:
-                bin_info = card.find("div", {"class": "bin-schedule-content-info"})
-                bin_name = bin_info.find_all("p")[0].text.strip() + " bin"
-                bin_date_str = bin_info.find_all("p")[1].text.split(":")[1].strip()
-                bin_date = datetime.strptime(bin_date_str, "%A, %B %d, %Y")
-                collections.append((bin_name, bin_date))
-
-            ordered_data = sorted(collections, key=lambda x: x[1])
-
-            for item in ordered_data:
-                dict_data = {
-                    "type": item[0].capitalize(),
-                    "collectionDate": item[1].strftime(date_format),
-                }
-                data["bins"].append(dict_data)
-
-            logging.info("Data extraction complete")
-            return data
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
 
-        except Exception as e:
-            logging.error(f"An error occurred: {e}")
-            raise
+    def parse_data(self, page: str, **kwargs) -> dict:
 
-        finally:
-            if driver:
-                driver.quit()
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        SESSION_URL = "https://my.cheshirewestandchester.gov.uk/authapi/isauthenticated?uri=https://my.cheshirewestandchester.gov.uk/fillform/?iframe_id=fillform-frame-1&db_id=&hostname=my.cheshirewestandchester.gov.uk&withCredentials=true"
+
+        API_URL = "https://my.cheshirewestandchester.gov.uk/apibroker/runLookup"
+
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+            "User-Agent": "Mozilla/5.0",
+            "X-Requested-With": "XMLHttpRequest",
+            "Referer": "https://mycouncil.milton-keynes.gov.uk/fillform/?iframe_id=fillform-frame-1&db_id=",
+        }
+        s = requests.session()
+        r = s.get(SESSION_URL)
+        r.raise_for_status()
+        session_data = r.json()
+        sid = session_data["auth-session"]
+        params = {
+            "id": "609b918c7dd6d",
+            "repeat_against": "",
+            "noRetry": "false",
+            "getOnlyTokens": "undefined",
+            "log_id": "",
+            "app_name": "AchieveForms",
+            # unix_timestamp
+            "_": str(int(time.time() * 1000)),
+            "sid": sid,
+        }
+
+        r = s.post(API_URL, headers=headers, params=params)
+        r.raise_for_status()
+
+        data = r.json()
+        rows_data = data["integration"]["transformed"]["rows_data"]["0"]
+        AuthenticateResponse = rows_data["AuthenticateResponse"]
+
+        params = {
+            "id": "6101d23110243",
+            "repeat_against": "",
+            "noRetry": "false",
+            "getOnlyTokens": "undefined",
+            "log_id": "",
+            "app_name": "AchieveForms",
+            # unix_timestamp
+            "_": str(int(time.time() * 1000)),
+            "sid": sid,
+        }
+
+        data = {
+            "formValues": {
+                "Section 1": {
+                    "UPRN": {
+                        "value": user_uprn,
+                    },
+                    "AuthenticateResponse": {
+                        "value": AuthenticateResponse,
+                    },
+                }
+            },
+        }
+
+        r = s.post(API_URL, json=data, headers=headers, params=params)
+        r.raise_for_status()
+
+        data = r.json()
+        rows_data = data["integration"]["transformed"]["rows_data"]
+        if not isinstance(rows_data, dict):
+            raise ValueError("Invalid data returned from API")
+
+        # Extract each service's relevant details for the bin schedule
+        for item in rows_data.values():
+            dict_data = {
+                "type": item["serviceType"],
+                "collectionDate": datetime.strptime(
+                    item["collectionDateTime"], "%Y-%m-%dT%H:%M:%S"
+                ).strftime(date_format),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/DerbyCityCouncil.py

@@ -0,0 +1,55 @@
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        URI = f"https://secure.derby.gov.uk/binday/Binday?search.PremisesId={user_uprn}"
+
+        # Make the GET request
+        session = requests.Session()
+        response = session.get(URI)
+
+        soup = BeautifulSoup(response.content, "html.parser")
+
+        # Find all divs with class "binresult" which contain the bin collection information
+        bin_results = soup.find_all("div", class_="binresult")
+
+        # Loop through each bin result to extract date and bin type
+        for result in bin_results:
+            # Find the collection date
+            date_text = result.find("p").strong.get_text(strip=True)
+
+            # Find the bin type by looking at the 'alt' attribute of the img tag
+            bin_type = result.find("img")["alt"]
+
+            if bin_type != "No bins":
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": datetime.strptime(
+                        date_text,
+                        "%A, %d %B %Y:",
+                    ).strftime(date_format),
+                }
+                bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/GraveshamBoroughCouncil.py

@@ -0,0 +1,122 @@
+import time
+
+import requests
+from dateutil.relativedelta import relativedelta
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        SESSION_URL = "https://my.gravesham.gov.uk/authapi/isauthenticated?uri=https%253A%252F%252Fmy.gravesham.gov.uk%252Fen%252FAchieveForms%252F%253Fform_uri%253Dsandbox-publish%253A%252F%252FAF-Process-22218d5c-c6d6-492f-b627-c713771126be%252FAF-Stage-905e87c1-144b-4a72-8932-5518ddd3e618%252Fdefinition.json%2526redirectlink%253D%25252Fen%2526cancelRedirectLink%253D%25252Fen%2526consentMessage%253Dyes&hostname=my.gravesham.gov.uk&withCredentials=true"
+
+        API_URL = "https://my.gravesham.gov.uk/apibroker/runLookup"
+
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+            "User-Agent": "Mozilla/5.0",
+            "X-Requested-With": "XMLHttpRequest",
+            "Referer": "https://my.gravesham.gov.uk/fillform/?iframe_id=fillform-frame-1&db_id=",
+        }
+        s = requests.session()
+        r = s.get(SESSION_URL)
+        r.raise_for_status()
+        session_data = r.json()
+        sid = session_data["auth-session"]
+        params = {
+            "id": "5ee8854759297",
+            "repeat_against": "",
+            "noRetry": "false",
+            "getOnlyTokens": "undefined",
+            "log_id": "",
+            "app_name": "AF-Renderer::Self",
+            # unix_timestamp
+            "_": str(int(time.time() * 1000)),
+            "sid": sid,
+        }
+        r = s.post(API_URL, headers=headers, params=params)
+        r.raise_for_status()
+
+        data = r.json()
+        rows_data = data["integration"]["transformed"]["rows_data"]["0"]
+        tokenString = rows_data["tokenString"]
+
+        # Get the current date and time
+        current_datetime = datetime.now()
+        future_datetime = current_datetime + relativedelta(months=1)
+
+        # Format it using strftime
+        current_datetime = current_datetime.strftime("%Y-%m-%dT%H:%M:%S")
+        future_datetime = future_datetime.strftime("%Y-%m-%dT%H:%M:%S")
+
+        data = {
+            "formValues": {
+                "Check your bin day": {
+                    "tokenString": {
+                        "value": tokenString,
+                    },
+                    "UPRNForAPI": {
+                        "value": user_uprn,
+                    },
+                    "formatDateToday": {
+                        "value": current_datetime,
+                    },
+                    "formatDateTo": {
+                        "value": future_datetime,
+                    },
+                }
+            },
+        }
+
+        params = {
+            "id": "5c8f869376376",
+            "repeat_against": "",
+            "noRetry": "false",
+            "getOnlyTokens": "undefined",
+            "log_id": "",
+            "app_name": "AF-Renderer::Self",
+            # unix_timestamp
+            "_": str(int(time.time() * 1000)),
+            "sid": sid,
+        }
+        r = s.post(API_URL, json=data, headers=headers, params=params)
+        r.raise_for_status()
+
+        data = r.json()
+
+        rows_data = data["integration"]["transformed"]["rows_data"]
+        if not isinstance(rows_data, dict):
+            raise ValueError("Invalid data returned from API")
+
+        # Extract each service's relevant details for the bin schedule
+        for item in rows_data.values():
+            if item["Name"]:
+                Bin_Types = item["Name"].split("Empty Bin ")
+                for Bin_Type in Bin_Types:
+                    if Bin_Type:
+                        dict_data = {
+                            "type": Bin_Type.strip(),
+                            "collectionDate": datetime.strptime(
+                                item["Date"], "%Y-%m-%dT%H:%M:%S"
+                            ).strftime(date_format),
+                        }
+                        bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+        return bindata

uk_bin_collection/uk_bin_collection/councils/HertsmereBoroughCouncil.py

@@ -0,0 +1,161 @@
+import re
+import time
+
+import requests
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_paon = kwargs.get("paon")
+        user_postcode = kwargs.get("postcode")
+        web_driver = kwargs.get("web_driver")
+        headless = kwargs.get("headless")
+        check_paon(user_paon)
+        check_postcode(user_postcode)
+        bindata = {"bins": []}
+
+        URI_1 = "https://www.hertsmere.gov.uk/Environment-Refuse-and-Recycling/Recycling--Waste/Bin-collections/Collections-and-calendar.aspx"
+        URI_2 = "https://hertsmere-services.onmats.com/w/webpage/round-search"
+
+        # Create Selenium webdriver
+        driver = create_webdriver(web_driver, headless, None, __name__)
+        driver.get(URI_1)
+
+        soup = BeautifulSoup(driver.page_source, "html.parser")
+
+        current_week = (soup.find("li", class_="current")).text.strip()
+
+        strong = soup.find_all("strong", text=re.compile(r"^Week"))
+
+        bin_weeks = []
+        for tag in strong:
+            parent = tag.parent
+            bin_type = (
+                (parent.text).split("-")[1].strip().replace("\xa0", " ").split(" and ")
+            )
+            for bin in bin_type:
+                dict_data = {
+                    "week": tag.text.replace("\xa0", " "),
+                    "bin_type": bin,
+                }
+                bin_weeks.append(dict_data)
+
+        driver.get(URI_2)
+
+        # Wait for the postcode field to appear then populate it
+        inputElement_postcode = WebDriverWait(driver, 30).until(
+            EC.presence_of_element_located(
+                (
+                    By.CLASS_NAME,
+                    "relation_path_type_ahead_search",
+                )
+            )
+        )
+        inputElement_postcode.send_keys(user_postcode)
+
+        WebDriverWait(driver, 10).until(
+            EC.element_to_be_clickable(
+                (
+                    By.XPATH,
+                    f"//ul[@class='result_list']/li[starts-with(@aria-label, '{user_paon}')]",
+                )
+            )
+        ).click()
+
+        WebDriverWait(driver, timeout=10).until(
+            EC.element_to_be_clickable(
+                (
+                    By.CSS_SELECTOR,
+                    "input.fragment_presenter_template_edit.btn.bg-primary.btn-medium[type='submit']",
+                )
+            )
+        ).click()
+
+        WebDriverWait(driver, timeout=10).until(
+            EC.presence_of_element_located(
+                (By.XPATH, "//h3[contains(text(), 'Collection days')]")
+            )
+        )
+
+        soup = BeautifulSoup(driver.page_source, "html.parser")
+
+        table = soup.find("table", class_="table listing table-striped")
+
+        # Check if the table was found
+        if table:
+            # Extract table rows and cells
+            table_data = []
+            for row in table.find("tbody").find_all("tr"):
+                # Extract cell data from each <td> tag
+                row_data = [cell.get_text(strip=True) for cell in row.find_all("td")]
+                table_data.append(row_data)
+
+        else:
+            print("Table not found.")
+
+        collection_day = (table_data[0])[1]
+
+        current_week_bins = [bin for bin in bin_weeks if bin["week"] == current_week]
+        next_week_bins = [bin for bin in bin_weeks if bin["week"] != current_week]
+
+        days_of_week = [
+            "Monday",
+            "Tuesday",
+            "Wednesday",
+            "Thursday",
+            "Friday",
+            "Saturday",
+            "Sunday",
+        ]
+
+        today = datetime.now()
+        today_idx = today.weekday()  # Monday is 0 and Sunday is 6
+        target_idx = days_of_week.index(collection_day)
+
+        days_until_target = (target_idx - today_idx) % 7
+        if days_until_target == 0:
+            next_day = today
+        else:
+            next_day = today + timedelta(days=days_until_target)
+
+        current_week_dates = get_dates_every_x_days(next_day, 14, 7)
+        next_week_date = next_day + timedelta(days=7)
+        next_week_dates = get_dates_every_x_days(next_week_date, 14, 7)
+
+        for date in current_week_dates:
+            for bin in current_week_bins:
+                dict_data = {
+                    "type": bin["bin_type"],
+                    "collectionDate": date,
+                }
+                bindata["bins"].append(dict_data)
+
+        for date in next_week_dates:
+            for bin in next_week_bins:
+                dict_data = {
+                    "type": bin["bin_type"],
+                    "collectionDate": date,
+                }
+                bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata