uk_bin_collection 0.141.0__tar.gz → 0.141.1__tar.gz

{uk_bin_collection-0.141.0 → uk_bin_collection-0.141.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.141.0
+Version: 0.141.1
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.141.0 → uk_bin_collection-0.141.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.141.0"
+version = "0.141.1"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

uk_bin_collection-0.141.1/uk_bin_collection/uk_bin_collection/councils/AshfordBoroughCouncil.py

@@ -0,0 +1,135 @@
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+        # Get and check UPRN
+            user_uprn = kwargs.get("uprn")
+            user_postcode = kwargs.get("postcode")
+            check_uprn(user_uprn)
+            check_postcode(user_postcode)
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            bindata = {"bins": []}
+
+            API_URL = "https://secure.ashford.gov.uk/waste/collectiondaylookup/"
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(API_URL)
+
+            # Wait for the postcode field to appear then populate it
+            inputElement_postcode = WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located(
+                    (By.ID, "ContentPlaceHolder1_CollectionDayLookup2_TextBox_PostCode")
+                )
+            )
+            inputElement_postcode.send_keys(user_postcode)
+
+            # Click search button
+            findAddress = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (
+                        By.ID,
+                        "ContentPlaceHolder1_CollectionDayLookup2_Button_PostCodeSearch",
+                    )
+                )
+            )
+            findAddress.click()
+
+            # Wait for the 'Select your property' dropdown to appear and select the first result
+            dropdown = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (
+                        By.ID,
+                        "ContentPlaceHolder1_CollectionDayLookup2_DropDownList_Addresses",
+                    )
+                )
+            )
+
+            # Create a 'Select' for it, then select the first address in the list
+            # (Index 0 is "Make a selection from the list")
+            dropdownSelect = Select(dropdown)
+            dropdownSelect.select_by_value(str(user_uprn))
+
+            # Click search button
+            findAddress = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.ID, "ContentPlaceHolder1_CollectionDayLookup2_Button_SelectAddress")
+                )
+            )
+            findAddress.click()
+
+            h4_element = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.XPATH, "//h4[contains(text(), 'Collection Dates')]")
+                )
+            )
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            bin_tables = soup.find_all("table")
+
+            for bin_table in bin_tables:
+                bin_text = bin_table.find("td", id=re.compile("CollectionDayLookup2_td_"))
+                if not bin_text:
+                    continue
+
+                bin_type_soup = bin_text.find("b")
+
+                if not bin_type_soup:
+                    continue
+                bin_type: str = bin_type_soup.text.strip().split(" (")[0]
+
+                date_soup = bin_text.find(
+                    "span", id=re.compile(r"CollectionDayLookup2_Label_\w*_Date")
+                )
+                if not date_soup or (
+                    " " not in date_soup.text.strip()
+                    and date_soup.text.strip().lower() != "today"
+                ):
+                    continue
+                date_str: str = date_soup.text.strip()
+                try:
+                    if date_soup.text.strip().lower() == "today":
+                        date = datetime.now().date()
+                    else:
+                        date = datetime.strptime(date_str.split(" ")[1], "%d/%m/%Y").date()
+
+                except ValueError:
+                    continue
+
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": date.strftime("%d/%m/%Y"),
+                }
+                bindata["bins"].append(dict_data)
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return bindata

uk_bin_collection-0.141.1/uk_bin_collection/uk_bin_collection/councils/EastHertsCouncil.py

@@ -0,0 +1,128 @@
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Get and check UPRN
+        driver = None
+        try:
+            user_postcode = kwargs.get("postcode")
+            user_paon = kwargs.get("paon")
+            check_paon(user_paon)
+            check_postcode(user_postcode)
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            bindata = {"bins": []}
+
+            API_URL = "https://uhte-wrp.whitespacews.com"
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(API_URL)
+
+            # Click Find my bin collection day button
+            collectionButton = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable((By.LINK_TEXT, "Find my bin collection day"))
+            )
+            collectionButton.click()
+
+            main_content = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.ID, "main-content"))
+            )
+
+            # Wait for the property number field to appear then populate it
+            inputElement_number = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (
+                        By.ID,
+                        "address_name_number",
+                    )
+                )
+            )
+            inputElement_number.send_keys(user_paon)
+
+            # Wait for the postcode field to appear then populate it
+            inputElement_postcode = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (
+                        By.ID,
+                        "address_postcode",
+                    )
+                )
+            )
+            inputElement_postcode.send_keys(user_postcode)
+
+            # Click search button
+            continueButton = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (
+                        By.ID,
+                        "Submit",
+                    )
+                )
+            )
+            continueButton.click()
+
+            # Wait for the 'Search Results' to appear and select the first result
+            property = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (
+                        By.CSS_SELECTOR,
+                        "li.app-subnav__section-item a",
+                        # "app-subnav__link govuk-link clicker colordarkblue fontfamilyArial fontsize12rem",
+                        # "//a[starts-with(@aria-label, '{user_paon}')]",
+                    )
+                )
+            )
+            property.click()
+
+            upcoming_scheduled_collections = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.ID, "upcoming-scheduled-collections"))
+            )
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            collections = []
+            for collection in soup.find_all(
+                "u1",
+                class_="displayinlineblock justifycontentleft alignitemscenter margin0 padding0",
+            ):
+                date = collection.find(
+                    "p", string=lambda text: text and "/" in text
+                ).text.strip()  # Extract date
+                service = collection.find(
+                    "p", string=lambda text: text and "Collection Service" in text
+                ).text.strip()  # Extract service type
+                collections.append({"date": date, "service": service})
+
+            # Print the parsed data
+            for item in collections:
+
+                dict_data = {
+                    "type": item["service"],
+                    "collectionDate": item["date"],
+                }
+                bindata["bins"].append(dict_data)
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return bindata

uk_bin_collection-0.141.1/uk_bin_collection/uk_bin_collection/councils/HertsmereBoroughCouncil.py

@@ -0,0 +1,171 @@
+import re
+import time
+
+import requests
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            user_paon = kwargs.get("paon")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_paon(user_paon)
+            check_postcode(user_postcode)
+            bindata = {"bins": []}
+
+            URI_1 = "https://www.hertsmere.gov.uk/Environment-Refuse-and-Recycling/Recycling--Waste/Bin-collections/Collections-and-calendar.aspx"
+            URI_2 = "https://hertsmere-services.onmats.com/w/webpage/round-search"
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(URI_1)
+
+            soup = BeautifulSoup(driver.page_source, "html.parser")
+
+            current_week = (soup.find("li", class_="current")).text.strip()
+
+            strong = soup.find_all("strong", text=re.compile(r"^Week"))
+
+            bin_weeks = []
+            for tag in strong:
+                parent = tag.parent
+                bin_type = (
+                    (parent.text).split("-")[1].strip().replace("\xa0", " ").split(" and ")
+                )
+                for bin in bin_type:
+                    dict_data = {
+                        "week": tag.text.replace("\xa0", " "),
+                        "bin_type": bin,
+                    }
+                    bin_weeks.append(dict_data)
+
+            driver.get(URI_2)
+
+            # Wait for the postcode field to appear then populate it
+            inputElement_postcode = WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located(
+                    (
+                        By.CLASS_NAME,
+                        "relation_path_type_ahead_search",
+                    )
+                )
+            )
+            inputElement_postcode.send_keys(user_postcode)
+
+            WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (
+                        By.XPATH,
+                        f"//ul[@class='result_list']/li[starts-with(@aria-label, '{user_paon}')]",
+                    )
+                )
+            ).click()
+
+            WebDriverWait(driver, timeout=10).until(
+                EC.element_to_be_clickable(
+                    (
+                        By.CSS_SELECTOR,
+                        "input.fragment_presenter_template_edit.btn.bg-primary.btn-medium[type='submit']",
+                    )
+                )
+            ).click()
+
+            WebDriverWait(driver, timeout=10).until(
+                EC.presence_of_element_located(
+                    (By.XPATH, "//h3[contains(text(), 'Collection days')]")
+                )
+            )
+
+            soup = BeautifulSoup(driver.page_source, "html.parser")
+
+            table = soup.find("table", class_="table listing table-striped")
+
+            # Check if the table was found
+            if table:
+                # Extract table rows and cells
+                table_data = []
+                for row in table.find("tbody").find_all("tr"):
+                    # Extract cell data from each <td> tag
+                    row_data = [cell.get_text(strip=True) for cell in row.find_all("td")]
+                    table_data.append(row_data)
+
+            else:
+                print("Table not found.")
+
+            collection_day = (table_data[0])[1]
+
+            current_week_bins = [bin for bin in bin_weeks if bin["week"] == current_week]
+            next_week_bins = [bin for bin in bin_weeks if bin["week"] != current_week]
+
+            days_of_week = [
+                "Monday",
+                "Tuesday",
+                "Wednesday",
+                "Thursday",
+                "Friday",
+                "Saturday",
+                "Sunday",
+            ]
+
+            today = datetime.now()
+            today_idx = today.weekday()  # Monday is 0 and Sunday is 6
+            target_idx = days_of_week.index(collection_day)
+
+            days_until_target = (target_idx - today_idx) % 7
+            if days_until_target == 0:
+                next_day = today
+            else:
+                next_day = today + timedelta(days=days_until_target)
+
+            current_week_dates = get_dates_every_x_days(next_day, 14, 7)
+            next_week_date = next_day + timedelta(days=7)
+            next_week_dates = get_dates_every_x_days(next_week_date, 14, 7)
+
+            for date in current_week_dates:
+                for bin in current_week_bins:
+                    dict_data = {
+                        "type": bin["bin_type"],
+                        "collectionDate": date,
+                    }
+                    bindata["bins"].append(dict_data)
+
+            for date in next_week_dates:
+                for bin in next_week_bins:
+                    dict_data = {
+                        "type": bin["bin_type"],
+                        "collectionDate": date,
+                    }
+                    bindata["bins"].append(dict_data)
+
+            bindata["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+            )
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return bindata

uk_bin_collection-0.141.1/uk_bin_collection/uk_bin_collection/councils/SwaleBoroughCouncil.py

@@ -0,0 +1,146 @@
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+def parse_collection_date(date_string) -> datetime:
+    now = datetime.now()
+    if date_string == "is due today":
+        return now
+
+    parsed_date = datetime.strptime(date_string, "%A, %d %B").replace(year=now.year)
+
+    if now.month == 12 and parsed_date.month < 12:
+        parsed_date = parsed_date.replace(year=(now.year + 1))
+
+    return parsed_date
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            # Get postcode and UPRN from kwargs
+            user_postcode = kwargs.get("postcode")
+            user_paon = kwargs.get("paon")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_postcode(user_postcode)
+            check_paon(user_paon)
+
+            # Build URL to parse
+            council_url = "https://swale.gov.uk/bins-littering-and-the-environment/bins/check-your-bin-day"
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(council_url)
+
+            # Wait for the postcode field to appear then populate it
+            try:
+                inputElement_postcode = WebDriverWait(driver, 10).until(
+                    EC.presence_of_element_located((By.ID, "q485476_q1"))
+                )
+                inputElement_postcode.send_keys(user_postcode)
+            except Exception:
+                print("Page failed to load. Probably due to Cloudflare robot check!")
+
+            # Click search button
+            findAddress = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.ID, "form_email_485465_submit"))
+            )
+            driver.execute_script("arguments[0].click();", findAddress)
+
+            # Wait for the 'Select address' dropdown to appear and select option matching the house name/number
+            WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (
+                        By.XPATH,
+                        "//select[@name='q485480:q1']//option[contains(., '"
+                        + user_paon
+                        + "')]",
+                    )
+                )
+            ).click()
+
+            # Click search button
+            getBins = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.ID, "form_email_485465_submit"))
+            )
+            driver.execute_script("arguments[0].click();", getBins)
+
+            BinTable = WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located((By.ID, "SBCYBDSummary"))
+            )
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            soup.prettify()
+
+            data = {"bins": []}
+
+            current_year = datetime.now().year
+            next_year = current_year + 1
+
+            next_collection_date = soup.find(
+                "strong", id="SBC-YBD-collectionDate"
+            ).text.strip()
+
+            # Extract bins for the next collection
+            next_bins = [li.text.strip() for li in soup.select("#SBCFirstBins ul li")]
+
+            # Extract future collection details
+            future_collection_date_tag = soup.find(
+                "p", text=lambda t: t and "starting from" in t
+            )
+            future_collection_date = (
+                future_collection_date_tag.text.split("starting from")[-1].strip()
+                if future_collection_date_tag
+                else "No future date found"
+            )
+
+            future_bins = [li.text.strip() for li in soup.select("#FirstFutureBins li")]
+
+            for bin in next_bins:
+                collection_date = datetime.strptime(next_collection_date, "%A, %d %B")
+                if (datetime.now().month == 12) and (collection_date.month == 1):
+                    collection_date = collection_date.replace(year=next_year)
+                else:
+                    collection_date = collection_date.replace(year=current_year)
+
+                dict_data = {
+                    "type": bin,
+                    "collectionDate": collection_date.strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+            for bin in future_bins:
+                collection_date = datetime.strptime(future_collection_date, "%A, %d %B")
+                if (datetime.now().month == 12) and (collection_date.month == 1):
+                    collection_date = collection_date.replace(year=next_year)
+                else:
+                    collection_date = collection_date.replace(year=current_year)
+                dict_data = {
+                    "type": bin,
+                    "collectionDate": collection_date.strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection-0.141.1/uk_bin_collection/uk_bin_collection/councils/WindsorAndMaidenheadCouncil.py

@@ -0,0 +1,62 @@
+from datetime import datetime
+
+import dateutil.parser
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            data = {"bins": []}
+            user_uprn = kwargs.get("uprn")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_uprn(user_uprn)
+
+            root_url = "https://forms.rbwm.gov.uk/bincollections?uprn="
+            api_url = root_url + user_uprn
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(api_url)
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            soup.prettify()
+
+            # Get collections div
+            next_collection_div = soup.find("div", {"class": "widget-bin-collections"})
+
+            for tbody in next_collection_div.find_all("tbody"):
+                for tr in tbody.find_all("tr"):
+                    td = tr.find_all("td")
+                    next_collection_type = td[0].get_text()
+                    next_collection_date = dateutil.parser.parse(td[1].get_text())
+                    print(next_collection_date)
+                    dict_data = {
+                        "type": next_collection_type,
+                        "collectionDate": next_collection_date.strftime("%d/%m/%Y"),
+                    }
+                    data["bins"].append(dict_data)
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data
+