uk_bin_collection 0.106.0__tar.gz → 0.107.0__tar.gz

{uk_bin_collection-0.106.0 → uk_bin_collection-0.107.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.106.0
+Version: 0.107.0
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.106.0 → uk_bin_collection-0.107.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.106.0"
+version = "0.107.0"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

{uk_bin_collection-0.106.0 → uk_bin_collection-0.107.0}/uk_bin_collection/tests/input.json

@@ -12,6 +12,13 @@
         "wiki_name": "Adur and Worthing Councils",
         "wiki_note": "Replace XXXXXXXX with UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
+    "ArdsAndNorthDownCouncil": {
+        "url": "https://www.ardsandnorthdown.gov.uk",
+        "wiki_command_url_override": "https://www.ardsandnorthdown.gov.uk",
+        "uprn": "187136177",
+        "wiki_name": "Ards and North Down Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "ArmaghBanbridgeCraigavonCouncil": {
         "url": "https://www.armaghbanbridgecraigavon.gov.uk/",
         "wiki_command_url_override": "https://www.armaghbanbridgecraigavon.gov.uk/",
@@ -27,6 +34,14 @@
         "web_driver": "http://selenium:4444",
         "wiki_name": "Arun Council"
     },
+    "AshfordBoroughCouncil": {
+        "url": "https://ashford.gov.uk",
+        "wiki_command_url_override": "https://ashford.gov.uk",
+        "postcode": "TN23 7SP",
+        "uprn": "100060777899",
+        "wiki_name": "Ashford Borough Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "AylesburyValeCouncil": {
         "skip_get_url": true,
         "uprn": "766252532",
@@ -391,6 +406,13 @@
         "wiki_name": "East Devon District Council",
         "wiki_note": "Replace XXXXXXXX with UPRN."
     },
+    "EastHertsCouncil": {
+        "house_number": "1",
+        "postcode": "CM20 2FZ",
+        "skip_get_url": true,
+        "url": "https://www.eastherts.gov.uk",
+        "wiki_name": "East Herts Council"
+    },
     "EastLindseyDistrictCouncil": {
         "house_number": "Raf Coningsby",
         "postcode": "LN4 4SY",
@@ -971,6 +993,14 @@
         "web_driver": "http://selenium:4444",
         "wiki_name": "Portsmouth City Council"
     },
+    "PowysCouncil": {
+        "house_number": "LANE COTTAGE",
+        "postcode": "HR3 5JS",
+        "skip_get_url": true,
+        "url": "https://www.powys.gov.uk",
+        "web_driver": "http://selenium:4444",
+        "wiki_name": "Powys Council"
+    },
     "PrestonCityCouncil": {
         "house_number": "Town Hall",
         "postcode": "PR1 2RL",
@@ -1519,6 +1549,13 @@
         "web_driver": "http://selenium:4444",
         "wiki_name": "Wokingham Borough Council"
     },
+    "WorcesterCityCouncil": {
+        "url": "https://www.Worcester.gov.uk",
+        "wiki_command_url_override": "https://www.Worcester.gov.uk",
+        "uprn": "100120650345",
+        "wiki_name": "Worcester City Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "WychavonDistrictCouncil": {
         "postcode": "WR3 7RU",
         "skip_get_url": true,

uk_bin_collection-0.107.0/uk_bin_collection/uk_bin_collection/councils/ArdsAndNorthDownCouncil.py

@@ -0,0 +1,103 @@
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Get and check UPRN
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        API_URL = (
+            "https://collections-ardsandnorthdown.azurewebsites.net/WSCollExternal.asmx"
+        )
+
+        # council seems to always be ARD no matter what the old council was
+        PAYLOAD = f"""<?xml version="1.0" encoding="utf-8" ?>
+        <soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+            xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+            xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
+            <soap:Body>
+                <getRoundCalendarForUPRN  xmlns="http://webaspx-collections.azurewebsites.net/">
+                    <council>ARD</council>
+                    <UPRN>{user_uprn}</UPRN>
+                    <from>Chtml</from>
+                </getRoundCalendarForUPRN >
+            </soap:Body>
+        </soap:Envelope>
+        """
+
+        r = requests.post(
+            API_URL,
+            data=PAYLOAD,
+            headers={"Content-Type": "text/xml; charset=utf-8"},
+        )
+        r.raise_for_status()
+
+        # html unescape text
+        text = (
+            (r.text.replace("&lt;", "<").replace("&gt;", ">").replace("&amp;", "&"))
+            .split("<getRoundCalendarForUPRNResult>")[-1]
+            .split("</getRoundCalendarForUPRNResult>")[0]
+        )
+
+        soup = BeautifulSoup(text, "html.parser")
+
+        # Initialize dictionary to store bin dates
+        bin_schedule = {}
+
+        # Define regex pattern to capture day and date (e.g., Tue 5 Nov)
+        date_pattern = re.compile(r"\b\w{3} \d{1,2} \w{3}\b")
+
+        current_year = datetime.now().year
+
+        # Find each bin collection line, parse date, and add to dictionary
+        for bin_info in soup.find_all("b"):
+            bin_type = bin_info.text.strip()
+            bin_details = bin_info.next_sibling.strip() if bin_info.next_sibling else ""
+            # Check for "Today" or "Tomorrow"
+            if "Today" in bin_details:
+                collection_date = datetime.now().strftime("%a %d %b")
+                bin_schedule[bin_type] = collection_date
+            elif "Tomorrow" in bin_details:
+                collection_date = (datetime.now() + timedelta(days=1)).strftime(
+                    "%a %d %b"
+                )
+                bin_schedule[bin_type] = collection_date
+            else:
+                # Extract date if it's a full date format
+                date_match = date_pattern.search(bin_details)
+                if date_match:
+                    bin_schedule[bin_type] = date_match.group()
+
+        # Display the parsed schedule with dates only
+        for bin_type, collection_date in bin_schedule.items():
+            date = datetime.strptime(collection_date, "%a %d %b")
+
+            if date.month == 1 and datetime.now().month > 1:
+                date = date.replace(year=current_year + 1)
+            else:
+                date = date.replace(year=current_year)
+
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": date.strftime("%d/%m/%Y"),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+        return bindata

uk_bin_collection-0.107.0/uk_bin_collection/uk_bin_collection/councils/AshfordBoroughCouncil.py

@@ -0,0 +1,124 @@
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Get and check UPRN
+        user_uprn = kwargs.get("uprn")
+        user_postcode = kwargs.get("postcode")
+        check_uprn(user_uprn)
+        check_postcode(user_postcode)
+        web_driver = kwargs.get("web_driver")
+        headless = kwargs.get("headless")
+        bindata = {"bins": []}
+
+        API_URL = "https://secure.ashford.gov.uk/waste/collectiondaylookup/"
+
+        # Create Selenium webdriver
+        driver = create_webdriver(web_driver, headless, None, __name__)
+        driver.get(API_URL)
+
+        # Wait for the postcode field to appear then populate it
+        inputElement_postcode = WebDriverWait(driver, 30).until(
+            EC.presence_of_element_located(
+                (By.ID, "ContentPlaceHolder1_CollectionDayLookup2_TextBox_PostCode")
+            )
+        )
+        inputElement_postcode.send_keys(user_postcode)
+
+        # Click search button
+        findAddress = WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located(
+                (
+                    By.ID,
+                    "ContentPlaceHolder1_CollectionDayLookup2_Button_PostCodeSearch",
+                )
+            )
+        )
+        findAddress.click()
+
+        # Wait for the 'Select your property' dropdown to appear and select the first result
+        dropdown = WebDriverWait(driver, 10).until(
+            EC.element_to_be_clickable(
+                (
+                    By.ID,
+                    "ContentPlaceHolder1_CollectionDayLookup2_DropDownList_Addresses",
+                )
+            )
+        )
+
+        # Create a 'Select' for it, then select the first address in the list
+        # (Index 0 is "Make a selection from the list")
+        dropdownSelect = Select(dropdown)
+        dropdownSelect.select_by_value(str(user_uprn))
+
+        # Click search button
+        findAddress = WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located(
+                (By.ID, "ContentPlaceHolder1_CollectionDayLookup2_Button_SelectAddress")
+            )
+        )
+        findAddress.click()
+
+        h4_element = WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located(
+                (By.XPATH, "//h4[contains(text(), 'Collection Dates')]")
+            )
+        )
+
+        soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+        bin_tables = soup.find_all("table")
+
+        for bin_table in bin_tables:
+            bin_text = bin_table.find("td", id=re.compile("CollectionDayLookup2_td_"))
+            if not bin_text:
+                continue
+
+            bin_type_soup = bin_text.find("b")
+
+            if not bin_type_soup:
+                continue
+            bin_type: str = bin_type_soup.text.strip().split(" (")[0]
+
+            date_soup = bin_text.find(
+                "span", id=re.compile(r"CollectionDayLookup2_Label_\w*_Date")
+            )
+            if not date_soup or (
+                " " not in date_soup.text.strip()
+                and date_soup.text.strip().lower() != "today"
+            ):
+                continue
+            date_str: str = date_soup.text.strip()
+            try:
+                if date_soup.text.strip().lower() == "today":
+                    date = datetime.now().date()
+                else:
+                    date = datetime.strptime(date_str.split(" ")[1], "%d/%m/%Y").date()
+
+            except ValueError:
+                continue
+
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": date.strftime("%d/%m/%Y"),
+            }
+            bindata["bins"].append(dict_data)
+
+        return bindata

{uk_bin_collection-0.106.0 → uk_bin_collection-0.107.0}/uk_bin_collection/uk_bin_collection/councils/CroydonCouncil.py

@@ -267,18 +267,12 @@ class CouncilClass(AbstractGetBinDataClass):
             data = {"bins": []}
 
             for e in collection_record_elements:
-                collection_type = e.find_all(
-                    "div", {"class": "fragment_presenter_template_show"}
-                )[0].text.strip()
-                collection_date = (
-                    e.find("div", {"class": "bin-collection-next"})
-                    .attrs["data-current_value"]
-                    .strip()
-                )
+                collection_type = e.find("h2").get_text()
+                collection_date = e.find("span", {"class": "value-as-text"}).get_text()
                 dict_data = {
                     "type": collection_type,
                     "collectionDate": datetime.strptime(
-                        collection_date, "%d/%m/%Y %H:%M"
+                        collection_date, "%A %d %B %Y"
                     ).strftime(date_format),
                 }
                 data["bins"].append(dict_data)

uk_bin_collection-0.107.0/uk_bin_collection/uk_bin_collection/councils/EastHertsCouncil.py

@@ -0,0 +1,117 @@
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Get and check UPRN
+        user_postcode = kwargs.get("postcode")
+        user_paon = kwargs.get("paon")
+        check_paon(user_paon)
+        check_postcode(user_postcode)
+        web_driver = kwargs.get("web_driver")
+        headless = kwargs.get("headless")
+        bindata = {"bins": []}
+
+        API_URL = "https://uhte-wrp.whitespacews.com"
+
+        # Create Selenium webdriver
+        driver = create_webdriver(web_driver, headless, None, __name__)
+        driver.get(API_URL)
+
+        # Click Find my bin collection day button
+        collectionButton = WebDriverWait(driver, 10).until(
+            EC.element_to_be_clickable((By.LINK_TEXT, "Find my bin collection day"))
+        )
+        collectionButton.click()
+
+        main_content = WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located((By.ID, "main-content"))
+        )
+
+        # Wait for the property number field to appear then populate it
+        inputElement_number = WebDriverWait(driver, 10).until(
+            EC.element_to_be_clickable(
+                (
+                    By.ID,
+                    "address_name_number",
+                )
+            )
+        )
+        inputElement_number.send_keys(user_paon)
+
+        # Wait for the postcode field to appear then populate it
+        inputElement_postcode = WebDriverWait(driver, 10).until(
+            EC.element_to_be_clickable(
+                (
+                    By.ID,
+                    "address_postcode",
+                )
+            )
+        )
+        inputElement_postcode.send_keys(user_postcode)
+
+        # Click search button
+        continueButton = WebDriverWait(driver, 10).until(
+            EC.element_to_be_clickable(
+                (
+                    By.ID,
+                    "Submit",
+                )
+            )
+        )
+        continueButton.click()
+
+        # Wait for the 'Search Results' to appear and select the first result
+        property = WebDriverWait(driver, 10).until(
+            EC.element_to_be_clickable(
+                (
+                    By.CSS_SELECTOR,
+                    "li.app-subnav__section-item a",
+                    # "app-subnav__link govuk-link clicker colordarkblue fontfamilyArial fontsize12rem",
+                    # "//a[starts-with(@aria-label, '{user_paon}')]",
+                )
+            )
+        )
+        property.click()
+
+        upcoming_scheduled_collections = WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located((By.ID, "upcoming-scheduled-collections"))
+        )
+
+        soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+        collections = []
+        for collection in soup.find_all(
+            "u1",
+            class_="displayinlineblock justifycontentleft alignitemscenter margin0 padding0",
+        ):
+            date = collection.find(
+                "p", string=lambda text: text and "/" in text
+            ).text.strip()  # Extract date
+            service = collection.find(
+                "p", string=lambda text: text and "Collection Service" in text
+            ).text.strip()  # Extract service type
+            collections.append({"date": date, "service": service})
+
+        # Print the parsed data
+        for item in collections:
+
+            dict_data = {
+                "type": item["service"],
+                "collectionDate": item["date"],
+            }
+            bindata["bins"].append(dict_data)
+
+        return bindata

{uk_bin_collection-0.106.0 → uk_bin_collection-0.107.0}/uk_bin_collection/uk_bin_collection/councils/ForestOfDeanDistrictCouncil.py

@@ -82,9 +82,16 @@ class CouncilClass(AbstractGetBinDataClass):
                 columns = row.find_all("td")
                 if columns:
                     container_type = row.find("th").text.strip()
-                    collection_day = re.sub(
-                        r"[^a-zA-Z0-9,\s]", "", columns[0].get_text()
-                    ).strip()
+                    if columns[0].get_text() == "Today":
+                        collection_day = datetime.now().strftime("%a, %d %B")
+                    elif columns[0].get_text() == "Tomorrow":
+                        collection_day = (datetime.now() + timedelta(days=1)).strftime(
+                            "%a, %d %B"
+                        )
+                    else:
+                        collection_day = re.sub(
+                            r"[^a-zA-Z0-9,\s]", "", columns[0].get_text()
+                        ).strip()
 
                     # Parse the date from the string
                     parsed_date = datetime.strptime(collection_day, "%a, %d %B")

uk_bin_collection-0.107.0/uk_bin_collection/uk_bin_collection/councils/PowysCouncil.py

@@ -0,0 +1,138 @@
+import time
+
+from bs4 import BeautifulSoup
+from dateutil.relativedelta import relativedelta
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        data = {"bins": []}
+        user_paon = kwargs.get("paon")
+        user_postcode = kwargs.get("postcode")
+        web_driver = kwargs.get("web_driver")
+        headless = kwargs.get("headless")
+        check_paon(user_paon)
+        check_postcode(user_postcode)
+
+        user_paon = user_paon.upper()
+
+        # Create Selenium webdriver
+        driver = create_webdriver(web_driver, headless, None, __name__)
+        driver.get("https://en.powys.gov.uk/binday")
+
+        accept_button = WebDriverWait(driver, timeout=10).until(
+            EC.element_to_be_clickable(
+                (
+                    By.NAME,
+                    "acceptall",
+                )
+            )
+        )
+        accept_button.click()
+
+        # Wait for the postcode field to appear then populate it
+        inputElement_postcode = WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located(
+                (By.ID, "BINDAYLOOKUP_ADDRESSLOOKUP_ADDRESSLOOKUPPOSTCODE")
+            )
+        )
+        inputElement_postcode.send_keys(user_postcode)
+
+        # Click search button
+        findAddress = WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located(
+                (By.ID, "BINDAYLOOKUP_ADDRESSLOOKUP_ADDRESSLOOKUPSEARCH")
+            )
+        )
+        findAddress.click()
+
+        # Wait for the 'Select address' dropdown to appear and select option matching the house name/number
+        WebDriverWait(driver, 10).until(
+            EC.element_to_be_clickable(
+                (
+                    By.XPATH,
+                    "//select[@id='BINDAYLOOKUP_ADDRESSLOOKUP_ADDRESSLOOKUPADDRESS']//option[contains(., '"
+                    + user_paon
+                    + "')]",
+                )
+            )
+        ).click()
+
+        # Wait for the submit button to appear, then click it to get the collection dates
+        WebDriverWait(driver, 30).until(
+            EC.element_to_be_clickable(
+                (By.ID, "BINDAYLOOKUP_ADDRESSLOOKUP_ADDRESSLOOKUPBUTTONS_NEXT")
+            )
+        ).click()
+
+        # Wait for the collections table to appear
+        WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located(
+                (By.ID, "BINDAYLOOKUP_COLLECTIONDATES_COLLECTIONDATES")
+            )
+        )
+
+        soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+        # General rubbish collection dates
+        general_rubbish_section = soup.find(
+            "h3", string="General Rubbish / Wheelie bin"
+        )
+        general_rubbish_dates = [
+            li.text for li in general_rubbish_section.find_next("ul").find_all("li")
+        ]
+
+        for date in general_rubbish_dates:
+            dict_data = {
+                "type": "General Rubbish / Wheelie bin",
+                "collectionDate": datetime.strptime(
+                    remove_ordinal_indicator_from_date_string(date), "%d %B %Y"
+                ).strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        # Recycling and food waste collection dates
+        recycling_section = soup.find("h3", string="Recycling and Food Waste")
+        recycling_dates = [
+            li.text for li in recycling_section.find_next("ul").find_all("li")
+        ]
+
+        for date in recycling_dates:
+            dict_data = {
+                "type": "Recycling and Food Waste",
+                "collectionDate": datetime.strptime(
+                    remove_ordinal_indicator_from_date_string(date), "%d %B %Y"
+                ).strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        # Garden waste collection dates
+        garden_waste_section = soup.find("h3", string="Garden Waste")
+        garden_waste_dates = [
+            li.text for li in garden_waste_section.find_next("ul").find_all("li")
+        ]
+        for date in garden_waste_dates:
+            dict_data = {
+                "type": "Garden Waste",
+                "collectionDate": datetime.strptime(
+                    remove_ordinal_indicator_from_date_string(date), "%d %B %Y"
+                ).strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data