uk_bin_collection 0.152.2__tar.gz → 0.152.3__tar.gz

{uk_bin_collection-0.152.2 → uk_bin_collection-0.152.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.152.2
+Version: 0.152.3
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.152.2 → uk_bin_collection-0.152.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.152.2"
+version = "0.152.3"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

{uk_bin_collection-0.152.2 → uk_bin_collection-0.152.3}/uk_bin_collection/tests/input.json

@@ -1693,9 +1693,10 @@
         "LAD24CD": "E06000012"
     },
     "NorthHertfordshireDistrictCouncil": {
-        "house_number": "2",
+        "house_number": "22",
         "postcode": "SG6 4BJ",
         "url": "https://www.north-herts.gov.uk",
+        "web_driver": "http://selenium:4444",
         "wiki_name": "North Hertfordshire",
         "wiki_note": "Pass the house number and postcode in their respective parameters.",
         "LAD24CD": "E07000099"
@@ -2180,6 +2181,7 @@
     },
     "SouthRibbleCouncil": {
         "uprn": "010013246384",
+        "postcode": "PR5 6DT",
         "url": "https://www.southribble.gov.uk",
         "wiki_command_url_override": "https://www.southribble.gov.uk",
         "wiki_name": "South Ribble",

uk_bin_collection-0.152.3/uk_bin_collection/uk_bin_collection/councils/AdurAndWorthingCouncils.py

@@ -0,0 +1,105 @@
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import (
+    date_format,
+    get_date_with_ordinal,
+)
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Handle both string and Response objects
+        page_content = page.text if hasattr(page, "text") else page
+
+        soup = BeautifulSoup(page_content, features="html.parser")
+
+        data = {"bins": []}
+        collections = []
+
+        # Find all bin collection rows
+        bin_rows = soup.find_all("div", class_="bin-collection-listing-row")
+
+        if not bin_rows:
+            raise ValueError("No bin collection rows found in HTML")
+
+        for bin_row in bin_rows:
+            try:
+                # Get bin type from h2
+                bin_type_elem = bin_row.find("h2")
+                if not bin_type_elem:
+                    continue
+                bin_type = bin_type_elem.text.strip()
+
+                # Find next collection date - look for all <p> tags
+                paragraphs = bin_row.find_all("p")
+
+                for p in paragraphs:
+                    # Check for both singular and plural "Next collection(s):"
+                    if p.get_text() and (
+                        "Next collection:" in p.get_text()
+                        or "Next collections:" in p.get_text()
+                    ):
+                        # Extract collection dates
+                        date_text = (
+                            p.get_text()
+                            .replace("Next collection:", "")
+                            .replace("Next collections:", "")
+                            .strip()
+                        )
+
+                        # Split multiple dates if comma-separated
+                        date_strings = [date.strip() for date in date_text.split(",")]
+
+                        for date_str in date_strings:
+                            try:
+                                # Extract day number from date string (e.g. "2" from "Tuesday 27th May")
+                                day_number = int("".join(filter(str.isdigit, date_str)))
+                                # Replace ordinal in date string with plain number
+                                date_str = date_str.replace(
+                                    get_date_with_ordinal(day_number), str(day_number)
+                                )
+
+                                # Parse date with full format
+                                bin_date = datetime.strptime(date_str, "%A %d %B")
+
+                                # Add current year since it's not in the date string
+                                current_year = datetime.now().year
+                                bin_date = bin_date.replace(year=current_year)
+
+                                # If the date is in the past, it's probably for next year
+                                if bin_date < datetime.now():
+                                    bin_date = bin_date.replace(year=current_year + 1)
+
+                                collections.append((bin_type, bin_date))
+                                print(
+                                    f"Successfully parsed date for {bin_type}: {bin_date}"
+                                )
+
+                            except ValueError as e:
+                                print(
+                                    f"Failed to parse date '{date_str}' for {bin_type}: {e}"
+                                )
+                                continue
+
+                        # Found and processed the collection dates, so break the loop
+                        break
+
+            except Exception as e:
+                print(f"Error processing bin row: {e}")
+                continue
+
+        if not collections:
+            raise ValueError("No valid collection dates found")
+
+        ordered_data = sorted(collections, key=lambda x: x[1])
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0],
+                "collectionDate": item[1].strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data

{uk_bin_collection-0.152.2 → uk_bin_collection-0.152.3}/uk_bin_collection/uk_bin_collection/councils/BradfordMDC.py

@@ -130,4 +130,22 @@ class CouncilClass(AbstractGetBinDataClass):
             key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
         )
 
+        data["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        # Deduplicate the bins based on type and collection date
+        # Feels a bit hacky, but fixes
+        # https://github.com/robbrad/UKBinCollectionData/issues/1436
+        unique_bins = []
+        seen = set()
+        for bin_item in data["bins"]:
+            # Create a unique identifier for each bin entry
+            bin_key = (bin_item["type"], bin_item["collectionDate"])
+            if bin_key not in seen:
+                seen.add(bin_key)
+                unique_bins.append(bin_item)
+
+        data["bins"] = unique_bins
+
         return data

{uk_bin_collection-0.152.2 → uk_bin_collection-0.152.3}/uk_bin_collection/uk_bin_collection/councils/EastleighBoroughCouncil.py

@@ -77,8 +77,13 @@ class CouncilClass(AbstractGetBinDataClass):
             return data
 
         except Exception as e:
-            print(f"Error fetching/parsing data: {str(e)}")
-            return {"bins": [{"type": "Error", "collectionDate": "2024-01-01"}]}
+            import traceback
+
+            error_message = f"Error fetching/parsing data for Eastleigh: {str(e)}\n{traceback.format_exc()}"
+            print(error_message)
+            # Use the correct date format for the error fallback
+            today = datetime.now().strftime("%d/%m/%Y")
+            return {"bins": [{"type": "Error", "collectionDate": today}]}
         finally:
             if "driver" in locals():
                 driver.quit()

uk_bin_collection-0.152.3/uk_bin_collection/uk_bin_collection/councils/NorthHertfordshireDistrictCouncil.py

@@ -0,0 +1,287 @@
+# direct URL works, but includes a token, so I'm using Selenium
+# https://waste.nc.north-herts.gov.uk/w/webpage/find-bin-collection-day-show-details?webpage_token=c7c7c3cbc2f0478735fc746ca985b8f4221dea31c24dde99e39fb1c556b07788&auth=YTc5YTAwZmUyMGQ3&id=1421457
+
+import re
+import time
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from dateutil.parser import parse
+from selenium.common.exceptions import NoSuchElementException, TimeoutException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            data = {"bins": []}
+
+            user_paon = kwargs.get("paon")
+            postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            url = "https://waste.nc.north-herts.gov.uk/w/webpage/find-bin-collection-day-input-address"
+
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(url)
+
+            WebDriverWait(driver, 10).until(
+                lambda d: d.execute_script("return document.readyState") == "complete"
+            )
+
+            # Define the wait variable
+            wait = WebDriverWait(
+                driver, 20
+            )  # Create the wait object with a 20-second timeout
+
+            # Enter postcode - try different approaches for reliability
+            # print("Looking for postcode input...")
+
+            postcode_input = wait.until(
+                EC.element_to_be_clickable(
+                    (
+                        By.CSS_SELECTOR,
+                        "input.relation_path_type_ahead_search.form-control",
+                    )
+                ),
+                message="Postcode input not found by class",
+            )
+            postcode_input.clear()
+            postcode_input.send_keys(postcode)
+            # print(f"Entered postcode: {postcode}")
+
+            # Wait for the dropdown to load
+            # print("Waiting for address list to populate...")
+            try:
+                # Wait for the results to appear
+                wait.until(
+                    EC.presence_of_element_located(
+                        (By.CSS_SELECTOR, ".relation_path_type_ahead_results_holder")
+                    ),
+                    message="Address results container not found",
+                )
+
+                # Wait for list items to appear
+                wait.until(
+                    EC.presence_of_all_elements_located(
+                        (By.CSS_SELECTOR, ".relation_path_type_ahead_results_holder li")
+                    ),
+                    message="No address items found in the list",
+                )
+                # print("Address list populated successfully")
+
+                # Search for user_paon in the address list using aria-label attribute
+                try:
+                    # Use XPath to look for aria-label containing user_paon
+                    address_xpath = (
+                        f"//li[@aria-label and contains(@aria-label, '{user_paon}')]"
+                    )
+                    matching_address = wait.until(
+                        EC.element_to_be_clickable((By.XPATH, address_xpath)),
+                        message=f"No address containing '{user_paon}' found in aria-label attributes",
+                    )
+                    # print(f"Found matching address: {matching_address.get_attribute('aria-label')}")
+                    matching_address.click()
+                    # print("Clicked on matching address")
+
+                    # Allow time for the selection to take effect
+                    time.sleep(2)
+
+                    # Find and click the "Select address and continue" button
+                    continue_button = wait.until(
+                        EC.element_to_be_clickable(
+                            (
+                                By.CSS_SELECTOR,
+                                "input.btn.bg-green[value='Select address and continue']",
+                            )
+                        ),
+                        message="Could not find 'Select address and continue' button",
+                    )
+                    # print("Found 'Select address and continue' button, clicking it...")
+                    continue_button.click()
+                    # print("Clicked on 'Select address and continue' button")
+
+                    # Allow time for the page to load after clicking the button
+                    time.sleep(3)
+                except TimeoutException as e:
+                    # print(f"Error finding address: {e}")
+                    raise
+            except TimeoutException as e:
+                # print(f"Error loading address list: {e}")
+                raise
+
+            # After pressing Next button and waiting for page to load
+            # print("Looking for schedule list...")
+
+            # Wait for the page to load - giving it extra time
+            time.sleep(5)
+
+            # Use only the selector that we know works
+            # print("Looking for bin type elements...")
+            try:
+                bin_type_selector = (
+                    By.CSS_SELECTOR,
+                    "div.formatting_bold.formatting_size_bigger.formatting span.value-as-text",
+                )
+                WebDriverWait(driver, 15).until(
+                    EC.presence_of_element_located(bin_type_selector)
+                )
+                # print(f"Found bin type elements with selector: {bin_type_selector}")
+            except TimeoutException:
+                # print("Could not find bin type elements. Taking screenshot for debugging...")
+                screenshot_path = f"bin_type_error_{int(time.time())}.png"
+                driver.save_screenshot(screenshot_path)
+                # print(f"Screenshot saved to {screenshot_path}")
+
+            # Create BS4 object from driver's page source
+            # print("Parsing page with BeautifulSoup...")
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            # Initialize data dictionary
+            data = {"bins": []}
+
+            # Looking for bin types in the exact HTML structure
+            bin_type_elements = soup.select(
+                "div.formatting_bold.formatting_size_bigger.formatting span.value-as-text"
+            )
+            # print(f"Found {len(bin_type_elements)} bin type elements")
+
+            # Look specifically for date elements with the exact structure
+            date_elements = soup.select("div.col-sm-12.font-xs-3xl span.value-as-text")
+            hidden_dates = soup.select(
+                "div.col-sm-12.font-xs-3xl input[type='hidden'][value*='/']"
+            )
+
+            # print(f"Found {len(bin_type_elements)} bin types and {len(date_elements)} date elements")
+
+            # We need a smarter way to match bin types with their dates
+            bin_count = 0
+
+            # Map of bin types to their collection dates
+            bin_date_map = {}
+
+            # Extract all date strings that look like actual dates
+            date_texts = []
+            date_pattern = re.compile(
+                r"(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)\s+\d+(?:st|nd|rd|th)?\s+\w+\s+\d{4}",
+                re.IGNORECASE,
+            )
+
+            for element in date_elements:
+                text = element.get_text(strip=True)
+                if date_pattern.search(text):
+                    date_texts.append(text)
+                    # print(f"Found valid date text: {text}")
+
+            # Find hidden date inputs with values in DD/MM/YYYY format
+            hidden_date_values = []
+            for hidden in hidden_dates:
+                value = hidden.get("value", "")
+                if re.match(r"\d{1,2}/\d{1,2}/\d{4}", value):
+                    hidden_date_values.append(value)
+                    # print(f"Found hidden date value: {value}")
+
+            # When filtering date elements
+            date_elements = soup.select("div.col-sm-12.font-xs-3xl span.value-as-text")
+            valid_date_elements = []
+
+            for element in date_elements:
+                text = element.get_text(strip=True)
+                if contains_date(text):
+                    valid_date_elements.append(element)
+                    # print(f"Found valid date element: {text}")
+                else:
+                    pass
+                    # print(f"Skipping non-date element: {text}")
+
+            # print(f"Found {len(bin_type_elements)} bin types and {len(valid_date_elements)} valid date elements")
+
+            # When processing each bin type
+            for i, bin_type_elem in enumerate(bin_type_elements):
+                bin_type = bin_type_elem.get_text(strip=True)
+
+                # Try to find a date for this bin type
+                date_text = None
+
+                # Look for a valid date element
+                if i < len(valid_date_elements):
+                    date_elem = valid_date_elements[i]
+                    date_text = date_elem.get_text(strip=True)
+
+                # If we don't have a valid date yet, try using the hidden input
+                if not date_text or not contains_date(date_text):
+                    if i < len(hidden_dates):
+                        date_value = hidden_dates[i].get("value")
+                        if contains_date(date_value):
+                            date_text = date_value
+
+                # Skip if we don't have a valid date
+                if not date_text or not contains_date(date_text):
+                    # print(f"No valid date found for bin type: {bin_type}")
+                    continue
+
+                # print(f"Found bin type: {bin_type} with date: {date_text}")
+
+                try:
+                    # Clean up the date text
+                    date_text = remove_ordinal_indicator_from_date_string(date_text)
+
+                    # Try to parse the date
+                    try:
+                        collection_date = datetime.strptime(
+                            date_text, "%A %d %B %Y"
+                        ).date()
+                    except ValueError:
+                        try:
+                            collection_date = datetime.strptime(
+                                date_text, "%d/%m/%Y"
+                            ).date()
+                        except ValueError:
+                            # Last resort
+                            collection_date = parse(date_text).date()
+
+                    # Create bin entry
+                    bin_entry = {
+                        "type": bin_type,
+                        "collectionDate": collection_date.strftime(date_format),
+                    }
+
+                    # Add to data
+                    data["bins"].append(bin_entry)
+                    bin_count += 1
+                    # print(f"Added bin entry: {bin_entry}")
+
+                except Exception as e:
+                    pass
+                    # print(f"Error parsing date '{date_text}': {str(e)}")
+
+            # print(f"Successfully parsed {bin_count} bin collections")
+
+            if not data["bins"]:
+                # print("No bin data found. Saving page for debugging...")
+                with open(f"debug_page_{int(time.time())}.html", "w") as f:
+                    f.write(driver.page_source)
+                driver.save_screenshot(f"final_error_screenshot_{int(time.time())}.png")
+                raise ValueError(
+                    "No bin collection data could be extracted from the page"
+                )
+
+            # Sort the bin collections by date
+            data["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+            )
+
+            return data
+
+        except Exception as e:
+            # print(f"Error parsing bin collection data: {e}")
+            raise

uk_bin_collection-0.152.2/uk_bin_collection/uk_bin_collection/councils/AdurAndWorthingCouncils.py

@@ -1,88 +0,0 @@
-from datetime import datetime
-
-from bs4 import BeautifulSoup
-
-from uk_bin_collection.uk_bin_collection.common import (
-    date_format,
-    get_date_with_ordinal,
-)
-from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
-
-
-class CouncilClass(AbstractGetBinDataClass):
-    def parse_data(self, page: str, **kwargs) -> dict:
-        # Handle both string and Response objects
-        page_content = page.text if hasattr(page, "text") else page
-
-        soup = BeautifulSoup(page_content, features="html.parser")
-
-        data = {"bins": []}
-        collections = []
-
-        # Find all bin collection rows
-        bin_rows = soup.find_all("div", class_="bin-collection-listing-row")
-
-        if not bin_rows:
-            raise ValueError("No bin collection rows found in HTML")
-
-        for bin_row in bin_rows:
-            try:
-                # Get bin type from h2
-                bin_type_elem = bin_row.find("h2")
-                if not bin_type_elem:
-                    continue
-                bin_type = bin_type_elem.text.strip()
-
-                # Find next collection date - look for all <p> tags
-                paragraphs = bin_row.find_all("p")
-
-                for p in paragraphs:
-                    if p.get_text() and "Next collection:" in p.get_text():
-                        date_str = p.get_text().replace("Next collection:", "").strip()
-                        # Extract day number from date string (e.g. "2" from "Friday 2nd May")
-                        day_number = int("".join(filter(str.isdigit, date_str)))
-                        # Replace ordinal in date string with plain number
-                        date_str = date_str.replace(
-                            get_date_with_ordinal(day_number), str(day_number)
-                        )
-
-                        try:
-                            # Parse date with full format
-                            bin_date = datetime.strptime(date_str, "%A %d %B")
-
-                            # Add current year since it's not in the date string
-                            current_year = datetime.now().year
-                            bin_date = bin_date.replace(year=current_year)
-
-                            # If the date is in the past, it's probably for next year
-                            if bin_date < datetime.now():
-                                bin_date = bin_date.replace(year=current_year + 1)
-
-                            collections.append((bin_type, bin_date))
-                            print(
-                                f"Successfully parsed date for {bin_type}: {bin_date}"
-                            )
-                            break
-
-                        except ValueError as e:
-                            print(
-                                f"Failed to parse date '{date_str}' for {bin_type}: {e}"
-                            )
-                            continue
-
-            except Exception as e:
-                print(f"Error processing bin row: {e}")
-                continue
-
-        if not collections:
-            raise ValueError("No valid collection dates found")
-
-        ordered_data = sorted(collections, key=lambda x: x[1])
-        for item in ordered_data:
-            dict_data = {
-                "type": item[0],
-                "collectionDate": item[1].strftime(date_format),
-            }
-            data["bins"].append(dict_data)
-
-        return data

uk_bin_collection-0.152.2/uk_bin_collection/uk_bin_collection/councils/NorthHertfordshireDistrictCouncil.py

@@ -1,93 +0,0 @@
-import requests
-from bs4 import BeautifulSoup
-
-from uk_bin_collection.uk_bin_collection.common import *
-from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
-
-
-# import the wonderful Beautiful Soup and the URL grabber
-class CouncilClass(AbstractGetBinDataClass):
-    """
-    Concrete classes have to implement all abstract operations of the
-    base class. They can also override some operations with a default
-    implementation.
-    """
-
-    def parse_data(self, page: str, **kwargs) -> dict:
-
-        user_postcode = kwargs.get("postcode")
-        user_paon = kwargs.get("paon")
-        check_postcode(user_postcode)
-        check_paon(user_paon)
-        bindata = {"bins": []}
-
-        URI = "https://uhtn-wrp.whitespacews.com/"
-
-        session = requests.Session()
-
-        # get link from first page as has some kind of unique hash
-        r = session.get(
-            URI,
-        )
-        r.raise_for_status()
-        soup = BeautifulSoup(r.text, features="html.parser")
-
-        alink = soup.find("a", text="Find my bin collection day")
-
-        if alink is None:
-            raise Exception("Initial page did not load correctly")
-
-        # greplace 'seq' query string to skip next step
-        nextpageurl = alink["href"].replace("seq=1", "seq=2")
-
-        data = {
-            "address_name_number": user_paon,
-            "address_postcode": user_postcode,
-        }
-
-        # get list of addresses
-        r = session.post(nextpageurl, data)
-        r.raise_for_status()
-
-        soup = BeautifulSoup(r.text, features="html.parser")
-
-        # get first address (if you don't enter enough argument values this won't find the right address)
-        alink = soup.find("div", id="property_list").find("a")
-
-        if alink is None:
-            raise Exception("Address not found")
-
-        nextpageurl = URI + alink["href"]
-
-        # get collection page
-        r = session.get(
-            nextpageurl,
-        )
-        r.raise_for_status()
-        soup = BeautifulSoup(r.text, features="html.parser")
-
-        if soup.find("span", id="waste-hint"):
-            raise Exception("No scheduled services at this address")
-
-        u1s = soup.find("section", id="scheduled-collections").find_all("u1")
-
-        for u1 in u1s:
-            lis = u1.find_all("li", recursive=False)
-
-            date = lis[1].text.replace("\n", "")
-            bin_type = lis[2].text.replace("\n", "")
-
-            dict_data = {
-                "type": bin_type,
-                "collectionDate": datetime.strptime(
-                    date,
-                    "%d/%m/%Y",
-                ).strftime(date_format),
-            }
-            bindata["bins"].append(dict_data)
-
-        bindata["bins"].sort(
-            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
-        )
-
-        return bindata