uk_bin_collection 0.152.9__tar.gz → 0.152.11__tar.gz

{uk_bin_collection-0.152.9 → uk_bin_collection-0.152.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.152.9
+Version: 0.152.11
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.152.9 → uk_bin_collection-0.152.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.152.9"
+version = "0.152.11"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

{uk_bin_collection-0.152.9 → uk_bin_collection-0.152.11}/uk_bin_collection/tests/input.json

@@ -641,6 +641,13 @@
         "wiki_note": "Pass the house number and postcode in their respective parameters. This parser requires a Selenium webdriver.",
         "LAD24CD": "E07000096"
     },
+    "DarlingtonBoroughCouncil": {
+        "uprn": "10003076924",
+        "url": "https://www.darlington.gov.uk/bins-waste-and-recycling/collection-day-lookup/",
+        "wiki_name": "Darlington Borough Council",
+        "wiki_note": "Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN.",
+        "LAD24CD": "E06000005"
+    },
     "DartfordBoroughCouncil": {
         "uprn": "100060861698",
         "url": "https://www.dartford.gov.uk/waste-recycling/collection-day",
@@ -761,13 +768,11 @@
     },
     "EastHertsCouncil": {
         "LAD24CD": "E07000097",
-        "house_number": "1",
-        "postcode": "CM20 2FZ",
         "skip_get_url": true,
-        "url": "https://www.eastherts.gov.uk",
-        "web_driver": "http://selenium:4444",
+        "uprn": "10023088183",
+        "url": "https://east-herts.co.uk/api/services/",
         "wiki_name": "East Herts Council",
-        "wiki_note": "Pass the house number and postcode in their respective parameters."
+        "wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search)."
     },
     "EastLindseyDistrictCouncil": {
         "house_number": "1",

{uk_bin_collection-0.152.9 → uk_bin_collection-0.152.11}/uk_bin_collection/uk_bin_collection/councils/BroxbourneCouncil.py

@@ -22,10 +22,25 @@ class CouncilClass(AbstractGetBinDataClass):
         check_postcode(user_postcode)
         
         bindata = {"bins": []}
-        driver = create_webdriver(web_driver, headless, None, __name__)
+        # Use a realistic user agent to help bypass Cloudflare
+        user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+        driver = create_webdriver(web_driver, headless, user_agent, __name__)
         
         try:
             driver.get("https://www.broxbourne.gov.uk/bin-collection-date")
+            
+            # Wait for Cloudflare challenge to complete
+            print("Waiting for page to load (Cloudflare check)...")
+            try:
+                WebDriverWait(driver, 45).until(
+                    lambda d: "Just a moment" not in d.title and d.title != "" and len(d.find_elements(By.TAG_NAME, "input")) > 0
+                )
+                print(f"Page loaded: {driver.title}")
+            except:
+                print(f"Timeout waiting for page load. Current title: {driver.title}")
+                # Try to continue anyway
+                pass
+            
             time.sleep(8)
             
             # Handle cookie banner with multiple attempts

{uk_bin_collection-0.152.9 → uk_bin_collection-0.152.11}/uk_bin_collection/uk_bin_collection/councils/CheltenhamBoroughCouncil.py

@@ -245,25 +245,39 @@ class CouncilClass(AbstractGetBinDataClass):
             # extract table body
             for row in table.find_all("tr")[1:]:
                 if row.find_all("td")[1].text.strip() == "Normal collection day":
-                    bh_dict[
-                        parse(
-                            row.find_all("td")[0].text.strip(),
-                            dayfirst=True,
-                            fuzzy=True,
-                        ).date()
-                    ] = parse(
-                        row.find_all("td")[0].text.strip(), dayfirst=True, fuzzy=True
-                    ).date()
+                    try:
+                        # Check for normal collection day (no change)
+                        if row.find_all("td")[0].text.strip() == "Normal collection":
+                            continue
+                        else:
+                            bh_dict[
+                                parse(
+                                    row.find_all("td")[0].text.strip(),
+                                    dayfirst=True,
+                                    fuzzy=True,
+                                ).date()
+                            ] = parse(
+                                row.find_all("td")[0].text.strip(), dayfirst=True, fuzzy=True
+                            ).date()
+                    except:
+                        continue
                 else:
-                    bh_dict[
-                        parse(
-                            row.find_all("td")[0].text.strip(),
-                            dayfirst=True,
-                            fuzzy=True,
-                        ).date()
-                    ] = parse(
-                        row.find_all("td")[1].text.strip(), dayfirst=True, fuzzy=True
-                    ).date()
+                    try:
+                        # Check for normal collection day (no change)
+                        if row.find_all("td")[1].text.strip() == "Normal collection":
+                            continue
+                        else:
+                            bh_dict[
+                                parse(
+                                    row.find_all("td")[0].text.strip(),
+                                    dayfirst=True,
+                                    fuzzy=True,
+                                ).date()
+                            ] = parse(
+                                row.find_all("td")[1].text.strip(), dayfirst=True, fuzzy=True
+                            ).date()
+                    except:
+                        continue
 
         for refuse_date in refuse_dates:
             collection_date = (datetime.strptime(refuse_date, "%d/%m/%Y") + timedelta(

uk_bin_collection-0.152.11/uk_bin_collection/uk_bin_collection/councils/DarlingtonBoroughCouncil.py

@@ -0,0 +1,68 @@
+import re
+
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        data = {"bins": []}
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        url = f"https://www.darlington.gov.uk/bins-waste-and-recycling/collection-day-lookup/?uprn={user_uprn}"
+
+        # Referrer: https://www.darlington.gov.uk/bins-waste-and-recycling/collection-day-lookup/
+        # X-Requested-With: XMLHttpRequest
+        headers = {
+            "Accept": "*/*",
+            "Accept-Encoding": "gzip, deflate, br, zstd",
+            "Accept-Language": "en-GB,en;q=0.5",
+            "Referer": "https://www.darlington.gov.uk/bins-waste-and-recycling/collection-day-lookup/",
+            "Sec-Detch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-origin",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.186 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+        # Make a BS4 object
+        page = requests.get(url, headers=headers)
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        # Loop over each date card
+        card_blocks = soup.select("#detailsDisplay .refuse-results")
+
+        for card in card_blocks:
+            bin_date_tag = card.select_one(".card-footer h3")
+            if not bin_date_tag:
+                continue
+
+            bin_type = card.select_one(".card-header h2").text.strip()
+            bin_date = bin_date_tag.text.strip()
+
+            # Remove any extra text from the date "(Today)", "(Tomorrow)"
+            cleaned_bin_date = re.sub(r"\s*\(.*?\)", "", bin_date).strip()
+
+            next_binfo = {
+                "type": bin_type,
+                "collectionDate": datetime.strptime(
+                    cleaned_bin_date, "%A %d %B %Y"
+                ).strftime(date_format),
+            }
+
+            data["bins"].append(next_binfo)
+
+        return data

uk_bin_collection-0.152.11/uk_bin_collection/uk_bin_collection/councils/EastHertsCouncil.py

@@ -0,0 +1,41 @@
+import json
+import requests
+from datetime import datetime
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+        
+        # Make API request
+        api_url = f"https://east-herts.co.uk/api/services/{user_uprn}"
+        response = requests.get(api_url)
+        response.raise_for_status()
+        
+        data = response.json()
+        today = datetime.now().date()
+        
+        for service in data.get("services", []):
+            collection_date_str = service.get("collectionDate")
+            if collection_date_str:
+                collection_date = datetime.strptime(collection_date_str, "%Y-%m-%d").date()
+                # Only include future dates
+                if collection_date >= today:
+                    dict_data = {
+                        "type": service.get("binType", ""),
+                        "collectionDate": collection_date.strftime("%d/%m/%Y"),
+                    }
+                    bindata["bins"].append(dict_data)
+        
+        return bindata

{uk_bin_collection-0.152.9 → uk_bin_collection-0.152.11}/uk_bin_collection/uk_bin_collection/councils/EastLothianCouncil.py

@@ -5,7 +5,6 @@ from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
-# import the wonderful Beautiful Soup and the URL grabber
 class CouncilClass(AbstractGetBinDataClass):
     """
     Concrete classes have to implement all abstract operations of the
@@ -14,70 +13,59 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page: str, **kwargs) -> dict:
-
         user_postcode = kwargs.get("postcode")
         user_paon = kwargs.get("paon")
         check_postcode(user_postcode)
         check_paon(user_paon)
         bindata = {"bins": []}
 
-        URI = "http://collectiondates.eastlothian.gov.uk/ajax/your-calendar/load-streets-spring-2024.asp"
-
-        payload = {
-            "postcode": user_postcode,
-        }
-
+        # Get address ID from the streets endpoint
+        streets_uri = "https://collectiondates.eastlothian.gov.uk/ajax/your-calendar/load-streets-summer-2025.asp"
         headers = {
-            "Referer": "http://collectiondates.eastlothian.gov.uk/your-calendar",
+            "Referer": "https://collectiondates.eastlothian.gov.uk/your-calendar",
             "User-Agent": "Mozilla/5.0",
         }
-
-        # Make the GET request
-        response = requests.get(URI, headers=headers, params=payload)
-
-        # Parse the HTML with BeautifulSoup
+        
+        response = requests.get(streets_uri, params={"postcode": user_postcode}, headers=headers)
         soup = BeautifulSoup(response.text, "html.parser")
-
-        # Find the select dropdown
+        
         select = soup.find("select", id="SelectStreet")
-
-        # Find the option that contains "Flat 1"
+        if not select:
+            raise ValueError(f"No streets found for postcode {user_postcode}")
+        
         address = select.find("option", string=lambda text: text and user_paon in text)
-
-        URI = "http://collectiondates.eastlothian.gov.uk/ajax/your-calendar/load-recycling-summer-2024.asp"
-
-        payload = {
-            "id": address["value"],
-        }
-
-        # Make the GET request
-        response = requests.get(URI, headers=headers, params=payload)
-
-        # Parse the HTML with BeautifulSoup
+        if not address:
+            raise ValueError(f"Address '{user_paon}' not found for postcode {user_postcode}")
+        
+        address_id = address["value"]
+        
+        # Get collection data using the correct endpoint
+        collections_uri = "https://collectiondates.eastlothian.gov.uk/ajax/your-calendar/load-recycling-summer-2025.asp"
+        response = requests.get(collections_uri, params={"id": address_id}, headers=headers)
+        
         soup = BeautifulSoup(response.text, "html.parser")
-
+        
         # Extract collection details
         calendar_items = soup.find_all("div", class_="calendar-item")
         for item in calendar_items:
             waste_label = item.find("div", class_="waste-label").text.strip()
             waste_value = item.find("div", class_="waste-value").find("h4").text.strip()
-
+            
             try:
                 collection_date = datetime.strptime(
                     remove_ordinal_indicator_from_date_string(waste_value),
                     "%A %d %B %Y",
                 )
+                
+                bindata["bins"].append({
+                    "type": waste_label.replace(" is:", ""),
+                    "collectionDate": collection_date.strftime(date_format),
+                })
             except ValueError:
                 continue
-
-            dict_data = {
-                "type": waste_label.replace(" is:", ""),
-                "collectionDate": collection_date.strftime(date_format),
-            }
-            bindata["bins"].append(dict_data)
-
+        
         bindata["bins"].sort(
             key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
         )
-
+        
         return bindata

uk_bin_collection-0.152.11/uk_bin_collection/uk_bin_collection/councils/EastRenfrewshireCouncil.py

@@ -0,0 +1,122 @@
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.support.ui import Select
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            data = {"bins": []}
+            user_paon = kwargs.get("paon")
+            user_postcode = kwargs.get("postcode")
+            user_uprn = kwargs.get("uprn")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_postcode(user_postcode)
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get("https://eastrenfrewshire.gov.uk/bin-days")
+
+            # Wait for the postcode field to appear then populate it
+            inputElement_postcode = WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located(
+                    (By.CSS_SELECTOR, "input[autocomplete='postal-code']")
+                )
+            )
+            inputElement_postcode.send_keys(user_postcode)
+
+            # Click search button
+            search_button = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, "//button[text()='Search']")
+                )
+            )
+            search_button.click()
+
+            # Wait for the addresses dropdown to appear
+            addresses_select = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.XPATH, "//label[text()='Addresses']/following-sibling::select")
+                )
+            )
+            
+            # Select the appropriate address based on UPRN or house number
+            select = Select(addresses_select)
+            if user_uprn:
+                # Select by UPRN value
+                select.select_by_value(user_uprn)
+            elif user_paon:
+                # Select by house number/name in the text
+                for option in select.options:
+                    if user_paon in option.text:
+                        select.select_by_visible_text(option.text)
+                        break
+            else:
+                # Select the first non-default option
+                select.select_by_index(1)
+
+            # Click the "Find my collection dates" button
+            find_dates_button = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, "//button[text()='Find my collection dates']")
+                )
+            )
+            find_dates_button.click()
+
+            # Wait for the results table to appear
+            WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (By.XPATH, "//th[text()='Bin Type']")
+                )
+            )
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            
+            # Find the table with bin collection data
+            table = soup.find("th", string="Bin Type").find_parent("table")
+            rows = table.find_all("tr")[1:]  # Skip header row
+            
+            for row in rows:
+                cells = row.find_all("td")
+                if len(cells) >= 3:
+                    date_cell = cells[0].get_text().strip()
+                    bin_type_cell = cells[2]
+                    
+                    # Only process rows that have a date
+                    if date_cell:
+                        # Get all text content including line breaks
+                        bin_type_text = bin_type_cell.get_text(separator='\n').strip()
+                        
+                        # Split multiple bin types that appear on separate lines
+                        bin_types = [bt.strip() for bt in bin_type_text.split('\n') if bt.strip()]
+                        
+                        for bin_type in bin_types:
+                            dict_data = {
+                                "type": bin_type,
+                                "collectionDate": date_cell,
+                            }
+                            data["bins"].append(dict_data)
+
+            data["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+            )
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            raise
+        finally:
+            if driver:
+                driver.quit()
+        return data

{uk_bin_collection-0.152.9 → uk_bin_collection-0.152.11}/uk_bin_collection/uk_bin_collection/councils/EnfieldCouncil.py

@@ -30,11 +30,31 @@ class CouncilClass(AbstractGetBinDataClass):
             check_paon(user_paon)
             headless = kwargs.get("headless")
             web_driver = kwargs.get("web_driver")
-            driver = create_webdriver(web_driver, headless, None, __name__)
+            # Use a realistic user agent to help bypass Cloudflare
+            user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+            driver = create_webdriver(web_driver, headless, user_agent, __name__)
             page = "https://www.enfield.gov.uk/services/rubbish-and-recycling/find-my-collection-day"
             driver.get(page)
 
-            time.sleep(5)
+            # Wait for Cloudflare challenge to complete
+            print("Waiting for page to load (Cloudflare check)...")
+            max_attempts = 3
+            for attempt in range(max_attempts):
+                try:
+                    WebDriverWait(driver, 60).until(
+                        lambda d: "Just a moment" not in d.title and d.title != "" and len(d.find_elements(By.TAG_NAME, "input")) > 1
+                    )
+                    print(f"Page loaded: {driver.title}")
+                    break
+                except:
+                    print(f"Attempt {attempt + 1}: Timeout waiting for page load. Current title: {driver.title}")
+                    if attempt < max_attempts - 1:
+                        time.sleep(10)
+                        driver.refresh()
+                    else:
+                        print("Failed to bypass Cloudflare after multiple attempts")
+            
+            time.sleep(8)
 
             try:
                 accept_cookies = WebDriverWait(driver, timeout=10).until(
@@ -47,23 +67,73 @@ class CouncilClass(AbstractGetBinDataClass):
                 )
                 pass
 
-            postcode_input = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located(
-                    (By.CSS_SELECTOR, '[aria-label="Enter your address"]')
-                )
-            )
+            # Check for multiple iframes and find the correct one
+            try:
+                iframes = driver.find_elements(By.TAG_NAME, "iframe")
+                
+                # Try each iframe to find the one with the bin collection form
+                for i, iframe in enumerate(iframes):
+                    try:
+                        driver.switch_to.frame(iframe)
+                        
+                        # Check if this iframe has the postcode input
+                        time.sleep(2)
+                        inputs = driver.find_elements(By.TAG_NAME, "input")
+                        
+                        # Look for address-related inputs
+                        for inp in inputs:
+                            aria_label = inp.get_attribute('aria-label') or ''
+                            placeholder = inp.get_attribute('placeholder') or ''
+                            if 'address' in aria_label.lower() or 'postcode' in placeholder.lower():
+                                break
+                        else:
+                            # This iframe doesn't have the form, try the next one
+                            driver.switch_to.default_content()
+                            continue
+                        
+                        # Found the right iframe, break out of the loop
+                        break
+                    except Exception as e:
+                        driver.switch_to.default_content()
+                        continue
+                else:
+                    # No suitable iframe found, stay in main content
+                    driver.switch_to.default_content()
+            except Exception as e:
+                pass
+
+            # Try multiple selectors for the postcode input
+            postcode_input = None
+            selectors = [
+                '[aria-label="Enter your address"]',
+                'input[placeholder*="postcode"]',
+                'input[placeholder*="address"]',
+                'input[type="text"]'
+            ]
+            
+            for selector in selectors:
+                try:
+                    postcode_input = WebDriverWait(driver, 5).until(
+                        EC.element_to_be_clickable((By.CSS_SELECTOR, selector))
+                    )
+                    break
+                except:
+                    continue
+            
+            if not postcode_input:
+                raise ValueError("Could not find postcode input field")
 
             postcode_input.send_keys(user_postcode)
 
             find_address_button = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located((By.ID, "submitButton0"))
+                EC.element_to_be_clickable((By.ID, "submitButton0"))
             )
             find_address_button.click()
 
             time.sleep(15)
             # Wait for address box to be visible
-            select_address_input = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located(
+            select_address_input = WebDriverWait(driver, 15).until(
+                EC.element_to_be_clickable(
                     (
                         By.CSS_SELECTOR,
                         '[aria-label="Select full address"]',

{uk_bin_collection-0.152.9 → uk_bin_collection-0.152.11}/uk_bin_collection/uk_bin_collection/councils/FermanaghOmaghDistrictCouncil.py

@@ -16,7 +16,7 @@ class CouncilClass(AbstractGetBinDataClass):
     implementation.
     """
 
-    base_url = "https://fermanaghomagh.isl-fusion.com/"
+    base_url = "https://fermanaghomagh.isl-fusion.com"
 
     def parse_data(self, page: str, **kwargs) -> dict:
         """