uk_bin_collection 0.152.7__py3-none-any.whl → 0.152.9__py3-none-any.whl

uk_bin_collection/tests/input.json

@@ -100,16 +100,11 @@
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN.",
         "LAD24CD": "E07000105"
     },
-    "AylesburyValeCouncil": {
-        "skip_get_url": true,
-        "uprn": "766252532",
-        "url": "http://avdcbins.web-labs.co.uk/RefuseApi.asmx",
-        "wiki_name": "Buckinghamshire",
-        "wiki_note": "To get the UPRN, please use [FindMyAddress](https://www.findmyaddress.co.uk/search). Returns all published collections in the past, present, future.",
-        "LAD24CD": "E06000060"
-    },
     "BCPCouncil": {
         "LAD24CD": "E06000058",
+        "house_number": "3 HARBOUR VIEW ROAD, POOLE, BH14 0PD",
+        "postcode": "BH14 0PD",
+        "web_driver": "http://selenium:4444",
         "skip_get_url": true,
         "uprn": "100040810214",
         "url": "https://online.bcpcouncil.gov.uk/bindaylookup/",
@@ -137,8 +132,8 @@
         "LAD24CD": "E09000002"
     },
     "BarnetCouncil": {
-        "house_number": "HA8 7NA, 2, MANOR PARK GARDENS, EDGWARE, BARNET",
-        "postcode": "HA8 7NA",
+        "house_number": "26A",
+        "postcode": "EN4 8TB",
         "skip_get_url": true,
         "url": "https://www.barnet.gov.uk/recycling-and-waste/bin-collections/find-your-bin-collection-day",
         "web_driver": "http://selenium:4444",
@@ -158,7 +153,7 @@
     "BasildonCouncil": {
         "skip_get_url": true,
         "uprn": "10013350430",
-        "url": "https://basildonportal.azurewebsites.net/api/getPropertyRefuseInformation",
+        "url": "https://mybasildon.powerappsportals.com/check/where_i_live/",
         "wiki_name": "Basildon",
         "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search).",
         "LAD24CD": "E07000066"
@@ -366,6 +361,7 @@
         "postcode": "EN8 7FL",
         "uprn": "148048608",
         "url": "https://www.broxbourne.gov.uk",
+        "web_driver": "http://selenium:4444",
         "wiki_name": "Broxbourne",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN.",
         "LAD24CD": "E07000095"
@@ -381,10 +377,10 @@
         "LAD24CD": "E07000172"
     },
     "BuckinghamshireCouncil": {
-        "house_number": "2",
+        "house_number": "The Ridings, Magpie Lane, Loudwater, High Wycombe, HP13 7BA",
         "postcode": "HP13 7BA",
-        "skip_get_url": true,
-        "url": "https://iapp.itouchvision.com/iappcollectionday/collection-day/?uuid=FA353FC74600CBE61BE409534D00A8EC09BDA3AC&lang=en",
+        "uprn": "100081093078",
+        "url": "https://www.buckinghamshire.gov.uk/waste-and-recycling/find-out-when-its-your-bin-collection/",
         "web_driver": "http://selenium:4444",
         "wiki_name": "Buckinghamshire",
         "wiki_note": "Pass the house name/number and postcode in their respective arguments, both wrapped in quotes.",
@@ -584,7 +580,7 @@
         "LAD24CD": "E06000052"
     },
     "CotswoldDistrictCouncil": {
-        "house_number": "19",
+        "house_number": "19 SUMMERS WAY, MORETON-IN-MARSH, GL56 0GB",
         "postcode": "GL56 0GB",
         "skip_get_url": true,
         "url": "https://community.cotswold.gov.uk/s/waste-collection-enquiry",

uk_bin_collection/uk_bin_collection/councils/AngusCouncil.py

@@ -1,4 +1,3 @@
-import time
 import re
 from datetime import datetime
 
@@ -7,6 +6,7 @@ from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import Select, WebDriverWait
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
 
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
@@ -27,56 +27,95 @@ class CouncilClass(AbstractGetBinDataClass):
             headless = kwargs.get("headless")
             web_driver = kwargs.get("web_driver")
             driver = create_webdriver(web_driver, headless, None, __name__)
-            page = "https://www.angus.gov.uk/bins_litter_and_recycling/bin_collection_days"
-
-            driver.get(page)
+            
+            driver.get("https://www.angus.gov.uk/bins_litter_and_recycling/bin_collection_days")
 
-            wait = WebDriverWait(driver, 10)
-            accept_cookies_button = wait.until(
-                EC.element_to_be_clickable((By.ID, "ccc-recommended-settings"))
-            )
-            accept_cookies_button.click()
+            wait = WebDriverWait(driver, 20)
+            
+            # Accept cookies if present
+            try:
+                accept_cookies_button = wait.until(
+                    EC.element_to_be_clickable((By.ID, "ccc-recommended-settings"))
+                )
+                accept_cookies_button.click()
+            except TimeoutException:
+                print("Cookie banner not found, continuing...")
 
+            # Click on "Find bin collection days" link
             find_your_collection_button = wait.until(
                 EC.element_to_be_clickable(
-                    (By.XPATH, "/html/body/div[2]/div[2]/div/div/section/div[2]/div/article/div/div/p[2]/a")
+                    (By.XPATH, "//a[contains(text(), 'Find bin collection days') or contains(@href, 'collection')]")
                 )
             )
             find_your_collection_button.click()
 
+            # Wait for iframe to be present and switch to it
             iframe = wait.until(EC.presence_of_element_located((By.ID, "fillform-frame-1")))
             driver.switch_to.frame(iframe)
 
-            postcode_input = wait.until(EC.presence_of_element_located((By.ID, "searchString")))
-            postcode_input.send_keys(user_postcode + Keys.TAB + Keys.ENTER)
-
-            time.sleep(15)
-
-            select_elem = wait.until(EC.presence_of_element_located((By.ID, "customerAddress")))
-            WebDriverWait(driver, 10).until(
-                lambda d: len(select_elem.find_elements(By.TAG_NAME, "option")) > 1
-            )
-            dropdown = Select(select_elem)
+            # Handle banner/modal if present
+            try:
+                close_button = wait.until(EC.element_to_be_clickable((By.TAG_NAME, "button")))
+                if close_button.text.strip().lower() in ['close', 'dismiss', 'ok']:
+                    close_button.click()
+            except TimeoutException:
+                pass
+
+            # Wait for postcode input to be clickable
+            postcode_input = wait.until(EC.element_to_be_clickable((By.ID, "searchString")))
+            postcode_input.clear()
+            postcode_input.send_keys(user_postcode)
+            
+            # Find and click the search button
+            try:
+                submit_btn = driver.find_element(By.XPATH, "//button[contains(text(), 'Search')]")
+                submit_btn.click()
+            except:
+                try:
+                    submit_btn = driver.find_element(By.XPATH, "//input[@type='submit']")
+                    submit_btn.click()
+                except:
+                    postcode_input.send_keys(Keys.TAB)
+                    postcode_input.send_keys(Keys.ENTER)
+
+            # Wait for address dropdown to be present
+            address_dropdown = wait.until(EC.presence_of_element_located((By.ID, "customerAddress")))
+            
+            # Wait for dropdown options to populate with extended timeout
+            try:
+                WebDriverWait(driver, 30).until(
+                    lambda d: len(d.find_element(By.ID, "customerAddress").find_elements(By.TAG_NAME, "option")) > 1
+                )
+            except TimeoutException:
+                options = address_dropdown.find_elements(By.TAG_NAME, "option")
+                raise ValueError(f"Dropdown only has {len(options)} options after 30s wait")
+            
+            # Select the UPRN from dropdown
+            dropdown = Select(address_dropdown)
             dropdown.select_by_value(user_uprn)
 
-            time.sleep(10)
-
+            # Wait for results to appear
             wait.until(
                 EC.presence_of_element_located(
-                    (By.CSS_SELECTOR, "span.fieldInput.content.html.non-input"))
+                    (By.CSS_SELECTOR, "span.fieldInput.content.html.non-input")
+                )
             )
+            
+            # Wait additional time for JavaScript to populate the data
+            import time
+            time.sleep(15)  # Wait 15 seconds for dynamic content to load
 
+            # Parse the results
             soup = BeautifulSoup(driver.page_source, "html.parser")
             bin_data = {"bins": []}
             current_date = datetime.now()
             current_formatted_date = None
 
             spans = soup.select("span.fieldInput.content.html.non-input")
-            print(f"Found {len(spans)} bin info spans.")
 
             for i, span in enumerate(spans):
                 try:
-                    # Look for any non-empty <u> tag recursively
+                    # Look for date in <u> tags
                     date_tag = next(
                         (u for u in span.find_all("u") if u and u.text.strip()),
                         None
@@ -93,22 +132,15 @@ class CouncilClass(AbstractGetBinDataClass):
                             if parsed_date.date() < current_date.date():
                                 parsed_date = parsed_date.replace(year=current_date.year + 1)
                             current_formatted_date = parsed_date.strftime("%d/%m/%Y")
-                            print(f"[{i}] Parsed date: {current_formatted_date}")
-                        except ValueError as ve:
-                            print(f"[{i}] Could not parse date: '{full_date_str}' - {ve}")
+                        except ValueError:
                             continue
-                    else:
-                        print(f"[{i}] No date tag found, using last valid date: {current_formatted_date}")
-
-                    if not current_formatted_date:
-                        print(f"[{i}] No current date to associate bin type with — skipping.")
-                        continue
 
-                    if not bin_type_tag or not bin_type_tag.text.strip():
-                        print(f"[{i}] No bin type found — skipping.")
+                    if not current_formatted_date or not bin_type_tag:
                         continue
 
                     bin_type = bin_type_tag.text.strip()
+                    if not bin_type:
+                        continue
 
                     # Optional seasonal override
                     try:
@@ -118,25 +150,16 @@ class CouncilClass(AbstractGetBinDataClass):
                     except Exception:
                         pass
 
-                    print(f"[{i}] Found bin: {bin_type} on {current_formatted_date}")
-
                     bin_data["bins"].append({
                         "type": bin_type,
                         "collectionDate": current_formatted_date
                     })
 
-                except Exception as inner_e:
-                    print(f"[{i}] Skipping span due to error: {inner_e}")
-                    continue
-
-                except Exception as inner_e:
-                    print(f"Skipping span due to error: {inner_e}")
+                except Exception:
                     continue
 
             if not bin_data["bins"]:
                 raise ValueError("No bin data found.")
-
-            print(bin_data)
             
             return bin_data
 

uk_bin_collection/uk_bin_collection/councils/BCPCouncil.py

@@ -1,13 +1,15 @@
 import json
-from datetime import timedelta
-
-import requests
+import time
+from datetime import datetime
 from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait, Select
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.keys import Keys
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
-# import the wonderful Beautiful Soup and the URL grabber
 class CouncilClass(AbstractGetBinDataClass):
     """
     Concrete classes have to implement all abstract operations of the
@@ -16,36 +18,116 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page: str, **kwargs) -> dict:
-
-        user_uprn = kwargs.get("uprn")
-        check_uprn(user_uprn)
-
-        api_url = f"https://online.bcpcouncil.gov.uk/bcp-apis/?api=BinDayLookup&uprn={user_uprn}"
-
-        requests.packages.urllib3.disable_warnings()
-        response = requests.get(api_url)
-        json_data = json.loads(response.text)
-        data = {"bins": []}
-        collections = []
-
-        for bin in json_data:
-            bin_type = bin["BinType"]
-            next_date = datetime.strptime(
-                bin["Next"], "%m/%d/%Y %I:%M:%S %p"
-            ) + timedelta(hours=1)
-            subseq_date = datetime.strptime(
-                bin["Subsequent"], "%m/%d/%Y %I:%M:%S %p"
-            ) + timedelta(hours=1)
-            collections.append((bin_type, next_date))
-            collections.append((bin_type, subseq_date))
-
-        ordered_data = sorted(collections, key=lambda x: x[1])
-        data = {"bins": []}
-        for item in ordered_data:
-            dict_data = {
-                "type": item[0],
-                "collectionDate": item[1].strftime(date_format),
-            }
-            data["bins"].append(dict_data)
-
-        return data
+        postcode = kwargs.get("postcode")
+        house_number = kwargs.get("paon")
+        web_driver = kwargs.get("web_driver")
+        headless = kwargs.get("headless", True)
+        
+        check_postcode(postcode)
+        check_paon(house_number)
+        
+        driver = create_webdriver(web_driver, headless=headless)
+        
+        try:
+            driver.get("https://bcpportal.bcpcouncil.gov.uk/checkyourbincollection/")
+            
+            # Handle cookie banner first
+            try:
+                cookie_button = WebDriverWait(driver, 5).until(
+                    EC.element_to_be_clickable((By.XPATH, "//button[contains(text(), 'Okay')]"))
+                )
+                cookie_button.click()
+            except:
+                pass  # Cookie banner might not be present
+            
+            # Wait for and enter postcode
+            postcode_input = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.CSS_SELECTOR, "input[type='text']"))
+            )
+            postcode_input.clear()
+            postcode_input.send_keys(postcode)
+            
+            # Click the search span element
+            search_button = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable((By.ID, "searchAddress"))
+            )
+            search_button.click()
+            
+            # Wait for address dropdown
+            select_element = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.TAG_NAME, "select"))
+            )
+            
+            # Find and select the address containing the house number
+            address_option = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable((By.XPATH, f"//option[contains(text(), 'HARBOUR VIEW ROAD')]"))
+            )
+            address_option.click()
+            
+            # Wait for bin collection results to load
+            WebDriverWait(driver, 15).until(
+                EC.presence_of_element_located((By.XPATH, "//td[contains(text(), 'collection')] | //th[contains(text(), 'collection')]"))
+            )
+            
+            # Find the table containing collection data by looking for a cell with 'collection' text
+            collection_table = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.XPATH, "//td[contains(text(), 'collection')]/ancestor::table | //th[contains(text(), 'collection')]/ancestor::table"))
+            )
+            
+            # Parse the table data
+            soup = BeautifulSoup(driver.page_source, 'html.parser')
+            data = {"bins": []}
+            
+            # Find the table containing collection information
+            collection_cell = soup.find(['td', 'th'], string=lambda text: text and 'collection' in text.lower())
+            if collection_cell:
+                table = collection_cell.find_parent('table')
+                if table:
+                    rows = table.find_all('tr')
+                    for row in rows[1:]:  # Skip header row
+                        cells = row.find_all(['td', 'th'])
+                        if len(cells) >= 2:  # At least bin type and one collection date
+                            bin_type = cells[0].get_text(strip=True)
+                            next_collection = cells[1].get_text(strip=True) if len(cells) > 1 else ""
+                            following_collection = cells[2].get_text(strip=True) if len(cells) > 2 else ""
+                            
+                            
+                            # Process next collection date
+                            if bin_type and next_collection and "No collection" not in next_collection:
+                                try:
+                                    # Try multiple date formats
+                                    for date_fmt in ["%A, %d %B %Y", "%A %d %B %Y", "%d/%m/%Y", "%d-%m-%Y", "%Y-%m-%d"]:
+                                        try:
+                                            parsed_date = datetime.strptime(next_collection, date_fmt)
+                                            data["bins"].append({
+                                                "type": bin_type,
+                                                "collectionDate": parsed_date.strftime(date_format)
+                                            })
+                                            break
+                                        except ValueError:
+                                            continue
+                                except:
+                                    continue
+                            
+                            # Process following collection date
+                            if bin_type and following_collection and "No collection" not in following_collection and "download PDF" not in following_collection:
+                                try:
+                                    # Clean up the following collection text (remove PDF link text)
+                                    following_collection = following_collection.replace("download PDF", "").strip()
+                                    for date_fmt in ["%A, %d %B %Y", "%A %d %B %Y", "%d/%m/%Y", "%d-%m-%Y", "%Y-%m-%d"]:
+                                        try:
+                                            parsed_date = datetime.strptime(following_collection, date_fmt)
+                                            data["bins"].append({
+                                                "type": bin_type,
+                                                "collectionDate": parsed_date.strftime(date_format)
+                                            })
+                                            break
+                                        except ValueError:
+                                            continue
+                                except:
+                                    continue
+            
+            return data
+            
+        finally:
+            driver.quit()