uk_bin_collection 0.152.11__py3-none-any.whl → 0.154.0__py3-none-any.whl

uk_bin_collection/tests/input.json

@@ -102,12 +102,9 @@
     },
     "BCPCouncil": {
         "LAD24CD": "E06000058",
-        "house_number": "3 HARBOUR VIEW ROAD, POOLE, BH14 0PD",
-        "postcode": "BH14 0PD",
-        "web_driver": "http://selenium:4444",
         "skip_get_url": true,
         "uprn": "100040810214",
-        "url": "https://online.bcpcouncil.gov.uk/bindaylookup/",
+        "url": "https://bcpportal.bcpcouncil.gov.uk/checkyourbincollection",
         "wiki_name": "Bournemouth, Christchurch and Poole",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
@@ -253,7 +250,7 @@
         "postcode": "BL1 5PQ",
         "skip_get_url": true,
         "uprn": "100010886936",
-        "url": "https://carehomes.bolton.gov.uk/bins.aspx",
+        "url": "https://web.bolton.gov.uk/bins.aspx",
         "web_driver": "http://selenium:4444",
         "wiki_name": "Bolton",
         "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search). Previously required a single field that was UPRN and full address; now requires UPRN and postcode as separate fields.",
@@ -377,13 +374,10 @@
         "LAD24CD": "E07000172"
     },
     "BuckinghamshireCouncil": {
-        "house_number": "The Ridings, Magpie Lane, Loudwater, High Wycombe, HP13 7BA",
-        "postcode": "HP13 7BA",
         "uprn": "100081093078",
         "url": "https://www.buckinghamshire.gov.uk/waste-and-recycling/find-out-when-its-your-bin-collection/",
-        "web_driver": "http://selenium:4444",
         "wiki_name": "Buckinghamshire",
-        "wiki_note": "Pass the house name/number and postcode in their respective arguments, both wrapped in quotes.",
+        "wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
         "LAD24CD": "E06000060"
     },
     "BurnleyBoroughCouncil": {
@@ -891,7 +885,7 @@
     "ErewashBoroughCouncil": {
         "skip_get_url": true,
         "uprn": "10003582028",
-        "url": "https://map.erewash.gov.uk/isharelive.web/myerewash.aspx",
+        "url": "https://www.erewash.gov.uk",
         "wiki_name": "Erewash",
         "wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
         "LAD24CD": "E07000036"
@@ -998,7 +992,7 @@
     },
     "GlasgowCityCouncil": {
         "uprn": "906700034497",
-        "url": "https://onlineservices.glasgow.gov.uk/forms/RefuseAndRecyclingWebApplication/AddressSearch.aspx",
+        "url": "https://onlineservices.glasgow.gov.uk/forms/refuseandrecyclingcalendar/AddressSearch.aspx",
         "skip_get_url": true,
         "wiki_name": "Glasgow City",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN.",
@@ -1446,7 +1440,7 @@
         "house_number": "71",
         "postcode": "ME16 8BT",
         "url": "https://my.maidstone.gov.uk/service/Find-your-bin-day",
-        "web_driver": "http://selenium:4444", 
+        "web_driver": "http://selenium:4444",
         "wiki_name": "Maidstone",
         "wiki_note": "Pass the house number and postcode in their respective parameters. This parser requires a Selenium webdriver.",
         "LAD24CD": "E07000110"
@@ -1782,21 +1776,22 @@
         "LAD24CD": "E06000065"
     },
     "NorthumberlandCouncil": {
-        "house_number": "22",
-        "postcode": "NE46 1UQ",
+        "uprn": "010096302588",
+        "postcode": "NE65 0ZP",
         "skip_get_url": true,
-        "url": "https://www.northumberland.gov.uk/Waste/Household-waste/Household-bin-collections/Bin-Calendars.aspx",
+        "url": "https://bincollection.northumberland.gov.uk/postcode",
         "web_driver": "http://selenium:4444",
         "wiki_name": "Northumberland",
-        "wiki_note": "Pass the house number and postcode in their respective parameters. This parser requires a Selenium webdriver.",
+        "wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
         "LAD24CD": "E06000057"
     },
     "NorwichCityCouncil": {
-        "uprn": "100090888980",
-        "url": "https://www.norwich.gov.uk",
-        "wiki_command_url_override": "https://www.norwich.gov.uk",
+        "house_number": "2",
+        "postcode": "NR2 3TT",
+        "url": "https://bnr-wrp.whitespacews.com",
+        "wiki_command_url_override": "hhttps://bnr-wrp.whitespacews.com",
         "wiki_name": "Norwich",
-        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN.",
+        "wiki_note": "Pass the house number and postcode in their respective parameters.",
         "LAD24CD": "E07000148"
     },
     "NottinghamCityCouncil": {
@@ -2807,4 +2802,4 @@
         "wiki_note": "Provide your UPRN.",
         "LAD24CD": "E06000014"
     }
-}
+}

uk_bin_collection/uk_bin_collection/councils/BCPCouncil.py

@@ -1,15 +1,13 @@
-import json
 import time
-from datetime import datetime
-from bs4 import BeautifulSoup
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support.ui import WebDriverWait, Select
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.common.keys import Keys
+
+import requests
+from dateutil.relativedelta import relativedelta
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
+# import the wonderful Beautiful Soup and the URL grabber
 class CouncilClass(AbstractGetBinDataClass):
     """
     Concrete classes have to implement all abstract operations of the
@@ -18,116 +16,43 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page: str, **kwargs) -> dict:
-        postcode = kwargs.get("postcode")
-        house_number = kwargs.get("paon")
-        web_driver = kwargs.get("web_driver")
-        headless = kwargs.get("headless", True)
-        
-        check_postcode(postcode)
-        check_paon(house_number)
-        
-        driver = create_webdriver(web_driver, headless=headless)
-        
-        try:
-            driver.get("https://bcpportal.bcpcouncil.gov.uk/checkyourbincollection/")
-            
-            # Handle cookie banner first
-            try:
-                cookie_button = WebDriverWait(driver, 5).until(
-                    EC.element_to_be_clickable((By.XPATH, "//button[contains(text(), 'Okay')]"))
-                )
-                cookie_button.click()
-            except:
-                pass  # Cookie banner might not be present
-            
-            # Wait for and enter postcode
-            postcode_input = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located((By.CSS_SELECTOR, "input[type='text']"))
-            )
-            postcode_input.clear()
-            postcode_input.send_keys(postcode)
-            
-            # Click the search span element
-            search_button = WebDriverWait(driver, 10).until(
-                EC.element_to_be_clickable((By.ID, "searchAddress"))
-            )
-            search_button.click()
-            
-            # Wait for address dropdown
-            select_element = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located((By.TAG_NAME, "select"))
-            )
-            
-            # Find and select the address containing the house number
-            address_option = WebDriverWait(driver, 10).until(
-                EC.element_to_be_clickable((By.XPATH, f"//option[contains(text(), 'HARBOUR VIEW ROAD')]"))
-            )
-            address_option.click()
-            
-            # Wait for bin collection results to load
-            WebDriverWait(driver, 15).until(
-                EC.presence_of_element_located((By.XPATH, "//td[contains(text(), 'collection')] | //th[contains(text(), 'collection')]"))
-            )
-            
-            # Find the table containing collection data by looking for a cell with 'collection' text
-            collection_table = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located((By.XPATH, "//td[contains(text(), 'collection')]/ancestor::table | //th[contains(text(), 'collection')]/ancestor::table"))
-            )
-            
-            # Parse the table data
-            soup = BeautifulSoup(driver.page_source, 'html.parser')
-            data = {"bins": []}
-            
-            # Find the table containing collection information
-            collection_cell = soup.find(['td', 'th'], string=lambda text: text and 'collection' in text.lower())
-            if collection_cell:
-                table = collection_cell.find_parent('table')
-                if table:
-                    rows = table.find_all('tr')
-                    for row in rows[1:]:  # Skip header row
-                        cells = row.find_all(['td', 'th'])
-                        if len(cells) >= 2:  # At least bin type and one collection date
-                            bin_type = cells[0].get_text(strip=True)
-                            next_collection = cells[1].get_text(strip=True) if len(cells) > 1 else ""
-                            following_collection = cells[2].get_text(strip=True) if len(cells) > 2 else ""
-                            
-                            
-                            # Process next collection date
-                            if bin_type and next_collection and "No collection" not in next_collection:
-                                try:
-                                    # Try multiple date formats
-                                    for date_fmt in ["%A, %d %B %Y", "%A %d %B %Y", "%d/%m/%Y", "%d-%m-%Y", "%Y-%m-%d"]:
-                                        try:
-                                            parsed_date = datetime.strptime(next_collection, date_fmt)
-                                            data["bins"].append({
-                                                "type": bin_type,
-                                                "collectionDate": parsed_date.strftime(date_format)
-                                            })
-                                            break
-                                        except ValueError:
-                                            continue
-                                except:
-                                    continue
-                            
-                            # Process following collection date
-                            if bin_type and following_collection and "No collection" not in following_collection and "download PDF" not in following_collection:
-                                try:
-                                    # Clean up the following collection text (remove PDF link text)
-                                    following_collection = following_collection.replace("download PDF", "").strip()
-                                    for date_fmt in ["%A, %d %B %Y", "%A %d %B %Y", "%d/%m/%Y", "%d-%m-%Y", "%Y-%m-%d"]:
-                                        try:
-                                            parsed_date = datetime.strptime(following_collection, date_fmt)
-                                            data["bins"].append({
-                                                "type": bin_type,
-                                                "collectionDate": parsed_date.strftime(date_format)
-                                            })
-                                            break
-                                        except ValueError:
-                                            continue
-                                except:
-                                    continue
-            
-            return data
-            
-        finally:
-            driver.quit()
+        # Make a BS4 object
+        uprn = kwargs.get("uprn")
+        # usrn = kwargs.get("paon")
+        check_uprn(uprn)
+        # check_usrn(usrn)
+        bindata = {"bins": []}
+
+        # uprn = uprn.zfill(12)
+
+        API_URL = "https://prod-17.uksouth.logic.azure.com/workflows/58253d7b7d754447acf9fe5fcf76f493/triggers/manual/paths/invoke?api-version=2016-06-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=TAvYIUFj6dzaP90XQCm2ElY6Cd34ze05I3ba7LKTiBs"
+
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "*/*",
+            "User-Agent": "Mozilla/5.0",
+            "Referer": "https://bcpportal.bcpcouncil.gov.uk/",
+        }
+        s = requests.session()
+        data = {
+            "uprn": uprn,
+        }
+
+        r = s.post(API_URL, json=data, headers=headers)
+        r.raise_for_status()
+
+        data = r.json()
+        rows_data = data["data"]
+        for row in rows_data:
+            bin_type = row["wasteContainerUsageTypeDescription"]
+            collections = row["scheduleDateRange"]
+            for collection in collections:
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": datetime.strptime(
+                        collection, "%Y-%m-%d"
+                    ).strftime(date_format),
+                }
+                bindata["bins"].append(dict_data)
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/BasingstokeCouncil.py

@@ -1,6 +1,8 @@
-from bs4 import BeautifulSoup
 from datetime import datetime
+
 import requests
+from bs4 import BeautifulSoup
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
@@ -10,6 +12,7 @@ COLLECTION_KINDS = {
     "glass": "rteelem_ctl03_pnlCollections_Glass",
     # Garden waste data is only returned if the property is subscribed to the Garden Waste service
     "garden": "rteelem_ctl03_pnlCollections_GardenWaste",
+    "food": "rteelem_ctl03_pnlCollections_Food",
 }
 
 

uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py

@@ -35,7 +35,7 @@ class CouncilClass(AbstractGetBinDataClass):
             data = {"bins": []}
 
             # Get our initial session running
-            page = "https://carehomes.bolton.gov.uk/bins.aspx"
+            page = "https://web.bolton.gov.uk/bins.aspx"
 
             driver = create_webdriver(web_driver, headless, None, __name__)
             driver.get(page)

uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py

@@ -72,7 +72,9 @@ class CouncilClass(AbstractGetBinDataClass):
                     break
 
             if not found:
-                raise Exception(f"Address containing '{user_paon}' not found in dropdown options")
+                raise Exception(
+                    f"Address containing '{user_paon}' not found in dropdown options"
+                )
 
             submit_btn = wait.until(
                 EC.presence_of_element_located(
@@ -84,7 +86,7 @@ class CouncilClass(AbstractGetBinDataClass):
 
             results = wait.until(
                 EC.presence_of_element_located(
-                    (By.XPATH, f'//span[contains(@class,"collection-sub")]')
+                    (By.XPATH, f'//div[contains(@class,"mx-name-listView1")]')
                 )
             )
 
@@ -96,44 +98,21 @@ class CouncilClass(AbstractGetBinDataClass):
             current_date = datetime.now()
 
             # Find all elements with class starting with 'mx-name-index-'
-            bins = soup.find_all(class_=lambda x: x and x.startswith("mx-name-index-"))
+            bin_view = soup.find(class_="mx-name-listView1")
+            bins = bin_view.find_all(
+                class_=lambda x: x and x.startswith("mx-name-index-")
+            )
 
             for bin_item in bins:
-                bin_type = bin_item.find(class_="collection-main").text.strip()
-                day_of_week_elements = bin_item.find_all(class_="collection-header")
-                bin_date = None
-
-                for elem in day_of_week_elements:
-                    if (
-                        elem.text.strip() != bin_type
-                    ):  # Avoid taking the bin type as the date
-                        next_sibling = elem.find_next_sibling()
-                        if next_sibling:
-                            bin_date_str = next_sibling.text.strip()
-                            try:
-                                # Try parsing the date string in the format 'dd Month' (e.g., '30 Dec', '5 January')
-                                bin_date = datetime.strptime(bin_date_str, "%d %b")
-                            except ValueError:
-                                try:
-                                    # If the above format fails, try 'dd MonthName' (e.g., '30 December', '5 January')
-                                    bin_date = datetime.strptime(bin_date_str, "%d %B")
-                                except ValueError:
-                                    pass
-
-                            if bin_date:
-                                # Set the year based on the logic provided
-                                if bin_date.month < current_date.month:
-                                    bin_date = bin_date.replace(
-                                        year=current_date.year + 1
-                                    )
-                                else:
-                                    bin_date = bin_date.replace(year=current_date.year)
-                                # Format the date to the desired format
-                                bin_date = bin_date.strftime("%d/%m/%Y")
-                                break
+                bin_type = bin_item.find(class_="mx-name-text31").text.strip()
+
+                bin_date_str = bin_item.find(class_="mx-name-text29").text.strip()
+
+                bin_date = datetime.strptime(bin_date_str, "%d %B %Y")
+                bin_date = bin_date.strftime(date_format)
+
                 dict_data = {"type": bin_type, "collectionDate": bin_date}
                 data["bins"].append(dict_data)
-                print(data)
         except Exception as e:
             # Here you can log the exception if needed
             print(f"An error occurred: {e}")

uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py

@@ -1,11 +1,26 @@
-from bs4 import BeautifulSoup
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.support.wait import WebDriverWait
+import json
+from dataclasses import asdict, dataclass
+from typing import Literal
 
-from uk_bin_collection.uk_bin_collection.common import *
+import requests
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import padding
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+
+from uk_bin_collection.uk_bin_collection.common import check_uprn
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
+key_hex = "F57E76482EE3DC3336495DEDEEF3962671B054FE353E815145E29C5689F72FEC"
+iv_hex = "2CBF4FC35C69B82362D393A4F0B9971A"
+
+
+@dataclass
+class BucksInput:
+    P_CLIENT_ID: Literal[152]
+    P_COUNCIL_ID: Literal[34505]
+    P_LANG_CODE: Literal["EN"]
+    P_UPRN: str
+
 
 class CouncilClass(AbstractGetBinDataClass):
     """
@@ -14,113 +29,73 @@ class CouncilClass(AbstractGetBinDataClass):
     implementation.
     """
 
-    def parse_data(self, page: str, **kwargs) -> dict:
-        driver = None
+    def encode_body(self, bucks_input: BucksInput):
+        key = bytes.fromhex(key_hex)
+        iv = bytes.fromhex(iv_hex)
+
+        json_data = json.dumps(asdict(bucks_input))
+        data_bytes = json_data.encode("utf-8")
+
+        padder = padding.PKCS7(128).padder()
+        padded_data = padder.update(data_bytes) + padder.finalize()
+
+        backend = default_backend()
+        cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=backend)
+        encryptor = cipher.encryptor()
+        ciphertext = encryptor.update(padded_data) + encryptor.finalize()
+
+        return ciphertext.hex()
+
+    def decode_response(self, hex_input: str):
+
+        key = bytes.fromhex(key_hex)
+        iv = bytes.fromhex(iv_hex)
+        ciphertext = bytes.fromhex(hex_input)
+
+        backend = default_backend()
+        cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=backend)
+        decryptor = cipher.decryptor()
+        decrypted_padded = decryptor.update(ciphertext) + decryptor.finalize()
+
+        unpadder = padding.PKCS7(128).unpadder()
+        plaintext_bytes = unpadder.update(decrypted_padded) + unpadder.finalize()
+        plaintext = plaintext_bytes.decode("utf-8")
+
+        return json.loads(plaintext)
+
+    def parse_data(self, _: str, **kwargs) -> dict:
         try:
-            data = {"bins": []}
-            user_paon = kwargs.get("paon")
-            user_postcode = kwargs.get("postcode")
-            user_uprn = kwargs.get("uprn")
-            web_driver = kwargs.get("web_driver")
-            headless = kwargs.get("headless")
-            check_paon(user_paon)
-            check_postcode(user_postcode)
-
-            # Create Selenium webdriver
-            driver = create_webdriver(web_driver, headless, None, __name__)
-            driver.get(kwargs.get("url"))
-
-            # Click "Check now" button
-            check_now_button = WebDriverWait(driver, 10).until(
-                EC.element_to_be_clickable((By.XPATH, "//a[contains(text(), 'Check now')]"))
+            user_uprn: str = kwargs.get("uprn") or ""
+            check_uprn(user_uprn)
+            bucks_input = BucksInput(
+                P_CLIENT_ID=152, P_COUNCIL_ID=34505, P_LANG_CODE="EN", P_UPRN=user_uprn
             )
-            check_now_button.click()
 
-            # Wait for the postcode field to appear then populate it
-            inputElement_postcode = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located((By.ID, "postcodeSearch"))
-            )
-            inputElement_postcode.send_keys(user_postcode)
+            encoded_input = self.encode_body(bucks_input)
 
-            # Click Find button
-            find_button = WebDriverWait(driver, 10).until(
-                EC.element_to_be_clickable((By.XPATH, "//button[contains(text(), 'Find')]"))
+            session = requests.Session()
+            response = session.post(
+                "https://itouchvision.app/portal/itouchvision/kmbd/collectionDay",
+                data=encoded_input,
             )
-            find_button.click()
-
-            # Wait for the address dropdown and select by UPRN
-            if user_uprn:
-                address_option = WebDriverWait(driver, 10).until(
-                    EC.element_to_be_clickable((By.XPATH, f"//option[@value='{user_uprn}']"))
-                )
-                address_option.click()
-            else:
-                # Fallback to selecting by address text
-                address_option = WebDriverWait(driver, 10).until(
-                    EC.element_to_be_clickable(
-                        (By.XPATH, f"//select[@id='addressSelect']//option[contains(., '{user_paon}')]")
-                    )
-                )
-                address_option.click()
 
-            # Wait a moment for the page to update after address selection
-            import time
-            time.sleep(2)
+            output = response.text
 
-            # Wait for collection information to appear - try multiple possible selectors
-            try:
-                WebDriverWait(driver, 15).until(
-                    EC.presence_of_element_located((By.XPATH, "//h2[contains(text(), 'Your next collections')]"))
-                )
-            except:
-                # Alternative wait for collection data structure
-                WebDriverWait(driver, 10).until(
-                    EC.presence_of_element_located((By.XPATH, "//div[contains(@class, 'ant-row') and contains(@class, 'd-flex')]//h3[@class='text-white']"))
+            decoded_bins = self.decode_response(output)
+            data: dict[str, list[dict[str, str]]] = {}
+            data["bins"] = list(
+                map(
+                    lambda a: {
+                        "type": a["binType"],
+                        "collectionDate": a["collectionDay"].replace("-", "/"),
+                    },
+                    decoded_bins["collectionDay"],
                 )
-
-            soup = BeautifulSoup(driver.page_source, features="html.parser")
-            
-            # Find all collection items with the specific structure - try multiple class patterns
-            collection_items = soup.find_all("div", class_=lambda x: x and "ant-col" in x and "ant-col-xs-12" in x)
-            if not collection_items:
-                # Fallback to finding items by structure
-                collection_items = soup.find_all("div", class_=lambda x: x and "p-2" in x and "d-flex" in x and "flex-column" in x)
-            
-            current_year = datetime.now().year
-            current_month = datetime.now().month
-
-            for item in collection_items:
-                # Extract bin type from h3 element
-                bin_type_elem = item.find("h3", class_="text-white")
-                # Extract date from div with specific classes
-                date_elem = item.find("div", class_="text-white fw-bold")
-                
-                if bin_type_elem and date_elem:
-                    bin_type = bin_type_elem.get_text().strip()
-                    date_text = date_elem.get_text().strip()
-                    
-                    try:
-                        collection_date = datetime.strptime(date_text, "%A %d %B")
-                        if (current_month > 10) and (collection_date.month < 3):
-                            collection_date = collection_date.replace(year=(current_year + 1))
-                        else:
-                            collection_date = collection_date.replace(year=current_year)
-
-                        dict_data = {
-                            "type": bin_type,
-                            "collectionDate": collection_date.strftime("%d/%m/%Y"),
-                        }
-                        data["bins"].append(dict_data)
-                    except ValueError:
-                        continue
+            )
 
         except Exception as e:
             # Here you can log the exception if needed
             print(f"An error occurred: {e}")
             # Optionally, re-raise the exception if you want it to propagate
             raise
-        finally:
-            # This block ensures that the driver is closed regardless of an exception
-            if driver:
-                driver.quit()
         return data