uk_bin_collection 0.148.3__py3-none-any.whl → 0.148.5__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/TorbayCouncil.py

@@ -1,51 +1,231 @@
+# This script pulls bin collection data from Barking and Dagenham Council
+# Example URL: https://www.lbbd.gov.uk/rubbish-recycling/household-bin-collection/check-your-bin-collection-days
+import time
+
+from bs4 import BeautifulSoup
+from dateutil.parser import parse
+from selenium.common.exceptions import NoSuchElementException, TimeoutException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
 class CouncilClass(AbstractGetBinDataClass):
-    """
-    Concrete classes have to implement all abstract operations of the
-    base class. They can also override some operations with a default
-    implementation.
-    """
 
     def parse_data(self, page: str, **kwargs) -> dict:
-        uprn = kwargs.get("uprn")
-        check_uprn(uprn)
-
-        headers = {
-            "Accept": "*/*",
-            "Accept-Encoding": "gzip, deflate, br",
-            "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
-            "Connection": "keep-alive",
-            "Host": "online.torbay.gov.uk",
-            "Origin": "https://www.torbay.gov.uk",
-            "Referer": "https://www.torbay.gov.uk/",
-            "sec-ch-ua": '"Chromium";v="110", "Not A(Brand";v="24", "Google Chrome";v="110"',
-            "sec-ch-ua-mobile": "?0",
-            "sec-ch-ua-platform": '"Windows"',
-            "Sec-Fetch-Dest": "empty",
-            "Sec-Fetch-Mode": "cors",
-            "Sec-Fetch-Site": "same-site",
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36",
-        }
-        requests.packages.urllib3.disable_warnings()
-        response = requests.get(
-            f"https://online.torbay.gov.uk/services.bartec/collections?uprn={uprn}",
-            headers=headers,
-        )
-        if response.status_code != 200:
-            raise ValueError("No bin data found for provided UPRN.")
-        json_data = json.loads(response.text)
-
-        data = {"bins": []}
-        for c in json_data:
-            dict_data = {
-                "type": c["Service"].replace("Empty ", "").strip(),
-                "collectionDate": datetime.strptime(
-                    c["NextCollection"].strip(), "%d %B %Y"
-                ).strftime(date_format),
-            }
-            data["bins"].append(dict_data)
+        driver = None
+        try:
+            data = {"bins": []}
+
+            user_uprn = kwargs.get("uprn")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            url = kwargs.get("url")
+
+            check_postcode(user_postcode)
+
+            print(
+                f"Starting parse_data with parameters: postcode={user_postcode}, uprn={user_uprn}"
+            )
+            print(
+                f"Creating webdriver with: web_driver={web_driver}, headless={headless}"
+            )
+
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            print(f"Navigating to URL: {url}")
+            driver.get(url)
+            print("Successfully loaded the page")
+
+            driver.maximize_window()
+
+            WebDriverWait(driver, 10).until(
+                lambda d: d.execute_script("return document.readyState") == "complete"
+            )
+
+            # Handle cookie banner if present
+            wait = WebDriverWait(driver, 60)
+            try:
+                cookie_button = wait.until(
+                    EC.element_to_be_clickable(
+                        (
+                            By.XPATH,
+                            "/html/body/div[1]/div/div[2]/button[1]",
+                        )
+                    ),
+                    message="Cookie banner not found",
+                )
+                cookie_button.click()
+                print("Cookie banner clicked.")
+                time.sleep(1)  # Brief pause to let banner disappear
+            except (TimeoutException, NoSuchElementException):
+                print("No cookie banner appeared or selector failed.")
+
+            bin_collection_button = wait.until(
+                EC.element_to_be_clickable(
+                    (
+                        By.XPATH,
+                        "/html/body/main/div[4]/div/div[1]/div/div/div/div/div[2]/div/div/div/p/a",
+                    )
+                ),
+            )
+            bin_collection_button.click()
+
+            # Save the original window
+            original_window = driver.current_window_handle
+
+            # Wait for the new window or tab
+            WebDriverWait(driver, 10).until(EC.number_of_windows_to_be(2))
+
+            # Loop through until we find a new window handle
+            for window_handle in driver.window_handles:
+                if window_handle != original_window:
+                    driver.switch_to.window(window_handle)
+                    break
+            # Now you're in the new tab and can interact with the postcode input
+            # Enter postcode
+            print("Looking for postcode input...")
+            wait.until(EC.presence_of_element_located((By.ID, "FF1168-text")))
+            post_code_input = wait.until(
+                EC.element_to_be_clickable((By.ID, "FF1168-text")),
+                message="Postcode input not clickable",
+            )
+            post_code_input.clear()
+            post_code_input.send_keys(user_postcode)
+            print(f"Entered postcode: {user_postcode}")
+
+            post_code_input.send_keys(Keys.TAB + Keys.ENTER)
+            # driver.switch_to.active_element.send_keys(Keys.TAB + Keys.ENTER)
+            print("Pressed ENTER on Search button")
+
+            # Wait for the dropdown to be clickable
+            address_select = wait.until(
+                EC.element_to_be_clickable((By.ID, "FF1168-list")),
+                message="Address dropdown not found",
+            )
+
+            # Click to focus the dropdown
+            address_select.click()
+            time.sleep(0.5)  # Brief pause to let the dropdown open
+
+            # Get all options
+            options = address_select.find_elements(By.TAG_NAME, "option")
+            print(f"Found {len(options)} options in dropdown")
+
+            # Print all options first for debugging
+            print("\nAvailable options:")
+            for opt in options:
+                value = opt.get_attribute("value")
+                text = opt.text
+                print(f"Value: '{value}', Text: '{text}'")
+
+            # Try to find our specific UPRN
+            target_uprn = f"U{user_uprn}|"
+            print(f"\nLooking for UPRN pattern: {target_uprn}")
+
+            found = False
+            for option in options:
+                value = option.get_attribute("value")
+                if value and target_uprn in value:
+                    print(f"Found matching address with value: {value}")
+                    option.click()
+                    found = True
+                    break
+
+            if not found:
+                print(f"No matching address found for UPRN: {user_uprn}")
+                return data
+
+            print("Address selected successfully")
+            time.sleep(1)  # Give time for the selection to take effect
+
+            # Wait for the address selection confirmation to appear
+            print("Waiting for address selection confirmation...")
+            WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.CLASS_NAME, "esbAddressSelected"))
+            )
+            print("Address selection confirmed")
+
+            # Click the Submit button
+            print("Clicking Submit button...")
+            submit_button = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable((By.ID, "submit-button"))
+            )
+            submit_button.click()
+            time.sleep(1)  # Brief pause to let the navigation start
+
+            # Wait for the collection details to appear
+            print("Waiting for collection details to load...")
+            try:
+                schedule_list = WebDriverWait(driver, 20).until(
+                    EC.presence_of_element_located((By.ID, "resiCollectionDetails"))
+                )
+                print("Collection details loaded successfully")
+            except TimeoutException:
+                print(
+                    "Timeout waiting for collection details - checking if page needs refresh"
+                )
+                driver.refresh()
+                time.sleep(2)
+                schedule_list = WebDriverWait(driver, 20).until(
+                    EC.presence_of_element_located((By.ID, "resiCollectionDetails"))
+                )
+                print("Collection details loaded after refresh")
+
+            # Make a BS4 object
+            print("Parsing page with BeautifulSoup...")
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            # Process collection details
+            print("Looking for collection details in the page...")
+
+            # Find all collection rows
+            collection_rows = soup.select("#resiCollectionDetails .row.fs-4")
+            print(f"\nProcessing {len(collection_rows)} collection rows...")
+
+            for row in collection_rows:
+                try:
+                    # Get the collection service type (e.g., "Domestic Collection Service")
+                    service_type = row.select_one("div.col:nth-child(3)").text.strip()
+
+                    # Get the date from the second column
+                    date_text = row.select_one("div[style*='width:360px']").text.strip()
+
+                    # Parse the date
+                    parsed_date = parse(date_text, fuzzy=True)
+                    bin_date = parsed_date.strftime("%d/%m/%Y")
+
+                    # Extract just the service type without " Collection Service"
+                    bin_type = service_type.replace(" Collection Service", "")
+
+                    # Add to data
+                    if bin_type and bin_date:
+                        dict_data = {
+                            "type": bin_type,
+                            "collectionDate": bin_date,
+                        }
+                        data["bins"].append(dict_data)
+                        print(f"Successfully added collection: {dict_data}")
+
+                except Exception as e:
+                    print(f"Error processing collection row: {e}")
+                    continue
+
+            # Debug: Print the complete dict_data
+            print("\nFinal bin collection data:")
+            print(data)
+
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            raise
+        finally:
+            print("Cleaning up webdriver...")
+            if driver:
+                driver.quit()
 
         return data

{uk_bin_collection-0.148.3.dist-info → uk_bin_collection-0.148.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.148.3
+Version: 0.148.5
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.148.3.dist-info → uk_bin_collection-0.148.5.dist-info}/RECORD

@@ -7,7 +7,7 @@ uk_bin_collection/tests/council_feature_input_parity.py,sha256=DO6Mk4ImYgM5ZCZ-c
 uk_bin_collection/tests/features/environment.py,sha256=VQZjJdJI_kZn08M0j5cUgvKT4k3iTw8icJge1DGOkoA,127
 uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=SJK-Vc737hrf03tssxxbeg_JIvAH-ddB8f6gU1LTbuQ,251
 uk_bin_collection/tests/generate_map_test_results.py,sha256=CKnGK2ZgiSXomRGkomX90DitgMP-X7wkHhyKORDcL2E,1144
-uk_bin_collection/tests/input.json,sha256=p_PfBWYOP3jsRCG0h9UTa_MlAzcrqMKAOy5P2N9EfwM,135010
+uk_bin_collection/tests/input.json,sha256=Zjpi_QnZFlIojCGBJfBkQ8Tn9O2zbi48w_yKr0wbSXU,131194
 uk_bin_collection/tests/output.schema,sha256=ZwKQBwYyTDEM4G2hJwfLUVM-5v1vKRvRK9W9SS1sd18,1086
 uk_bin_collection/tests/step_defs/step_helpers/file_handler.py,sha256=Ygzi4V0S1MIHqbdstUlIqtRIwnynvhu4UtpweJ6-5N8,1474
 uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=VZ0a81sioJULD7syAYHjvK_-nT_Rd36tUyzPetSA0gk,3475
@@ -295,7 +295,7 @@ uk_bin_collection/uk_bin_collection/councils/ThanetDistrictCouncil.py,sha256=-op
 uk_bin_collection/uk_bin_collection/councils/ThreeRiversDistrictCouncil.py,sha256=RHt3e9oeKzwxjjY-M8aC0nk-ZXhHIoyC81JzxkPVxsE,5531
 uk_bin_collection/uk_bin_collection/councils/ThurrockCouncil.py,sha256=vAZMm6mcsdEcOkP15xwxWy9gdXpmLYQFH7qRifurNoY,2935
 uk_bin_collection/uk_bin_collection/councils/TonbridgeAndMallingBC.py,sha256=UlgnHDoi8ecav2H5-HqKNDpqW1J3RN-c___5c08_Q7I,4859
-uk_bin_collection/uk_bin_collection/councils/TorbayCouncil.py,sha256=JW_BS7wkfxFsmx6taQtPAQWdBp1AfLrxs0XRQ2XZcSw,2029
+uk_bin_collection/uk_bin_collection/councils/TorbayCouncil.py,sha256=fpyr5f5x8JSsusdMaEH248HINPl6yQv14McD1-xLnqk,9269
 uk_bin_collection/uk_bin_collection/councils/TorridgeDistrictCouncil.py,sha256=qyxzMFaQp4ymDsnvoySqmJonQ_MLo4jYBbKmREECur4,4777
 uk_bin_collection/uk_bin_collection/councils/TunbridgeWellsCouncil.py,sha256=s8Nm9Ef-4561mEXPa0ylYHrXyYIulgCcNV2uAnrXyZk,2846
 uk_bin_collection/uk_bin_collection/councils/UttlesfordDistrictCouncil.py,sha256=GSELWbSn5jtznv6FSLIMxK6CyQ27MW9FoY_m5jhTEBA,4175
@@ -337,8 +337,8 @@ uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py,sha256=I2kBYMlsD4bId
 uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=QD4v4xpsEE0QheR_fGaNOIRMc2FatcUfKkkhAhseyVU,1159
 uk_bin_collection/uk_bin_collection/create_new_council.py,sha256=m-IhmWmeWQlFsTZC4OxuFvtw5ZtB8EAJHxJTH4O59lQ,1536
 uk_bin_collection/uk_bin_collection/get_bin_data.py,sha256=YvmHfZqanwrJ8ToGch34x-L-7yPe31nB_x77_Mgl_vo,4545
-uk_bin_collection-0.148.3.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
-uk_bin_collection-0.148.3.dist-info/METADATA,sha256=GnD4UvGYQsS6G8rNA5Idbr7vemGqQYZZvMdRWWqjh_o,20914
-uk_bin_collection-0.148.3.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-uk_bin_collection-0.148.3.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
-uk_bin_collection-0.148.3.dist-info/RECORD,,
+uk_bin_collection-0.148.5.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
+uk_bin_collection-0.148.5.dist-info/METADATA,sha256=QFI4MOygjxoWUqezFNR-eFxcJM105HMe1volyN_IQjk,20914
+uk_bin_collection-0.148.5.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+uk_bin_collection-0.148.5.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
+uk_bin_collection-0.148.5.dist-info/RECORD,,