uk_bin_collection 0.98.0__tar.gz → 0.98.1__tar.gz

{uk_bin_collection-0.98.0 → uk_bin_collection-0.98.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.98.0
+Version: 0.98.1
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.98.0 → uk_bin_collection-0.98.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.98.0"
+version = "0.98.1"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

{uk_bin_collection-0.98.0 → uk_bin_collection-0.98.1}/uk_bin_collection/tests/input.json

@@ -49,7 +49,7 @@
         "wiki_name": "Barnsley Metropolitan Borough Council",
         "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search)."
     },
-"BasildonCouncil": {
+    "BasildonCouncil": {
         "skip_get_url": true,
         "uprn": "10013350430",
         "url": "https://basildonportal.azurewebsites.net/api/getPropertyRefuseInformation",

{uk_bin_collection-0.98.0 → uk_bin_collection-0.98.1}/uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py

@@ -70,10 +70,19 @@ class CouncilClass(AbstractGetBinDataClass):
             headless = kwargs.get("headless")
             web_driver = kwargs.get("web_driver")
             driver = create_webdriver(web_driver, headless, None, __name__)
-            page = "https://account.barnet.gov.uk/Forms/Home/Redirector/Index/?id=6a2ac067-3322-46e5-96e4-16c0c214454a&mod=OA&casetype=BAR&formname=BNTCOLDATE"
+            page = "https://www.barnet.gov.uk/recycling-and-waste/bin-collections/find-your-bin-collection-day"
+
             driver.get(page)
 
-            time.sleep(5)
+            # Wait for the element to be clickable
+            find_your_collection_button = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, '//a[contains(text(), "Find your household collection day")]')
+                )
+            )
+
+            # Click the element
+            find_your_collection_button.click()
 
             try:
                 accept_cookies = WebDriverWait(driver, timeout=10).until(

uk_bin_collection-0.98.1/uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py

@@ -0,0 +1,147 @@
+import time
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+# import the wonderful Beautiful Soup and the URL grabber
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            page = "https://waste.bexley.gov.uk/waste"
+
+            data = {"bins": []}
+
+            user_uprn = kwargs.get("uprn")
+            user_paon = kwargs.get("paon")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(page)
+
+            wait = WebDriverWait(driver, 10)
+
+            inputElement_postcodesearch = wait.until(
+                EC.element_to_be_clickable((By.ID, "pc"))
+            )
+            inputElement_postcodesearch.send_keys(user_postcode)
+
+
+
+            find_address_btn = wait.until(
+                EC.element_to_be_clickable((By.XPATH, '//*[@id="sub"]'))
+            )
+            find_address_btn.click()
+
+            dropdown_options = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, '//*[@id="address"]')
+                )
+            )
+            time.sleep(2)
+            dropdown_options.click()
+            time.sleep(1)
+
+            # Wait for the element to be clickable
+            address = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, f'//li[contains(text(), "{user_paon}")]')
+                )
+            )
+
+            # Click the element
+            address.click()
+
+
+            submit_address = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, '//*[@id="go"]')
+                )
+            )
+            time.sleep(2)
+            submit_address.click()
+
+            results_found = wait.until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, '//h1[contains(text(), "Your bin days")]')
+                )           
+                )
+
+            final_page = wait.until(
+                EC.presence_of_element_located(
+                    (By.CLASS_NAME, "waste__collections")
+                )
+            )
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            # Find all waste services
+
+            # Initialize the data dictionary
+            data = {"bins": []}
+            bin_sections = soup.find_all("h3", class_="waste-service-name")
+
+            # Loop through each bin field
+            for bin_section in bin_sections:
+                # Extract the bin type (e.g., "Brown Caddy", "Green Wheelie Bin", etc.)
+                bin_type = bin_section.get_text(strip=True).split("\n")[0]  # The first part is the bin type
+
+                # Find the next sibling <dl> tag that contains the next collection information
+                summary_list = bin_section.find_next("dl", class_="govuk-summary-list")
+
+                if summary_list:
+                    # Now, instead of finding by class, we'll search by text within the dt element
+                    next_collection_dt = summary_list.find("dt", string=lambda text: "Next collection" in text)
+
+                    if next_collection_dt:
+                        # Find the sibling <dd> tag for the collection date
+                        next_collection = next_collection_dt.find_next_sibling("dd").get_text(strip=True)
+
+                        if next_collection:
+                            try:
+                                # Parse the next collection date (assuming the format is like "Tuesday 15 October 2024")
+                                parsed_date = datetime.strptime(next_collection, "%A %d %B %Y")
+
+                                # Add the bin information to the data dictionary
+                                data["bins"].append({
+                                    "type": bin_type,
+                                    "collectionDate": parsed_date.strftime(date_format),
+                                })
+                            except ValueError as e:
+                                print(f"Error parsing date for {bin_type}: {e}")
+                        else:
+                            print(f"No next collection date found for {bin_type}")
+                    else:
+                        print(f"No 'Next collection' text found for {bin_type}")
+                else:
+                    print(f"No summary list found for {bin_type}")
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection-0.98.0/uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py

@@ -1,147 +0,0 @@
-import time
-from datetime import datetime
-
-from bs4 import BeautifulSoup
-from selenium.webdriver.common.by import By
-from selenium.webdriver.common.keys import Keys
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.support.ui import Select
-from selenium.webdriver.support.wait import WebDriverWait
-
-from uk_bin_collection.uk_bin_collection.common import *
-from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
-
-# import the wonderful Beautiful Soup and the URL grabber
-
-
-class CouncilClass(AbstractGetBinDataClass):
-    """
-    Concrete classes have to implement all abstract operations of the
-    base class. They can also override some operations with a default
-    implementation.
-    """
-
-    def parse_data(self, page: str, **kwargs) -> dict:
-        driver = None
-        try:
-            page = "https://mybexley.bexley.gov.uk/service/When_is_my_collection_day"
-
-            data = {"bins": []}
-
-            user_uprn = kwargs.get("uprn")
-            user_paon = kwargs.get("paon")
-            user_postcode = kwargs.get("postcode")
-            web_driver = kwargs.get("web_driver")
-            headless = kwargs.get("headless")
-
-            # Create Selenium webdriver
-            driver = create_webdriver(web_driver, headless, None, __name__)
-            driver.get(page)
-
-            # If you bang in the house number (or property name) and postcode in the box it should find your property
-
-            iframe_presense = WebDriverWait(driver, 30).until(
-                EC.presence_of_element_located((By.ID, "fillform-frame-1"))
-            )
-
-            driver.switch_to.frame(iframe_presense)
-            wait = WebDriverWait(driver, 60)
-            start_btn = wait.until(
-                EC.element_to_be_clickable(
-                    (By.XPATH, "//button/span[contains(text(), 'Next')]")
-                )
-            )
-
-            start_btn.click()
-
-            inputElement_postcodesearch = wait.until(
-                EC.element_to_be_clickable((By.ID, "postcode_search"))
-            )
-            inputElement_postcodesearch.send_keys(user_postcode)
-
-            find_address_btn = wait.until(
-                EC.element_to_be_clickable((By.XPATH, '//*[@id="search"]'))
-            )
-            find_address_btn.click()
-
-            dropdown_options = wait.until(
-                EC.presence_of_element_located(
-                    (By.XPATH, '//*[@id="select2-chosen-1"]')
-                )
-            )
-            time.sleep(2)
-            dropdown_options.click()
-            time.sleep(1)
-            dropdown_input = wait.until(
-                EC.presence_of_element_located(
-                    (By.XPATH, '//*[@id="s2id_autogen1_search"]')
-                )
-            )
-            time.sleep(1)
-            dropdown_input.send_keys(user_paon)
-            dropdown_input.send_keys(Keys.ENTER)
-
-            results_found = wait.until(
-                EC.presence_of_element_located((By.CLASS_NAME, "found-content"))
-            )
-            finish_btn = wait.until(
-                EC.element_to_be_clickable(
-                    (By.XPATH, "//button/span[contains(text(), 'Next')]")
-                )
-            )
-            finish_btn.click()
-            final_page = wait.until(
-                EC.presence_of_element_located(
-                    (By.CLASS_NAME, "waste-header-container")
-                )
-            )
-
-            soup = BeautifulSoup(driver.page_source, features="html.parser")
-
-            bin_fields = soup.find_all("div", class_="waste-panel-container")
-            # Define your XPath
-
-            for bin in bin_fields:
-                # Extract h3 text from the current element
-                h3_text = (
-                    bin.find("h3", class_="container-name").get_text(strip=True)
-                    if bin.find("h3", class_="container-name")
-                    else None
-                )
-
-                date_text = (
-                    bin.find("p", class_="container-status").get_text(strip=True)
-                    if bin.find("p", class_="container-status")
-                    else None
-                )
-
-                if h3_text and date_text:
-                    # Parse the date using the appropriate format
-                    parsed_date = datetime.strptime(date_text, "%A %d %B")
-
-                    # Assuming the current year is used for the collection date
-                    current_year = datetime.now().year
-
-                    # If the parsed date is in the past, assume it's for the next year
-                    if parsed_date < datetime.now():
-                        current_year += 1
-
-                    data["bins"].append(
-                        {
-                            "type": h3_text,
-                            "collectionDate": parsed_date.replace(
-                                year=current_year
-                            ).strftime("%d/%m/%Y"),
-                        }
-                    )
-
-        except Exception as e:
-            # Here you can log the exception if needed
-            print(f"An error occurred: {e}")
-            # Optionally, re-raise the exception if you want it to propagate
-            raise
-        finally:
-            # This block ensures that the driver is closed regardless of an exception
-            if driver:
-                driver.quit()
-        return data