uk_bin_collection 0.80.0__tar.gz → 0.81.0__tar.gz

{uk_bin_collection-0.80.0 → uk_bin_collection-0.81.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.80.0
+Version: 0.81.0
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.80.0 → uk_bin_collection-0.81.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.80.0"
+version = "0.81.0"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

{uk_bin_collection-0.80.0 → uk_bin_collection-0.81.0}/uk_bin_collection/tests/input.json

@@ -1041,13 +1041,13 @@
         "wiki_name": "Torridge District Council"
     },
     "UttlesfordDistrictCouncil": {
-        "url": "https://bins.uttlesford.gov.uk/",
-        "wiki_name": "UttlesfordDistrictCouncil",
-        "skip_get_url": true,
+        "house_number": "72, Birchanger Lane",
         "postcode": "CM23 5QF",
+        "skip_get_url": true,
         "uprn": "100090643434",
-        "house_number": "72, Birchanger Lane",
-        "web_driver": "http://selenium:4444"
+        "url": "https://bins.uttlesford.gov.uk/",
+        "web_driver": "http://selenium:4444",
+        "wiki_name": "UttlesfordDistrictCouncil"
     },
     "ValeofGlamorganCouncil": {
         "skip_get_url": true,
@@ -1175,6 +1175,14 @@
         "wiki_name": "Woking Borough Council/Joint Waste Solutions",
         "wiki_note": "Works with all collection areas that use Joint Waste Solutions. Just use the correct URL."
     },
+    "WychavonDistrictCouncil": {
+        "url": "https://selfservice.wychavon.gov.uk/wdcroundlookup/wdc_search.jsp",
+        "wiki_name": "WychavonDistrictCouncil",
+        "web_driver": "http://selenium:4444",
+        "skip_get_url": true,
+        "postcode": "WR3 7RU", 
+        "uprn": "100120716273"
+    },
     "WyreCouncil": {
         "postcode": "FY6 8HG",
         "skip_get_url": true,

{uk_bin_collection-0.80.0 → uk_bin_collection-0.81.0}/uk_bin_collection/uk_bin_collection/councils/BirminghamCityCouncil.py

@@ -1,7 +1,14 @@
+from typing import Dict, List, Any, Optional
 from bs4 import BeautifulSoup
 from dateutil.relativedelta import relativedelta
-
-from uk_bin_collection.uk_bin_collection.common import *
+import requests
+import logging
+import re
+from datetime import datetime
+from uk_bin_collection.common import *
+from dateutil.parser import parse
+
+from uk_bin_collection.uk_bin_collection.common import check_uprn, check_postcode
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
@@ -24,7 +31,7 @@ class CouncilClass(AbstractGetBinDataClass):
     implementation.
     """
 
-    def get_data(self, url) -> str:
+    def get_data(self, url: str) -> str:
         """This method makes the request to the council
 
         Keyword arguments:
@@ -45,21 +52,27 @@ class CouncilClass(AbstractGetBinDataClass):
             full_page = session.get(url)
             return full_page
         except requests.exceptions.HTTPError as errh:
-            _LOGGER.error(f"Http Error: {errh}")
+            logging.error(f"Http Error: {errh}")
             raise
         except requests.exceptions.ConnectionError as errc:
-            _LOGGER.error(f"Error Connecting: {errc}")
+            logging.error(f"Error Connecting: {errc}")
             raise
         except requests.exceptions.Timeout as errt:
-            _LOGGER.error(f"Timeout Error: {errt}")
+            logging.error(f"Timeout Error: {errt}")
             raise
         except requests.exceptions.RequestException as err:
-            _LOGGER.error(f"Oops: Something Else {err}")
+            logging.error(f"Oops: Something Else {err}")
             raise
 
-    def parse_data(self, page: str, **kwargs) -> dict:
-        uprn = kwargs.get("uprn")
-        postcode = kwargs.get("postcode")
+    def parse_data(self, page: str, **kwargs: Any) -> Dict[str, List[Dict[str, str]]]:
+        uprn: Optional[str] = kwargs.get("uprn")
+        postcode: Optional[str] = kwargs.get("postcode")
+
+        if uprn is None:
+            raise ValueError("UPRN is required and must be a non-empty string.")
+        if postcode is None:
+            raise ValueError("Postcode is required and must be a non-empty string.")
+
         check_uprn(uprn)
         check_postcode(postcode)
 
@@ -85,7 +98,7 @@ class CouncilClass(AbstractGetBinDataClass):
         rows = soup.find("table").find_all("tr")
 
         # Form a JSON wrapper
-        data = {"bins": []}
+        data: Dict[str, List[Dict[str, str]]] = {"bins": []}
 
         # Loops the Rows
         for row in rows:
@@ -94,13 +107,13 @@ class CouncilClass(AbstractGetBinDataClass):
                 bin_type = cells[0].get_text(strip=True)
                 collection_next = cells[1].get_text(strip=True)
 
-                collection_date = re.findall("\(.*?\)", collection_next)
+                collection_date = re.findall(r"\(.*?\)", collection_next)
 
                 if len(collection_date) != 1:
                     continue
 
                 collection_date_obj = parse(
-                    re.sub("[()]", "", collection_date[0])
+                    re.sub(r"[()]", "", collection_date[0])
                 ).date()
 
                 # since we only have the next collection day, if the parsed date is in the past,

{uk_bin_collection-0.80.0 → uk_bin_collection-0.81.0}/uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py

@@ -1,17 +1,14 @@
 # This script pulls (in one hit) the data from Bromley Council Bins Data
 import datetime
-import time
 from datetime import datetime
 
 from bs4 import BeautifulSoup
 from dateutil.relativedelta import relativedelta
 from selenium.webdriver.common.by import By
-from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.support.ui import Select
-from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.support.ui import WebDriverWait
 
-from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.common import create_webdriver
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
@@ -28,7 +25,6 @@ class CouncilClass(AbstractGetBinDataClass):
         driver = None
         try:
             bin_data_dict = {"bins": []}
-            collections = []
             web_driver = kwargs.get("web_driver")
             headless = kwargs.get("headless")
 
@@ -39,18 +35,14 @@ class CouncilClass(AbstractGetBinDataClass):
             driver.get(kwargs.get("url"))
 
             wait = WebDriverWait(driver, 30)
-            results = wait.until(
+            wait.until(
                 EC.presence_of_element_located((By.CLASS_NAME, "waste-service-image"))
             )
-            # Search for the specific bins in the table using BS
-            # Parse the HTML content
-            # Find all elements with the class 'container-name' to extract bin types
+
             # Parse the HTML content
             soup = BeautifulSoup(driver.page_source, "html.parser")
-            soup.prettify
 
             # Find all elements with class 'govuk-summary-list'
-            bin_info = []
             waste_services = soup.find_all(
                 "h3", class_="govuk-heading-m waste-service-name"
             )
@@ -58,7 +50,7 @@ class CouncilClass(AbstractGetBinDataClass):
             for service in waste_services:
                 service_title = service.get_text(strip=True)
                 next_collection = service.find_next_sibling().find(
-                    "dt", text="Next collection"
+                    "dt", string="Next collection"
                 )
 
                 if next_collection:
@@ -69,8 +61,7 @@ class CouncilClass(AbstractGetBinDataClass):
                     next_collection_date_parse = next_collection_date.split(",")[
                         1
                     ].strip()
-                    day = next_collection_date_parse.split()[0]
-                    month = next_collection_date_parse.split()[1]
+                    day, month = next_collection_date_parse.split()[:2]
 
                     # Remove the suffix (e.g., 'th', 'nd', 'rd', 'st') from the day
                     if day.endswith(("th", "nd", "rd", "st")):

{uk_bin_collection-0.80.0 → uk_bin_collection-0.81.0}/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py

@@ -1,11 +1,11 @@
 import time
-
 import pandas as pd
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support.ui import Select
+from io import StringIO
 
-from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.common import create_webdriver
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
@@ -16,7 +16,7 @@ class CouncilClass(AbstractGetBinDataClass):
     implementation.
     """
 
-    def get_data(self, df) -> dict:
+    def get_data(self, df: pd.DataFrame) -> dict:
         # Create dictionary of data to be returned
         data = {"bins": []}
 
@@ -69,7 +69,7 @@ class CouncilClass(AbstractGetBinDataClass):
             time.sleep(4)
 
             # Submit address information and wait
-            inputElement_bn = driver.find_element(
+            driver.find_element(
                 By.ID, "COPYOFECHOCOLLECTIONDATES_ADDRESSSELECTION_NAV1_NEXT"
             ).click()
 
@@ -79,8 +79,10 @@ class CouncilClass(AbstractGetBinDataClass):
             table = driver.find_element(
                 By.ID, "COPYOFECHOCOLLECTIONDATES_PAGE1_DATES2"
             ).get_attribute("outerHTML")
-            df = pd.read_html(table, header=[1])
-            df = df[0]
+
+            # Wrap the HTML table in a StringIO object to address the warning
+            table_io = StringIO(table)
+            df = pd.read_html(table_io, header=[1])[0]
 
             # Parse data into dict
             data = self.get_data(df)

{uk_bin_collection-0.80.0 → uk_bin_collection-0.81.0}/uk_bin_collection/uk_bin_collection/councils/HaringeyCouncil.py

@@ -1,6 +1,10 @@
 from bs4 import BeautifulSoup
+import requests
+import logging
+import re
+from typing import Dict, List, Any, Optional
 
-from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.common import check_uprn
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
@@ -11,10 +15,14 @@ class CouncilClass(AbstractGetBinDataClass):
     implementation.
     """
 
-    def parse_data(self, page: str, **kwargs) -> dict:
-        data = {"bins": []}
+    def parse_data(self, page: str, **kwargs: Any) -> Dict[str, List[Dict[str, str]]]:
+        data: Dict[str, List[Dict[str, str]]] = {"bins": []}
+
+        uprn: Optional[str] = kwargs.get("uprn")
+
+        if uprn is None:
+            raise ValueError("UPRN is required and must be a non-empty string.")
 
-        uprn = kwargs.get("uprn")
         check_uprn(uprn)  # Assuming check_uprn() raises an exception if UPRN is invalid
 
         try:
@@ -42,7 +50,7 @@ class CouncilClass(AbstractGetBinDataClass):
 
                 if service_name_element and next_service_element:
                     service = service_name_element.text
-                    next_collection = next_service_element.find(text=date_regex)
+                    next_collection = next_service_element.find(string=date_regex)
 
                     if next_collection:
                         dict_data = {

{uk_bin_collection-0.80.0 → uk_bin_collection-0.81.0}/uk_bin_collection/uk_bin_collection/councils/WokingBoroughCouncil.py

@@ -29,7 +29,7 @@ class CouncilClass(AbstractGetBinDataClass):
         start = BeautifulSoup(req.text, features="html.parser")
         start.prettify()
         base_link = start.select(
-            "#menu-content > div > div:nth-child(1) > p.govuk-body.govuk-\!-margin-bottom-0.colorblue.lineheight15 > a"
+            "#menu-content > div > div:nth-child(1) > p.govuk-body.govuk-\\!-margin-bottom-0.colorblue.lineheight15 > a"
         )[0].attrs.get("href")
 
         # We need to reorder the query parts from the unique URL, so split them up to make it easier

uk_bin_collection-0.81.0/uk_bin_collection/uk_bin_collection/councils/WychavonDistrictCouncil.py

@@ -0,0 +1,156 @@
+import logging
+import pickle
+import time
+
+import requests
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+from uk_bin_collection.uk_bin_collection.common import *
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+# Set up logging
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
+
+
+class CouncilClass(AbstractGetBinDataClass):
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            data = {"bins": []}
+            collections = []
+            user_uprn = kwargs.get("uprn")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_postcode(user_postcode)
+            url = "https://selfservice.wychavon.gov.uk/wdcroundlookup/wdc_search.jsp"
+
+            # Get session cookies using requests
+
+            user_agent = """general.useragent.override", "userAgent=Mozilla/5.0 
+            (iPhone; CPU iPhone OS 15_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like 
+            Gecko) CriOS/101.0.4951.44 Mobile/15E148 Safari/604.1"""
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless, user_agent, __name__)
+
+            # Navigate to the page first
+            driver.get(url)
+
+            wait = WebDriverWait(driver, 60)
+
+            logging.info("Accepting cookies")
+
+            try:
+                logging.info("Cookies")
+                cookie_window = wait.until(
+                    EC.presence_of_element_located(
+                        (By.XPATH, '//div[@id="ccc-content"]')
+                    )
+                )
+                time.sleep(2)
+                accept_cookies = WebDriverWait(driver, timeout=10).until(
+                    EC.presence_of_element_located(
+                        (By.XPATH, '//button[@id="ccc-recommended-settings"]')
+                    )
+                )
+                accept_cookies.send_keys(Keys.ENTER)
+                accept_cookies.click()
+                accept_cookies_close = WebDriverWait(driver, timeout=10).until(
+                    EC.presence_of_element_located(
+                        (By.XPATH, '//button[@id="ccc-close"]')
+                    )
+                )
+                accept_cookies_close.send_keys(Keys.ENTER)
+                accept_cookies_close.click()
+            except:
+                print(
+                    "Accept cookies banner not found or clickable within the specified time."
+                )
+                pass
+
+            logging.info("Entering postcode")
+            input_element_postcode = wait.until(
+                EC.presence_of_element_located((By.XPATH, '//input[@id="alAddrtxt"]'))
+            )
+
+            input_element_postcode.send_keys(user_postcode)
+
+            logging.info("Searching for postcode")
+            input_element_postcode_btn = wait.until(
+                EC.element_to_be_clickable((By.XPATH, '//button[@id="alAddrbtn"]'))
+            )
+
+            input_element_postcode_btn.click()
+
+            logging.info("Waiting for address dropdown")
+            input_element_postcode_dropdown = wait.until(
+                EC.presence_of_element_located((By.XPATH, '//select[@id="alAddrsel"]'))
+            )
+
+            logging.info("Selecting address")
+            drop_down_values = Select(input_element_postcode_dropdown)
+            option_element = wait.until(
+                EC.presence_of_element_located(
+                    (By.CSS_SELECTOR, f'option[value="{str(user_uprn)}"]')
+                )
+            )
+
+            driver.execute_script("arguments[0].scrollIntoView();", option_element)
+            drop_down_values.select_by_value(str(user_uprn))
+
+            input_element_address_btn = wait.until(
+                EC.element_to_be_clickable((By.XPATH, '//input[@id="btnSubmit"]'))
+            )
+
+            input_element_address_btn.click()
+
+            logging.info("Waiting for bin collection page")
+            strong_element = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, "//strong[contains(text(), 'Upcoming collections')]")
+                )
+            )
+
+            logging.info("Extracting bin collection data")
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            bins = []
+
+            rows = soup.select("table tbody tr")
+            for row in rows:
+                bin_type = row.select_one("td:nth-of-type(2)").contents[0].strip()
+                date_elements = row.select("td:nth-of-type(3) strong")
+                if date_elements:
+                    dates = [date.get_text(strip=True) for date in date_elements]
+                else:
+                    dates = ["Not applicable"]
+
+                for date in dates:
+                    if date != "Not applicable":
+                        # Format the date to dd/mm/yyyy
+                        formatted_date = re.search(r"\d{2}/\d{2}/\d{4}", date).group(0)
+                        bins.append(
+                            {"type": bin_type, "collectionDate": formatted_date}
+                        )
+
+            bin_data = {"bins": bins}
+
+            return bin_data
+
+        except Exception as e:
+            logging.error(f"An error occurred: {e}")
+            raise
+
+        finally:
+            if driver:
+                driver.quit()