uk_bin_collection 0.125.2__tar.gz → 0.126.0__tar.gz

{uk_bin_collection-0.125.2 → uk_bin_collection-0.126.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.125.2
+Version: 0.126.0
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com
@@ -115,7 +115,7 @@ If you miss this on the first setup you can reconfigure it.
     "color": "blue"     
   }
 }
-
+```
 ---
 
 ## Standalone Usage

{uk_bin_collection-0.125.2 → uk_bin_collection-0.126.0}/README.md

@@ -92,7 +92,7 @@ If you miss this on the first setup you can reconfigure it.
     "color": "blue"     
   }
 }
-
+```
 ---
 
 ## Standalone Usage

{uk_bin_collection-0.125.2 → uk_bin_collection-0.126.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.125.2"
+version = "0.126.0"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

{uk_bin_collection-0.125.2 → uk_bin_collection-0.126.0}/uk_bin_collection/tests/input.json

@@ -1993,8 +1993,8 @@
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
     "WestLindseyDistrictCouncil": {
-        "house_number": "PRIVATE ACCOMMODATION",
-        "postcode": "LN8 2AR",
+        "house_number": "35",
+        "postcode": "LN8 3AX",
         "skip_get_url": true,
         "url": "https://www.west-lindsey.gov.uk/",
         "wiki_name": "West Lindsey District Council",

{uk_bin_collection-0.125.2 → uk_bin_collection-0.126.0}/uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py

@@ -21,7 +21,6 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page: str, **kwargs) -> dict:
-        # Make a BS4 object
         driver = None
         try:
             bin_data_dict = {"bins": []}
@@ -76,12 +75,13 @@ class CouncilClass(AbstractGetBinDataClass):
                     # Get the current year
                     current_year = datetime.now().year
 
+                    # Append the year to the date
+                    date_with_year = date_object.replace(year=current_year)
+
                     # Check if the parsed date is in the past compared to the current date
                     if date_object < datetime.now():
                         # If the parsed date is in the past, assume it's for the next year
                         current_year += 1
-                    # Append the year to the date
-                    date_with_year = date_object.replace(year=current_year)
 
                     # Format the date with the year
                     date_with_year_formatted = date_with_year.strftime(

uk_bin_collection-0.126.0/uk_bin_collection/uk_bin_collection/councils/KirkleesCouncil.py

@@ -0,0 +1,143 @@
+import time
+from datetime import datetime
+from typing import Optional
+
+from bs4 import BeautifulSoup
+from selenium.common import TimeoutException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.remote.webdriver import WebDriver
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.wait import WebDriverWait
+from webdriver_manager.drivers.chrome import ChromeDriver
+
+from selenium import webdriver
+
+from uk_bin_collection.uk_bin_collection.common import create_webdriver
+from uk_bin_collection.uk_bin_collection.common import date_format
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+def wait_for_element(driver, element_type, element: str, timeout: int = 5):
+    element_present = EC.presence_of_element_located((element_type, element))
+    wait_for_element_conditions(driver, element_present, timeout=timeout)
+
+
+def wait_for_element_conditions(driver, conditions, timeout: int = 5):
+    try:
+        WebDriverWait(driver, timeout).until(conditions)
+    except TimeoutException:
+        print("Timed out waiting for page to load")
+        raise
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def __init__(self):
+        self._driver: Optional[WebDriver] = None
+
+    def parse_data(self, *args, **kwargs) -> dict:
+        try:
+            return self._parse_data(*args, **kwargs)
+        finally:
+            if self._driver:
+                self._driver.quit()
+
+    def _parse_data(self, page: str, **kwargs) -> dict:
+        """
+        Process:
+
+        - Use a house number and postcode that is known to be domestic and resolves to a
+          single unique address. When the address search form is submitted with
+          those details, a session is created
+
+        - Now a session exists, navigate to the calendar URL, specifying the UPRN
+
+        - Extract info from the 'alt' attribute of the images on that page
+        """
+        data = {"bins": []}
+        collections = []
+
+        user_paon = kwargs["paon"]
+        user_postcode = kwargs["postcode"]
+
+        self._driver = driver = webdriver.Chrome()
+        # self._driver = driver = create_webdriver(
+        #     web_driver=kwargs["web_driver"],
+        #     headless=kwargs.get("headless", True),
+        #     session_name=__name__,
+        # )
+        driver.implicitly_wait(1)
+
+        driver.get(
+            "https://my.kirklees.gov.uk/service/Bins_and_recycling___Manage_your_bins"
+        )
+
+        time.sleep(5)
+
+        # Switch to iframe
+        iframe = driver.find_element(By.CSS_SELECTOR, "#fillform-frame-1")
+        driver.switch_to.frame(iframe)
+
+        wait_for_element(
+            driver, By.ID, "mandatory_Postcode", timeout=10
+        )
+
+        postcode_input = driver.find_element(
+            By.ID, "Postcode"
+        )
+        postcode_input.send_keys(user_postcode)
+
+        wait_for_element(driver, By.ID, "List")
+        time.sleep(2)
+
+        WebDriverWait(driver, 10).until(
+            EC.element_to_be_clickable(
+                (
+                    By.XPATH,
+                    "//select[@name='List']//option[contains(., '"
+                    + user_paon
+                    + "')]",
+                )
+            )
+        ).click()
+
+        time.sleep(10)
+
+        # For whatever reason, the page sometimes automatically goes to the next step
+        next_button = driver.find_element(By.XPATH, '/html/body/div/div/section/form/div/nav/div[2]/button')
+        if next_button.is_displayed():
+            next_button.click()
+
+
+        time.sleep(5)
+
+        soup = BeautifulSoup(self._driver.page_source, features="html.parser")
+        soup.prettify()
+
+        radio_button_text = soup.find_all("label", {"class": "radio-label"})
+        for label in radio_button_text:
+            parsed_text = label.text.split("x ")
+            row = parsed_text[1].lower().split("collection date: ")
+            bin_type = row[0].split("(")[0].strip()
+            date_text = row[1].strip().replace(")", "")
+            if date_text == "today":
+                bin_date = datetime.now()
+            else:
+                bin_date = datetime.strptime(date_text, "%A %d %B %Y")
+            collections.append((bin_type, bin_date))
+
+        ordered_data = sorted(collections, key=lambda x: x[1])
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0].replace("standard ", "").capitalize(),
+                "collectionDate": item[1].strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data

{uk_bin_collection-0.125.2 → uk_bin_collection-0.126.0}/uk_bin_collection/uk_bin_collection/councils/NorwichCityCouncil.py

@@ -48,15 +48,13 @@ class CouncilClass(AbstractGetBinDataClass):
             alternateCheck = False
 
         strong = soup.find_all("strong")
+        collections = []
 
         if alternateCheck:
             bin_types = strong[2].text.strip().replace(".", "").split(" and ")
             for bin in bin_types:
-                dict_data = {
-                    "type": bin,
-                    "collectionDate": strong[1].text.strip(),
-                }
-                bindata["bins"].append(dict_data)
+                collections.append((bin.capitalize(), datetime.strptime(strong[1].text.strip(), date_format)))
+
         else:
             p_tag = soup.find_all("p")
             i = 1
@@ -65,11 +63,18 @@ class CouncilClass(AbstractGetBinDataClass):
                     p.text.split("Your ")[1].split(" is collected")[0].split(" and ")
                 )
                 for bin in bin_types:
-                    dict_data = {
-                        "type": bin,
-                        "collectionDate": strong[i].text.strip(),
-                    }
-                    bindata["bins"].append(dict_data)
+                    collections.append((bin.capitalize(), datetime.strptime(strong[1].text.strip(), date_format)))
                 i += 2
 
+        if len(strong) > 3:
+            collections.append(("Garden", datetime.strptime(strong[4].text.strip(), date_format)))
+
+        ordered_data = sorted(collections, key=lambda x: x[1])
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0] + " bin",
+                "collectionDate": item[1].strftime(date_format),
+            }
+            bindata["bins"].append(dict_data)
+
         return bindata

{uk_bin_collection-0.125.2 → uk_bin_collection-0.126.0}/uk_bin_collection/uk_bin_collection/councils/RugbyBoroughCouncil.py

@@ -13,11 +13,6 @@ class CouncilClass(AbstractGetBinDataClass):
 
     def parse_data(self, page: str, **kwargs) -> dict:
         data = {"bins": []}
-        bin_types = {
-            "240L RUBBISH BIN": "Black bin",
-            "240L GARDEN BIN": "Green bin",
-            "180L RECYCLING BIN": "Blue lid bin",
-        }
         collections = []
 
         user_postcode = kwargs.get("postcode")
@@ -73,7 +68,8 @@ class CouncilClass(AbstractGetBinDataClass):
 
         for row in table_rows:
             row_text = row.text.strip().split("\n")
-            bin_type = bin_types.get(row_text[0])
+            bin_text = row_text[0].split(" ")
+            bin_type = ' '.join(bin_text[1:]).capitalize()
             collections.append(
                 (bin_type, datetime.strptime(row_text[1], "%A %d %b %Y"))
             )

{uk_bin_collection-0.125.2 → uk_bin_collection-0.126.0}/uk_bin_collection/uk_bin_collection/councils/SwaleBoroughCouncil.py

@@ -6,8 +6,18 @@ from selenium.webdriver.support.wait import WebDriverWait
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
-# import the wonderful Beautiful Soup and the URL grabber
 
+def parse_collection_date(date_string) -> datetime:
+    now = datetime.now()
+    if date_string == "is due today":
+        return now
+
+    parsed_date = datetime.strptime(date_string, "%A, %d %B").replace(year=now.year)
+
+    if now.month == 12 and parsed_date.month < 12:
+        parsed_date = parsed_date.replace(year=(now.year + 1))
+
+    return parsed_date
 
 class CouncilClass(AbstractGetBinDataClass):
     """

{uk_bin_collection-0.125.2 → uk_bin_collection-0.126.0}/uk_bin_collection/uk_bin_collection/councils/WalsallCouncil.py

@@ -28,21 +28,25 @@ class CouncilClass(AbstractGetBinDataClass):
         response = requests.get(URI, headers=headers)
 
         soup = BeautifulSoup(response.text, "html.parser")
-        # Extract links to collection shedule pages and iterate through the pages
-        schedule_links = soup.findAll("a", {"class": "nav-link"}, href=True)
+        # Extract links to collection schedule pages and iterate through the pages
+        schedule_links = soup.findAll("td")
+
         for item in schedule_links:
-            if "roundname" in item["href"]:
+            if "roundname" in item.contents[1]["href"]:
                 # get bin colour
-                bincolour = item["href"].split("=")[-1].split("%")[0].upper()
-                binURL = "https://cag.walsall.gov.uk" + item["href"]
-                r = requests.get(binURL, headers=headers)
+                bin_colour = item.contents[1]["href"].split("=")[-1].split("%")[0].upper()
+                bin_url = "https://cag.walsall.gov.uk" + item.contents[1]["href"]
+                r = requests.get(bin_url, headers=headers)
+                if r.status_code != 200:
+                    print(f"Collection details for {bin_colour.lower()} bin could not be retrieved.")
+                    break
                 soup = BeautifulSoup(r.text, "html.parser")
                 table = soup.findAll("tr")
                 for tr in table:
                     td = tr.findAll("td")
                     if td:
                         dict_data = {
-                            "type": bincolour,
+                            "type": bin_colour.capitalize() + " bin",
                             "collectionDate": datetime.strptime(
                                 td[1].text.strip(), "%d/%m/%Y"
                             ).strftime("%d/%m/%Y"),
@@ -50,7 +54,7 @@ class CouncilClass(AbstractGetBinDataClass):
                         bindata["bins"].append(dict_data)
 
         bindata["bins"].sort(
-            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
         )
 
         return bindata

{uk_bin_collection-0.125.2 → uk_bin_collection-0.126.0}/uk_bin_collection/uk_bin_collection/councils/WestBerkshireCouncil.py

@@ -99,7 +99,7 @@ class CouncilClass(AbstractGetBinDataClass):
                 ).replace(year=datetime.now().year)
 
             food_div = soup.find(
-                "div", {"id": "FINDYOURBINDAYS_RECYCLINGDATE_OUTERDIV"}
+                "div", {"id": "FINDYOURBINDAYS_FOODWASTEDATE_OUTERDIV"}
             )
             food_date = food_div.find_all("div")[2]
             if food_date.text == "Today":

{uk_bin_collection-0.125.2 → uk_bin_collection-0.126.0}/uk_bin_collection/uk_bin_collection/councils/WestLindseyDistrictCouncil.py

@@ -32,7 +32,7 @@ class CouncilClass(AbstractGetBinDataClass):
 
         # Strip data and parse the JSON
         address_data = json.loads(
-            re.sub("getAddressesCallback\d+\(", "", address_data)[:-2]
+            re.sub(r"getAddressesCallback\d+\(", "", address_data)[:-2]
         )
 
         if address_data["TotalHits"] == 0:
@@ -48,9 +48,7 @@ class CouncilClass(AbstractGetBinDataClass):
         address_x = address_data["Locations"][0]["X"]
         address_y = address_data["Locations"][0]["Y"]
 
-        stage2_url = "https://wlnk.statmap.co.uk/map/Cluster.svc/getpage?script=\Cluster\Cluster.AuroraScript$&taskId=bins&format=js&updateOnly=true&query=x%3D{}%3By%3D{}%3Bid%3D{}".format(
-            address_x, address_y, address_id
-        )
+        stage2_url = fr"https://wlnk.statmap.co.uk/map/Cluster.svc/getpage?script=\Cluster\Cluster.AuroraScript$&taskId=bins&format=js&updateOnly=true&query=x%3D{address_x}%3By%3D{address_y}%3Bid%3D{address_id}"
 
         bin_query = requests.get(stage2_url).text
 
@@ -61,7 +59,7 @@ class CouncilClass(AbstractGetBinDataClass):
             )
 
         # Return only the HTML contained within the Javascript function payload.
-        pattern = 'document\.getElementById\("DR1"\)\.innerHTML="(.+)";'
+        pattern = r'document\.getElementById\("DR1"\)\.innerHTML="(.+)";'
 
         bin_html = re.findall(pattern, bin_query)
 
@@ -86,19 +84,20 @@ class CouncilClass(AbstractGetBinDataClass):
 
             # Get bin date
             bin_date_text = row.text
-            pattern = "\d+\/\d+"
+            pattern = r"\d+\/\d+"
             bin_dates = re.findall(pattern, bin_date_text)
 
             for bin_date in bin_dates:
                 # Split the bin date into day and month and build a full date with the current year
                 split_date = bin_date.split("/")
+                if len(split_date[0]) < 1:
+                    raise ValueError("Error parsing dates retrieved from website")
                 full_date = datetime(
                     datetime.now().year, int(split_date[1]), int(split_date[0])
                 )
-
                 # If the current month is December and one of the next collections is in January, increment the year
-                if datetime.now().month == 12 and int(split_date[1]) == 1:
-                    full_date = bin_date + relativedelta(years=1)
+                if datetime.now().month == 12 and int(split_date[1]) < 12:
+                    full_date = datetime(year=datetime.now().year + 1, month=int(split_date[1]), day=int(split_date[0]))
 
                 # Since data in unordered, add to a tuple
                 collections.append((bin_type.title(), full_date))
@@ -116,3 +115,4 @@ class CouncilClass(AbstractGetBinDataClass):
             data["bins"].append(dict_data)
 
         return data
+

uk_bin_collection-0.125.2/uk_bin_collection/uk_bin_collection/councils/KirkleesCouncil.py

@@ -1,132 +0,0 @@
-from datetime import datetime
-from typing import Optional
-
-from selenium.common import TimeoutException
-from selenium.webdriver.common.by import By
-from selenium.webdriver.common.keys import Keys
-from selenium.webdriver.remote.webdriver import WebDriver
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.support.wait import WebDriverWait
-
-from uk_bin_collection.uk_bin_collection.common import create_webdriver
-from uk_bin_collection.uk_bin_collection.common import date_format
-from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
-
-
-def wait_for_element(driver, element_type, element: str, timeout: int = 5):
-    element_present = EC.presence_of_element_located((element_type, element))
-    wait_for_element_conditions(driver, element_present, timeout=timeout)
-
-
-def wait_for_element_conditions(driver, conditions, timeout: int = 5):
-    try:
-        WebDriverWait(driver, timeout).until(conditions)
-    except TimeoutException:
-        print("Timed out waiting for page to load")
-        raise
-
-
-class CouncilClass(AbstractGetBinDataClass):
-    """
-    Concrete classes have to implement all abstract operations of the
-    base class. They can also override some operations with a default
-    implementation.
-    """
-
-    def __init__(self):
-        self._driver: Optional[WebDriver] = None
-
-    def parse_data(self, *args, **kwargs) -> dict:
-        try:
-            return self._parse_data(*args, **kwargs)
-        finally:
-            if self._driver:
-                self._driver.quit()
-
-    def _parse_data(self, page: str, **kwargs) -> dict:
-        """
-        Process:
-
-        - Use a house number and postcode that is known to be domestic and resolves to a
-          single unique address. When the address search form is submitted with
-          those details, a session is created
-
-        - Now a session exists, navigate to the calendar URL, specifying the UPRN
-
-        - Extract info from the 'alt' attribute of the images on that page
-        """
-        bins = []
-
-        user_paon = kwargs["paon"]
-        user_postcode = kwargs["postcode"]
-
-        self._driver = driver = create_webdriver(
-            web_driver=kwargs["web_driver"],
-            headless=kwargs.get("headless", True),
-            session_name=__name__,
-        )
-        driver.implicitly_wait(1)
-
-        driver.get(
-            "https://www.kirklees.gov.uk/beta/your-property-bins-recycling/your-bins/default.aspx"
-        )
-
-        wait_for_element(
-            driver, By.ID, "cphPageBody_cphContent_thisGeoSearch_txtGeoPremises"
-        )
-
-        house_input = driver.find_element(
-            By.ID, "cphPageBody_cphContent_thisGeoSearch_txtGeoPremises"
-        )
-        house_input.send_keys(user_paon)
-
-        postcode_input = driver.find_element(
-            By.ID, "cphPageBody_cphContent_thisGeoSearch_txtGeoSearch"
-        )
-        postcode_input.send_keys(user_postcode)
-
-        # submit address search
-        driver.find_element(By.ID, "butGeoSearch").send_keys(Keys.RETURN)
-
-        wait_for_element(
-            driver,
-            By.ID,
-            "cphPageBody_cphContent_wtcDomestic240__lnkAccordionAnchor",
-            # submitting can be slow
-            timeout=30,
-        )
-
-        # Open the panel
-        driver.find_element(
-            By.ID, "cphPageBody_cphContent_wtcDomestic240__lnkAccordionAnchor"
-        ).click()
-
-        # Domestic waste calendar
-        wait_for_element(
-            driver, By.ID, "cphPageBody_cphContent_wtcDomestic240__LnkCalendar"
-        )
-        calendar_link = driver.find_element(
-            By.ID, "cphPageBody_cphContent_wtcDomestic240__LnkCalendar"
-        )
-        driver.execute_script("arguments[0].click();", calendar_link)
-
-        # <img alt="Recycling                      collection date 14 March 2024"
-        # <img alt="Domestic                       collection date 21 March 2024
-        date_strings = driver.find_elements(
-            By.CSS_SELECTOR, 'img[alt*="collection date"]'
-        )
-
-        for date in date_strings:
-            bin_type, _, _, day, month, year = date.get_attribute("alt").split()
-            collection_date = datetime.strptime(
-                f"{day} {month} {year}", "%d %B %Y"
-            ).strftime(date_format)
-
-            bins.append(
-                {
-                    "type": bin_type,
-                    "collectionDate": collection_date,
-                }
-            )
-
-        return {"bins": bins}