uk_bin_collection 0.151.0__py3-none-any.whl → 0.152.1__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/DoverDistrictCouncil.py

@@ -1,12 +1,27 @@
-from bs4 import BeautifulSoup
-from datetime import datetime
 import re
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+
 from uk_bin_collection.uk_bin_collection.common import *  # Consider specific imports
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
 class CouncilClass(AbstractGetBinDataClass):
     def parse_data(self, page: str, **kwargs) -> dict:
+
+        try:
+            user_uprn = kwargs.get("uprn")
+            check_uprn(user_uprn)
+            url = f"https://collections.dover.gov.uk/property/{user_uprn}"
+            if not user_uprn:
+                # This is a fallback for if the user stored a URL in old system. Ensures backwards compatibility.
+                url = kwargs.get("url")
+        except Exception as e:
+            raise ValueError(f"Error getting identifier: {str(e)}")
+
+        # Make a BS4 object
+        page = requests.get(url)
         soup = BeautifulSoup(page.text, "html.parser")
 
         bins_data = {"bins": []}

uk_bin_collection/uk_bin_collection/councils/EastDevonDC.py

@@ -3,7 +3,8 @@ from datetime import datetime
 
 import pandas as pd
 from bs4 import BeautifulSoup
-from uk_bin_collection.uk_bin_collection.common import date_format
+
+from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
@@ -15,7 +16,19 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page: str, **kwargs) -> dict:
+
+        try:
+            user_uprn = kwargs.get("uprn")
+            check_uprn(user_uprn)
+            url = f"https://eastdevon.gov.uk/recycling-and-waste/recycling-waste-information/when-is-my-bin-collected/future-collections-calendar/?UPRN={user_uprn}"
+            if not user_uprn:
+                # This is a fallback for if the user stored a URL in old system. Ensures backwards compatibility.
+                url = kwargs.get("url")
+        except Exception as e:
+            raise ValueError(f"Error getting identifier: {str(e)}")
+
         # Make a BS4 object
+        page = requests.get(url)
         soup = BeautifulSoup(page.text, features="html.parser")
         soup.prettify()
 

uk_bin_collection/uk_bin_collection/councils/EastbourneBoroughCouncil.py

@@ -0,0 +1,76 @@
+# Lewes Borough Council uses the same script.
+
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        try:
+            user_uprn = kwargs.get("uprn")
+            check_uprn(user_uprn)
+            url = f"https://environmentfirst.co.uk/house.php?uprn={user_uprn}"
+            if not user_uprn:
+                # This is a fallback for if the user stored a URL in old system. Ensures backwards compatibility.
+                url = kwargs.get("url")
+        except Exception as e:
+            raise ValueError(f"Error getting identifier: {str(e)}")
+
+        # Make a BS4 object
+        page = requests.get(url)
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        # Get the paragraph lines from the page
+        data = {"bins": []}
+        page_text = soup.find("div", {"class": "collect"}).find_all("p")
+
+        # Parse the correct lines (find them, remove the ordinal indicator and make them the correct format date) and
+        # then add them to the dictionary
+        rubbish_day = datetime.strptime(
+            remove_ordinal_indicator_from_date_string(
+                page_text[2].find_next("strong").text
+            ),
+            "%d %B %Y",
+        ).strftime(date_format)
+        dict_data = {
+            "type": "Rubbish",
+            "collectionDate": rubbish_day,
+        }
+        data["bins"].append(dict_data)
+        recycling_day = datetime.strptime(
+            remove_ordinal_indicator_from_date_string(
+                page_text[4].find_next("strong").text
+            ),
+            "%d %B %Y",
+        ).strftime(date_format)
+        dict_data = {
+            "type": "Recycling",
+            "collectionDate": recycling_day,
+        }
+        data["bins"].append(dict_data)
+
+        if len(page_text) > 5:
+            garden_day = datetime.strptime(
+                remove_ordinal_indicator_from_date_string(
+                    page_text[6].find_next("strong").text
+                ),
+                "%d %B %Y",
+            ).strftime(date_format)
+            dict_data = {
+                "type": "Garden",
+                "collectionDate": garden_day,
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/EastleighBoroughCouncil.py

@@ -1,4 +1,8 @@
 from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select, WebDriverWait
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
@@ -12,59 +16,69 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page: str, **kwargs) -> dict:
-        uprn = kwargs.get("uprn")
-        # Check the UPRN is valid
-        check_uprn(uprn)
+        try:
+            uprn = kwargs.get("uprn")
+            # Check the UPRN is valid
+            check_uprn(uprn)
+            headless = kwargs.get("headless")
+            web_driver = kwargs.get("web_driver")
+            url = f"https://www.eastleigh.gov.uk/waste-bins-and-recycling/collection-dates/your-waste-bin-and-recycling-collections?uprn={uprn}"
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(url)
 
-        # Request URL
-        url = f"https://www.eastleigh.gov.uk/waste-bins-and-recycling/collection-dates/your-waste-bin-and-recycling-collections?uprn={uprn}"
+            wait = WebDriverWait(driver, 10)
+            bin_content = wait.until(
+                EC.presence_of_element_located((By.CLASS_NAME, "dl-horizontal"))
+            )
 
-        # Make Request
-        requests.packages.urllib3.disable_warnings()
-        page = requests.get(url)
+            # Make a BS4 object from driver's page source
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
 
-        # Make a BS4 object
-        soup = BeautifulSoup(page.text, features="html.parser")
-        soup.prettify()
+            # Data to return
+            data = {"bins": []}
 
-        # Data to return
-        data = {"bins": []}
+            # Valid bin types
+            binTypes = [
+                "Household Waste Bin",
+                "Recycling Bin",
+                "Food Waste Bin",
+                "Glass Box and Batteries",
+                "Garden Waste Bin",
+            ]
 
-        # Valid bin types
-        binTypes = [
-            "Household Waste Bin",
-            "Recycling Bin",
-            "Food Waste Bin",
-            "Glass Box and Batteries",
-            "Garden Waste Bin",
-        ]
+            # Value to create dict for DL values
+            keys, values = [], []
 
-        # Value to create dict for DL values
-        keys, values = [], []
+            # Loop though DT and DD for DL containing bins
+            dl = soup.find("dl", {"class": "dl-horizontal"})
+            for dt in dl.find_all("dt"):
+                keys.append(dt.text.strip())
+            for dd in dl.find_all("dd"):
+                values.append(dd.text.strip())
 
-        # Loop though DT and DD for DL containing bins
-        dl = soup.find("dl", {"class": "dl-horizontal"})
-        for dt in dl.find_all("dt"):
-            keys.append(dt.text.strip())
-        for dd in dl.find_all("dd"):
-            values.append(dd.text.strip())
+            # Create dict for bin name and string dates
+            binDict = dict(zip(keys, values))
 
-        # Create dict for bin name and string dates
-        binDict = dict(zip(keys, values))
+            # Process dict for valid bin types
+            for bin in list(binDict):
+                if bin in binTypes:
+                    if not binDict[bin].startswith("You haven't yet signed up for"):
+                        # Convert date
+                        date = datetime.strptime(binDict[bin], "%a, %d %b %Y")
 
-        # Process dict for valid bin types
-        for bin in list(binDict):
-            if bin in binTypes:
-                if not binDict[bin].startswith("You haven't yet signed up for"):
-                    # Convert date
-                    date = datetime.strptime(binDict[bin], "%a, %d %b %Y")
+                        # Set bin data
+                        dict_data = {
+                            "type": bin,
+                            "collectionDate": date.strftime(date_format),
+                        }
+                        data["bins"].append(dict_data)
 
-                    # Set bin data
-                    dict_data = {
-                        "type": bin,
-                        "collectionDate": date.strftime(date_format),
-                    }
-                    data["bins"].append(dict_data)
+            # Return bin data
+            return data
 
-        # Return bin data
-        return data
+        except Exception as e:
+            print(f"Error fetching/parsing data: {str(e)}")
+            return {"bins": [{"type": "Error", "collectionDate": "2024-01-01"}]}
+        finally:
+            if "driver" in locals():
+                driver.quit()

uk_bin_collection/uk_bin_collection/councils/EnvironmentFirst.py

@@ -1,3 +1,5 @@
+# Legacy script. Copied to Lewes and Eastbourne.
+
 from bs4 import BeautifulSoup
 
 from uk_bin_collection.uk_bin_collection.common import *

uk_bin_collection/uk_bin_collection/councils/EppingForestDistrictCouncil.py

@@ -1,13 +1,15 @@
+from datetime import datetime
+
 from bs4 import BeautifulSoup
-from uk_bin_collection.uk_bin_collection.common import *
-from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 from selenium import webdriver
-from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.common.by import By
-from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as EC
-from datetime import datetime
+from selenium.webdriver.support.ui import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.common import date_format
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
 class CouncilClass(AbstractGetBinDataClass):
@@ -15,27 +17,57 @@ class CouncilClass(AbstractGetBinDataClass):
         postcode = kwargs.get("postcode", "")
         web_driver = kwargs.get("web_driver")
         headless = kwargs.get("headless")
-
-        options = webdriver.ChromeOptions()
-        if headless:
-            options.add_argument("--headless")
-        driver = create_webdriver(web_driver, headless)
+        data = {"bins": []}
 
         try:
-            driver.get(
-                f"https://eppingforestdc.maps.arcgis.com/apps/instant/lookup/index.html?appid=bfca32b46e2a47cd9c0a84f2d8cdde17&find={postcode}"
+            # Initialize webdriver with logging
+            print(f"Initializing webdriver with: {web_driver}, headless: {headless}")
+            driver = create_webdriver(web_driver, headless, None, __name__)
+
+            # Format and load URL
+            page_url = f"https://eppingforestdc.maps.arcgis.com/apps/instant/lookup/index.html?appid=bfca32b46e2a47cd9c0a84f2d8cdde17&find={postcode}"
+            print(f"Accessing URL: {page_url}")
+            driver.get(page_url)
+
+            # Wait for initial page load
+            wait = WebDriverWait(driver, 20)  # Reduced timeout to fail faster if issues
+
+            # First wait for any loading indicators to disappear
+            try:
+                print("Waiting for loading spinner to disappear...")
+                wait.until(
+                    EC.invisibility_of_element_located(
+                        (By.CSS_SELECTOR, ".esri-widget--loader-container")
+                    )
+                )
+            except Exception as e:
+                print(f"Loading spinner wait failed (may be normal): {str(e)}")
+
+            # Then wait for the content container
+            print("Waiting for content container...")
+            wait.until(
+                EC.presence_of_element_located(
+                    (By.CSS_SELECTOR, ".esri-feature-content")
+                )
             )
-            wait = WebDriverWait(driver, 10)
-            WebDriverWait(driver, 10).until(
+
+            # Finally wait for actual content
+            print("Waiting for content to be visible...")
+            content = wait.until(
                 EC.visibility_of_element_located(
                     (By.CSS_SELECTOR, ".esri-feature-content")
                 )
             )
+
+            # Check if content is actually present
+            if not content:
+                raise ValueError("Content element found but empty")
+
+            print("Content found, getting page source...")
             html_content = driver.page_source
 
             soup = BeautifulSoup(html_content, "html.parser")
             bin_info_divs = soup.select(".esri-feature-content p")
-            data = {"bins": []}
             for div in bin_info_divs:
                 if "collection day is" in div.text:
                     bin_type, date_str = div.text.split(" collection day is ")

uk_bin_collection/uk_bin_collection/councils/FermanaghOmaghDistrictCouncil.py

@@ -0,0 +1,102 @@
+import difflib
+from datetime import date, datetime
+
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    base_url = "https://fermanaghomagh.isl-fusion.com/"
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        """
+        This function will make a request to the search endpoint with the postcode, extract the
+        house numbers from the responses, then retrieve the ID of the entry with the house number that matches,
+        to then retrieve the bin schedule.
+
+        The API here is a weird combination of HTML in json responses.
+        """
+        postcode = kwargs.get("postcode")
+        paon = kwargs.get("paon")
+
+        if not postcode:
+            raise ValueError("Must provide a postcode")
+
+        if not paon:
+            raise ValueError("Must provide a house number")
+
+        search_url = f"{self.base_url}/address/{postcode}"
+
+        requests.packages.urllib3.disable_warnings()
+        s = requests.Session()
+        response = s.get(search_url)
+        response.raise_for_status()
+
+        address_data = response.json()
+
+        address_list = address_data["html"]
+
+        soup = BeautifulSoup(address_list, features="html.parser")
+
+        address_by_id = {}
+
+        for li in soup.find_all("li"):
+            link = li.find_all("a")[0]
+            address_id = link.attrs["href"]
+            address = link.text
+
+            address_by_id[address_id] = address
+
+        addresses = list(address_by_id.values())
+
+        common = difflib.SequenceMatcher(
+            a=addresses[0], b=addresses[1]
+        ).find_longest_match()
+        extra_bit = addresses[0][common.a : common.a + common.size]
+
+        ids_by_paon = {
+            a.replace(extra_bit, ""): a_id.replace("/view/", "").replace("/", "")
+            for a_id, a in address_by_id.items()
+        }
+
+        property_id = ids_by_paon.get(paon)
+        if not property_id:
+            raise ValueError(
+                f"Invalid house number, valid values are {', '.join(ids_by_paon.keys())}"
+            )
+
+        today = date.today()
+        calendar_url = (
+            f"{self.base_url}/calendar/{property_id}/{today.strftime('%Y-%m-%d')}"
+        )
+        response = s.get(calendar_url)
+        response.raise_for_status()
+        calendar_data = response.json()
+        next_collections = calendar_data["nextCollections"]
+
+        collections = list(next_collections["collections"].values())
+
+        data = {"bins": []}
+
+        for collection in collections:
+            collection_date = datetime.strptime(collection["date"], "%Y-%m-%d")
+            bins = [c["name"] for c in collection["collections"].values()]
+
+            for bin in bins:
+                data["bins"].append(
+                    {
+                        "type": bin,
+                        "collectionDate": collection_date.strftime(date_format),
+                    }
+                )
+        return data

uk_bin_collection/uk_bin_collection/councils/GlasgowCityCouncil.py

@@ -14,7 +14,19 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page: str, **kwargs) -> dict:
-        # Parse the page
+
+        try:
+            user_uprn = kwargs.get("uprn")
+            check_uprn(user_uprn)
+            url = f"https://onlineservices.glasgow.gov.uk/forms/RefuseAndRecyclingWebApplication/CollectionsCalendar.aspx?UPRN={user_uprn}"
+            if not user_uprn:
+                # This is a fallback for if the user stored a URL in old system. Ensures backwards compatibility.
+                url = kwargs.get("url")
+        except Exception as e:
+            raise ValueError(f"Error getting identifier: {str(e)}")
+
+        # Make a BS4 object
+        page = requests.get(url, verify=False)
         soup = BeautifulSoup(page.text, features="html.parser")
         soup.prettify()
 

uk_bin_collection/uk_bin_collection/councils/GuildfordCouncil.py

@@ -27,9 +27,8 @@ class CouncilClass(AbstractGetBinDataClass):
     def parse_data(self, page: str, **kwargs) -> dict:
         driver = None
         try:
-            uprn = kwargs.get("uprn")
             postcode = kwargs.get("postcode")
-            full_address = kwargs.get("paon")
+            house_number = kwargs.get("paon")
 
             url = "https://my.guildford.gov.uk/customers/s/view-bin-collections"
 
@@ -60,7 +59,7 @@ class CouncilClass(AbstractGetBinDataClass):
                 EC.presence_of_element_located(
                     (
                         By.XPATH,
-                        f"//lightning-base-formatted-text[contains(text(), '{full_address}')]",
+                        f"//lightning-base-formatted-text[contains(text(), '{house_number}')]",
                     )
                 )
             )

uk_bin_collection/uk_bin_collection/councils/HerefordshireCouncil.py

@@ -15,9 +15,20 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page: str, **kwargs) -> dict:
+        try:
+            user_uprn = kwargs.get("uprn")
+            check_uprn(user_uprn)
+            url = f"https://www.herefordshire.gov.uk/rubbish-recycling/check-bin-collection-day?blpu_uprn={user_uprn}"
+            if not user_uprn:
+                # This is a fallback for if the user stored a URL in old system. Ensures backwards compatibility.
+                url = kwargs.get("url")
+        except Exception as e:
+            raise ValueError(f"Error getting identifier: {str(e)}")
+
         # Make a BS4 object
-        soup = BeautifulSoup(page.text, features="html.parser")
-        soup.prettify()
+        page = requests.get(url)
+        soup = BeautifulSoup(page.text, "html.parser")
+        soup.prettify
 
         data = {"bins": []}
 

uk_bin_collection/uk_bin_collection/councils/HuntingdonDistrictCouncil.py

@@ -2,10 +2,12 @@
 
 # This script pulls (in one hit) the data from
 # Huntingdon District Council District Council Bins Data
+from datetime import datetime
+
 from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
-from uk_bin_collection.uk_bin_collection.common import date_format
-from datetime import datetime
 
 
 # import the wonderful Beautiful Soup and the URL grabber
@@ -17,9 +19,21 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page, **kwargs) -> None:
+
+        try:
+            user_uprn = kwargs.get("uprn")
+            check_uprn(user_uprn)
+            url = f"http://www.huntingdonshire.gov.uk/refuse-calendar/{user_uprn}"
+            if not user_uprn:
+                # This is a fallback for if the user stored a URL in old system. Ensures backwards compatibility.
+                url = kwargs.get("url")
+        except Exception as e:
+            raise ValueError(f"Error getting identifier: {str(e)}")
+
         # Make a BS4 object
-        soup = BeautifulSoup(page.text, features="html.parser")
-        soup.prettify()
+        page = requests.get(url)
+        soup = BeautifulSoup(page.text, "html.parser")
+        soup.prettify
 
         data = {"bins": []}
 

uk_bin_collection/uk_bin_collection/councils/LewesDistrictCouncil.py

@@ -0,0 +1,76 @@
+# Eastbourne uses the same script.
+
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        try:
+            user_uprn = kwargs.get("uprn")
+            check_uprn(user_uprn)
+            url = f"https://environmentfirst.co.uk/house.php?uprn={user_uprn}"
+            if not user_uprn:
+                # This is a fallback for if the user stored a URL in old system. Ensures backwards compatibility.
+                url = kwargs.get("url")
+        except Exception as e:
+            raise ValueError(f"Error getting identifier: {str(e)}")
+
+        # Make a BS4 object
+        page = requests.get(url)
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        # Get the paragraph lines from the page
+        data = {"bins": []}
+        page_text = soup.find("div", {"class": "collect"}).find_all("p")
+
+        # Parse the correct lines (find them, remove the ordinal indicator and make them the correct format date) and
+        # then add them to the dictionary
+        rubbish_day = datetime.strptime(
+            remove_ordinal_indicator_from_date_string(
+                page_text[2].find_next("strong").text
+            ),
+            "%d %B %Y",
+        ).strftime(date_format)
+        dict_data = {
+            "type": "Rubbish",
+            "collectionDate": rubbish_day,
+        }
+        data["bins"].append(dict_data)
+        recycling_day = datetime.strptime(
+            remove_ordinal_indicator_from_date_string(
+                page_text[4].find_next("strong").text
+            ),
+            "%d %B %Y",
+        ).strftime(date_format)
+        dict_data = {
+            "type": "Recycling",
+            "collectionDate": recycling_day,
+        }
+        data["bins"].append(dict_data)
+
+        if len(page_text) > 5:
+            garden_day = datetime.strptime(
+                remove_ordinal_indicator_from_date_string(
+                    page_text[6].find_next("strong").text
+                ),
+                "%d %B %Y",
+            ).strftime(date_format)
+            dict_data = {
+                "type": "Garden",
+                "collectionDate": garden_day,
+            }
+            data["bins"].append(dict_data)
+
+        return data