uk_bin_collection 0.97.1__tar.gz → 0.98.1__tar.gz

{uk_bin_collection-0.97.1 → uk_bin_collection-0.98.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.97.1
+Version: 0.98.1
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.97.1 → uk_bin_collection-0.98.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.97.1"
+version = "0.98.1"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

{uk_bin_collection-0.97.1 → uk_bin_collection-0.98.1}/uk_bin_collection/tests/input.json

@@ -5,6 +5,13 @@
         "wiki_name": "Adur and Worthing Councils",
         "wiki_note": "Replace XXXXXXXX with UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
+    "ArmaghBanbridgeCraigavonCouncil": {
+        "url": "https://www.armaghbanbridgecraigavon.gov.uk/",
+        "wiki_command_url_override": "https://www.armaghbanbridgecraigavon.gov.uk/",
+        "uprn": "185625284",
+        "wiki_name": "Armagh Banbridge Craigavon Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "ArunCouncil": {
         "house_number": "1",
         "postcode": "BN16 4DA",
@@ -42,7 +49,7 @@
         "wiki_name": "Barnsley Metropolitan Borough Council",
         "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search)."
     },
-"BasildonCouncil": {
+    "BasildonCouncil": {
         "skip_get_url": true,
         "uprn": "10013350430",
         "url": "https://basildonportal.azurewebsites.net/api/getPropertyRefuseInformation",
@@ -606,6 +613,13 @@
         "web_driver": "http://selenium:4444",
         "wiki_name": "Leeds City Council"
     },
+    "LichfieldDistrictCouncil": {
+        "url": "https://www.lichfielddc.gov.uk",
+        "wiki_command_url_override": "https://www.lichfielddc.gov.uk",
+        "uprn": "100031694085",
+        "wiki_name": "Lichfield District Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "LisburnCastlereaghCityCouncil": {
         "house_number": "97",
         "postcode": "BT28 1JN",
@@ -1216,6 +1230,13 @@
         "wiki_name": "Wakefield City Council",
         "wiki_note": "Follow the instructions [here](https://www.wakefield.gov.uk/where-i-live/) until you get the page that includes a \"Bin Collections\" section then copy the URL and replace the URL in the command."
     },
+    "WalsallCouncil": {
+        "url": "https://cag.walsall.gov.uk/",
+        "wiki_command_url_override": "https://cag.walsall.gov.uk/",
+        "uprn": "100071080513",
+        "wiki_name": "Walsall Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "WalthamForest": {
         "house_number": "17 Chingford Road, Walthamstow",
         "postcode": "E17 4PW",
@@ -1278,6 +1299,13 @@
         "wiki_name": "West Lothian Council",
         "wiki_note": "Pass the house name/number in the house number parameter, wrapped in double quotes"
     },
+    "WestMorlandAndFurness": {
+        "url": "https://www.westmorlandandfurness.gov.uk/",
+        "wiki_command_url_override": "https://www.westmorlandandfurness.gov.uk/",
+        "uprn": "100110353478",
+        "wiki_name": "West Morland And Furness Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "WestNorthamptonshireCouncil": {
         "postcode": "NN3 2JB",
         "skip_get_url": true,
@@ -1322,6 +1350,13 @@
         "url": "https://forms.rbwm.gov.uk/bincollections?uprn=",
         "wiki_name": "Windsor and Maidenhead Council"
     },
+    "WirralCouncil": {
+        "url": "https://www.wirral.gov.uk",
+        "wiki_command_url_override": "https://www.wirral.gov.uk",
+        "uprn": "Vernon Avenue,Seacombe",
+        "wiki_name": "Wirral Council",
+        "wiki_note": "Please use the UPRN field to enter your street name and suburb separated by a comma, for example 'Vernon Avenue,Seacombe'"
+    },
     "WokingBoroughCouncil": {
         "house_number": "2",
         "postcode": "GU214JY",

uk_bin_collection-0.98.1/uk_bin_collection/uk_bin_collection/councils/ArmaghBanbridgeCraigavonCouncil.py

@@ -0,0 +1,72 @@
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        # Function to extract bin collection information
+        def extract_bin_schedule(soup, heading_class):
+            collections = []
+
+            # Find the relevant section based on the heading class
+            section_heading = soup.find("div", class_=heading_class)
+            if section_heading:
+                # Find all the bin collection dates in that section
+                collection_dates = section_heading.find_next(
+                    "div", class_="col-sm-12 col-md-9"
+                ).find_all("h4")
+                for date in collection_dates:
+                    # Clean and add the date to the list
+                    collections.append(date.get_text(strip=True))
+
+            return collections
+
+        # URL for bin collection schedule
+        url = f"https://www.armaghbanbridgecraigavon.gov.uk/resident/binday-result/?address={user_uprn}"
+
+        # Send a GET request to fetch the page content
+        response = requests.get(url)
+
+        # Check if the request was successful
+        if response.status_code == 200:
+            # Parse the page content using BeautifulSoup
+            soup = BeautifulSoup(response.text, "html.parser")
+
+            # Extract bin collection schedules by their sections
+            domestic_collections = extract_bin_schedule(soup, "heading bg-black")
+            for collection in domestic_collections:
+                bindata["bins"].append(
+                    {"collectionDate": collection, "type": "Domestic"}
+                )
+            recycling_collections = extract_bin_schedule(soup, "heading bg-green")
+            for collection in recycling_collections:
+                bindata["bins"].append(
+                    {"collectionDate": collection, "type": "Recycling"}
+                )
+            garden_collections = extract_bin_schedule(soup, "heading bg-brown")
+            for collection in garden_collections:
+                bindata["bins"].append({"collectionDate": collection, "type": "Garden"})
+
+        else:
+            print(f"Failed to retrieve data. Status code: {response.status_code}")
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

{uk_bin_collection-0.97.1 → uk_bin_collection-0.98.1}/uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py

@@ -70,10 +70,19 @@ class CouncilClass(AbstractGetBinDataClass):
             headless = kwargs.get("headless")
             web_driver = kwargs.get("web_driver")
             driver = create_webdriver(web_driver, headless, None, __name__)
-            page = "https://account.barnet.gov.uk/Forms/Home/Redirector/Index/?id=6a2ac067-3322-46e5-96e4-16c0c214454a&mod=OA&casetype=BAR&formname=BNTCOLDATE"
+            page = "https://www.barnet.gov.uk/recycling-and-waste/bin-collections/find-your-bin-collection-day"
+
             driver.get(page)
 
-            time.sleep(5)
+            # Wait for the element to be clickable
+            find_your_collection_button = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, '//a[contains(text(), "Find your household collection day")]')
+                )
+            )
+
+            # Click the element
+            find_your_collection_button.click()
 
             try:
                 accept_cookies = WebDriverWait(driver, timeout=10).until(

uk_bin_collection-0.98.1/uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py

@@ -0,0 +1,147 @@
+import time
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+# import the wonderful Beautiful Soup and the URL grabber
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            page = "https://waste.bexley.gov.uk/waste"
+
+            data = {"bins": []}
+
+            user_uprn = kwargs.get("uprn")
+            user_paon = kwargs.get("paon")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(page)
+
+            wait = WebDriverWait(driver, 10)
+
+            inputElement_postcodesearch = wait.until(
+                EC.element_to_be_clickable((By.ID, "pc"))
+            )
+            inputElement_postcodesearch.send_keys(user_postcode)
+
+
+
+            find_address_btn = wait.until(
+                EC.element_to_be_clickable((By.XPATH, '//*[@id="sub"]'))
+            )
+            find_address_btn.click()
+
+            dropdown_options = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, '//*[@id="address"]')
+                )
+            )
+            time.sleep(2)
+            dropdown_options.click()
+            time.sleep(1)
+
+            # Wait for the element to be clickable
+            address = WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, f'//li[contains(text(), "{user_paon}")]')
+                )
+            )
+
+            # Click the element
+            address.click()
+
+
+            submit_address = wait.until(
+                EC.presence_of_element_located(
+                    (By.XPATH, '//*[@id="go"]')
+                )
+            )
+            time.sleep(2)
+            submit_address.click()
+
+            results_found = wait.until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, '//h1[contains(text(), "Your bin days")]')
+                )           
+                )
+
+            final_page = wait.until(
+                EC.presence_of_element_located(
+                    (By.CLASS_NAME, "waste__collections")
+                )
+            )
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            # Find all waste services
+
+            # Initialize the data dictionary
+            data = {"bins": []}
+            bin_sections = soup.find_all("h3", class_="waste-service-name")
+
+            # Loop through each bin field
+            for bin_section in bin_sections:
+                # Extract the bin type (e.g., "Brown Caddy", "Green Wheelie Bin", etc.)
+                bin_type = bin_section.get_text(strip=True).split("\n")[0]  # The first part is the bin type
+
+                # Find the next sibling <dl> tag that contains the next collection information
+                summary_list = bin_section.find_next("dl", class_="govuk-summary-list")
+
+                if summary_list:
+                    # Now, instead of finding by class, we'll search by text within the dt element
+                    next_collection_dt = summary_list.find("dt", string=lambda text: "Next collection" in text)
+
+                    if next_collection_dt:
+                        # Find the sibling <dd> tag for the collection date
+                        next_collection = next_collection_dt.find_next_sibling("dd").get_text(strip=True)
+
+                        if next_collection:
+                            try:
+                                # Parse the next collection date (assuming the format is like "Tuesday 15 October 2024")
+                                parsed_date = datetime.strptime(next_collection, "%A %d %B %Y")
+
+                                # Add the bin information to the data dictionary
+                                data["bins"].append({
+                                    "type": bin_type,
+                                    "collectionDate": parsed_date.strftime(date_format),
+                                })
+                            except ValueError as e:
+                                print(f"Error parsing date for {bin_type}: {e}")
+                        else:
+                            print(f"No next collection date found for {bin_type}")
+                    else:
+                        print(f"No 'Next collection' text found for {bin_type}")
+                else:
+                    print(f"No summary list found for {bin_type}")
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data

uk_bin_collection-0.98.1/uk_bin_collection/uk_bin_collection/councils/LichfieldDistrictCouncil.py

@@ -0,0 +1,60 @@
+import re
+
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        def solve(s):
+            return re.sub(r"(\d)(st|nd|rd|th)", r"\1", s)
+
+        URI = f"https://www.lichfielddc.gov.uk/homepage/6/bin-collection-dates?uprn={user_uprn}"
+
+        # Make the GET request
+        response = requests.get(URI)
+
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        bins = soup.find_all("h3", class_="bin-collection-tasks__heading")
+        dates = soup.find_all("p", class_="bin-collection-tasks__date")
+
+        current_year = datetime.now().year
+        current_month = datetime.now().month
+
+        for i in range(len(dates)):
+            bint = " ".join(bins[i].text.split()[2:4])
+            date = dates[i].text
+
+            date = datetime.strptime(
+                solve(date),
+                "%d %B",
+            )
+
+            if (current_month > 10) and (date.month < 3):
+                date = date.replace(year=(current_year + 1))
+            else:
+                date = date.replace(year=current_year)
+
+            dict_data = {
+                "type": bint,
+                "collectionDate": date.strftime("%d/%m/%Y"),
+            }
+            bindata["bins"].append(dict_data)
+
+        return bindata

uk_bin_collection-0.98.1/uk_bin_collection/uk_bin_collection/councils/WalsallCouncil.py

@@ -0,0 +1,56 @@
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        URI = f"https://cag.walsall.gov.uk/BinCollections/GetBins?uprn={user_uprn}"
+
+        headers = {
+            "user-agent": "Mozilla/5.0",
+        }
+
+        response = requests.get(URI, headers=headers)
+
+        soup = BeautifulSoup(response.text, "html.parser")
+        # Extract links to collection shedule pages and iterate through the pages
+        schedule_links = soup.findAll("a", {"class": "nav-link"}, href=True)
+        for item in schedule_links:
+            if "roundname" in item["href"]:
+                # get bin colour
+                bincolour = item["href"].split("=")[-1].split("%")[0].upper()
+                binURL = "https://cag.walsall.gov.uk" + item["href"]
+                r = requests.get(binURL, headers=headers)
+                soup = BeautifulSoup(r.text, "html.parser")
+                table = soup.findAll("tr")
+                for tr in table:
+                    td = tr.findAll("td")
+                    if td:
+                        dict_data = {
+                            "type": bincolour,
+                            "collectionDate": datetime.strptime(
+                                td[1].text.strip(), "%d/%m/%Y"
+                            ).strftime("%d/%m/%Y"),
+                        }
+                        bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

uk_bin_collection-0.98.1/uk_bin_collection/uk_bin_collection/councils/WestMorlandAndFurness.py

@@ -0,0 +1,65 @@
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        URI = f"https://www.westmorlandandfurness.gov.uk/bins-recycling-and-street-cleaning/waste-collection-schedule/view/{user_uprn}"
+
+        headers = {
+            "user-agent": "Mozilla/5.0",
+        }
+
+        current_year = datetime.now().year
+        current_month = datetime.now().month
+
+        response = requests.get(URI)
+
+        soup = BeautifulSoup(response.text, "html.parser")
+        # Extract links to collection shedule pages and iterate through the pages
+        schedule = soup.findAll("div", {"class": "waste-collection__month"})
+        for month in schedule:
+            collectionmonth = datetime.strptime(month.find("h3").text, "%B")
+            collectionmonth = collectionmonth.month
+            collectiondays = month.findAll("li", {"class": "waste-collection__day"})
+            for collectionday in collectiondays:
+                day = collectionday.find(
+                    "span", {"class": "waste-collection__day--day"}
+                ).text.strip()
+                collectiondate = datetime.strptime(day, "%d")
+                collectiondate = collectiondate.replace(month=collectionmonth)
+                bintype = collectionday.find(
+                    "span", {"class": "waste-collection__day--colour"}
+                ).text
+
+                if (current_month > 9) and (collectiondate.month < 4):
+                    collectiondate = collectiondate.replace(year=(current_year + 1))
+                else:
+                    collectiondate = collectiondate.replace(year=current_year)
+
+                dict_data = {
+                    "type": bintype,
+                    "collectionDate": collectiondate.strftime("%d/%m/%Y"),
+                }
+                bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

uk_bin_collection-0.98.1/uk_bin_collection/uk_bin_collection/councils/WirralCouncil.py

@@ -0,0 +1,64 @@
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        bindata = {"bins": []}
+
+        street = user_uprn.split(",")[0]
+        suburb = user_uprn.split(",")[1]
+
+        if not street:
+            print("Street name could not be parsed"),
+            return
+        if not suburb:
+            print("Street name could not be parsed"),
+            return
+
+        wastes = {
+            "Non recycleable (green bin)",
+            "Garden waste (brown bin)",
+            "Paper and packaging (grey bin)",
+        }
+        DATE_REGEX = "^([0-9]{1,2} [A-Za-z]+ [0-9]{4})"
+
+        s = requests.Session()
+        # Loop through waste types
+        for waste in wastes:
+            r = s.get(
+                f"https://ww3.wirral.gov.uk//recycling/detailContentDru7.asp?s={street}&t={suburb}&c={waste}"
+            )
+            # extract dates
+            soup = BeautifulSoup(r.text, "html.parser")
+            dates = soup.findAll("li")
+            if len(dates) != 0:
+                for item in dates:
+                    match = re.match(DATE_REGEX, item.text)
+                    if match:
+                        dict_data = {
+                            "type": waste,
+                            "collectionDate": datetime.strptime(
+                                match.group(1),
+                                "%d %B %Y",
+                            ).strftime("%d/%m/%Y"),
+                        }
+                        bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata