uk_bin_collection 0.114.5__tar.gz → 0.115.0__tar.gz

{uk_bin_collection-0.114.5 → uk_bin_collection-0.115.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.114.5
+Version: 0.115.0
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.114.5 → uk_bin_collection-0.115.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.114.5"
+version = "0.115.0"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

{uk_bin_collection-0.114.5 → uk_bin_collection-0.115.0}/uk_bin_collection/tests/input.json

@@ -12,6 +12,12 @@
         "wiki_name": "Adur and Worthing Councils",
         "wiki_note": "Replace XXXXXXXX with your UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find it."
     },
+    "AntrimAndNewtonabbeyCouncil": {
+        "url": "https://antrimandnewtownabbey.gov.uk/residents/bins-recycling/bins-schedule/?Id=643",
+        "wiki_command_url_override": "https://antrimandnewtownabbey.gov.uk/residents/bins-recycling/bins-schedule/?Id=XXXX",
+        "wiki_name": "Antrim & Newtonabbey Council",
+        "wiki_note": "Navigate to [https://antrimandnewtownabbey.gov.uk/residents/bins-recycling/bins-schedule] and search for your street name. Use the URL with the ID to replace XXXXXXXX with your specific ID."
+    },
     "ArdsAndNorthDownCouncil": {
         "url": "https://www.ardsandnorthdown.gov.uk",
         "wiki_command_url_override": "https://www.ardsandnorthdown.gov.uk",
@@ -230,6 +236,13 @@
         "wiki_name": "Bromsgrove District Council",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
+    "BroxbourneCouncil": {
+        "url": "https://www.broxbourne.gov.uk",
+        "uprn": "148048608",
+        "postcode": "EN8 7FL",
+        "wiki_name": "Broxbourne Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "BroxtoweBoroughCouncil": {
         "postcode": "NG16 2LY",
         "skip_get_url": true,
@@ -736,6 +749,15 @@
         "wiki_name": "Harrogate Borough Council",
         "wiki_note": "Pass the UPRN, which can be found at [this site](https://secure.harrogate.gov.uk/inmyarea). URL doesn't need to be passed."
     },
+    "HertsmereBoroughCouncil": {
+        "house_number": "1",
+        "postcode": "WD7 9HZ",
+        "skip_get_url": true,
+        "url": "https://www.hertsmere.gov.uk",
+        "web_driver": "http://selenium:4444",
+        "wiki_name": "Hertsmere Borough Council",
+        "wiki_note": "Provide your house number in the `house_number` parameter and postcode in the `postcode` parameter."
+    },
     "HighlandCouncil": {
         "url": "https://www.highland.gov.uk",
         "wiki_command_url_override": "https://www.highland.gov.uk",
@@ -1664,6 +1686,13 @@
         "wiki_name": "Waltham Forest",
         "wiki_note": "Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN."
     },
+    "WarringtonBoroughCouncil": {
+        "url": "https://www.warrington.gov.uk",
+        "wiki_command_url_override": "https://www.warrington.gov.uk",
+        "uprn": "10094964379",
+        "wiki_name": "Warrington Borough Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "WarwickDistrictCouncil": {
         "url": "https://estates7.warwickdc.gov.uk/PropertyPortal/Property/Recycling/100070263793",
         "wiki_command_url_override": "https://estates7.warwickdc.gov.uk/PropertyPortal/Property/Recycling/XXXXXXXX",
@@ -1715,6 +1744,13 @@
         "wiki_name": "West Berkshire Council",
         "wiki_note": "Provide your house number in the `house_number` parameter and postcode in the `postcode` parameter."
     },
+    "WestLancashireBoroughCouncil": {
+        "url": "https://www.westlancs.gov.uk",
+        "uprn": "10012343339",
+        "postcode": "WN8 0HR",
+        "wiki_name": "West Lancashire Borough Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "WestLindseyDistrictCouncil": {
         "house_number": "PRIVATE ACCOMMODATION",
         "postcode": "LN8 2AR",

uk_bin_collection-0.115.0/uk_bin_collection/uk_bin_collection/councils/AntrimAndNewtonabbeyCouncil.py

@@ -0,0 +1,53 @@
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        bindata = {"bins": []}
+
+        soup = BeautifulSoup(page.content, "html.parser")
+        soup.prettify
+
+        collection_divs = soup.select("div.feature-box.bins")
+        if not collection_divs:
+            raise Exception("No collections found")
+
+        for collection_div in collection_divs:
+            date_p = collection_div.select_one("p.date")
+            if not date_p:
+                continue
+
+            # Thu 22 Aug, 2024
+            date_ = datetime.strptime(date_p.text.strip(), "%a %d %b, %Y").strftime(
+                "%d/%m/%Y"
+            )
+            bins = collection_div.select("li")
+            if not bins:
+                continue
+            for bin in bins:
+                if not bin.text.strip():
+                    continue
+                bin_type = bin.text.strip()
+
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": date_,
+                }
+                bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

{uk_bin_collection-0.114.5 → uk_bin_collection-0.115.0}/uk_bin_collection/uk_bin_collection/councils/BradfordMDC.py

@@ -1,3 +1,5 @@
+import re
+
 import requests
 from bs4 import BeautifulSoup
 
@@ -89,12 +91,40 @@ class CouncilClass(AbstractGetBinDataClass):
                     )
                 ).strftime(date_format)
 
-        # Build data dict for each entry
-        dict_data = {
-            "type": bin_type,
-            "collectionDate": bin_date,
-        }
-        data["bins"].append(dict_data)
+            # Build data dict for each entry
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": bin_date,
+            }
+            data["bins"].append(dict_data)
+
+        for bin in soup.find_all(attrs={"id": re.compile(r"CTID-D0TUYGxO-\d+-A")}):
+            dict_data = {
+                "type": "General Waste",
+                "collectionDate": datetime.strptime(
+                    bin.text.strip(),
+                    "%a %b %d %Y",
+                ).strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+        for bin in soup.find_all(attrs={"id": re.compile(r"CTID-d3gapLk-\d+-A")}):
+            dict_data = {
+                "type": "Recycling Waste",
+                "collectionDate": datetime.strptime(
+                    bin.text.strip(),
+                    "%a %b %d %Y",
+                ).strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+        for bin in soup.find_all(attrs={"id": re.compile(r"CTID-L8OidMPA-\d+-A")}):
+            dict_data = {
+                "type": "Garden Waste (Subscription Only)",
+                "collectionDate": datetime.strptime(
+                    bin.text.strip(),
+                    "%a %b %d %Y",
+                ).strftime(date_format),
+            }
+            data["bins"].append(dict_data)
 
         data["bins"].sort(
             key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)

uk_bin_collection-0.115.0/uk_bin_collection/uk_bin_collection/councils/BroxbourneCouncil.py

@@ -0,0 +1,71 @@
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        user_postcode = kwargs.get("postcode")
+        check_uprn(user_uprn)
+        check_postcode(user_postcode)
+        bindata = {"bins": []}
+
+        API_URL = "https://www.broxbourne.gov.uk/xfp/form/205"
+
+        post_data = {
+            "page": "490",
+            "locale": "en_GB",
+            "qacf7e570cf99fae4cb3a2e14d5a75fd0d6561058_0_0": user_postcode,
+            "qacf7e570cf99fae4cb3a2e14d5a75fd0d6561058_1_0": user_uprn,
+            "next": "Next",
+        }
+
+        r = requests.post(API_URL, data=post_data)
+        r.raise_for_status()
+
+        soup = BeautifulSoup(r.content, features="html.parser")
+        soup.prettify()
+
+        form__instructions = soup.find(attrs={"class": "form__instructions"})
+        table = form__instructions.find("table")
+
+        rows = table.find_all("tr")
+
+        current_year = datetime.now().year
+
+        # Process each row into a list of dictionaries
+        for row in rows[1:]:  # Skip the header row
+            columns = row.find_all("td")
+            collection_date = (
+                columns[0].get_text(separator=" ").replace("\xa0", " ").strip()
+            )
+            service = columns[1].get_text(separator=" ").replace("\xa0", " ").strip()
+
+            collection_date = datetime.strptime(collection_date, "%a %d %b")
+
+            if collection_date.month == 1:
+                collection_date = collection_date.replace(year=current_year + 1)
+            else:
+                collection_date = collection_date.replace(year=current_year)
+
+            dict_data = {
+                "type": service,
+                "collectionDate": (collection_date).strftime(date_format),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+        return bindata

uk_bin_collection-0.115.0/uk_bin_collection/uk_bin_collection/councils/HertsmereBoroughCouncil.py

@@ -0,0 +1,161 @@
+import re
+import time
+
+import requests
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_paon = kwargs.get("paon")
+        user_postcode = kwargs.get("postcode")
+        web_driver = kwargs.get("web_driver")
+        headless = kwargs.get("headless")
+        check_paon(user_paon)
+        check_postcode(user_postcode)
+        bindata = {"bins": []}
+
+        URI_1 = "https://www.hertsmere.gov.uk/Environment-Refuse-and-Recycling/Recycling--Waste/Bin-collections/Collections-and-calendar.aspx"
+        URI_2 = "https://hertsmere-services.onmats.com/w/webpage/round-search"
+
+        # Create Selenium webdriver
+        driver = create_webdriver(web_driver, headless, None, __name__)
+        driver.get(URI_1)
+
+        soup = BeautifulSoup(driver.page_source, "html.parser")
+
+        current_week = (soup.find("li", class_="current")).text.strip()
+
+        strong = soup.find_all("strong", text=re.compile(r"^Week"))
+
+        bin_weeks = []
+        for tag in strong:
+            parent = tag.parent
+            bin_type = (
+                (parent.text).split("-")[1].strip().replace("\xa0", " ").split(" and ")
+            )
+            for bin in bin_type:
+                dict_data = {
+                    "week": tag.text.replace("\xa0", " "),
+                    "bin_type": bin,
+                }
+                bin_weeks.append(dict_data)
+
+        driver.get(URI_2)
+
+        # Wait for the postcode field to appear then populate it
+        inputElement_postcode = WebDriverWait(driver, 30).until(
+            EC.presence_of_element_located(
+                (
+                    By.CLASS_NAME,
+                    "relation_path_type_ahead_search",
+                )
+            )
+        )
+        inputElement_postcode.send_keys(user_postcode)
+
+        WebDriverWait(driver, 10).until(
+            EC.element_to_be_clickable(
+                (
+                    By.XPATH,
+                    f"//ul[@class='result_list']/li[starts-with(@aria-label, '{user_paon}')]",
+                )
+            )
+        ).click()
+
+        WebDriverWait(driver, timeout=10).until(
+            EC.element_to_be_clickable(
+                (
+                    By.CSS_SELECTOR,
+                    "input.fragment_presenter_template_edit.btn.bg-primary.btn-medium[type='submit']",
+                )
+            )
+        ).click()
+
+        WebDriverWait(driver, timeout=10).until(
+            EC.presence_of_element_located(
+                (By.XPATH, "//h3[contains(text(), 'Collection days')]")
+            )
+        )
+
+        soup = BeautifulSoup(driver.page_source, "html.parser")
+
+        table = soup.find("table", class_="table listing table-striped")
+
+        # Check if the table was found
+        if table:
+            # Extract table rows and cells
+            table_data = []
+            for row in table.find("tbody").find_all("tr"):
+                # Extract cell data from each <td> tag
+                row_data = [cell.get_text(strip=True) for cell in row.find_all("td")]
+                table_data.append(row_data)
+
+        else:
+            print("Table not found.")
+
+        collection_day = (table_data[0])[1]
+
+        current_week_bins = [bin for bin in bin_weeks if bin["week"] == current_week]
+        next_week_bins = [bin for bin in bin_weeks if bin["week"] != current_week]
+
+        days_of_week = [
+            "Monday",
+            "Tuesday",
+            "Wednesday",
+            "Thursday",
+            "Friday",
+            "Saturday",
+            "Sunday",
+        ]
+
+        today = datetime.now()
+        today_idx = today.weekday()  # Monday is 0 and Sunday is 6
+        target_idx = days_of_week.index(collection_day)
+
+        days_until_target = (target_idx - today_idx) % 7
+        if days_until_target == 0:
+            next_day = today
+        else:
+            next_day = today + timedelta(days=days_until_target)
+
+        current_week_dates = get_dates_every_x_days(next_day, 14, 7)
+        next_week_date = next_day + timedelta(days=7)
+        next_week_dates = get_dates_every_x_days(next_week_date, 14, 7)
+
+        for date in current_week_dates:
+            for bin in current_week_bins:
+                dict_data = {
+                    "type": bin["bin_type"],
+                    "collectionDate": date,
+                }
+                bindata["bins"].append(dict_data)
+
+        for date in next_week_dates:
+            for bin in next_week_bins:
+                dict_data = {
+                    "type": bin["bin_type"],
+                    "collectionDate": date,
+                }
+                bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

{uk_bin_collection-0.114.5 → uk_bin_collection-0.115.0}/uk_bin_collection/uk_bin_collection/councils/NuneatonBedworthBoroughCouncil.py

@@ -36,7 +36,7 @@ class CouncilClass(AbstractGetBinDataClass):
                     for date in v:
                         dict_data = {
                             "type": k,
-                            "collectionDate": date
+                            "collectionDate": datetime.strptime(date, "%Y-%m-%d").strftime(date_format)
                         }
                         data["bins"].append(dict_data)
     

uk_bin_collection-0.115.0/uk_bin_collection/uk_bin_collection/councils/WarringtonBoroughCouncil.py

@@ -0,0 +1,50 @@
+import requests
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        URI = f"https://www.warrington.gov.uk/bin-collections/get-jobs/{user_uprn}"
+
+        # Make the GET request
+        response = requests.get(URI)
+
+        # Parse the JSON response
+        bin_collection = response.json()
+
+        # Loop through each collection in bin_collection
+        for collection in bin_collection["schedule"]:
+            bin_type = collection["Name"]
+            collection_dates = collection["ScheduledStart"]
+
+            print(f"Bin Type: {bin_type}")
+            print(f"Collection Date: {collection_dates}")
+
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": datetime.strptime(
+                    collection_dates,
+                    "%Y-%m-%dT%H:%M:%S",
+                ).strftime(date_format),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

uk_bin_collection-0.115.0/uk_bin_collection/uk_bin_collection/councils/WestLancashireBoroughCouncil.py

@@ -0,0 +1,114 @@
+import re
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        user_postcode = kwargs.get("postcode")
+        check_uprn(user_uprn)
+        check_postcode(user_postcode)
+        bindata = {"bins": []}
+
+        user_postcode = user_postcode.replace(" ", "+")
+
+        API_URL = (
+            f"https://your.westlancs.gov.uk/yourwestlancs.aspx?address={user_postcode}"
+        )
+
+        session = requests.Session()
+        response = session.get(API_URL)
+        soup = BeautifulSoup(response.content, "html.parser")
+
+        soup = BeautifulSoup(response.content, features="html.parser")
+        soup.prettify()
+
+        pattern = r"SELECT\$\d+"
+
+        # Loop through each row to find the one with the target UPRN
+        for row in soup.find("table", class_="striped-table").find_all("tr"):
+            cells = row.find_all("td")
+            if len(cells) > 2 and cells[2].get_text(strip=True) == user_uprn:
+                link = row.find("a", href=True)
+                if link:
+                    match = re.search(pattern, link["href"])
+
+                    # Extract important form data like __VIEWSTATE and __EVENTVALIDATION
+                    viewstate = soup.find("input", {"name": "__VIEWSTATE"})["value"]
+                    eventvalidation = soup.find("input", {"name": "__EVENTVALIDATION"})[
+                        "value"
+                    ]
+
+                    # Parameters for the "click" - usually __EVENTTARGET and __EVENTARGUMENT
+                    post_data = {
+                        "__VIEWSTATE": viewstate,
+                        "__EVENTVALIDATION": eventvalidation,
+                        "__EVENTTARGET": "ctl00$MainContent$GridView1",
+                        "__EVENTARGUMENT": match.group(
+                            0
+                        ),  # Modify as needed for the specific link
+                    }
+
+                    post_response = session.post(API_URL, data=post_data)
+
+                    soup = BeautifulSoup(post_response.text, features="html.parser")
+                    StreetSceneTable = soup.find("table", {"id": "StreetSceneTable"})
+
+                    if StreetSceneTable:
+
+                        # Extract each collection date or information by locating the span elements
+                        refuse_collection = soup.find(
+                            "span", id="ctl00_MainContent_lbNextDomRoundZones"
+                        ).text.strip()
+                        recycling_collection = soup.find(
+                            "span", id="ctl00_MainContent_lbNextRecRoundZones"
+                        ).text.strip()
+                        garden_waste_collection = soup.find(
+                            "span", id="ctl00_MainContent_lbNextGardenRoundZones"
+                        ).text.strip()
+
+                        # Structure the extracted data in a dictionary
+                        bin_schedule = [
+                            {
+                                "Service": "Refuse Collection",
+                                "Date": refuse_collection,
+                            },
+                            {
+                                "Service": "Recycling Collection",
+                                "Date": recycling_collection,
+                            },
+                            {
+                                "Service": "Garden Waste Collection",
+                                "Date": garden_waste_collection,
+                            },
+                        ]
+
+                        if bin_schedule:
+                            for service in bin_schedule:
+                                if service["Date"] != "Not subscribed":
+                                    dict_data = {
+                                        "type": service["Service"],
+                                        "collectionDate": service["Date"],
+                                    }
+                                    bindata["bins"].append(dict_data)
+
+                else:
+                    print("No link found in the row with the target UPRN.")
+                    break
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+        return bindata