uk_bin_collection 0.154.0__tar.gz → 0.158.0__tar.gz

{uk_bin_collection-0.154.0 → uk_bin_collection-0.158.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.154.0
+Version: 0.158.0
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.154.0 → uk_bin_collection-0.158.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.154.0"
+version = "0.158.0"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]
@@ -65,9 +65,13 @@ tabulate = "^0.9.0"
 icalevents = "^0.2.1"
 
 [tool.commitizen]
-major_version_zero = true
+name = "cz_conventional_commits"
 version_provider = "poetry"
 version_scheme = "semver"
+major_version_zero = true
+tag_format = "$version"
+update_changelog_on_bump = true
+annotated_tag = true
 version_files = [
     "custom_components/uk_bin_collection/manifest.json:version",
     "custom_components/uk_bin_collection/manifest.json:requirements",

{uk_bin_collection-0.154.0 → uk_bin_collection-0.158.0}/uk_bin_collection/tests/input.json

@@ -760,6 +760,13 @@
         "wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
         "LAD24CD": "E07000040"
     },
+    "EastDunbartonshireCouncil": {
+        "uprn": "132027197",
+        "url": "https://www.eastdunbarton.gov.uk/",
+        "wiki_name": "East Dunbartonshire",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN.",
+        "LAD24CD": "S12000045"
+    },
     "EastHertsCouncil": {
         "LAD24CD": "E07000097",
         "skip_get_url": true,
@@ -1608,7 +1615,7 @@
         "postcode": "SO41 0GJ",
         "skip_get_url": true,
         "uprn": "100060482345",
-        "url": "https://forms.newforest.gov.uk/id/FIND_MY_COLLECTION",
+        "url": "https://forms.newforest.gov.uk/ufs/FIND_MY_BIN_BAR.eb",
         "web_driver": "http://selenium:4444",
         "wiki_name": "New Forest",
         "wiki_note": "Pass the postcode and UPRN. This parser requires a Selenium webdriver.",
@@ -1647,10 +1654,11 @@
     "NewportCityCouncil": {
         "postcode": "NP20 4HE",
         "skip_get_url": true,
-        "uprn": "100100688837",
+        "house_number": "6",
         "url": "https://www.newport.gov.uk/",
+        "web_driver": "http://selenium:4444",
         "wiki_name": "Newport",
-        "wiki_note": "Pass the postcode and UPRN. You can find the UPRN using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
+        "wiki_note": "Pass the postcode and house number in their respective arguments, both wrapped in quotes.",
         "LAD24CD": "W06000022"
     },
     "NorthAyrshireCouncil": {
@@ -2087,10 +2095,11 @@
     "SomersetCouncil": {
         "postcode": "TA6 4AA",
         "skip_get_url": true,
-        "uprn": "10090857775",
+        "house_number": "5",
         "url": "https://www.somerset.gov.uk/",
+        "web_driver": "http://selenium:4444",
         "wiki_name": "Somerset",
-        "wiki_note": "Provide your UPRN and postcode. Find your UPRN using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
+        "wiki_note": "Provide your house number and postcode",
         "LAD24CD": "E06000066"
     },
     "SouthAyrshireCouncil": {
@@ -2122,7 +2131,7 @@
     "SouthGloucestershireCouncil": {
         "skip_get_url": true,
         "uprn": "566419",
-        "url": "https://beta.southglos.gov.uk/waste-and-recycling-collection-date",
+        "url": "https://api.southglos.gov.uk/wastecomp/GetCollectionDetails",
         "wiki_name": "South Gloucestershire",
         "wiki_note": "Provide your UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
         "LAD24CD": "E06000025"
@@ -2415,12 +2424,13 @@
         "LAD24CD": "E07000076"
     },
     "TestValleyBoroughCouncil": {
-        "postcode": "SO51 9ZD",
+        "postcode": "SO51 0BY",
         "skip_get_url": true,
-        "uprn": "200010012019",
-        "url": "https://testvalley.gov.uk/wasteandrecycling/when-are-my-bins-collected",
+        "house_number": "2",
+        "url": "https://testvalley.gov.uk/wasteandrecycling/when-are-my-bins-collected/when-are-my-bins-collected",
+        "web_driver": "http://selenium:4444",
         "wiki_name": "Test Valley",
-        "wiki_note": "Provide your UPRN and postcode. Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN.",
+        "wiki_note": "Provide your house number and postcode",
         "LAD24CD": "E07000093"
     },
     "ThanetDistrictCouncil": {
@@ -2463,6 +2473,7 @@
         "skip_get_url": true,
         "uprn": "10000016984",
         "postcode": "TQ1 1AG",
+        "web_driver": "http://selenium:4444",
         "url": "https://www.torbay.gov.uk/recycling/bin-collections/",
         "wiki_name": "Torbay",
         "wiki_note": "Provide your UPRN. Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find it.",

{uk_bin_collection-0.154.0 → uk_bin_collection-0.158.0}/uk_bin_collection/uk_bin_collection/councils/AberdeenCityCouncil.py

@@ -5,7 +5,6 @@ import requests
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
-
 # import the wonderful Beautiful Soup and the URL grabber
 class CouncilClass(AbstractGetBinDataClass):
     """

{uk_bin_collection-0.154.0 → uk_bin_collection-0.158.0}/uk_bin_collection/uk_bin_collection/councils/DacorumBoroughCouncil.py

@@ -76,19 +76,28 @@ class CouncilClass(AbstractGetBinDataClass):
             )
 
             for Collection in NextCollections:
-                BinType = Collection.find("strong").text.strip()
-                if BinType:
-                    CollectionDate = datetime.strptime(
-                        Collection.find_all("div", {"style": "display:table-cell;"})[1]
-                        .get_text()
-                        .strip(),
-                        "%a, %d %b %Y",
-                    )
-                    dict_data = {
-                        "type": BinType,
-                        "collectionDate": CollectionDate.strftime("%d/%m/%Y"),
-                    }
-                    data["bins"].append(dict_data)
+                strong_element = Collection.find("strong")
+                if strong_element:
+                    BinType = strong_element.text.strip()
+                    # Skip if this is not a bin type (e.g., informational text)
+                    if BinType and not any(skip_text in BinType.lower() for skip_text in 
+                                         ["please note", "we may collect", "bank holiday", "different day"]):
+                        date_cells = Collection.find_all("div", {"style": "display:table-cell;"})
+                        if len(date_cells) > 1:
+                            date_text = date_cells[1].get_text().strip()
+                            if date_text:
+                                try:
+                                    CollectionDate = datetime.strptime(date_text, "%a, %d %b %Y")
+                                    dict_data = {
+                                        "type": BinType,
+                                        "collectionDate": CollectionDate.strftime("%d/%m/%Y"),
+                                    }
+                                    # Check for duplicates before adding
+                                    if dict_data not in data["bins"]:
+                                        data["bins"].append(dict_data)
+                                except ValueError:
+                                    # Skip if date parsing fails
+                                    continue
 
         except Exception as e:
             # Here you can log the exception if needed

uk_bin_collection-0.158.0/uk_bin_collection/uk_bin_collection/councils/EastDunbartonshireCouncil.py

@@ -0,0 +1,52 @@
+import requests
+from bs4 import BeautifulSoup, Tag
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        URI = f"https://www.eastdunbarton.gov.uk/services/a-z-of-services/bins-waste-and-recycling/bins-and-recycling/collections/?uprn={user_uprn}"
+
+        # Make the GET request
+        response = requests.get(URI)
+
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        table = soup.find("table", {"class": "bin-table"})
+
+        tbody = table.find("tbody")
+
+        trs = tbody.find_all("tr")
+
+        for tr in trs:
+            tds = tr.find_all("td")
+            bin_type = tds[0].get_text()
+            collection_date_str = tds[1].find("span").get_text()
+
+            collection_date = datetime.strptime(collection_date_str, "%A, %d %B %Y")
+
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": collection_date.strftime(date_format),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        return bindata

{uk_bin_collection-0.154.0 → uk_bin_collection-0.158.0}/uk_bin_collection/uk_bin_collection/councils/IslingtonCouncil.py

@@ -17,11 +17,14 @@ class CouncilClass(AbstractGetBinDataClass):
 
         data = {"bins": []}
 
-        waste_table = (
-            soup.find(string="Waste and recycling collections")
-            .find_next("div", class_="m-toggle-content")
-            .find("table")
-        )
+        # Find the waste and recycling section with proper null checking
+        waste_section = soup.find(string="Waste and recycling collections")
+        waste_table = None
+        
+        if waste_section:
+            toggle_content = waste_section.find_next("div", class_="m-toggle-content")
+            if toggle_content:
+                waste_table = toggle_content.find("table")
 
         if waste_table:
             rows = waste_table.find_all("tr")

{uk_bin_collection-0.154.0 → uk_bin_collection-0.158.0}/uk_bin_collection/uk_bin_collection/councils/LancasterCityCouncil.py

@@ -57,17 +57,30 @@ class CouncilClass(AbstractGetBinDataClass):
         response = session.get(addr_link)
         new_soup = BeautifulSoup(response.text, features="html.parser")
         services = new_soup.find("section", {"id": "scheduled-collections"})
+        
+        if services is None:
+            raise Exception("Could not find scheduled collections section on the page")
+            
         services_sub = services.find_all("li")
+        if not services_sub:
+            raise Exception("No collection services found")
+            
         for i in range(0, len(services_sub), 3):
-            dt = datetime.strptime(services_sub[i + 1].text.strip(), "%d/%m/%Y").date()
-            bin_type = BeautifulSoup(services_sub[i + 2].text, features="lxml").find(
-                "p"
-            )
-            data["bins"].append(
-                {
-                    "type": bin_type.text.strip().removesuffix(" Collection Service"),
-                    "collectionDate": dt.strftime(date_format),
-                }
-            )
+            if i + 2 < len(services_sub):
+                date_text = services_sub[i + 1].text.strip() if services_sub[i + 1] else None
+                if date_text:
+                    try:
+                        dt = datetime.strptime(date_text, "%d/%m/%Y").date()
+                        bin_type_element = BeautifulSoup(services_sub[i + 2].text, features="lxml").find("p")
+                        if bin_type_element and bin_type_element.text:
+                            data["bins"].append(
+                                {
+                                    "type": bin_type_element.text.strip().removesuffix(" Collection Service"),
+                                    "collectionDate": dt.strftime(date_format),
+                                }
+                            )
+                    except (ValueError, AttributeError) as e:
+                        # Skip invalid date or missing elements
+                        continue
 
         return data

uk_bin_collection-0.158.0/uk_bin_collection/uk_bin_collection/councils/LondonBoroughSutton.py

@@ -0,0 +1,90 @@
+import requests
+from bs4 import BeautifulSoup
+from datetime import datetime
+import re
+from time import sleep
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+def remove_ordinal_indicator_from_date_string(date_str):
+    return re.sub(r'(\d+)(st|nd|rd|th)', r'\1', date_str)
+
+class CouncilClass(AbstractGetBinDataClass):
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        bindata = {"bins": []}
+
+        URI = f"https://waste-services.sutton.gov.uk/waste/{user_uprn}"
+
+        s = requests.Session()
+        r = s.get(URI)
+        while "Loading your bin days..." in r.text:
+            sleep(2)
+            r = s.get(URI)
+        r.raise_for_status()
+
+        soup = BeautifulSoup(r.content, "html.parser")
+        current_year = datetime.now().year
+        next_year = current_year + 1
+
+        # Find all h3 headers (bin types)
+        services = soup.find_all("h3")
+        for service in services:
+            bin_type = service.get_text(strip=True)
+            if "Bulky Waste" in bin_type:
+                continue
+
+            # Find the next element (next sibling) which is likely a paragraph with date info
+            next_sib = service.find_next_sibling()
+            while next_sib and getattr(next_sib, 'name', None) not in [None, 'p']:
+                next_sib = next_sib.find_next_sibling()
+
+            next_coll = None
+            if next_sib:
+                text = next_sib.get_text() if hasattr(next_sib, 'get_text') else str(next_sib)
+                match = re.search(r"Next collection\s*([A-Za-z]+,? \d{1,2}(?:st|nd|rd|th)? [A-Za-z]+)", text)
+                if match:
+                    next_coll = match.group(1)
+                else:
+                    # Sometimes the text may be attached without a space after 'Next collection'
+                    match = re.search(r"Next collection([A-Za-z]+,? \d{1,2}(?:st|nd|rd|th)? [A-Za-z]+)", text)
+                    if match:
+                        next_coll = match.group(1)
+
+            # Try several siblings forward if not found
+            if not next_coll:
+                sib_try = service
+                for _ in range(3):
+                    if sib_try:
+                        sib_try = sib_try.find_next_sibling()
+                    else:
+                        break
+                    if sib_try:
+                        text = sib_try.get_text() if hasattr(sib_try, 'get_text') else str(sib_try)
+                        match = re.search(r"Next collection\s*([A-Za-z]+,? \d{1,2}(?:st|nd|rd|th)? [A-Za-z]+)", text)
+                        if match:
+                            next_coll = match.group(1)
+                            break
+
+            if next_coll:
+                next_coll = remove_ordinal_indicator_from_date_string(next_coll)
+                try:
+                    next_collection = datetime.strptime(next_coll, "%A, %d %B")
+                except ValueError:
+                    continue
+
+                if (datetime.now().month == 12 and next_collection.month == 1):
+                    next_collection = next_collection.replace(year=next_year)
+                else:
+                    next_collection = next_collection.replace(year=current_year)
+
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": next_collection.strftime("%d/%m/%Y"),
+                }
+                bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y"))
+        return bindata

uk_bin_collection-0.158.0/uk_bin_collection/uk_bin_collection/councils/MidSuffolkDistrictCouncil.py

@@ -0,0 +1,174 @@
+import re
+import time
+
+import holidays
+import requests
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        collection_day = kwargs.get("paon")
+        garden_collection_week = kwargs.get("postcode")
+        garden_collection_day = kwargs.get("uprn")
+        bindata = {"bins": []}
+
+        days_of_week = [
+            "Monday",
+            "Tuesday",
+            "Wednesday",
+            "Thursday",
+            "Friday",
+            "Saturday",
+            "Sunday",
+        ]
+
+        garden_week = ["Week 1", "Week 2"]
+
+        refusestartDate = datetime(2024, 11, 11)
+        recyclingstartDate = datetime(2024, 11, 4)
+
+        offset_days = days_of_week.index(collection_day)
+        offset_days_garden = days_of_week.index(garden_collection_day)
+        if garden_collection_week:
+            garden_collection = garden_week.index(garden_collection_week)
+
+        refuse_dates = get_dates_every_x_days(refusestartDate, 14, 28)
+        recycling_dates = get_dates_every_x_days(recyclingstartDate, 14, 28)
+
+        # Generate bank holidays dynamically using the holidays library
+        def get_bank_holidays_set():
+            """Get set of bank holiday dates for quick lookup."""
+            current_year = datetime.now().year
+            uk_holidays = holidays.UK(years=range(current_year - 1, current_year + 3))
+            return set(uk_holidays.keys())
+
+        def find_next_collection_day(original_date):
+            """Find the next valid collection day, avoiding weekends and bank holidays."""
+            bank_holiday_dates = get_bank_holidays_set()
+            check_date = datetime.strptime(original_date, "%d/%m/%Y")
+
+            # Safety limit to prevent infinite loops
+            max_attempts = 10
+            attempts = 0
+
+            # Keep moving forward until we find a valid collection day
+            while attempts < max_attempts:
+                attempts += 1
+
+                # Check if it's a weekend (Saturday=5, Sunday=6)
+                if check_date.weekday() >= 5:
+                    check_date += timedelta(days=1)
+                    continue
+
+                # Check if it's a bank holiday
+                if check_date.date() in bank_holiday_dates:
+                    # Major holidays (Christmas/New Year) get bigger delays
+                    holiday_name = str(holidays.UK().get(check_date.date(), ''))
+                    is_major_holiday = (
+                        'Christmas' in holiday_name or
+                        'Boxing' in holiday_name or
+                        'New Year' in holiday_name
+                    )
+                    delay_days = 2 if is_major_holiday else 1
+                    check_date += timedelta(days=delay_days)
+                    continue
+
+                # Found a valid collection day
+                break
+
+            # If we've exhausted attempts, return the original date as fallback
+            if attempts >= max_attempts:
+                return original_date
+
+            return check_date.strftime("%d/%m/%Y")
+
+        bank_holidays = []  # No longer needed - using smart date calculation
+
+        for refuseDate in refuse_dates:
+            # Calculate initial collection date
+            initial_date = (
+                datetime.strptime(refuseDate, "%d/%m/%Y") + timedelta(days=offset_days)
+            ).strftime("%d/%m/%Y")
+
+            # Find the next valid collection day (handles weekends + cascading holidays)
+            collection_date = find_next_collection_day(initial_date)
+
+            dict_data = {
+                "type": "Refuse Bin",
+                "collectionDate": collection_date,
+            }
+            bindata["bins"].append(dict_data)
+
+        for recyclingDate in recycling_dates:
+            # Calculate initial collection date
+            initial_date = (
+                datetime.strptime(recyclingDate, "%d/%m/%Y")
+                + timedelta(days=offset_days)
+            ).strftime("%d/%m/%Y")
+
+            # Find the next valid collection day (handles weekends + cascading holidays)
+            collection_date = find_next_collection_day(initial_date)
+
+            dict_data = {
+                "type": "Recycling Bin",
+                "collectionDate": collection_date,
+            }
+            bindata["bins"].append(dict_data)
+
+        if garden_collection_week:
+            if garden_collection == 0:
+                gardenstartDate = datetime(2024, 11, 11)
+            elif garden_collection == 1:
+                gardenstartDate = datetime(2024, 11, 4)
+
+            garden_dates = get_dates_every_x_days(gardenstartDate, 14, 28)
+
+            def is_christmas_period(date_obj):
+                """Check if date is in Christmas/New Year skip period for garden collections."""
+                if date_obj.month == 12 and date_obj.day >= 23:
+                    return True
+                if date_obj.month == 1 and date_obj.day <= 3:
+                    return True
+                return False
+
+            for gardenDate in garden_dates:
+                # Calculate initial collection date
+                initial_date = (
+                    datetime.strptime(gardenDate, "%d/%m/%Y")
+                    + timedelta(days=offset_days_garden)
+                )
+
+                # Skip garden collections during Christmas/New Year period
+                if is_christmas_period(initial_date):
+                    continue
+
+                # Find the next valid collection day (handles weekends + holidays)
+                collection_date = find_next_collection_day(initial_date.strftime("%d/%m/%Y"))
+
+                dict_data = {
+                    "type": "Garden Bin",
+                    "collectionDate": collection_date,
+                }
+                bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata