uk_bin_collection 0.158.1__py3-none-any.whl → 0.159.0__py3-none-any.whl

uk_bin_collection/tests/input.json

@@ -1757,12 +1757,12 @@
         "LAD24CD": "E06000024"
     },
     "NorthTynesideCouncil": {
-        "postcode": "NE26 2TG",
         "skip_get_url": true,
         "uprn": "47097627",
-        "url": "https://my.northtyneside.gov.uk/category/81/bin-collection-dates",
-        "wiki_name": "North Tyneside",
-        "wiki_note": "Pass the postcode and UPRN. You can find the UPRN using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
+        "url": "https://www.northtyneside.gov.uk/waste-collection-schedule",
+        "wiki_command_url_override": "https://www.northtyneside.gov.uk/waste-collection-schedule/view/XXXXXXXX",
+      "wiki_name": "North Tyneside",
+        "wiki_note": "Pass only the UPRN (no postcode). You can find the UPRN using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
         "LAD24CD": "E08000022"
     },
     "NorthWestLeicestershire": {
@@ -2155,13 +2155,11 @@
         "LAD24CD": "E07000140"
     },
     "SouthKestevenDistrictCouncil": {
-        "house_number": "2 Althorpe Close, Market Deeping, PE6 8BL",
         "postcode": "PE68BL",
         "skip_get_url": true,
-        "url": "https://pre.southkesteven.gov.uk/BinSearch.aspx",
-        "web_driver": "http://selenium:4444",
-        "wiki_name": "South Kesteven",
-        "wiki_note": "Provide your full address in the `house_number` parameter and your postcode in the `postcode` parameter.",
+        "url": "https://pre.southkesteven.gov.uk/skdcNext/tempforms/checkmybin.aspx",
+        "wiki_name": "South Kesteven District Council",
+        "wiki_note": "Provide your postcode in the `postcode` parameter. The scraper uses requests-based form submission and OCR to parse calendar images for accurate bin type determination and green bin collection patterns.",
         "LAD24CD": "E07000141"
     },
     "SouthLanarkshireCouncil": {

uk_bin_collection/uk_bin_collection/councils/NorthTynesideCouncil.py

@@ -1,138 +1,115 @@
-import math
-from datetime import *
-
-import requests
+import logging
+from datetime import datetime
 from bs4 import BeautifulSoup
 
-from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.common import check_uprn, date_format
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
+logger = logging.getLogger(__name__)
+
 
 class CouncilClass(AbstractGetBinDataClass):
+    """
+    North Tyneside Council bin collection schedule parser
+    """
 
     def parse_data(self, page: str, **kwargs) -> dict:
+        """
+        Parse waste collection schedule data for a given UPRN.
+
+        Args:
+            page (str): Unused parameter (required by parent class interface).
+            **kwargs: Keyword arguments containing:
+                - uprn (str): The Unique Property Reference Number for the property.
+
+        Returns:
+            dict: A dictionary containing:
+                - bins (list): A list of dictionaries, each containing:
+                    - type (str): Bin type and colour in format "Type (Colour)"
+                                 (e.g., "Recycling (Grey)")
+                    - collectionDate (str): Collection date in the format specified
+                                           by date_format
+
+        Raises:
+            ValueError: If no waste collection schedule is found on the page, indicating
+                       the page structure may have changed.
+            requests.HTTPError: If the HTTP request to fetch the schedule fails.
+
+        Notes:
+            - The method handles bank holiday notifications that may appear in the
+              collection type field, extracting only the direct text content.
+            - Invalid or unparsable collection entries are logged and skipped.
+            - Results are sorted by collection date in ascending order.
+        """
+        # `page` is unused because we construct the view URL directly.
+        del page
 
         user_uprn = kwargs.get("uprn")
-        user_postcode = kwargs.get("postcode")
         check_uprn(user_uprn)
-        check_postcode(user_postcode)
 
-        requests.packages.urllib3.disable_warnings()
-        s = requests.Session()
+        # Fetch the schedule page (includes UA, verify=False, timeout)
+        view_url = f"https://www.northtyneside.gov.uk/waste-collection-schedule/view/{user_uprn}"
+        response = self.get_data(view_url)
 
-        # Get the first form
-        response = s.get(
-            "https://my.northtyneside.gov.uk/category/81/bin-collection-dates",
-            verify=False,
-        )
+        # Fail fast on HTTP errors
+        if getattr(response, "raise_for_status", None):
+            response.raise_for_status()
 
-        # Find the form ID and submit with a postcode
-        soup = BeautifulSoup(response.text, features="html.parser")
-        form_build_id = soup.find("input", {"name": "form_build_id"})["value"]
-        response = s.post(
-            "https://my.northtyneside.gov.uk/category/81/bin-collection-dates",
-            data={
-                "postcode": user_postcode,
-                "op": "Find",
-                "form_build_id": form_build_id,
-                "form_id": "ntc_address_wizard",
-            },
-            verify=False,
-        )
-
-        # Find the form ID and submit with the UPRN
-        soup = BeautifulSoup(response.text, features="html.parser")
-        form_build_id = soup.find("input", {"name": "form_build_id"})["value"]
-        response = s.post(
-            "https://my.northtyneside.gov.uk/category/81/bin-collection-dates",
-            data={
-                "house_number": f"0000{user_uprn}",
-                "op": "Use",
-                "form_build_id": form_build_id,
-                "form_id": "ntc_address_wizard",
-            },
-            verify=False,
-        )
 
         # Parse form page and get the day of week and week offsets
         soup = BeautifulSoup(response.text, features="html.parser")
-        info_section = soup.find("section", {"class": "block block-ntc-bins clearfix"})
-
-        regular_day, garden_day, special_day = None, None, None
-        # Get day of week and week label for refuse, garden and special collections.
-        # Week label is A or B.  Convert that to an int to use as an offset.
-        for anchor in info_section.findAll("a"):
-            if anchor.text.startswith("Refuse and Recycling"):
-                regular_day = anchor.text.strip().split()[-3]
-                if anchor.text.strip().split()[-1] == "A":
-                    regular_week = 0
-                else:
-                    regular_week = 1
-            elif anchor.text.startswith("Garden Waste"):
-                garden_day = anchor.text.strip().split()[-3]
-                if anchor.text.strip().split()[-1] == "A":
-                    garden_week = 0
-                else:
-                    garden_week = 1
-        for para in info_section.findAll("p"):
-            if para.text.startswith("Your special collections day"):
-                special_day = para.find("strong").text.strip()
-
-        # The regular calendar only shows until end of March 2026, work out how many weeks that is
-        weeks_total = math.floor((datetime(2026, 4, 1) - datetime.now()).days / 7)
-
-        # The garden calendar only shows until end of November 2025, work out how many weeks that is
-        garden_weeks_total = math.floor(
-            (datetime(2025, 12, 1) - datetime.now()).days / 7
-        )
-
-        regular_collections, garden_collections, special_collections = [], [], []
-        # Convert day text to series of dates using previous calculation
-        if regular_day is not None:
-            regular_collections = get_weekday_dates_in_period(
-                datetime.today(),
-                days_of_week.get(regular_day.capitalize()),
-                amount=weeks_total,
-            )
-        if garden_day is not None:
-            garden_collections = get_weekday_dates_in_period(
-                datetime.today(),
-                days_of_week.get(garden_day.capitalize()),
-                amount=garden_weeks_total,
-            )
-        if special_day is not None:
-            special_collections = get_weekday_dates_in_period(
-                datetime.today(),
-                days_of_week.get(special_day.capitalize()),
-                amount=weeks_total,
-            )
+        schedule = soup.find("div", {"class": "waste-collection__schedule"})
+
+        if schedule is None:
+            raise ValueError("No waste-collection schedule found. The page structure may have changed.")
+
+
+        # Find days of form:
+        #
+        # <li class="waste-collection__day">
+        #   <span class="waste-collection__day--day"><time datetime="2025-11-13">13</time></span>
+        #   <span class="waste-collection__day--type">Recycling</span>
+        #   <span class="waste-collection__day--colour waste-collection__day--grey">Grey</span>
+        # </li>
+        #
+        #
+        # Note that on back holidays the collection type is of form:
+        # ...
+        # <span class="waste-collection__day--type">Recycling
+        #    <span>
+        #      Public holiday - services may be affected. Check service updates on <a href="/household-rubbish-and-recycling/household-bin-collections/bank-holiday-bin-collections">our website</a>
+        #    </span>
+        # </span>
+        # ...
 
         collections = []
 
-        # Add regular collections, and differentiate between regular and recycling bins
-        for item in regular_collections:
-            item_as_date = datetime.strptime(item, date_format)
-            # Check if holiday (calendar only has one day that's a holiday, and it's moved to the next day)
-            if is_holiday(item_as_date, Region.ENG):
-                item_as_date += timedelta(days=1)
-            # Use the isoweek number to separate collections based on week label.
-            if (item_as_date.date().isocalendar()[1] % 2) == regular_week:
-                collections.append(("Refuse (green)", item_as_date))
-            else:
-                collections.append(("Recycling (grey)", item_as_date))
-
-        # Add garden collections
-        for item in garden_collections:
-            item_as_date = datetime.strptime(item, date_format)
-            # Garden collections do not move for bank holidays
-            if (item_as_date.date().isocalendar()[1] % 2) == garden_week:
-                collections.append(("Garden Waste (brown)", item_as_date))
-
-        # Add special collections
-        collections += [
-            ("Special Collection (bookable)", datetime.strptime(item, date_format))
-            for item in special_collections
-        ]
+        for day in schedule.find_all("li", {"class": "waste-collection__day"}):
+            try:
+                time_el = day.find("time")
+                if not time_el or not time_el.get("datetime"):
+                    logger.warning("Skipping day: missing time/datetime")
+                    continue
+                collection_date = datetime.strptime(time_el["datetime"], "%Y-%m-%d")
+
+                type_span = day.find("span", {"class": "waste-collection__day--type"})
+                # Direct text only (exclude nested spans, e.g., bank-holiday note)
+                bin_type_text = type_span.find(text=True, recursive=False) if type_span else None
+                if not bin_type_text:
+                    logger.warning("Skipping day: missing type")
+                    continue
+                bin_type = bin_type_text.strip()
+
+                colour_span = day.find("span", {"class": "waste-collection__day--colour"})
+                if not colour_span:
+                    logger.warning("Skipping day: missing colour")
+                    continue
+                bin_colour = colour_span.get_text(strip=True)
+
+                collections.append((f"{bin_type} ({bin_colour})", collection_date))
+            except (AttributeError, KeyError, TypeError, ValueError) as e:
+                logger.warning(f"Skipping unparsable day node: {e}")
+                continue
 
         return {
             "bins": [