uk_bin_collection 0.152.0__py3-none-any.whl → 0.152.2__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py

@@ -1,7 +1,10 @@
 import time
 
 from bs4 import BeautifulSoup
+from selenium.common.exceptions import TimeoutException
 from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
 
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
@@ -42,28 +45,61 @@ class CouncilClass(AbstractGetBinDataClass):
             driver = create_webdriver(web_driver, headless, None, __name__)
             driver.get(page)
 
-            time.sleep(1)
+            # Create wait object
+            wait = WebDriverWait(driver, 20)
 
-            # Press the cookie accept - wait is to let the JS load it up
-            driver.find_element(By.ID, "ccc-notify-accept").click()
-
-            inputElement_hn = driver.find_element(
-                By.ID,
-                "p_lt_ctl04_pageplaceholder_p_lt_ctl02_WasteCollectionCalendars_NCCAddressLookup_txtHouse",
+            # Wait for and click cookie button
+            cookie_button = wait.until(
+                EC.element_to_be_clickable((By.ID, "ccc-notify-accept"))
+            )
+            cookie_button.click()
+
+            # Wait for and find house number input
+            inputElement_hn = wait.until(
+                EC.presence_of_element_located(
+                    (
+                        By.ID,
+                        "p_lt_ctl04_pageplaceholder_p_lt_ctl02_WasteCollectionCalendars_NCCAddressLookup_txtHouse",
+                    )
+                )
             )
-            inputElement_pc = driver.find_element(
-                By.ID,
-                "p_lt_ctl04_pageplaceholder_p_lt_ctl02_WasteCollectionCalendars_NCCAddressLookup_txtPostcode",
+
+            # Wait for and find postcode input
+            inputElement_pc = wait.until(
+                EC.presence_of_element_located(
+                    (
+                        By.ID,
+                        "p_lt_ctl04_pageplaceholder_p_lt_ctl02_WasteCollectionCalendars_NCCAddressLookup_txtPostcode",
+                    )
+                )
             )
 
+            # Enter details
             inputElement_pc.send_keys(user_postcode)
             inputElement_hn.send_keys(user_paon)
 
-            driver.find_element(
-                By.ID,
-                "p_lt_ctl04_pageplaceholder_p_lt_ctl02_WasteCollectionCalendars_NCCAddressLookup_butLookup",
-            ).click()
+            # Click lookup button and wait for results
+            lookup_button = wait.until(
+                EC.element_to_be_clickable(
+                    (
+                        By.ID,
+                        "p_lt_ctl04_pageplaceholder_p_lt_ctl02_WasteCollectionCalendars_NCCAddressLookup_butLookup",
+                    )
+                )
+            )
+            lookup_button.click()
+
+            # Wait for results to load
+            route_summary = wait.until(
+                EC.presence_of_element_located(
+                    (
+                        By.ID,
+                        "p_lt_ctl04_pageplaceholder_p_lt_ctl02_WasteCollectionCalendars_spanRouteSummary",
+                    )
+                )
+            )
 
+            # Get page source after everything has loaded
             soup = BeautifulSoup(driver.page_source, features="html.parser")
 
             # Work out which bins can be collected for this address. Glass bins are only on some houses due to pilot programme.

uk_bin_collection/uk_bin_collection/councils/SouthRibbleCouncil.py

@@ -1,12 +1,29 @@
-import time
-
+from typing import Dict, List, Any, Optional
+from bs4 import BeautifulSoup
+from dateutil.relativedelta import relativedelta
 import requests
-
+import logging
+import re
+from datetime import datetime
 from uk_bin_collection.uk_bin_collection.common import *
+from dateutil.parser import parse
+
+from uk_bin_collection.uk_bin_collection.common import check_uprn, check_postcode
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
-# import the wonderful Beautiful Soup and the URL grabber
+def get_token(page) -> str:
+    """
+    Get a __token to include in the form data
+        :param page: Page html
+        :return: Form __token
+    """
+    soup = BeautifulSoup(page.text, features="html.parser")
+    soup.prettify()
+    token = soup.find("input", {"name": "__token"}).get("value")
+    return token
+
+
 class CouncilClass(AbstractGetBinDataClass):
     """
     Concrete classes have to implement all abstract operations of the
@@ -14,70 +31,103 @@ class CouncilClass(AbstractGetBinDataClass):
     implementation.
     """
 
-    def parse_data(self, page: str, **kwargs) -> dict:
+    def get_data(self, url: str) -> str:
+        """This method makes the request to the council
 
-        user_uprn = kwargs.get("uprn")
-        check_uprn(user_uprn)
-        bindata = {"bins": []}
+        Keyword arguments:
+        url -- the url to get the data from
+        """
+        # Set a user agent so we look like a browser ;-)
+        user_agent = (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/108.0.0.0 Safari/537.36"
+        )
+        headers = {"User-Agent": user_agent}
+        requests.packages.urllib3.disable_warnings()
 
-        SESSION_URL = "https://southribble-ss.achieveservice.com/authapi/isauthenticated?uri=https%253A%252F%252Fsouthribble-ss.achieveservice.com%252FAchieveForms%252F%253Fmode%253Dfill%2526form_uri%253Dsandbox-publish%25253A%252F%252FAF-Process-d3971054-2e93-4a74-8375-494347dd13fd%252FAF-Stage-2693df73-8a5f-4e24-86b8-456ef81dd4a1%252Fdefinition.json%2526process%253D1%2526process_uri%253Dsandbox-processes%25253A%252F%252FAF-Process-d3971054-2e93-4a74-8375-494347dd13fd%2526process_id%253DAF-Process-d3971054-2e93-4a74-8375-494347dd13fd%2526accept%253Dyes%2526consentMessageIds%25255B%25255D%253D3%2526accept%253Dyes%2526consentMessageIds%255B%255D%253D3&hostname=southribble-ss.achieveservice.com&withCredentials=true"
+        # Make the Request - change the URL - find out your property number
+        try:
+            session = requests.Session()
+            session.headers.update(headers)
+            full_page = session.get(url)
+            return full_page
+        except requests.exceptions.HTTPError as errh:
+            logging.error(f"Http Error: {errh}")
+            raise
+        except requests.exceptions.ConnectionError as errc:
+            logging.error(f"Error Connecting: {errc}")
+            raise
+        except requests.exceptions.Timeout as errt:
+            logging.error(f"Timeout Error: {errt}")
+            raise
+        except requests.exceptions.RequestException as err:
+            logging.error(f"Oops: Something Else {err}")
+            raise
 
-        API_URL = "https://southribble-ss.achieveservice.com/apibroker/runLookup"
+    def parse_data(self, page: str, **kwargs: Any) -> Dict[str, List[Dict[str, str]]]:
+        uprn: Optional[str] = kwargs.get("uprn")
+        postcode: Optional[str] = kwargs.get("postcode")
 
-        data = {
-            "formValues": {
-                "Your Collections": {
-                    "PickupDate": {"value": datetime.now().strftime("%Y-%m-%d")},
-                },
-                "Your details": {
-                    "LLPGUPRN": {"value": user_uprn},
-                },
-            },
-        }
+        if uprn is None:
+            raise ValueError("UPRN is required and must be a non-empty string.")
+        if postcode is None:
+            raise ValueError("Postcode is required and must be a non-empty string.")
 
-        headers = {
-            "Content-Type": "application/json",
-            "Accept": "application/json",
-            "User-Agent": "Mozilla/5.0",
-            "X-Requested-With": "XMLHttpRequest",
-            "Referer": "https://southribble-ss.achieveservice.com/fillform/?iframe_id=fillform-frame-1&db_id=",
-        }
-        s = requests.session()
-        r = s.get(SESSION_URL)
-        r.raise_for_status()
-        session_data = r.json()
-        sid = session_data["auth-session"]
-        params = {
-            "id": "58ab44ef078bd",
-            "repeat_against": "",
-            "noRetry": "false",
-            "getOnlyTokens": "undefined",
-            "log_id": "",
-            "app_name": "AF-Renderer::Self",
-            # unix_timestamp
-            "_": str(int(time.time() * 1000)),
-            "sid": sid,
+        check_uprn(uprn)
+        check_postcode(postcode)
+
+        values = {
+            "__token": get_token(page),
+            "page": "491",
+            "locale": "en_GB",
+            "q1f8ccce1d1e2f58649b4069712be6879a839233f_0_0": postcode,
+            "q1f8ccce1d1e2f58649b4069712be6879a839233f_1_0": uprn,
+            "next": "Next",
         }
-        r = s.post(API_URL, json=data, headers=headers, params=params)
-        r.raise_for_status()
-        data = r.json()
-        rows_data = data["integration"]["transformed"]["rows_data"]["0"]
-        if not isinstance(rows_data, dict):
-            raise ValueError("Invalid data returned from API")
-        BIN_TYPES = [
-            ("RCNextCollectionDate", "Household Waste (Non-Recyclable Waste)"),
-            ("RENextCollectionDate", "Blue/Green Recyclable Waste"),
-            ("GWNextCollectionDate", "Garden Waste Collection"),
-            ("FWNextCollectionDate", "Food Waste"),
-        ]
-        bin_type_dict = dict(BIN_TYPES)
-
-        for row in rows_data.items():
-            if row[0].endswith("NextCollectionDate"):
-                if row[1]:
-                    bin_type = bin_type_dict.get(row[0], row[0])
-                    collection_date = row[1]
-                    dict_data = {"type": bin_type, "collectionDate": collection_date}
-                    bindata["bins"].append(dict_data)
-
-        return bindata
+        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64)"}
+        requests.packages.urllib3.disable_warnings()
+        response = requests.request(
+            "POST",
+            "https://forms.chorleysouthribble.gov.uk/xfp/form/70",
+            headers=headers,
+            data=values,
+        )
+
+        soup = BeautifulSoup(response.text, features="html.parser")
+
+        rows = soup.find("table").find_all("tr")
+
+        # Form a JSON wrapper
+        data: Dict[str, List[Dict[str, str]]] = {"bins": []}
+
+        # Loops the Rows
+        for row in rows:
+            cells = row.find_all("td")
+            if cells:
+                bin_type = cells[0].get_text(strip=True)
+                collection_next = cells[1].get_text(strip=True)
+
+                collection_date = re.findall(r"\(.*?\)", collection_next)
+
+                if len(collection_date) != 1:
+                    continue
+
+                collection_date_obj = parse(
+                    re.sub(r"[()]", "", collection_date[0])
+                ).date()
+
+                # since we only have the next collection day, if the parsed date is in the past,
+                # assume the day is instead next month
+                if collection_date_obj < datetime.now().date():
+                    collection_date_obj += relativedelta(months=1)
+
+                # Make each Bin element in the JSON
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": collection_date_obj.strftime(date_format),
+                }
+
+                # Add data to the main JSON Wrapper
+                data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/StokeOnTrentCityCouncil.py

@@ -54,14 +54,19 @@ class CouncilClass(AbstractGetBinDataClass):
                 bin_type = bin_types.get(
                     item.find_next("Bin").text.replace("EMPTY BINS", "").strip()
                 )
-                bin_date = datetime.strptime(
-                    item.find_next("DateTime").text, "%d/%m/%Y %H:%M:%S"
-                )
+                date_text = item.find_next("DateTime").text.strip()
+
+                # Handle inconsistent date formats
+                if " " in date_text:  # Date and time present
+                    bin_date = datetime.strptime(date_text, "%d/%m/%Y %H:%M:%S")
+                else:  # Only date present
+                    bin_date = datetime.strptime(date_text, "%d/%m/%Y")
+
                 if bin_date >= datetime.now():
                     collections.append((bin_type, bin_date))
-            except:
+            except Exception as e:
                 raise SystemError(
-                    "Error has been encountered parsing API. Please try again later and if the issue "
+                    f"Error has been encountered parsing API: {e}. Please try again later and if the issue "
                     "persists, open a GitHub ticket!"
                 )
 

uk_bin_collection/uk_bin_collection/councils/TorbayCouncil.py

@@ -1,5 +1,3 @@
-# This script pulls bin collection data from Barking and Dagenham Council
-# Example URL: https://www.lbbd.gov.uk/rubbish-recycling/household-bin-collection/check-your-bin-collection-days
 import time
 
 from bs4 import BeautifulSoup
@@ -39,7 +37,7 @@ class CouncilClass(AbstractGetBinDataClass):
 
             driver = create_webdriver(web_driver, headless, None, __name__)
             print(f"Navigating to URL: {url}")
-            driver.get(url)
+            driver.get("https://www.torbay.gov.uk/recycling/bin-collections/")
             print("Successfully loaded the page")
 
             driver.maximize_window()

uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py

@@ -21,6 +21,9 @@ class CouncilClass(AbstractGetBinDataClass):
             )
             driver.get(kwargs.get("url"))
 
+            # This URL format also works:
+            # https://www.wakefield.gov.uk/where-i-live?a=115%20Elizabeth%20Drive
+
             # Make a BS4 object
             soup = BeautifulSoup(driver.page_source, features="html.parser")
             soup.prettify()

{uk_bin_collection-0.152.0.dist-info → uk_bin_collection-0.152.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.152.0
+Version: 0.152.2
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com
@@ -398,6 +398,184 @@ Open the map viewer in VS Code:
 ![Test Results Map](test_results_map.png)
 
 ---
+## ICS Calendar Generation
+
+You can convert bin collection data to an ICS calendar file that can be imported into calendar applications like Google Calendar, Apple Calendar, Microsoft Outlook, etc.
+
+### Overview
+
+The `bin_to_ics.py` script allows you to:
+- Convert JSON output from bin collection data into ICS calendar events
+- Group multiple bin collections on the same day into a single event
+- Create all-day events (default) or timed events
+- Add optional reminders/alarms to events
+- Customize the calendar name
+
+### Requirements
+
+- Python 3.6 or higher
+- The `icalendar` package, which can be installed with:
+  ```bash
+  pip install icalendar
+  ```
+
+### Basic Usage
+
+```bash
+# Basic usage with stdin input and default output file (bin.ics)
+python bin_to_ics.py < bin_data.json
+
+# Specify input and output files
+python bin_to_ics.py -i bin_data.json -o my_calendar.ics
+
+# Custom calendar name
+python bin_to_ics.py -i bin_data.json -o my_calendar.ics -n "My Bin Collections"
+```
+
+### Options
+
+```
+--input, -i        Input JSON file (if not provided, read from stdin)
+--output, -o       Output ICS file (default: bin.ics)
+--name, -n         Calendar name (default: Bin Collections)
+--alarms, -a       Comma-separated list of alarm times before event (e.g., "1d,2h,30m")
+--no-all-day       Create timed events instead of all-day events
+```
+
+### Examples
+
+#### Adding Reminders (Alarms)
+
+Add reminders 1 day and 2 hours before each collection:
+
+```bash
+python bin_to_ics.py -i bin_data.json -a "1d,2h"
+```
+
+The time format supports:
+- Days: `1d`, `2day`, `3days`
+- Hours: `1h`, `2hour`, `3hours`
+- Minutes: `30m`, `45min`, `60mins`, `90minutes`
+
+#### Creating Timed Events
+
+By default, events are created as all-day events. To create timed events instead (default time: 7:00 AM):
+
+```bash
+python bin_to_ics.py -i bin_data.json --no-all-day
+```
+
+### Integration with Bin Collection Data Retriever
+
+You can pipe the output from the bin collection data retriever directly to the ICS generator. The required parameters (postcode, house number, UPRN, etc.) depend on the specific council implementation - refer to the [Quickstart](#quickstart) section above or check the [project wiki](https://github.com/robbrad/UKBinCollectionData/wiki) for details about your council.
+
+```bash
+python uk_bin_collection/uk_bin_collection/collect_data.py CouncilName "URL" [OPTIONS] | 
+  python bin_to_ics.py [OPTIONS]
+```
+
+#### Complete Example for a Council
+
+```bash
+python uk_bin_collection/uk_bin_collection/collect_data.py CouncilName \
+  "council_url" \
+  -p "YOUR_POSTCODE" \
+  -n "YOUR_HOUSE_NUMBER" \
+  -w "http://localhost:4444/wd/hub" |
+  python bin_to_ics.py \
+    --name "My Bin Collections" \
+    --output my_bins.ics \
+    --alarms "1d,12h"
+```
+
+This will:
+1. Fetch bin collection data for your address from your council's website
+2. Convert it to an ICS file named "my_bins.ics"
+3. Set the calendar name to "My Bin Collections"
+4. Add reminders 1 day and 12 hours before each collection
+
+For postcode lookup and UPRN information, please check the [UPRN Finder](#uprn-finder) section above.
+
+### Using the Calendar
+
+You have two options for using the generated ICS file:
+
+#### 1. Importing the Calendar
+
+You can directly import the ICS file into your calendar application:
+
+- **Google Calendar**: Go to Settings > Import & export > Import
+- **Apple Calendar**: File > Import
+- **Microsoft Outlook**: File > Open & Export > Import/Export > Import an iCalendar (.ics)
+
+Note: Importing creates a static copy of the calendar events. If bin collection dates change, you'll need to re-import the calendar.
+
+#### 2. Subscribing to the Calendar
+
+If you host the ICS file on a publicly accessible web server, you can subscribe to it as an internet calendar:
+
+- **Google Calendar**: Go to "Other calendars" > "+" > "From URL" > Enter the URL of your hosted ICS file
+- **Apple Calendar**: File > New Calendar Subscription > Enter the URL
+- **Microsoft Outlook**: File > Account Settings > Internet Calendars > New > Enter the URL
+
+Benefits of subscribing:
+- Calendar automatically updates when the source file changes
+- No need to manually re-import when bin collection dates change
+- Easily share the calendar with household members
+
+You can set up a cron job or scheduled task to regularly:
+1. Retrieve the latest bin collection data
+2. Generate a fresh ICS file
+3. Publish it to a web-accessible location
+
+### Additional Examples and Use Cases
+
+#### Automation with Cron Jobs
+
+Create a weekly update script on a Linux/Mac system:
+
+```bash
+#!/bin/bash
+# File: update_bin_calendar.sh
+
+# Set variables
+COUNCIL="YourCouncilName"
+COUNCIL_URL="https://your-council-website.gov.uk/bins"
+POSTCODE="YOUR_POSTCODE"
+HOUSE_NUMBER="YOUR_HOUSE_NUMBER"
+OUTPUT_DIR="/var/www/html/calendars"  # Web-accessible directory
+CALENDAR_NAME="Household Bins"
+
+# Ensure output directory exists
+mkdir -p $OUTPUT_DIR
+
+# Run the collector and generate the calendar
+cd /path/to/UKBinCollectionData && \
+python uk_bin_collection/uk_bin_collection/collect_data.py $COUNCIL "$COUNCIL_URL" \
+  -p "$POSTCODE" -n "$HOUSE_NUMBER" | \
+python bin_to_ics.py --name "$CALENDAR_NAME" --output "$OUTPUT_DIR/bins.ics" --alarms "1d,6h"
+
+# Add timestamp to show last update time
+echo "Calendar last updated: $(date)" > "$OUTPUT_DIR/last_update.txt"
+```
+
+Make the script executable:
+```bash
+chmod +x update_bin_calendar.sh
+```
+
+Add to crontab to run weekly (every Monday at 2 AM):
+```bash
+0 2 * * 1 /path/to/update_bin_calendar.sh
+```
+
+**Google Assistant/Alexa Integration**
+
+If you have your calendar connected to Google Calendar or Outlook, you can ask your smart assistant about upcoming bin collections:
+
+- "Hey Google, when is my next bin collection?"
+- "Alexa, what's on my calendar tomorrow?" (will include bin collections)
+
 ## Docker API Server
 We have created an API for this located under [uk_bin_collection_api_server](https://github.com/robbrad/UKBinCollectionData/uk_bin_collection_api_server)