uk_bin_collection 0.139.0__py3-none-any.whl → 0.140.0__py3-none-any.whl

uk_bin_collection/tests/input.json

@@ -19,8 +19,8 @@
         "wiki_note": "Replace XXXXXXXX with your UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find it."
     },
     "AmberValleyBoroughCouncil": {
-        "url": "https://ambervalley.gov.uk",
         "uprn": "100030026621",
+        "url": "https://ambervalley.gov.uk",
         "wiki_name": "Amber Valley Borough Council",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
@@ -71,8 +71,8 @@
     "AshfordBoroughCouncil": {
         "postcode": "TN23 7SP",
         "uprn": "100060777899",
-        "web_driver": "http://selenium:4444",
         "url": "https://ashford.gov.uk",
+        "web_driver": "http://selenium:4444",
         "wiki_command_url_override": "https://ashford.gov.uk",
         "wiki_name": "Ashford Borough Council",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
@@ -203,8 +203,8 @@
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
     "BolsoverCouncil": {
-        "url": "https://bolsover.gov.uk",
         "uprn": "100030066827",
+        "url": "https://bolsover.gov.uk",
         "wiki_name": "Bolsover Council",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
@@ -282,6 +282,15 @@
         "wiki_name": "Bristol City Council",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
+    "BroadlandDistrictCouncil": {
+        "skip_get_url": true,
+        "house_number": "1 Park View, Horsford, Norfolk, NR10 3FD",
+        "postcode": "NR10 3FD",
+        "url": "https://area.southnorfolkandbroadland.gov.uk/FindAddress",
+        "web_driver": "http://selenium:4444",
+        "wiki_name": "Broadland District Council",
+        "wiki_note": "Use the full address as it appears on the drop-down on the site when you search by postcode."
+    },
     "BromleyBoroughCouncil": {
         "url": "https://recyclingservices.bromley.gov.uk/waste/6087017",
         "web_driver": "http://selenium:4444",
@@ -386,13 +395,13 @@
         "wiki_name": "Castlepoint District Council",
         "wiki_note": "For this council, 'uprn' is actually a 4-digit code for your street. Go [here](https://apps.castlepoint.gov.uk/cpapps/index.cfm?fa=wastecalendar) and inspect the source of the dropdown box to find the 4-digit number for your street."
     },
-    "CeredigionCountyCouncil":{
-        "house_number":"BLAEN CWMMAGWR, TRISANT, CEREDIGION, SY23 4RQ",
-        "postcode":"SY23 4RQ",
-        "url":"https://www.ceredigion.gov.uk/resident/bins-recycling/",
-        "web_driver":"http://selenium:4444",
-        "wiki_name":"Ceredigion County Council",
-        "wiki_note":"House Number is the full address as it appears on the drop-down on the site when you search by postcode. This parser requires a Selenium webdriver."
+    "CeredigionCountyCouncil": {
+        "house_number": "BLAEN CWMMAGWR, TRISANT, CEREDIGION, SY23 4RQ",
+        "postcode": "SY23 4RQ",
+        "url": "https://www.ceredigion.gov.uk/resident/bins-recycling/",
+        "web_driver": "http://selenium:4444",
+        "wiki_name": "Ceredigion County Council",
+        "wiki_note": "House Number is the full address as it appears on the drop-down on the site when you search by postcode. This parser requires a Selenium webdriver."
     },
     "CharnwoodBoroughCouncil": {
         "url": "https://my.charnwood.gov.uk/location?put=cbc10070067259&rememberme=0&redirect=%2F",
@@ -409,9 +418,9 @@
         "wiki_note": "Follow the instructions [here](https://www.chelmsford.gov.uk/myhome/) until you get the page listing your address, then copy the entire address text and use that in the house number field."
     },
     "CheltenhamBoroughCouncil": {
-        "uprn": "100120372027",
         "postcode": "GL51 3NA",
         "skip_get_url": true,
+        "uprn": "100120372027",
         "url": "https://www.cheltenham.gov.uk",
         "wiki_name": "Cheltenham Borough Council",
         "wiki_note": "Pass the UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search)."
@@ -639,8 +648,8 @@
         "house_number": "1",
         "postcode": "CM20 2FZ",
         "skip_get_url": true,
-        "web_driver": "http://selenium:4444",
         "url": "https://www.eastherts.gov.uk",
+        "web_driver": "http://selenium:4444",
         "wiki_name": "East Herts Council",
         "wiki_note": "Pass the house number and postcode in their respective parameters."
     },
@@ -733,8 +742,8 @@
     },
     "EppingForestDistrictCouncil": {
         "postcode": "IG9 6EP",
-        "web_driver": "http://selenium:4444",
         "url": "https://eppingforestdc.maps.arcgis.com/apps/instant/lookup/index.html?appid=bfca32b46e2a47cd9c0a84f2d8cdde17&find=IG9%206EP",
+        "web_driver": "http://selenium:4444",
         "wiki_name": "Epping Forest District Council",
         "wiki_note": "Replace the postcode in the URL with your own."
     },
@@ -848,12 +857,12 @@
         "wiki_note": "Pass the house number, postcode, and UPRN in their respective parameters. This parser requires a Selenium webdriver."
     },
     "GooglePublicCalendarCouncil": {
-        "wiki_name": "Google Calendar (Public)",
-        "url": "https://calendar.google.com/calendar/ical/c_bca6c975545213622b3958bddab567e1e7398709c63e377b6fe9a9140bd1caf8%40group.calendar.google.com/public/basic.ics",
         "skip_get_url": true,
+        "url": "https://calendar.google.com/calendar/ical/c_bca6c975545213622b3958bddab567e1e7398709c63e377b6fe9a9140bd1caf8%40group.calendar.google.com/public/basic.ics",
+        "wiki_name": "Google Calendar (Public)",
         "wiki_note": "The URL should be the public ics file URL for the public Google calendar. See https://support.google.com/calendar/answer/37083?sjid=7202815583021446882-EU. Councils that currently need this are Trafford."
     },
-        "GraveshamBoroughCouncil": {
+    "GraveshamBoroughCouncil": {
         "skip_get_url": true,
         "uprn": "100060927046",
         "url": "https://www.gravesham.gov.uk",
@@ -1134,8 +1143,8 @@
     "LondonBoroughOfRichmondUponThames": {
         "house_number": "March Road",
         "skip_get_url": true,
-        "web_driver": "http://selenium:4444",
         "url": "https://www.richmond.gov.uk/services/waste_and_recycling/collection_days/",
+        "web_driver": "http://selenium:4444",
         "wiki_name": "London Borough Of Richmond Upon Thames",
         "wiki_note": "Pass the name of the street ONLY in the house number parameter, unfortunately post code's are not allowed. "
     },
@@ -1217,14 +1226,6 @@
         "wiki_name": "Mid Devon Council",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
-    "MiddlesbroughCouncil": {
-        "house_number": "12 Constantine Court Park Road North, Middlesbrough",
-        "skip_get_url": true,
-        "url": "https://www.midsussex.gov.uk/waste-recycling/bin-collection/",
-        "web_driver": "http://selenium:4444",
-        "wiki_name": "Middlesbrough Council",
-        "wiki_note": "Pass the entire address without postcode as it appears when you type it on the website. This parser requires a Selenium webdriver."
-    },
     "MidSuffolkDistrictCouncil": {
         "house_number": "Monday",
         "postcode": "Week 2",
@@ -1243,6 +1244,14 @@
         "wiki_name": "Mid Sussex District Council",
         "wiki_note": "Pass the name of the street with the house number parameter, wrapped in double quotes. This parser requires a Selenium webdriver."
     },
+    "MiddlesbroughCouncil": {
+        "house_number": "12 Constantine Court Park Road North, Middlesbrough",
+        "skip_get_url": true,
+        "url": "https://www.midsussex.gov.uk/waste-recycling/bin-collection/",
+        "web_driver": "http://selenium:4444",
+        "wiki_name": "Middlesbrough Council",
+        "wiki_note": "Pass the entire address without postcode as it appears when you type it on the website. This parser requires a Selenium webdriver."
+    },
     "MidlothianCouncil": {
         "house_number": "52",
         "postcode": "EH19 2EB",
@@ -1335,11 +1344,11 @@
         "wiki_name": "North Ayrshire Council",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
-    "NorthDevonCountyCouncil":{
+    "NorthDevonCountyCouncil": {
         "house_number": "1",
-        "uprn": "100040249471",
         "postcode": "EX31 2LE",
         "skip_get_url": true,
+        "uprn": "100040249471",
         "url": "https://my.northdevon.gov.uk/service/WasteRecyclingCollectionCalendar",
         "web_driver": "http://selenium:4444",
         "wiki_name": "North Devon County Council",
@@ -1690,13 +1699,6 @@
         "wiki_name": "Somerset Council",
         "wiki_note": "Provide your UPRN and postcode. Find your UPRN using [FindMyAddress](https://www.findmyaddress.co.uk/search)."
     },
-    "SouthamptonCityCouncil": {
-        "skip_get_url": true,
-        "uprn": "100060731893",
-        "url": "https://www.southampton.gov.uk",
-        "wiki_name": "Southampton City Council",
-        "wiki_note": "Pass the UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search)."
-    },
     "SouthAyrshireCouncil": {
         "postcode": "KA19 7BN",
         "skip_get_url": true,
@@ -1736,8 +1738,8 @@
     "SouthHollandDistrictCouncil": {
         "house_number": "1",
         "postcode": "PE6 0HE",
-        "uprn": "100030872493",
         "skip_get_url": true,
+        "uprn": "100030872493",
         "url": "https://www.sholland.gov.uk/mycollections",
         "web_driver": "http://selenium:4444",
         "wiki_name": "South Holland District Council",
@@ -1793,6 +1795,13 @@
         "wiki_name": "South Tyneside Council",
         "wiki_note": "Provide your house number in the `house_number` parameter and postcode in the `postcode` parameter."
     },
+    "SouthamptonCityCouncil": {
+        "skip_get_url": true,
+        "uprn": "100060731893",
+        "url": "https://www.southampton.gov.uk",
+        "wiki_name": "Southampton City Council",
+        "wiki_note": "Pass the UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search)."
+    },
     "SouthwarkCouncil": {
         "uprn": "200003469271",
         "url": "https://services.southwark.gov.uk/bins/lookup/",
@@ -2289,4 +2298,4 @@
         "wiki_name": "York Council",
         "wiki_note": "Provide your UPRN."
     }
-}
+}

uk_bin_collection/uk_bin_collection/councils/BroadlandDistrictCouncil.py

@@ -0,0 +1,185 @@
+# This script pulls (in one hit) the data from Broadland District Council Bins Data
+# Working command line:
+# python collect_data.py BroadlandDistrictCouncil "https://area.southnorfolkandbroadland.gov.uk/FindAddress" -p "NR10 3FD" -n "1 Park View, Horsford, Norfolk, NR10 3FD"
+
+import re
+import time
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            data = {"bins": []}
+            headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64)"}
+
+            uprn = kwargs.get("uprn")
+            user_paon = kwargs.get("paon")
+            postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            url = kwargs.get("url")
+
+            print(
+                f"Starting parse_data with parameters: postcode={postcode}, paon={user_paon}"
+            )
+            print(
+                f"Creating webdriver with: web_driver={web_driver}, headless={headless}"
+            )
+
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            print(f"Navigating to URL: {url}")
+            driver.get(url)
+            print("Successfully loaded the page")
+
+            # Handle cookie confirmation dialog
+            try:
+                # Adjust the selector depending on the site's button
+                accept_button = WebDriverWait(driver, 10).until(
+                    EC.element_to_be_clickable(
+                        (By.ID, "CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll")
+                    )
+                )
+                accept_button.click()
+                print("Cookie banner clicked.")
+            except:
+                print("No cookie banner appeared or selector failed.")
+
+            wait = WebDriverWait(driver, 60)
+            post_code_search = wait.until(
+                EC.presence_of_element_located((By.ID, "Postcode"))
+            )
+            post_code_search.send_keys(postcode)
+
+            # Click the Find address button
+            print("Looking for 'Find address' button...")
+            submit_btn = wait.until(
+                EC.presence_of_element_located(
+                    (
+                        By.XPATH,
+                        "//input[@type='submit' and @class='button button--secondary']",
+                    )
+                )
+            )
+            print("Clicking button...")
+            submit_btn.send_keys(Keys.ENTER)
+
+            # Wait for the address dropdown to appear
+            print("Waiting for address dropdown to appear...")
+            address_dropdown = wait.until(
+                EC.presence_of_element_located((By.ID, "UprnAddress"))
+            )
+            print("Found address dropdown")
+
+            # Create a Select object for the dropdown
+            dropdown_select = Select(address_dropdown)
+
+            # Search for the exact address
+            print(f"Looking for address: {user_paon}")
+
+            # Select the address by visible text
+            dropdown_select.select_by_visible_text(user_paon)
+            print(f"Selected address: {user_paon}")
+
+            print("Looking for submit button after address selection...")
+            submit_btn = wait.until(
+                EC.element_to_be_clickable((By.XPATH, "//input[@type='submit']"))
+            )
+            print("Clicking button...")
+            submit_btn.send_keys(Keys.ENTER)
+
+            print("Waiting for collection details to appear...")
+            address_dropdown = wait.until(
+                EC.presence_of_element_located(
+                    (
+                        By.XPATH,
+                        "//div[contains(@class, 'card-body')]//h4[contains(text(), 'Your next collections')]",
+                    )
+                )
+            )
+
+            # Make a BS4 object
+            print("Parsing page with BeautifulSoup...")
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            # Initialize current date
+            current_date = datetime.now()
+
+            # Process collection details
+            print("Looking for collection details in the page...")
+
+            # Find the card-body div that contains the bin collection information
+            card_body = soup.find("div", class_="card-body")
+
+            if card_body:
+                # Find the "Your next collections" heading
+                next_collections_heading = card_body.find(
+                    "h4", string="Your next collections"
+                )
+
+                if next_collections_heading:
+                    # Find all bin collection divs (each with class "my-2")
+                    bin_divs = next_collections_heading.find_next_siblings(
+                        "div", class_="my-2"
+                    )
+
+                    print(f"Found {len(bin_divs)} bin collection divs")
+
+                    for bin_div in bin_divs:
+                        # Find the bin type (in a strong tag)
+                        bin_type_elem = bin_div.find("strong")
+                        bin_type = None
+
+                        if bin_type_elem:
+                            bin_type = bin_type_elem.text.strip().replace(
+                                " (if applicable)", ""
+                            )
+
+                            # Get the parent element that contains both the bin type and date
+                            text_container = bin_type_elem.parent
+                            if text_container:
+                                # Extract the full text and remove the bin type to get the date part
+                                full_text = text_container.get_text(strip=True)
+                                date_text = full_text.replace(bin_type, "").strip()
+                                print(f"Unparsed collection date: {date_text}")
+
+                                # Parse the date
+                                # First, remove any ordinal indicators (1st, 2nd, 3rd, etc.)
+                                cleaned_date_text = (
+                                    remove_ordinal_indicator_from_date_string(date_text)
+                                )
+
+                                from dateutil.parser import parse
+
+                                parsed_date = parse(cleaned_date_text, fuzzy=True)
+                                bin_date = parsed_date.strftime("%d/%m/%Y")
+
+                                # Only process if we have both bin_type and bin_date
+                                if bin_type and bin_date:
+                                    dict_data = {
+                                        "type": bin_type,
+                                        "collectionDate": bin_date,
+                                    }
+                                    data["bins"].append(dict_data)
+                                    print(f"Added bin data: {dict_data}")
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            raise
+        finally:
+            print("Cleaning up webdriver...")
+            if driver:
+                driver.quit()
+
+        return data

{uk_bin_collection-0.139.0.dist-info → uk_bin_collection-0.140.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.139.0
+Version: 0.140.0
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.139.0.dist-info → uk_bin_collection-0.140.0.dist-info}/RECORD

@@ -3,7 +3,7 @@ uk_bin_collection/tests/check_selenium_url_in_input.json.py,sha256=Iecdja0I3XIiY
 uk_bin_collection/tests/council_feature_input_parity.py,sha256=DO6Mk4ImYgM5ZCZ-cutwz5RoYYWZRLYx2tr6zIs_9Rc,3843
 uk_bin_collection/tests/features/environment.py,sha256=VQZjJdJI_kZn08M0j5cUgvKT4k3iTw8icJge1DGOkoA,127
 uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=SJK-Vc737hrf03tssxxbeg_JIvAH-ddB8f6gU1LTbuQ,251
-uk_bin_collection/tests/input.json,sha256=FlKli01VYh9oXZ9Fy9I6arVV4kH1-VsNdBN79OKgmFs,119598
+uk_bin_collection/tests/input.json,sha256=b6MsNMMQbs1-lW6g1_dYz5WTiwt8Puuu3q_BEopiulk,120060
 uk_bin_collection/tests/output.schema,sha256=ZwKQBwYyTDEM4G2hJwfLUVM-5v1vKRvRK9W9SS1sd18,1086
 uk_bin_collection/tests/step_defs/step_helpers/file_handler.py,sha256=Ygzi4V0S1MIHqbdstUlIqtRIwnynvhu4UtpweJ6-5N8,1474
 uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=VZ0a81sioJULD7syAYHjvK_-nT_Rd36tUyzPetSA0gk,3475
@@ -50,6 +50,7 @@ uk_bin_collection/uk_bin_collection/councils/BrecklandCouncil.py,sha256=PX6A_pDv
 uk_bin_collection/uk_bin_collection/councils/BrentCouncil.py,sha256=ucwokxvASYi_KiOYSOVdaGfC1kfUbII0r6Zl2NE1hnU,4208
 uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py,sha256=k6qt4cds-Ejd97Z-__pw2BYvGVbFdc9SUfF73PPrTNA,5823
 uk_bin_collection/uk_bin_collection/councils/BristolCityCouncil.py,sha256=kJmmDJz_kQ45DHmG7ocrUpNJonEn0kuXYEDQyZaf9ks,5576
+uk_bin_collection/uk_bin_collection/councils/BroadlandDistrictCouncil.py,sha256=aelqhh503dx6O2EEmC3AT5tnY39Dc53qcouH8T-mek8,7613
 uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py,sha256=dii85JLmYU1uMidCEsWVo3stTcq_QqyC65DxG8u1UmE,4302
 uk_bin_collection/uk_bin_collection/councils/BromsgroveDistrictCouncil.py,sha256=PUfxP8j5Oh9wFHkdjbrJzQli9UzMHZzwrZ2hkThrvhI,1781
 uk_bin_collection/uk_bin_collection/councils/BroxbourneCouncil.py,sha256=mcRtkFc9g3YNN17OQfhzYJtNYeWrZPP_e7m7goEhz5I,3012
@@ -324,8 +325,8 @@ uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py,sha256=I2kBYMlsD4bId
 uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=EQWRhZ2pEejlvm0fPyOTsOHKvUZmPnxEYO_OWRGKTjs,1158
 uk_bin_collection/uk_bin_collection/create_new_council.py,sha256=m-IhmWmeWQlFsTZC4OxuFvtw5ZtB8EAJHxJTH4O59lQ,1536
 uk_bin_collection/uk_bin_collection/get_bin_data.py,sha256=YvmHfZqanwrJ8ToGch34x-L-7yPe31nB_x77_Mgl_vo,4545
-uk_bin_collection-0.139.0.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
-uk_bin_collection-0.139.0.dist-info/METADATA,sha256=G_dFWc1G250I1qFCLJO74t7HOcIwC2HCitNWkJPrZGg,19851
-uk_bin_collection-0.139.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-uk_bin_collection-0.139.0.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
-uk_bin_collection-0.139.0.dist-info/RECORD,,
+uk_bin_collection-0.140.0.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
+uk_bin_collection-0.140.0.dist-info/METADATA,sha256=3Dw3sim4P-LwGOmRbcX7jSK15SFpT16G2lTDl_wcuj8,19851
+uk_bin_collection-0.140.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+uk_bin_collection-0.140.0.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
+uk_bin_collection-0.140.0.dist-info/RECORD,,