uk_bin_collection 0.141.4__py3-none-any.whl → 0.143.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -100,6 +100,15 @@
100
100
  "wiki_name": "Babergh District Council",
101
101
  "wiki_note": "Use the House Number field to pass the DAY of the week for your NORMAL collections. [Monday/Tuesday/Wednesday/Thursday/Friday]. [OPTIONAL] Use the 'postcode' field to pass the WEEK for your garden collection. [Week 1/Week 2]. [OPTIONAL] Use the 'uprn' field to pass the DAY for your garden collection. [Monday/Tuesday/Wednesday/Thursday/Friday]"
102
102
  },
103
+ "BarkingDagenham": {
104
+ "house_number": "19 KELLY WAY, CHADWELL HEATH, RM6 6XH",
105
+ "postcode": "RM6 6XH",
106
+ "skip_get_url": true,
107
+ "web_driver": "http://selenium:4444",
108
+ "url": "https://www.lbbd.gov.uk/rubbish-recycling/household-bin-collection/check-your-bin-collection-days",
109
+ "wiki_name": "BarkingDagenham",
110
+ "wiki_note": "Use the full address as it appears on the drop-down on the site when you search by postcode."
111
+ },
103
112
  "BarnetCouncil": {
104
113
  "house_number": "HA8 7NA, 2, MANOR PARK GARDENS, EDGWARE, BARNET",
105
114
  "postcode": "HA8 7NA",
@@ -1861,6 +1870,15 @@
1861
1870
  "wiki_name": "Stevenage Borough Council",
1862
1871
  "wiki_note": "Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN."
1863
1872
  },
1873
+ "StirlingCouncil": {
1874
+ "house_number": "5, SUNNYLAW ROAD, BRIDGE OF ALLAN, STIRLING, FK9 4QA",
1875
+ "postcode": "FK9 4QA",
1876
+ "skip_get_url": true,
1877
+ "url": "https://www.stirling.gov.uk/bins-and-recycling/bin-collection-dates-search/",
1878
+ "web_driver": "http://selenium:4444",
1879
+ "wiki_name": "Stirling Council",
1880
+ "wiki_note": "Use the full address as it appears on the drop-down on the site when you search by postcode."
1881
+ },
1864
1882
  "StockportBoroughCouncil": {
1865
1883
  "url": "https://myaccount.stockport.gov.uk/bin-collections/show/100011434401",
1866
1884
  "wiki_command_url_override": "https://myaccount.stockport.gov.uk/bin-collections/show/XXXXXXXX",
@@ -0,0 +1,150 @@
1
+ # This script pulls bin collection data from Barking and Dagenham Council
2
+ # Example URL: https://www.lbbd.gov.uk/rubbish-recycling/household-bin-collection/check-your-bin-collection-days
3
+ import time
4
+
5
+ from bs4 import BeautifulSoup
6
+ from dateutil.parser import parse
7
+ from selenium.common.exceptions import NoSuchElementException, TimeoutException
8
+ from selenium.webdriver.common.by import By
9
+ from selenium.webdriver.common.keys import Keys
10
+ from selenium.webdriver.support import expected_conditions as EC
11
+ from selenium.webdriver.support.ui import Select
12
+ from selenium.webdriver.support.wait import WebDriverWait
13
+
14
+ from uk_bin_collection.uk_bin_collection.common import *
15
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
16
+
17
+
18
+ class CouncilClass(AbstractGetBinDataClass):
19
+
20
+ def parse_data(self, page: str, **kwargs) -> dict:
21
+ driver = None
22
+ try:
23
+ data = {"bins": []}
24
+
25
+ user_paon = kwargs.get("paon")
26
+ postcode = kwargs.get("postcode")
27
+ web_driver = kwargs.get("web_driver")
28
+ headless = kwargs.get("headless")
29
+ url = kwargs.get("url")
30
+
31
+ print(
32
+ f"Starting parse_data with parameters: postcode={postcode}, paon={user_paon}"
33
+ )
34
+ print(
35
+ f"Creating webdriver with: web_driver={web_driver}, headless={headless}"
36
+ )
37
+
38
+ driver = create_webdriver(web_driver, headless, None, __name__)
39
+ print(f"Navigating to URL: {url}")
40
+ driver.get(url)
41
+ print("Successfully loaded the page")
42
+
43
+ WebDriverWait(driver, 10).until(
44
+ lambda d: d.execute_script("return document.readyState") == "complete"
45
+ )
46
+
47
+ # Close popup if it exists
48
+ driver.switch_to.active_element.send_keys(Keys.ESCAPE)
49
+
50
+ # Handle cookie banner if present
51
+ wait = WebDriverWait(driver, 60)
52
+ try:
53
+ cookie_button = wait.until(
54
+ EC.element_to_be_clickable(
55
+ (
56
+ By.CSS_SELECTOR,
57
+ ".agree-button.eu-cookie-compliance-secondary-button.button.button--small",
58
+ )
59
+ ),
60
+ message="Cookie banner not found",
61
+ )
62
+ cookie_button.click()
63
+ print("Cookie banner clicked.")
64
+ time.sleep(1) # Brief pause to let banner disappear
65
+ except (TimeoutException, NoSuchElementException):
66
+ print("No cookie banner appeared or selector failed.")
67
+
68
+ # Enter postcode
69
+ print("Looking for postcode input...")
70
+ post_code_input = wait.until(
71
+ EC.element_to_be_clickable((By.ID, "postcode")),
72
+ message="Postcode input not found",
73
+ )
74
+ post_code_input.clear()
75
+ post_code_input.send_keys(postcode)
76
+ print(f"Entered postcode: {postcode}")
77
+
78
+ driver.switch_to.active_element.send_keys(Keys.TAB + Keys.ENTER)
79
+ print("Pressed ENTER on Search button")
80
+
81
+ # Wait for and select address
82
+ print("Waiting for address dropdown...")
83
+ address_select = wait.until(
84
+ EC.element_to_be_clickable((By.ID, "address")),
85
+ message="Address dropdown not found",
86
+ )
87
+ dropdown = Select(address_select)
88
+
89
+ dropdown.select_by_visible_text(user_paon)
90
+ print("Address selected successfully")
91
+
92
+ driver.switch_to.active_element.send_keys(Keys.TAB + Keys.ENTER)
93
+ print("Pressed ENTER on Next button")
94
+
95
+ print("Looking for schedule list...")
96
+ schedule_list = WebDriverWait(driver, 10).until(
97
+ EC.presence_of_element_located((By.CLASS_NAME, "bin--container"))
98
+ )
99
+
100
+ # Make a BS4 object
101
+ print("Parsing page with BeautifulSoup...")
102
+ soup = BeautifulSoup(driver.page_source, features="html.parser")
103
+
104
+ # Process collection details
105
+ print("Looking for collection details in the page...")
106
+
107
+ bin_rows = soup.select("div.bin--row:not(:first-child)") # Skip header row
108
+ print(f"\nProcessing {len(bin_rows)} bin rows...")
109
+
110
+ for row in bin_rows:
111
+ try:
112
+ # Extract bin type from first column
113
+ bin_type = row.select_one("div.col-md-3").text.strip()
114
+
115
+ # Get the collection dates column
116
+ collection_dates_div = row.select("div.col-md-3")[1] # Third column
117
+
118
+ # Get only the immediate text content before any <p> tags
119
+ next_collection_text = "".join(
120
+ collection_dates_div.find_all(text=True, recursive=False)
121
+ ).strip()
122
+
123
+ # Parse the date
124
+ cleaned_date_text = remove_ordinal_indicator_from_date_string(
125
+ next_collection_text
126
+ )
127
+ parsed_date = parse(cleaned_date_text, fuzzy=True)
128
+ bin_date = parsed_date.strftime("%d/%m/%Y")
129
+
130
+ # Add to data
131
+ if bin_type and bin_date:
132
+ dict_data = {
133
+ "type": bin_type,
134
+ "collectionDate": bin_date,
135
+ }
136
+ data["bins"].append(dict_data)
137
+ print(f"Successfully added collection: {dict_data}")
138
+
139
+ except Exception as e:
140
+ print(f"Error processing item: {e}")
141
+ continue
142
+ except Exception as e:
143
+ print(f"An error occurred: {e}")
144
+ raise
145
+ finally:
146
+ print("Cleaning up webdriver...")
147
+ if driver:
148
+ driver.quit()
149
+
150
+ return data
@@ -0,0 +1,163 @@
1
+ # This script pulls bin collection data from Stirling Council
2
+ # Example URL: https://www.stirling.gov.uk/bins-and-recycling/bin-collection-dates-search/
3
+ import time
4
+
5
+ from bs4 import BeautifulSoup
6
+ from dateutil.parser import parse
7
+ from selenium.common.exceptions import NoSuchElementException, TimeoutException
8
+ from selenium.webdriver.common.by import By
9
+ from selenium.webdriver.common.keys import Keys
10
+ from selenium.webdriver.support import expected_conditions as EC
11
+ from selenium.webdriver.support.ui import Select
12
+ from selenium.webdriver.support.wait import WebDriverWait
13
+
14
+ from uk_bin_collection.uk_bin_collection.common import *
15
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
16
+
17
+
18
+ class CouncilClass(AbstractGetBinDataClass):
19
+
20
+ def parse_data(self, page: str, **kwargs) -> dict:
21
+ driver = None
22
+ try:
23
+ data = {"bins": []}
24
+
25
+ user_paon = kwargs.get("paon")
26
+ postcode = kwargs.get("postcode")
27
+ web_driver = kwargs.get("web_driver")
28
+ headless = kwargs.get("headless")
29
+ url = kwargs.get("url")
30
+
31
+ print(
32
+ f"Starting parse_data with parameters: postcode={postcode}, paon={user_paon}"
33
+ )
34
+ print(
35
+ f"Creating webdriver with: web_driver={web_driver}, headless={headless}"
36
+ )
37
+
38
+ driver = create_webdriver(web_driver, headless, None, __name__)
39
+ print(f"Navigating to URL: {url}")
40
+ driver.get(url)
41
+ print("Successfully loaded the page")
42
+
43
+ # Handle cookie confirmation dialog
44
+ wait = WebDriverWait(driver, 60)
45
+
46
+ # Handle cookie banner if present
47
+ try:
48
+ cookie_button = wait.until(
49
+ EC.element_to_be_clickable((By.ID, "ccc-recommended-settings")),
50
+ message="Cookie banner not found",
51
+ )
52
+ cookie_button.click()
53
+ print("Cookie banner clicked.")
54
+ time.sleep(1) # Brief pause to let banner disappear
55
+ except (TimeoutException, NoSuchElementException):
56
+ print("No cookie banner appeared or selector failed.")
57
+
58
+ # Enter postcode
59
+ print("Looking for postcode input...")
60
+ post_code_input = wait.until(
61
+ EC.element_to_be_clickable((By.ID, "js-postcode-lookup-postcode")),
62
+ message="Postcode input not found",
63
+ )
64
+ post_code_input.clear()
65
+ post_code_input.send_keys(postcode)
66
+ print(f"Entered postcode: {postcode}")
67
+
68
+ driver.switch_to.active_element.send_keys(Keys.TAB + Keys.ENTER)
69
+ print("Pressed ENTER on Find")
70
+
71
+ # Wait for and select address
72
+ print("Waiting for address dropdown...")
73
+ address_select = wait.until(
74
+ EC.element_to_be_clickable((By.CLASS_NAME, "select__input")),
75
+ message="Address dropdown not found",
76
+ )
77
+ dropdown = Select(address_select)
78
+
79
+ dropdown.select_by_visible_text(user_paon)
80
+ print("Address selected successfully")
81
+
82
+ driver.switch_to.active_element.send_keys(Keys.TAB * 2 + Keys.ENTER)
83
+ print("Pressed ENTER on Next button")
84
+
85
+ print("Looking for schedule list...")
86
+ schedule_list = WebDriverWait(driver, 10).until(
87
+ EC.presence_of_element_located((By.CLASS_NAME, "schedule__list"))
88
+ )
89
+
90
+ # Make a BS4 object
91
+ print("Parsing page with BeautifulSoup...")
92
+ soup = BeautifulSoup(driver.page_source, features="html.parser")
93
+
94
+ # Process collection details
95
+ print("Looking for collection details in the page...")
96
+
97
+ schedule_items = []
98
+ selectors = [
99
+ "li.schedule__item",
100
+ ]
101
+
102
+ for selector in selectors:
103
+ items = soup.select(selector)
104
+ if items:
105
+ print(f"Found {len(items)} items using selector: {selector}")
106
+ schedule_items = items
107
+ break
108
+
109
+ print(f"\nProcessing {len(schedule_items)} schedule items...")
110
+
111
+ for item in schedule_items:
112
+ try:
113
+ # Try multiple selectors for bin type
114
+ title = item.find("h2", class_="schedule__title")
115
+
116
+ bin_type = title.text.strip()
117
+
118
+ summary = item.find("p", class_="schedule__summary")
119
+
120
+ # Extract date text
121
+ summary_text = summary.get_text(strip=True)
122
+ print(f"Found summary text: {summary_text}")
123
+
124
+ # Try different date formats
125
+ date_text = None
126
+ for splitter in ["Then every", "then every", "Every"]:
127
+ if splitter in summary_text:
128
+ date_text = summary_text.split(splitter)[0].strip()
129
+ break
130
+
131
+ if not date_text:
132
+ date_text = summary_text # Use full text if no splitter found
133
+
134
+ print(f"Extracted date text: {date_text}")
135
+
136
+ # Parse the date
137
+ cleaned_date_text = remove_ordinal_indicator_from_date_string(
138
+ date_text
139
+ )
140
+ parsed_date = parse(cleaned_date_text, fuzzy=True)
141
+ bin_date = parsed_date.strftime("%d/%m/%Y")
142
+
143
+ # Add only the next collection
144
+ if bin_type and bin_date:
145
+ dict_data = {
146
+ "type": bin_type,
147
+ "collectionDate": bin_date,
148
+ }
149
+ data["bins"].append(dict_data)
150
+ print(f"Successfully added collection: {dict_data}")
151
+
152
+ except Exception as e:
153
+ print(f"Error processing item: {e}")
154
+ continue
155
+ except Exception as e:
156
+ print(f"An error occurred: {e}")
157
+ raise
158
+ finally:
159
+ print("Cleaning up webdriver...")
160
+ if driver:
161
+ driver.quit()
162
+
163
+ return data
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: uk_bin_collection
3
- Version: 0.141.4
3
+ Version: 0.143.0
4
4
  Summary: Python Lib to collect UK Bin Data
5
5
  Author: Robert Bradley
6
6
  Author-email: robbrad182@gmail.com
@@ -3,7 +3,7 @@ uk_bin_collection/tests/check_selenium_url_in_input.json.py,sha256=Iecdja0I3XIiY
3
3
  uk_bin_collection/tests/council_feature_input_parity.py,sha256=DO6Mk4ImYgM5ZCZ-cutwz5RoYYWZRLYx2tr6zIs_9Rc,3843
4
4
  uk_bin_collection/tests/features/environment.py,sha256=VQZjJdJI_kZn08M0j5cUgvKT4k3iTw8icJge1DGOkoA,127
5
5
  uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=SJK-Vc737hrf03tssxxbeg_JIvAH-ddB8f6gU1LTbuQ,251
6
- uk_bin_collection/tests/input.json,sha256=NQy_rR9oaERmyOMOwk02BTJqAafV1F-Nk1aNjJFTRag,120526
6
+ uk_bin_collection/tests/input.json,sha256=yjo1upzl2zK_UgPqSSZY3mjyY-ItLPJ8CzfL1fDB7vk,121472
7
7
  uk_bin_collection/tests/output.schema,sha256=ZwKQBwYyTDEM4G2hJwfLUVM-5v1vKRvRK9W9SS1sd18,1086
8
8
  uk_bin_collection/tests/step_defs/step_helpers/file_handler.py,sha256=Ygzi4V0S1MIHqbdstUlIqtRIwnynvhu4UtpweJ6-5N8,1474
9
9
  uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=VZ0a81sioJULD7syAYHjvK_-nT_Rd36tUyzPetSA0gk,3475
@@ -27,6 +27,7 @@ uk_bin_collection/uk_bin_collection/councils/AshfordBoroughCouncil.py,sha256=4eb
27
27
  uk_bin_collection/uk_bin_collection/councils/AylesburyValeCouncil.py,sha256=LouqjspEMt1TkOGqWHs2zkxwOETIy3n7p64uKIlAgUg,2401
28
28
  uk_bin_collection/uk_bin_collection/councils/BCPCouncil.py,sha256=W7QBx6Mgso8RYosuXsaYo3GGNAu-tiyBSmuYxr1JSOU,1707
29
29
  uk_bin_collection/uk_bin_collection/councils/BaberghDistrictCouncil.py,sha256=1eXdST58xFRMdYl8AGNG_EwyQeLa31WSWUe882hQ2ec,6329
30
+ uk_bin_collection/uk_bin_collection/councils/BarkingDagenham.py,sha256=8lYnRazbh_iOcmciSOJIhF44e8z3ZFlTZ0vnZKEfwJI,5973
30
31
  uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py,sha256=Sd4-pbv0QZsR7soxvXYqsfdOUIqZqS6notyoZthG77s,9182
31
32
  uk_bin_collection/uk_bin_collection/councils/BarnsleyMBCouncil.py,sha256=hqzVKEqwYmjcQjCYBletcCt9_pE96qQ3kn7eDroJeNk,4764
32
33
  uk_bin_collection/uk_bin_collection/councils/BasildonCouncil.py,sha256=NymPmq5pud0PJ8ePcc2r1SKED4EHQ0EY2l71O-Metxc,3313
@@ -265,6 +266,7 @@ uk_bin_collection/uk_bin_collection/councils/StHelensBC.py,sha256=X9dvnQTNn7QUO8
265
266
  uk_bin_collection/uk_bin_collection/councils/StaffordBoroughCouncil.py,sha256=9Qj4HJI7Dbiqb2mVSG2UtkBe27Y7wvQ5SYFTwGzJ5g0,2292
266
267
  uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py,sha256=vdSnDbiKLQIdvvqVXDi2butZXRBMxv401aIrkLkKgy4,4370
267
268
  uk_bin_collection/uk_bin_collection/councils/StevenageBoroughCouncil.py,sha256=EiDIyOlHhdiJ-YYjo7T5uA5sN2jzNoysu6FctjuAjBI,3549
269
+ uk_bin_collection/uk_bin_collection/councils/StirlingCouncil.py,sha256=QaUw5oP_mmFAXo98EdHhI6lWo4OjF2E8zY2M7HQo2bk,6308
268
270
  uk_bin_collection/uk_bin_collection/councils/StockportBoroughCouncil.py,sha256=v0HmioNVRoU1-9OnLJl2V3M5pVR1aVu1BgOLHFR1Sf4,1429
269
271
  uk_bin_collection/uk_bin_collection/councils/StocktonOnTeesCouncil.py,sha256=obaBgsmIJ95Ah7KaTNWdU107tZDPVuuJox0mGUoGjNk,6070
270
272
  uk_bin_collection/uk_bin_collection/councils/StokeOnTrentCityCouncil.py,sha256=KM0EgWeO7mk8lkozX0RCTfMchXdjrfBqIjCiOtB09aM,2884
@@ -326,8 +328,8 @@ uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py,sha256=I2kBYMlsD4bId
326
328
  uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=EQWRhZ2pEejlvm0fPyOTsOHKvUZmPnxEYO_OWRGKTjs,1158
327
329
  uk_bin_collection/uk_bin_collection/create_new_council.py,sha256=m-IhmWmeWQlFsTZC4OxuFvtw5ZtB8EAJHxJTH4O59lQ,1536
328
330
  uk_bin_collection/uk_bin_collection/get_bin_data.py,sha256=YvmHfZqanwrJ8ToGch34x-L-7yPe31nB_x77_Mgl_vo,4545
329
- uk_bin_collection-0.141.4.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
330
- uk_bin_collection-0.141.4.dist-info/METADATA,sha256=3InKR2PGDz8pWtSdL27kM0OwKZAZV0dxHHhZaK06LWs,19851
331
- uk_bin_collection-0.141.4.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
332
- uk_bin_collection-0.141.4.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
333
- uk_bin_collection-0.141.4.dist-info/RECORD,,
331
+ uk_bin_collection-0.143.0.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
332
+ uk_bin_collection-0.143.0.dist-info/METADATA,sha256=t5XZR0LERoiQknGYzJMdNIHGH0w2cF2ZGaru0yjlGBA,19851
333
+ uk_bin_collection-0.143.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
334
+ uk_bin_collection-0.143.0.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
335
+ uk_bin_collection-0.143.0.dist-info/RECORD,,