uk_bin_collection 0.138.1__py3-none-any.whl → 0.140.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- uk_bin_collection/tests/input.json +63 -26
- uk_bin_collection/uk_bin_collection/councils/AberdeenCityCouncil.py +2 -1
- uk_bin_collection/uk_bin_collection/councils/AberdeenshireCouncil.py +1 -0
- uk_bin_collection/uk_bin_collection/councils/ArdsAndNorthDownCouncil.py +1 -0
- uk_bin_collection/uk_bin_collection/councils/BarnsleyMBCouncil.py +1 -0
- uk_bin_collection/uk_bin_collection/councils/BroadlandDistrictCouncil.py +185 -0
- uk_bin_collection/uk_bin_collection/councils/BroxbourneCouncil.py +7 -3
- uk_bin_collection/uk_bin_collection/councils/CeredigionCountyCouncil.py +157 -0
- uk_bin_collection/uk_bin_collection/councils/CheltenhamBoroughCouncil.py +95 -61
- uk_bin_collection/uk_bin_collection/councils/CheshireEastCouncil.py +1 -0
- uk_bin_collection/uk_bin_collection/councils/CoventryCityCouncil.py +4 -1
- uk_bin_collection/uk_bin_collection/councils/ForestOfDeanDistrictCouncil.py +52 -41
- uk_bin_collection/uk_bin_collection/councils/GooglePublicCalendarCouncil.py +3 -4
- uk_bin_collection/uk_bin_collection/councils/LondonBoroughOfRichmondUponThames.py +11 -9
- uk_bin_collection/uk_bin_collection/councils/MiddlesbroughCouncil.py +13 -4
- uk_bin_collection/uk_bin_collection/councils/MonmouthshireCountyCouncil.py +5 -1
- uk_bin_collection/uk_bin_collection/councils/NewForestCouncil.py +1 -3
- uk_bin_collection/uk_bin_collection/councils/NorthDevonCountyCouncil.py +159 -0
- uk_bin_collection/uk_bin_collection/councils/NorwichCityCouncil.py +15 -3
- uk_bin_collection/uk_bin_collection/councils/NuneatonBedworthBoroughCouncil.py +873 -871
- uk_bin_collection/uk_bin_collection/councils/RugbyBoroughCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py +3 -6
- uk_bin_collection/uk_bin_collection/councils/SouthHollandDistrictCouncil.py +136 -0
- uk_bin_collection/uk_bin_collection/councils/WalsallCouncil.py +6 -2
- uk_bin_collection/uk_bin_collection/councils/WalthamForest.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/WestLindseyDistrictCouncil.py +6 -3
- uk_bin_collection/uk_bin_collection/councils/WychavonDistrictCouncil.py +1 -0
- {uk_bin_collection-0.138.1.dist-info → uk_bin_collection-0.140.0.dist-info}/METADATA +1 -1
- {uk_bin_collection-0.138.1.dist-info → uk_bin_collection-0.140.0.dist-info}/RECORD +32 -28
- {uk_bin_collection-0.138.1.dist-info → uk_bin_collection-0.140.0.dist-info}/LICENSE +0 -0
- {uk_bin_collection-0.138.1.dist-info → uk_bin_collection-0.140.0.dist-info}/WHEEL +0 -0
- {uk_bin_collection-0.138.1.dist-info → uk_bin_collection-0.140.0.dist-info}/entry_points.txt +0 -0
@@ -36,21 +36,29 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
36
36
|
s = requests.session()
|
37
37
|
|
38
38
|
# Ask for a new SessionId from the server
|
39
|
-
session_id_url =
|
40
|
-
"
|
39
|
+
session_id_url = (
|
40
|
+
"https://maps.cheltenham.gov.uk/map/Aurora.svc/"
|
41
|
+
"RequestSession?userName=guest+CBC&password=&"
|
41
42
|
"script=%5CAurora%5CCBC+Waste+Streets.AuroraScript%24"
|
43
|
+
)
|
42
44
|
session_id_response = s.get(session_id_url)
|
43
45
|
session_id_response.raise_for_status()
|
44
46
|
session_id = session_id_response.json().get("Session").get("SessionId")
|
45
47
|
|
46
48
|
# Ask what tasks we can do within the session
|
47
|
-
tasks_url =
|
49
|
+
tasks_url = (
|
50
|
+
f"https://maps.cheltenham.gov.uk/map/Aurora.svc/"
|
48
51
|
f"GetWorkflow?sessionId={session_id}&workflowId=wastestreet"
|
52
|
+
)
|
49
53
|
tasks_response = s.get(tasks_url)
|
50
54
|
tasks_response.raise_for_status()
|
51
55
|
# JSON response contained a BOM marker
|
52
56
|
tasks = json.loads(tasks_response.text[1:])
|
53
|
-
retrieve_results_task_id, initialise_map_task_id, drilldown_task_id =
|
57
|
+
retrieve_results_task_id, initialise_map_task_id, drilldown_task_id = (
|
58
|
+
None,
|
59
|
+
None,
|
60
|
+
None,
|
61
|
+
)
|
54
62
|
# Pull out the ID's of the tasks we will need
|
55
63
|
for task in tasks.get("Tasks"):
|
56
64
|
if task.get("$type") == "StatMap.Aurora.FetchResultSetTask, StatMapService":
|
@@ -59,64 +67,86 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
59
67
|
initialise_map_task_id = task.get("Id")
|
60
68
|
elif task.get("$type") == "StatMap.Aurora.DrillDownTask, StatMapService":
|
61
69
|
drilldown_task_id = task.get("Id")
|
62
|
-
if not all(
|
70
|
+
if not all(
|
71
|
+
[retrieve_results_task_id, initialise_map_task_id, drilldown_task_id]
|
72
|
+
):
|
63
73
|
raise ValueError("Not all task ID's found")
|
64
74
|
|
65
75
|
# Find the X / Y coordinates for the requested postcode
|
66
|
-
postcode_search_url =
|
76
|
+
postcode_search_url = (
|
77
|
+
"https://maps.cheltenham.gov.uk/map/Aurora.svc/FindLocation?"
|
67
78
|
f"sessionId={session_id}&address={postcode}&limit=1000"
|
79
|
+
)
|
68
80
|
postcode_search_response = s.get(postcode_search_url)
|
69
81
|
postcode_search_response.raise_for_status()
|
70
82
|
if len(locations_list := postcode_search_response.json().get("Locations")) == 0:
|
71
83
|
raise ValueError("Address locations empty")
|
72
84
|
for location in locations_list:
|
73
|
-
location_search_url =
|
85
|
+
location_search_url = (
|
86
|
+
"https://maps.cheltenham.gov.uk/map/Aurora.svc/FindLocation?"
|
74
87
|
f"sessionId={session_id}&locationId={location.get('Id')}"
|
88
|
+
)
|
75
89
|
location_search_response = s.get(location_search_url)
|
76
90
|
location_search_response.raise_for_status()
|
77
91
|
if not (location_list := location_search_response.json().get("Locations")):
|
78
92
|
raise KeyError("Locations wasn't present in results")
|
79
93
|
if not (location_detail := location_list[0].get("Details")):
|
80
94
|
raise KeyError("Details wasn't present in location")
|
81
|
-
location_uprn = [
|
82
|
-
|
95
|
+
location_uprn = [
|
96
|
+
detail.get("Value")
|
97
|
+
for detail in location_detail
|
98
|
+
if detail.get("Name") == "UPRN"
|
99
|
+
][0]
|
83
100
|
if str(location_uprn) == uprn:
|
84
|
-
location_usrn = str(
|
85
|
-
|
101
|
+
location_usrn = str(
|
102
|
+
[
|
103
|
+
detail.get("Value")
|
104
|
+
for detail in location_detail
|
105
|
+
if detail.get("Name") == "USRN"
|
106
|
+
][0]
|
107
|
+
)
|
86
108
|
location_x = location_list[0].get("X")
|
87
109
|
location_y = location_list[0].get("Y")
|
88
110
|
break
|
89
111
|
|
90
112
|
# Needed to initialise the server to allow follow on call
|
91
|
-
open_map_url =
|
113
|
+
open_map_url = (
|
114
|
+
"https://maps.cheltenham.gov.uk/map/Aurora.svc/OpenScriptMap?"
|
92
115
|
f"sessionId={session_id}"
|
116
|
+
)
|
93
117
|
if res := s.get(open_map_url):
|
94
118
|
res.raise_for_status()
|
95
119
|
|
96
120
|
# Needed to initialise the server to allow follow on call
|
97
|
-
save_state_map_url =
|
98
|
-
|
99
|
-
"
|
121
|
+
save_state_map_url = (
|
122
|
+
"https://maps.cheltenham.gov.uk/map/Aurora.svc/ExecuteTaskJob?"
|
123
|
+
f"sessionId={session_id}&taskId={initialise_map_task_id}&job="
|
124
|
+
"%7BTask%3A+%7B+%24type%3A+%27StatMap.Aurora.SaveStateTask%2C"
|
100
125
|
"+StatMapService%27+%7D%7D"
|
126
|
+
)
|
101
127
|
if res := s.get(save_state_map_url):
|
102
128
|
res.raise_for_status()
|
103
129
|
|
104
130
|
# Start search for address given by x / y coord
|
105
|
-
drilldown_map_url =
|
106
|
-
|
107
|
-
f"
|
108
|
-
"
|
131
|
+
drilldown_map_url = (
|
132
|
+
"https://maps.cheltenham.gov.uk/map/Aurora.svc/ExecuteTaskJob?"
|
133
|
+
f"sessionId={session_id}&taskId={drilldown_task_id}&job=%7B%22"
|
134
|
+
f"QueryX%22%3A{location_x}%2C%22QueryY%22%3A{location_y}%2C%22"
|
135
|
+
"Task%22%3A%7B%22Type%22%3A%22StatMap.Aurora.DrillDownTask%2C"
|
109
136
|
"+StatMapService%22%7D%7D"
|
137
|
+
)
|
110
138
|
if res := s.get(drilldown_map_url):
|
111
139
|
res.raise_for_status()
|
112
140
|
|
113
141
|
# Get results from search for address given by x / y coord
|
114
|
-
address_details_url =
|
115
|
-
|
116
|
-
f"&
|
117
|
-
f"
|
118
|
-
"
|
142
|
+
address_details_url = (
|
143
|
+
"https://maps.cheltenham.gov.uk/map/Aurora.svc/ExecuteTaskJob?"
|
144
|
+
f"sessionId={session_id}&taskId={retrieve_results_task_id}"
|
145
|
+
f"&job=%7B%22QueryX%22%3A{location_x}%2C%22QueryY%22%3A"
|
146
|
+
f"{location_y}%2C%22Task%22%3A%7B%22Type%22%3A%22"
|
147
|
+
"StatMap.Aurora.FetchResultSetTask%2C+StatMapService"
|
119
148
|
"%22%2C%22ResultSetName%22%3A%22inspection%22%7D%7D"
|
149
|
+
)
|
120
150
|
address_details_response = s.get(address_details_url)
|
121
151
|
address_details_response.raise_for_status()
|
122
152
|
# JSON response contained a BOM marker, skip first character
|
@@ -150,7 +180,9 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
150
180
|
# After we've got the correct result, pull out the week number each bin type is taken on
|
151
181
|
if (refuse_week_raw := result_dict.get("New_Refuse_Week".upper())) is not None:
|
152
182
|
refuse_week = int(refuse_week_raw)
|
153
|
-
if (
|
183
|
+
if (
|
184
|
+
recycling_week_raw := result_dict.get("New_Recycling_Week".upper())
|
185
|
+
) is not None:
|
154
186
|
recycling_week = int(recycling_week_raw)
|
155
187
|
if (garden_week_raw := result_dict.get("Garden_Bin_Week".upper())) is not None:
|
156
188
|
garden_week = int(garden_week_raw)
|
@@ -169,13 +201,17 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
169
201
|
]
|
170
202
|
|
171
203
|
refuse_day_offset = days_of_week.index(
|
172
|
-
str(result_dict.get("New_Refuse_Day_internal".upper())).upper()
|
204
|
+
str(result_dict.get("New_Refuse_Day_internal".upper())).upper()
|
205
|
+
)
|
173
206
|
recycling_day_offset = days_of_week.index(
|
174
|
-
str(result_dict.get("New_Recycling_Day".upper())).upper()
|
207
|
+
str(result_dict.get("New_Recycling_Day".upper())).upper()
|
208
|
+
)
|
175
209
|
garden_day_offset = days_of_week.index(
|
176
|
-
str(result_dict.get("New_Garden_Day".upper())).upper()
|
210
|
+
str(result_dict.get("New_Garden_Day".upper())).upper()
|
211
|
+
)
|
177
212
|
food_day_offset = days_of_week.index(
|
178
|
-
str(result_dict.get("New_Food_Day".upper())).upper()
|
213
|
+
str(result_dict.get("New_Food_Day".upper())).upper()
|
214
|
+
)
|
179
215
|
|
180
216
|
# Initialise WEEK-1/WEEK-2 based on known details
|
181
217
|
week_1_epoch = datetime(2025, 1, 13)
|
@@ -186,27 +222,20 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
186
222
|
# If there's an even number of weeks between the week-1
|
187
223
|
# epoch and this week, then this week is of type week-1
|
188
224
|
if (((this_week - week_1_epoch).days // 7)) % 2 == 0:
|
189
|
-
week = {
|
190
|
-
1: this_week,
|
191
|
-
2: this_week + timedelta(days=7)
|
192
|
-
}
|
225
|
+
week = {1: this_week, 2: this_week + timedelta(days=7)}
|
193
226
|
else:
|
194
|
-
week = {
|
195
|
-
1: this_week - timedelta(days=7),
|
196
|
-
2: this_week
|
197
|
-
}
|
227
|
+
week = {1: this_week - timedelta(days=7), 2: this_week}
|
198
228
|
|
199
|
-
refuse_dates: list[str] = get_dates_every_x_days(
|
200
|
-
week[refuse_week], 14, 28)
|
229
|
+
refuse_dates: list[str] = get_dates_every_x_days(week[refuse_week], 14, 28)
|
201
230
|
recycling_dates: list[str] = get_dates_every_x_days(
|
202
|
-
week[recycling_week], 14, 28
|
203
|
-
|
204
|
-
|
231
|
+
week[recycling_week], 14, 28
|
232
|
+
)
|
233
|
+
garden_dates: list[str] = get_dates_every_x_days(week[garden_week], 14, 28)
|
205
234
|
|
206
235
|
for refuse_date in refuse_dates:
|
207
236
|
collection_date = (
|
208
|
-
datetime.strptime(refuse_date, "%d/%m/%Y")
|
209
|
-
timedelta(days=refuse_day_offset)
|
237
|
+
datetime.strptime(refuse_date, "%d/%m/%Y")
|
238
|
+
+ timedelta(days=refuse_day_offset)
|
210
239
|
).strftime("%d/%m/%Y")
|
211
240
|
|
212
241
|
dict_data = {
|
@@ -218,8 +247,8 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
218
247
|
for recycling_date in recycling_dates:
|
219
248
|
|
220
249
|
collection_date = (
|
221
|
-
datetime.strptime(recycling_date, "%d/%m/%Y")
|
222
|
-
timedelta(days=recycling_day_offset)
|
250
|
+
datetime.strptime(recycling_date, "%d/%m/%Y")
|
251
|
+
+ timedelta(days=recycling_day_offset)
|
223
252
|
).strftime("%d/%m/%Y")
|
224
253
|
|
225
254
|
dict_data = {
|
@@ -231,8 +260,8 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
231
260
|
for garden_date in garden_dates:
|
232
261
|
|
233
262
|
collection_date = (
|
234
|
-
datetime.strptime(garden_date, "%d/%m/%Y")
|
235
|
-
timedelta(days=garden_day_offset)
|
263
|
+
datetime.strptime(garden_date, "%d/%m/%Y")
|
264
|
+
+ timedelta(days=garden_day_offset)
|
236
265
|
).strftime("%d/%m/%Y")
|
237
266
|
|
238
267
|
dict_data = {
|
@@ -241,15 +270,18 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
241
270
|
}
|
242
271
|
bindata["bins"].append(dict_data)
|
243
272
|
|
244
|
-
if (
|
245
|
-
|
273
|
+
if (
|
274
|
+
food_waste_week := str(
|
275
|
+
result_dict.get("FOOD_WASTE_WEEK_EXTERNAL", "")
|
276
|
+
).upper()
|
277
|
+
) == "weekly".upper():
|
246
278
|
food_dates: list[str] = get_dates_every_x_days(week[1], 7, 56)
|
247
279
|
|
248
280
|
for food_date in food_dates:
|
249
281
|
|
250
282
|
collection_date = (
|
251
|
-
datetime.strptime(food_date, "%d/%m/%Y")
|
252
|
-
timedelta(days=food_day_offset)
|
283
|
+
datetime.strptime(food_date, "%d/%m/%Y")
|
284
|
+
+ timedelta(days=food_day_offset)
|
253
285
|
).strftime("%d/%m/%Y")
|
254
286
|
|
255
287
|
dict_data = {
|
@@ -266,21 +298,24 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
266
298
|
first_week = int(first_week.strip())
|
267
299
|
|
268
300
|
second_week_day, _, second_week_number = second_week_detail.partition(
|
269
|
-
"WEEK"
|
301
|
+
"WEEK"
|
302
|
+
)
|
270
303
|
second_week_number = int(second_week_number.strip())
|
271
304
|
second_week_day: str = second_week_day.strip()[:3]
|
272
305
|
|
273
306
|
food_dates_first: list[str] = get_dates_every_x_days(
|
274
|
-
week[first_week], 14, 28
|
307
|
+
week[first_week], 14, 28
|
308
|
+
)
|
275
309
|
food_dates_second: list[str] = get_dates_every_x_days(
|
276
|
-
week[second_week_number], 14, 28
|
310
|
+
week[second_week_number], 14, 28
|
311
|
+
)
|
277
312
|
second_week_offset = days_of_week.index(second_week_day)
|
278
313
|
|
279
314
|
for food_date in food_dates_first:
|
280
315
|
|
281
316
|
collection_date = (
|
282
|
-
datetime.strptime(food_date, "%d/%m/%Y")
|
283
|
-
timedelta(days=food_day_offset)
|
317
|
+
datetime.strptime(food_date, "%d/%m/%Y")
|
318
|
+
+ timedelta(days=food_day_offset)
|
284
319
|
).strftime("%d/%m/%Y")
|
285
320
|
|
286
321
|
dict_data = {
|
@@ -291,8 +326,8 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
291
326
|
for food_date in food_dates_second:
|
292
327
|
|
293
328
|
collection_date = (
|
294
|
-
datetime.strptime(food_date, "%d/%m/%Y")
|
295
|
-
timedelta(days=second_week_offset)
|
329
|
+
datetime.strptime(food_date, "%d/%m/%Y")
|
330
|
+
+ timedelta(days=second_week_offset)
|
296
331
|
).strftime("%d/%m/%Y")
|
297
332
|
|
298
333
|
dict_data = {
|
@@ -302,7 +337,6 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
302
337
|
bindata["bins"].append(dict_data)
|
303
338
|
|
304
339
|
bindata["bins"].sort(
|
305
|
-
key=lambda x: datetime.strptime(
|
306
|
-
x.get("collectionDate", ""), "%d/%m/%Y")
|
340
|
+
key=lambda x: datetime.strptime(x.get("collectionDate", ""), "%d/%m/%Y")
|
307
341
|
)
|
308
342
|
return bindata
|
@@ -6,6 +6,7 @@ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataC
|
|
6
6
|
This module provides bin collection data for Cheshire East Council.
|
7
7
|
"""
|
8
8
|
|
9
|
+
|
9
10
|
class CouncilClass(AbstractGetBinDataClass):
|
10
11
|
"""
|
11
12
|
A class to fetch and parse bin collection data for Cheshire East Council.
|
@@ -20,7 +20,10 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
20
20
|
curr_date = datetime.today()
|
21
21
|
|
22
22
|
soup = BeautifulSoup(page.content, features="html.parser")
|
23
|
-
button = soup.find(
|
23
|
+
button = soup.find(
|
24
|
+
"a",
|
25
|
+
text="Find out which bin will be collected when and sign up for a free email reminder.",
|
26
|
+
)
|
24
27
|
|
25
28
|
if button["href"]:
|
26
29
|
URI = button["href"]
|
@@ -12,6 +12,7 @@ from uk_bin_collection.uk_bin_collection.common import *
|
|
12
12
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
13
13
|
|
14
14
|
# import the wonderful Beautiful Soup and the URL grabber
|
15
|
+
import re
|
15
16
|
|
16
17
|
|
17
18
|
class CouncilClass(AbstractGetBinDataClass):
|
@@ -42,72 +43,82 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
42
43
|
wait = WebDriverWait(driver, 60)
|
43
44
|
address_entry_field = wait.until(
|
44
45
|
EC.presence_of_element_located(
|
45
|
-
(By.XPATH, '//*[@
|
46
|
+
(By.XPATH, '//*[@placeholder="Search Properties..."]')
|
46
47
|
)
|
47
48
|
)
|
48
49
|
|
49
50
|
address_entry_field.send_keys(str(full_address))
|
50
51
|
|
51
52
|
address_entry_field = wait.until(
|
52
|
-
EC.element_to_be_clickable((By.XPATH, '//*[@
|
53
|
+
EC.element_to_be_clickable((By.XPATH, f'//*[@title="{full_address}"]'))
|
53
54
|
)
|
54
55
|
address_entry_field.click()
|
55
|
-
address_entry_field.send_keys(Keys.BACKSPACE)
|
56
|
-
address_entry_field.send_keys(str(full_address[len(full_address) - 1]))
|
57
56
|
|
58
|
-
|
57
|
+
next_button = wait.until(
|
59
58
|
EC.element_to_be_clickable(
|
60
|
-
(By.XPATH,
|
59
|
+
(By.XPATH, "//lightning-button/button[contains(text(), 'Next')]")
|
61
60
|
)
|
62
61
|
)
|
62
|
+
next_button.click()
|
63
63
|
|
64
|
-
|
65
|
-
# Wait for the 'Select your property' dropdown to appear and select the first result
|
66
|
-
next_btn = wait.until(
|
67
|
-
EC.element_to_be_clickable((By.XPATH, "//lightning-button/button"))
|
68
|
-
)
|
69
|
-
next_btn.click()
|
70
|
-
bin_data = wait.until(
|
64
|
+
result = wait.until(
|
71
65
|
EC.presence_of_element_located(
|
72
|
-
(
|
66
|
+
(
|
67
|
+
By.XPATH,
|
68
|
+
'//table[@class="slds-table slds-table_header-fixed slds-table_bordered slds-table_edit slds-table_resizable-cols"]',
|
69
|
+
)
|
73
70
|
)
|
74
71
|
)
|
75
72
|
|
76
|
-
|
73
|
+
# Make a BS4 object
|
74
|
+
soup = BeautifulSoup(
|
75
|
+
result.get_attribute("innerHTML"), features="html.parser"
|
76
|
+
) # Wait for the 'Select your property' dropdown to appear and select the first result
|
77
77
|
|
78
|
+
data = {"bins": []}
|
79
|
+
today = datetime.now()
|
80
|
+
current_year = today.year
|
81
|
+
|
82
|
+
# Find all bin rows in the table
|
78
83
|
rows = soup.find_all("tr", class_="slds-hint-parent")
|
79
|
-
current_year = datetime.now().year
|
80
84
|
|
81
85
|
for row in rows:
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
else:
|
92
|
-
collection_day = re.sub(
|
93
|
-
r"[^a-zA-Z0-9,\s]", "", columns[0].get_text()
|
94
|
-
).strip()
|
95
|
-
|
96
|
-
# Parse the date from the string
|
97
|
-
parsed_date = datetime.strptime(collection_day, "%a, %d %B")
|
98
|
-
if parsed_date < datetime(
|
99
|
-
parsed_date.year, parsed_date.month, parsed_date.day
|
100
|
-
):
|
101
|
-
parsed_date = parsed_date.replace(year=current_year + 1)
|
102
|
-
else:
|
103
|
-
parsed_date = parsed_date.replace(year=current_year)
|
104
|
-
# Format the date as %d/%m/%Y
|
105
|
-
formatted_date = parsed_date.strftime("%d/%m/%Y")
|
86
|
+
try:
|
87
|
+
bin_type_cell = row.find("th")
|
88
|
+
date_cell = row.find("td")
|
89
|
+
|
90
|
+
if not bin_type_cell or not date_cell:
|
91
|
+
continue
|
92
|
+
|
93
|
+
container_type = bin_type_cell.get("data-cell-value", "").strip()
|
94
|
+
raw_date_text = date_cell.get("data-cell-value", "").strip()
|
106
95
|
|
107
|
-
#
|
96
|
+
# Handle relative values like "Today" or "Tomorrow"
|
97
|
+
if "today" in raw_date_text.lower():
|
98
|
+
parsed_date = today
|
99
|
+
elif "tomorrow" in raw_date_text.lower():
|
100
|
+
parsed_date = today + timedelta(days=1)
|
101
|
+
else:
|
102
|
+
# Expected format: "Thu, 10 April"
|
103
|
+
# Strip any rogue characters and try parsing
|
104
|
+
cleaned_date = re.sub(r"[^\w\s,]", "", raw_date_text)
|
105
|
+
try:
|
106
|
+
parsed_date = datetime.strptime(cleaned_date, "%a, %d %B")
|
107
|
+
parsed_date = parsed_date.replace(year=current_year)
|
108
|
+
if parsed_date < today:
|
109
|
+
# Date has passed this year, must be next year
|
110
|
+
parsed_date = parsed_date.replace(year=current_year + 1)
|
111
|
+
except Exception as e:
|
112
|
+
print(f"Could not parse date '{cleaned_date}': {e}")
|
113
|
+
continue
|
114
|
+
|
115
|
+
formatted_date = parsed_date.strftime(date_format)
|
108
116
|
data["bins"].append(
|
109
117
|
{"type": container_type, "collectionDate": formatted_date}
|
110
118
|
)
|
119
|
+
|
120
|
+
except Exception as e:
|
121
|
+
print(f"Error processing row: {e}")
|
111
122
|
except Exception as e:
|
112
123
|
# Here you can log the exception if needed
|
113
124
|
print(f"An error occurred: {e}")
|
@@ -30,9 +30,8 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
30
30
|
except Exception:
|
31
31
|
continue
|
32
32
|
|
33
|
-
bindata["bins"].append(
|
34
|
-
"type": event.name,
|
35
|
-
|
36
|
-
})
|
33
|
+
bindata["bins"].append(
|
34
|
+
{"type": event.name, "collectionDate": collection_date}
|
35
|
+
)
|
37
36
|
|
38
37
|
return bindata
|
@@ -50,12 +50,18 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
50
50
|
for index, bin_type in enumerate(bin_types):
|
51
51
|
# currently only handled weekly and garden collection, special collections like Christmas Day need to be added
|
52
52
|
if index == WEEKLY_COLLECTION:
|
53
|
-
next_collection_date = get_next_day_of_week(
|
53
|
+
next_collection_date = get_next_day_of_week(
|
54
|
+
collection_days[index].text.strip(), date_format
|
55
|
+
)
|
54
56
|
elif index == GARDEN_COLLECTION:
|
55
57
|
split_date_part = collection_days[index].text.split("More dates")[0]
|
56
|
-
next_collection_date = datetime.strptime(
|
58
|
+
next_collection_date = datetime.strptime(
|
59
|
+
split_date_part.strip(), "%d %B %Y"
|
60
|
+
).strftime(date_format)
|
57
61
|
else:
|
58
|
-
next_collection_date = datetime.strptime(
|
62
|
+
next_collection_date = datetime.strptime(
|
63
|
+
collection_days[index].text.strip(), "%d %B %Y"
|
64
|
+
).strftime(date_format)
|
59
65
|
|
60
66
|
dict_data = {
|
61
67
|
"type": bin_type.text.strip(),
|
@@ -83,16 +89,12 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
83
89
|
|
84
90
|
def input_street_name(self, street_name, wait):
|
85
91
|
input_element_postcodesearch = wait.until(
|
86
|
-
EC.visibility_of_element_located(
|
87
|
-
(By.ID, "Street")
|
88
|
-
)
|
92
|
+
EC.visibility_of_element_located((By.ID, "Street"))
|
89
93
|
)
|
90
94
|
input_element_postcodesearch.send_keys(street_name)
|
91
95
|
|
92
96
|
def dismiss_cookie_banner(self, wait):
|
93
97
|
cookie_banner = wait.until(
|
94
|
-
EC.visibility_of_element_located(
|
95
|
-
(By.ID, "ccc-dismiss-button")
|
96
|
-
)
|
98
|
+
EC.visibility_of_element_located((By.ID, "ccc-dismiss-button"))
|
97
99
|
)
|
98
100
|
cookie_banner.send_keys(Keys.ENTER)
|
@@ -12,6 +12,7 @@ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataC
|
|
12
12
|
|
13
13
|
import re
|
14
14
|
|
15
|
+
|
15
16
|
class CouncilClass(AbstractGetBinDataClass):
|
16
17
|
def parse_data(self, page: str, **kwargs) -> dict:
|
17
18
|
try:
|
@@ -63,19 +64,27 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
63
64
|
|
64
65
|
# **Regex to match "Wednesday, February 19" format**
|
65
66
|
match = re.match(r"([A-Za-z]+), ([A-Za-z]+) (\d{1,2})", raw_date)
|
66
|
-
|
67
|
+
|
67
68
|
if match:
|
68
|
-
day_name, month_name, day_number =
|
69
|
+
day_name, month_name, day_number = (
|
70
|
+
match.groups()
|
71
|
+
) # Extract components
|
69
72
|
extracted_month = datetime.strptime(month_name, "%B").month
|
70
73
|
extracted_day = int(day_number)
|
71
74
|
|
72
75
|
# Handle Dec-Jan rollover: If month is before the current month, assume next year
|
73
|
-
inferred_year =
|
76
|
+
inferred_year = (
|
77
|
+
current_year + 1
|
78
|
+
if extracted_month < current_month
|
79
|
+
else current_year
|
80
|
+
)
|
74
81
|
|
75
82
|
# **Correct the raw_date format before parsing**
|
76
83
|
raw_date = f"{day_name}, {month_name} {day_number}, {inferred_year}"
|
77
84
|
|
78
|
-
print(
|
85
|
+
print(
|
86
|
+
f"DEBUG: Final raw_date before parsing -> {raw_date}"
|
87
|
+
) # Debugging output
|
79
88
|
|
80
89
|
# Convert to required format (%d/%m/%Y)
|
81
90
|
try:
|
@@ -43,7 +43,11 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
43
43
|
|
44
44
|
# Extract collection date (e.g., "Monday 9th December")
|
45
45
|
date_tag = panel.find("p")
|
46
|
-
if
|
46
|
+
if (
|
47
|
+
date_tag
|
48
|
+
and "Your next collection date is"
|
49
|
+
in date_tag.text.strip().replace("\r", "").replace("\n", "")
|
50
|
+
):
|
47
51
|
collection_date = date_tag.find("strong").text.strip()
|
48
52
|
else:
|
49
53
|
continue
|
@@ -127,9 +127,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
127
127
|
# Garden waste
|
128
128
|
garden_waste = soup.find("div", class_="eb-2HIpCnWC-Override-EditorInput")
|
129
129
|
if garden_waste:
|
130
|
-
match = re.search(
|
131
|
-
r"(\d{2}/\d{2}/\d{4})", garden_waste.text
|
132
|
-
)
|
130
|
+
match = re.search(r"(\d{2}/\d{2}/\d{4})", garden_waste.text)
|
133
131
|
if match:
|
134
132
|
bins.append(
|
135
133
|
{"type": "Garden waste", "collectionDate": match.group(1)}
|