uk_bin_collection 0.134.1__py3-none-any.whl → 0.135.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- uk_bin_collection/tests/check_selenium_url_in_input.json.py +209 -0
- uk_bin_collection/tests/input.json +55 -5
- uk_bin_collection/uk_bin_collection/councils/AmberValleyBoroughCouncil.py +60 -0
- uk_bin_collection/uk_bin_collection/councils/BolsoverCouncil.py +298 -0
- uk_bin_collection/uk_bin_collection/councils/CherwellDistrictCouncil.py +75 -0
- uk_bin_collection/uk_bin_collection/councils/ConwyCountyBorough.py +11 -3
- uk_bin_collection/uk_bin_collection/councils/CotswoldDistrictCouncil.py +3 -5
- uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py +54 -50
- uk_bin_collection/uk_bin_collection/councils/EpsomandEwellBoroughCouncil.py +86 -0
- uk_bin_collection/uk_bin_collection/councils/GloucesterCityCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py +2 -1
- uk_bin_collection/uk_bin_collection/councils/MiddlesbroughCouncil.py +100 -0
- uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py +2 -0
- uk_bin_collection/uk_bin_collection/councils/NorthYorkshire.py +17 -15
- uk_bin_collection/uk_bin_collection/councils/RedcarandClevelandCouncil.py +108 -0
- uk_bin_collection/uk_bin_collection/councils/RunnymedeBoroughCouncil.py +54 -0
- uk_bin_collection/uk_bin_collection/councils/SunderlandCityCouncil.py +21 -15
- uk_bin_collection/uk_bin_collection/councils/TendringDistrictCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/TorridgeDistrictCouncil.py +1 -35
- {uk_bin_collection-0.134.1.dist-info → uk_bin_collection-0.135.2.dist-info}/METADATA +1 -1
- {uk_bin_collection-0.134.1.dist-info → uk_bin_collection-0.135.2.dist-info}/RECORD +24 -16
- {uk_bin_collection-0.134.1.dist-info → uk_bin_collection-0.135.2.dist-info}/LICENSE +0 -0
- {uk_bin_collection-0.134.1.dist-info → uk_bin_collection-0.135.2.dist-info}/WHEEL +0 -0
- {uk_bin_collection-0.134.1.dist-info → uk_bin_collection-0.135.2.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,298 @@
|
|
1
|
+
import time
|
2
|
+
|
3
|
+
import requests
|
4
|
+
|
5
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
6
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
7
|
+
|
8
|
+
|
9
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
10
|
+
class CouncilClass(AbstractGetBinDataClass):
|
11
|
+
"""
|
12
|
+
Concrete classes have to implement all abstract operations of the
|
13
|
+
base class. They can also override some operations with a default
|
14
|
+
implementation.
|
15
|
+
"""
|
16
|
+
|
17
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
18
|
+
|
19
|
+
user_uprn = kwargs.get("uprn")
|
20
|
+
check_uprn(user_uprn)
|
21
|
+
bindata = {"bins": []}
|
22
|
+
|
23
|
+
SESSION_URL = "https://selfservice.bolsover.gov.uk/authapi/isauthenticated?uri=https%253A%252F%252Fselfservice.bolsover.gov.uk%252Fservice%252FCheck_your_Bin_Day&hostname=selfservice.bolsover.gov.uk&withCredentials=true"
|
24
|
+
|
25
|
+
API_URL = "https://selfservice.bolsover.gov.uk/apibroker/runLookup"
|
26
|
+
|
27
|
+
data = {
|
28
|
+
"formValues": {"Bin Collection": {"uprnLoggedIn": {"value": user_uprn}}},
|
29
|
+
}
|
30
|
+
headers = {
|
31
|
+
"Content-Type": "application/json",
|
32
|
+
"Accept": "application/json",
|
33
|
+
"User-Agent": "Mozilla/5.0",
|
34
|
+
"X-Requested-With": "XMLHttpRequest",
|
35
|
+
"Referer": "https://selfservice.bolsover.gov.uk/fillform/?iframe_id=fillform-frame-1&db_id=",
|
36
|
+
}
|
37
|
+
s = requests.session()
|
38
|
+
r = s.get(SESSION_URL)
|
39
|
+
r.raise_for_status()
|
40
|
+
session_data = r.json()
|
41
|
+
sid = session_data["auth-session"]
|
42
|
+
params = {
|
43
|
+
"id": "6023d37e037c3",
|
44
|
+
"repeat_against": "",
|
45
|
+
"noRetry": "true",
|
46
|
+
"getOnlyTokens": "undefined",
|
47
|
+
"log_id": "",
|
48
|
+
"app_name": "AF-Renderer::Self",
|
49
|
+
# unix_timestamp
|
50
|
+
"_": str(int(time.time() * 1000)),
|
51
|
+
"sid": sid,
|
52
|
+
}
|
53
|
+
|
54
|
+
r = s.post(API_URL, json=data, headers=headers, params=params)
|
55
|
+
r.raise_for_status()
|
56
|
+
|
57
|
+
data = r.json()
|
58
|
+
rows_data = data["integration"]["transformed"]["rows_data"]["0"]
|
59
|
+
if not isinstance(rows_data, dict):
|
60
|
+
raise ValueError("Invalid data returned from API")
|
61
|
+
|
62
|
+
# print(rows_data)
|
63
|
+
|
64
|
+
route = rows_data["Route"]
|
65
|
+
|
66
|
+
# print(route)
|
67
|
+
|
68
|
+
def get_route_number(route):
|
69
|
+
if route[:2] == "Mo":
|
70
|
+
return 0
|
71
|
+
elif route[:2] == "Tu":
|
72
|
+
return 1
|
73
|
+
elif route[:2] == "We":
|
74
|
+
return 2
|
75
|
+
elif route[:2] == "Th":
|
76
|
+
return 3
|
77
|
+
elif route[:2] == "Fr":
|
78
|
+
return 4
|
79
|
+
else:
|
80
|
+
return None # Default case if none of the conditions match
|
81
|
+
|
82
|
+
dayOfCollectionAsNumber = get_route_number(route)
|
83
|
+
# print(dayOfCollectionAsNumber)
|
84
|
+
|
85
|
+
def calculate_collection_date(
|
86
|
+
dayOfCollectionAsNumber,
|
87
|
+
currentDayAsNumber,
|
88
|
+
today,
|
89
|
+
dayDiffPlus,
|
90
|
+
dayDiffMinus,
|
91
|
+
):
|
92
|
+
if dayOfCollectionAsNumber == currentDayAsNumber:
|
93
|
+
return today
|
94
|
+
elif dayOfCollectionAsNumber > currentDayAsNumber:
|
95
|
+
return today + timedelta(days=dayDiffPlus)
|
96
|
+
else:
|
97
|
+
return today + timedelta(days=dayDiffMinus)
|
98
|
+
|
99
|
+
# Example usage
|
100
|
+
today = datetime.today() # Current date
|
101
|
+
currentDayAsNumber = today.weekday()
|
102
|
+
dayDiffPlus = dayOfCollectionAsNumber - currentDayAsNumber
|
103
|
+
dayDiffMinus = dayOfCollectionAsNumber - currentDayAsNumber + 7
|
104
|
+
|
105
|
+
week1 = calculate_collection_date(
|
106
|
+
dayOfCollectionAsNumber,
|
107
|
+
currentDayAsNumber,
|
108
|
+
today,
|
109
|
+
dayDiffPlus,
|
110
|
+
dayDiffMinus,
|
111
|
+
)
|
112
|
+
week2 = week1 + timedelta(days=7)
|
113
|
+
week3 = week2 + timedelta(days=7)
|
114
|
+
week4 = week3 + timedelta(days=7)
|
115
|
+
|
116
|
+
# print(week1.strftime(date_format))
|
117
|
+
# print(week2.strftime(date_format))
|
118
|
+
# print(week3.strftime(date_format))
|
119
|
+
# print(week4.strftime(date_format))
|
120
|
+
|
121
|
+
greenSusStart = datetime.strptime("2024-11-08", "%Y-%m-%d")
|
122
|
+
greenSusEnd = datetime.strptime("2025-03-18", "%Y-%m-%d")
|
123
|
+
|
124
|
+
def is_within_green_sus(dtDay0, greenSusStart, greenSusEnd):
|
125
|
+
return "Yes" if greenSusStart <= dtDay0 < greenSusEnd else "No"
|
126
|
+
|
127
|
+
week1InSus = is_within_green_sus(week1, greenSusStart, greenSusEnd)
|
128
|
+
week2InSus = is_within_green_sus(week2, greenSusStart, greenSusEnd)
|
129
|
+
week3InSus = is_within_green_sus(week3, greenSusStart, greenSusEnd)
|
130
|
+
week4InSus = is_within_green_sus(week4, greenSusStart, greenSusEnd)
|
131
|
+
|
132
|
+
# print(week1InSus)
|
133
|
+
# print(week2InSus)
|
134
|
+
# print(week3InSus)
|
135
|
+
# print(week4InSus)
|
136
|
+
|
137
|
+
WeekBlack = rows_data["WeekBlack"]
|
138
|
+
WeekBandG = rows_data["WeekBandG"]
|
139
|
+
|
140
|
+
if WeekBlack == "1":
|
141
|
+
WeekBandG = ""
|
142
|
+
if WeekBandG == "1":
|
143
|
+
WeekBlack = ""
|
144
|
+
|
145
|
+
def determine_bin_collection_week1(
|
146
|
+
txtBlack, txtBurgGreen, dtDay0, today, week1InSus
|
147
|
+
):
|
148
|
+
# Check for empty values
|
149
|
+
if txtBlack == "" and txtBurgGreen == "":
|
150
|
+
return ""
|
151
|
+
|
152
|
+
# Black Bin Collection
|
153
|
+
if txtBlack == "1" and dtDay0 >= today:
|
154
|
+
return "Black Bin"
|
155
|
+
|
156
|
+
# Burgundy Bin Collection
|
157
|
+
if txtBurgGreen == "1" and dtDay0 > today:
|
158
|
+
if week1InSus == "Yes":
|
159
|
+
return "Burgundy Bin"
|
160
|
+
elif week1InSus == "No":
|
161
|
+
return "Burgundy Bin & Green Bin"
|
162
|
+
|
163
|
+
# Default cases based on week1InSus
|
164
|
+
if txtBlack == "" and dtDay0 >= today:
|
165
|
+
if week1InSus == "Yes":
|
166
|
+
return "Burgundy Bin"
|
167
|
+
elif week1InSus == "No":
|
168
|
+
return "Burgundy Bin & Green Bin"
|
169
|
+
|
170
|
+
return "" # Default empty case
|
171
|
+
|
172
|
+
def determine_bin_collection_week2(
|
173
|
+
txtBlack, txtBurgGreen, dtDay7, today, week2InSus
|
174
|
+
):
|
175
|
+
# Check for empty values
|
176
|
+
if txtBlack == "" and txtBurgGreen == "":
|
177
|
+
return ""
|
178
|
+
|
179
|
+
# Black Bin Collection
|
180
|
+
if txtBlack == "" and dtDay7 >= today:
|
181
|
+
return "Black Bin"
|
182
|
+
|
183
|
+
# Burgundy Bin Collection (week2InSus check)
|
184
|
+
if txtBurgGreen == "1" and dtDay7 > today:
|
185
|
+
if week2InSus == "Yes":
|
186
|
+
return "Burgundy Bin"
|
187
|
+
elif week2InSus == "No":
|
188
|
+
return "Burgundy Bin & Green Bin"
|
189
|
+
|
190
|
+
# Burgundy Bin Collection for txtBlack = '1'
|
191
|
+
if txtBlack == "1" and dtDay7 >= today:
|
192
|
+
if week2InSus == "Yes":
|
193
|
+
return "Burgundy Bin"
|
194
|
+
elif week2InSus == "No":
|
195
|
+
return "Burgundy Bin & Green Bin"
|
196
|
+
|
197
|
+
return "" # Default empty case
|
198
|
+
|
199
|
+
def determine_bin_collection_week3(
|
200
|
+
txtBlack, txtBurgGreen, dtDay14, today, week3InSus
|
201
|
+
):
|
202
|
+
# Check for empty values
|
203
|
+
if txtBlack == "" and txtBurgGreen == "":
|
204
|
+
return ""
|
205
|
+
|
206
|
+
# Black Bin Collection
|
207
|
+
if txtBlack == "1" and dtDay14 >= today:
|
208
|
+
return "Black Bin"
|
209
|
+
|
210
|
+
# Burgundy Bin Collection (week3InSus check)
|
211
|
+
if txtBurgGreen == "1" and dtDay14 > today:
|
212
|
+
if week3InSus == "Yes":
|
213
|
+
return "Burgundy Bin"
|
214
|
+
elif week3InSus == "No":
|
215
|
+
return "Burgundy Bin & Green Bin"
|
216
|
+
|
217
|
+
# Burgundy Bin Collection for txtBlack = ''
|
218
|
+
if txtBlack == "" and dtDay14 >= today:
|
219
|
+
if week3InSus == "Yes":
|
220
|
+
return "Burgundy Bin"
|
221
|
+
elif week3InSus == "No":
|
222
|
+
return "Burgundy Bin & Green Bin"
|
223
|
+
|
224
|
+
return "" # Default empty case
|
225
|
+
|
226
|
+
def determine_bin_collection_week4(
|
227
|
+
txtBlack, txtBurgGreen, dtDay21, today, week4InSus
|
228
|
+
):
|
229
|
+
# Check for empty values
|
230
|
+
if txtBlack == "" and txtBurgGreen == "":
|
231
|
+
return ""
|
232
|
+
|
233
|
+
# Black Bin Collection
|
234
|
+
if txtBlack == "" and dtDay21 >= today:
|
235
|
+
return "Black Bin"
|
236
|
+
|
237
|
+
# Burgundy Bin Collection (week4InSus check)
|
238
|
+
if txtBurgGreen == "1" and dtDay21 > today:
|
239
|
+
if week4InSus == "Yes":
|
240
|
+
return "Burgundy Bin"
|
241
|
+
elif week4InSus == "No":
|
242
|
+
return "Burgundy Bin & Green Bin"
|
243
|
+
|
244
|
+
# Burgundy Bin Collection for txtBlack = '1'
|
245
|
+
if txtBlack == "1" and dtDay21 >= today:
|
246
|
+
if week4InSus == "Yes":
|
247
|
+
return "Burgundy Bin"
|
248
|
+
elif week4InSus == "No":
|
249
|
+
return "Burgundy Bin & Green Bin"
|
250
|
+
|
251
|
+
return "" # Default empty case
|
252
|
+
|
253
|
+
week1Text = determine_bin_collection_week1(
|
254
|
+
WeekBlack, WeekBandG, week1, today, week1InSus
|
255
|
+
)
|
256
|
+
week2Text = determine_bin_collection_week2(
|
257
|
+
WeekBlack, WeekBandG, week2, today, week2InSus
|
258
|
+
)
|
259
|
+
week3Text = determine_bin_collection_week3(
|
260
|
+
WeekBlack, WeekBandG, week3, today, week3InSus
|
261
|
+
)
|
262
|
+
week4Text = determine_bin_collection_week4(
|
263
|
+
WeekBlack, WeekBandG, week4, today, week4InSus
|
264
|
+
)
|
265
|
+
|
266
|
+
# print(week1Text)
|
267
|
+
# print(week2Text)
|
268
|
+
# print(week3Text)
|
269
|
+
# print(week4Text)
|
270
|
+
|
271
|
+
week_data = [
|
272
|
+
(week1Text, week1),
|
273
|
+
(week2Text, week2),
|
274
|
+
(week3Text, week3),
|
275
|
+
(week4Text, week4),
|
276
|
+
]
|
277
|
+
|
278
|
+
# print(week_data)
|
279
|
+
|
280
|
+
# Iterate through the array
|
281
|
+
for week_text, week_date in week_data:
|
282
|
+
# Check if '&' exists and split
|
283
|
+
if "&" in week_text:
|
284
|
+
split_texts = [text.strip() for text in week_text.split("&")]
|
285
|
+
for text in split_texts:
|
286
|
+
dict_data = {
|
287
|
+
"type": text,
|
288
|
+
"collectionDate": week_date.strftime(date_format),
|
289
|
+
}
|
290
|
+
bindata["bins"].append(dict_data)
|
291
|
+
else:
|
292
|
+
dict_data = {
|
293
|
+
"type": week_text,
|
294
|
+
"collectionDate": week_date.strftime(date_format),
|
295
|
+
}
|
296
|
+
bindata["bins"].append(dict_data)
|
297
|
+
|
298
|
+
return bindata
|
@@ -0,0 +1,75 @@
|
|
1
|
+
from datetime import datetime, timedelta
|
2
|
+
|
3
|
+
import requests
|
4
|
+
from bs4 import BeautifulSoup
|
5
|
+
|
6
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
7
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
8
|
+
|
9
|
+
|
10
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
11
|
+
class CouncilClass(AbstractGetBinDataClass):
|
12
|
+
"""
|
13
|
+
Concrete classes have to implement all abstract operations of the
|
14
|
+
base class. They can also override some operations with a default
|
15
|
+
implementation.
|
16
|
+
"""
|
17
|
+
|
18
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
19
|
+
|
20
|
+
user_uprn = kwargs.get("uprn")
|
21
|
+
check_uprn(user_uprn)
|
22
|
+
bindata = {"bins": []}
|
23
|
+
|
24
|
+
URI = f"https://www.cherwell.gov.uk/homepage/129/bin-collection-search?uprn={user_uprn}"
|
25
|
+
|
26
|
+
# Make the GET request
|
27
|
+
response = requests.get(URI)
|
28
|
+
|
29
|
+
soup = BeautifulSoup(response.text, "html.parser")
|
30
|
+
|
31
|
+
def get_full_date(date_str):
|
32
|
+
# Get the current year
|
33
|
+
current_year = datetime.today().year
|
34
|
+
|
35
|
+
date_str = remove_ordinal_indicator_from_date_string(date_str)
|
36
|
+
|
37
|
+
# Convert the input string to a datetime object (assuming the current year first)
|
38
|
+
date_obj = datetime.strptime(f"{date_str} {current_year}", "%d %B %Y")
|
39
|
+
|
40
|
+
# If the date has already passed this year, use next year
|
41
|
+
if date_obj < datetime.today():
|
42
|
+
date_obj = datetime.strptime(
|
43
|
+
f"{date_str} {current_year + 1}", "%d %B %Y"
|
44
|
+
)
|
45
|
+
|
46
|
+
return date_obj.strftime(date_format) # Return in YYYY-MM-DD format
|
47
|
+
|
48
|
+
# print(soup)
|
49
|
+
|
50
|
+
div = soup.find("div", class_="bin-collection-results__tasks")
|
51
|
+
|
52
|
+
for item in div.find_all("li", class_="list__item"):
|
53
|
+
# Extract bin type
|
54
|
+
bin_type_tag = item.find("h3", class_="bin-collection-tasks__heading")
|
55
|
+
bin_type = (
|
56
|
+
"".join(bin_type_tag.find_all(text=True, recursive=False)).strip()
|
57
|
+
if bin_type_tag
|
58
|
+
else "Unknown Bin"
|
59
|
+
)
|
60
|
+
|
61
|
+
# Extract collection date
|
62
|
+
date_tag = item.find("p", class_="bin-collection-tasks__date")
|
63
|
+
collection_date = date_tag.text.strip() if date_tag else "Unknown Date"
|
64
|
+
|
65
|
+
dict_data = {
|
66
|
+
"type": bin_type,
|
67
|
+
"collectionDate": get_full_date(collection_date),
|
68
|
+
}
|
69
|
+
bindata["bins"].append(dict_data)
|
70
|
+
|
71
|
+
bindata["bins"].sort(
|
72
|
+
key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
|
73
|
+
)
|
74
|
+
|
75
|
+
return bindata
|
@@ -1,12 +1,20 @@
|
|
1
|
+
from datetime import datetime
|
2
|
+
|
3
|
+
import requests
|
1
4
|
from bs4 import BeautifulSoup
|
2
|
-
|
5
|
+
|
3
6
|
from uk_bin_collection.uk_bin_collection.common import *
|
4
|
-
from
|
7
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
5
8
|
|
6
9
|
|
7
10
|
class CouncilClass(AbstractGetBinDataClass):
|
8
11
|
def parse_data(self, page: str, **kwargs) -> dict:
|
9
|
-
|
12
|
+
user_uprn = kwargs.get("uprn")
|
13
|
+
check_uprn(user_uprn)
|
14
|
+
uri = f"https://www.conwy.gov.uk/Contensis-Forms/erf/collection-result-soap-xmas2025.asp?ilangid=1&uprn={user_uprn}"
|
15
|
+
|
16
|
+
response = requests.get(uri)
|
17
|
+
soup = BeautifulSoup(response.content, features="html.parser")
|
10
18
|
data = {"bins": []}
|
11
19
|
|
12
20
|
for bin_section in soup.select('div[class*="containererf"]'):
|
@@ -41,15 +41,13 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
41
41
|
# If you bang in the house number (or property name) and postcode in the box it should find your property
|
42
42
|
wait = WebDriverWait(driver, 60)
|
43
43
|
address_entry_field = wait.until(
|
44
|
-
EC.
|
45
|
-
(By.XPATH, '//*[@id="combobox-input-20"]')
|
46
|
-
)
|
44
|
+
EC.element_to_be_clickable((By.XPATH, '//*[@id="combobox-input-22"]'))
|
47
45
|
)
|
48
46
|
|
49
47
|
address_entry_field.send_keys(str(full_address))
|
50
48
|
|
51
49
|
address_entry_field = wait.until(
|
52
|
-
EC.element_to_be_clickable((By.XPATH, '//*[@id="combobox-input-
|
50
|
+
EC.element_to_be_clickable((By.XPATH, '//*[@id="combobox-input-22"]'))
|
53
51
|
)
|
54
52
|
address_entry_field.click()
|
55
53
|
address_entry_field.send_keys(Keys.BACKSPACE)
|
@@ -57,7 +55,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
57
55
|
|
58
56
|
first_found_address = wait.until(
|
59
57
|
EC.element_to_be_clickable(
|
60
|
-
(By.XPATH, '//*[@id="dropdown-element-
|
58
|
+
(By.XPATH, '//*[@id="dropdown-element-22"]/ul')
|
61
59
|
)
|
62
60
|
)
|
63
61
|
|
@@ -1,8 +1,5 @@
|
|
1
|
+
import requests
|
1
2
|
from bs4 import BeautifulSoup
|
2
|
-
from selenium.webdriver.common.by import By
|
3
|
-
from selenium.webdriver.support import expected_conditions as EC
|
4
|
-
from selenium.webdriver.support.ui import Select
|
5
|
-
from selenium.webdriver.support.wait import WebDriverWait
|
6
3
|
|
7
4
|
from uk_bin_collection.uk_bin_collection.common import *
|
8
5
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
@@ -19,63 +16,70 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
19
16
|
def parse_data(self, page: str, **kwargs) -> dict:
|
20
17
|
driver = None
|
21
18
|
try:
|
22
|
-
|
19
|
+
uri = "https://selfserve.derbyshiredales.gov.uk/renderform.aspx?t=103&k=9644C066D2168A4C21BCDA351DA2642526359DFF"
|
23
20
|
|
24
|
-
|
21
|
+
bindata = {"bins": []}
|
25
22
|
|
26
23
|
user_uprn = kwargs.get("uprn")
|
27
24
|
user_postcode = kwargs.get("postcode")
|
28
|
-
web_driver = kwargs.get("web_driver")
|
29
|
-
headless = kwargs.get("headless")
|
30
25
|
check_uprn(user_uprn)
|
31
26
|
check_postcode(user_postcode)
|
32
27
|
|
33
|
-
#
|
34
|
-
|
35
|
-
driver.get(page)
|
28
|
+
# Start a session
|
29
|
+
session = requests.Session()
|
36
30
|
|
37
|
-
|
38
|
-
inputElement_postcode = driver.find_element(
|
39
|
-
By.ID,
|
40
|
-
"ctl00_ContentPlaceHolder1_FF2924TB",
|
41
|
-
)
|
42
|
-
inputElement_postcode.send_keys(user_postcode)
|
43
|
-
|
44
|
-
# Click search button
|
45
|
-
driver.find_element(
|
46
|
-
By.ID,
|
47
|
-
"ctl00_ContentPlaceHolder1_FF2924BTN",
|
48
|
-
).click()
|
49
|
-
|
50
|
-
# Wait for the 'Select address' dropdown to appear and select option matching UPRN
|
51
|
-
dropdown = WebDriverWait(driver, 10).until(
|
52
|
-
EC.presence_of_element_located(
|
53
|
-
(By.ID, "ctl00_ContentPlaceHolder1_FF2924DDL")
|
54
|
-
)
|
55
|
-
)
|
56
|
-
# Create a 'Select' for it, then select the matching URPN option
|
57
|
-
dropdownSelect = Select(dropdown)
|
58
|
-
dropdownSelect.select_by_value("U" + user_uprn)
|
59
|
-
|
60
|
-
# Wait for the submit button to appear, then click it to get the collection dates
|
61
|
-
submit = WebDriverWait(driver, 10).until(
|
62
|
-
EC.presence_of_element_located(
|
63
|
-
(By.ID, "ctl00_ContentPlaceHolder1_btnSubmit")
|
64
|
-
)
|
65
|
-
)
|
66
|
-
submit.click()
|
31
|
+
response = session.get(uri)
|
67
32
|
|
68
|
-
soup = BeautifulSoup(
|
33
|
+
soup = BeautifulSoup(response.content, features="html.parser")
|
69
34
|
|
70
|
-
|
71
|
-
|
72
|
-
.find("
|
73
|
-
|
74
|
-
|
35
|
+
# Function to extract hidden input values
|
36
|
+
def get_hidden_value(soup, name):
|
37
|
+
element = soup.find("input", {"name": name})
|
38
|
+
return element["value"] if element else None
|
39
|
+
|
40
|
+
# Extract the required values
|
41
|
+
data = {
|
42
|
+
"__RequestVerificationToken": get_hidden_value(
|
43
|
+
soup, "__RequestVerificationToken"
|
44
|
+
),
|
45
|
+
"FormGuid": get_hidden_value(soup, "FormGuid"),
|
46
|
+
"ObjectTemplateID": get_hidden_value(soup, "ObjectTemplateID"),
|
47
|
+
"Trigger": "submit",
|
48
|
+
"CurrentSectionID": get_hidden_value(soup, "CurrentSectionID"),
|
49
|
+
"TriggerCtl": "",
|
50
|
+
"FF2924": "U" + user_uprn,
|
51
|
+
"FF2924lbltxt": "Collection address",
|
52
|
+
"FF2924-text": user_postcode,
|
53
|
+
}
|
54
|
+
|
55
|
+
# Print extracted data
|
56
|
+
# print("Extracted Data:", data)
|
57
|
+
|
58
|
+
# Step 2: Submit the extracted data via a POST request
|
59
|
+
headers = {
|
60
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
61
|
+
"Referer": uri,
|
62
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
63
|
+
}
|
64
|
+
|
65
|
+
URI = "https://selfserve.derbyshiredales.gov.uk/renderform/Form"
|
66
|
+
|
67
|
+
# Make the POST request
|
68
|
+
post_response = session.post(URI, data=data, headers=headers)
|
69
|
+
|
70
|
+
soup = BeautifulSoup(post_response.content, features="html.parser")
|
71
|
+
|
72
|
+
# print(soup)
|
73
|
+
|
74
|
+
bin_rows = soup.find("div", {"class": "ss_confPanel"})
|
75
|
+
|
76
|
+
bin_rows = bin_rows.find_all("div", {"class": "row"})
|
75
77
|
if bin_rows:
|
76
78
|
for bin_row in bin_rows:
|
77
79
|
bin_data = bin_row.find_all("div")
|
78
80
|
if bin_data and bin_data[0] and bin_data[1]:
|
81
|
+
if bin_data[0].get_text(strip=True) == "Your Collections":
|
82
|
+
continue
|
79
83
|
collection_date = datetime.strptime(
|
80
84
|
bin_data[0].get_text(strip=True), "%A%d %B, %Y"
|
81
85
|
)
|
@@ -83,9 +87,9 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
83
87
|
"type": bin_data[1].get_text(strip=True),
|
84
88
|
"collectionDate": collection_date.strftime(date_format),
|
85
89
|
}
|
86
|
-
|
90
|
+
bindata["bins"].append(dict_data)
|
87
91
|
|
88
|
-
|
92
|
+
bindata["bins"].sort(
|
89
93
|
key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
|
90
94
|
)
|
91
95
|
except Exception as e:
|
@@ -97,4 +101,4 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
97
101
|
# This block ensures that the driver is closed regardless of an exception
|
98
102
|
if driver:
|
99
103
|
driver.quit()
|
100
|
-
return
|
104
|
+
return bindata
|
@@ -0,0 +1,86 @@
|
|
1
|
+
from datetime import datetime, timedelta
|
2
|
+
|
3
|
+
import requests
|
4
|
+
from bs4 import BeautifulSoup
|
5
|
+
|
6
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
7
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
8
|
+
|
9
|
+
|
10
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
11
|
+
class CouncilClass(AbstractGetBinDataClass):
|
12
|
+
"""
|
13
|
+
Concrete classes have to implement all abstract operations of the
|
14
|
+
base class. They can also override some operations with a default
|
15
|
+
implementation.
|
16
|
+
"""
|
17
|
+
|
18
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
19
|
+
|
20
|
+
user_uprn = kwargs.get("uprn")
|
21
|
+
check_uprn(user_uprn)
|
22
|
+
bindata = {"bins": []}
|
23
|
+
|
24
|
+
URI = f"https://maps.epsom-ewell.gov.uk/myeebc.aspx?action=SetAddress&UniqueId={user_uprn}"
|
25
|
+
|
26
|
+
# Make the GET request
|
27
|
+
response = requests.get(URI)
|
28
|
+
|
29
|
+
soup = BeautifulSoup(response.text, "html.parser")
|
30
|
+
|
31
|
+
# print(soup)
|
32
|
+
|
33
|
+
div = soup.find_all("div", class_="atPanelContent atAlt1 atLast")
|
34
|
+
|
35
|
+
# print(div[1])
|
36
|
+
|
37
|
+
panels = div[1].find_all("div", class_="atPanelData")
|
38
|
+
|
39
|
+
# print(panels)
|
40
|
+
|
41
|
+
def get_full_date(date_str):
|
42
|
+
# Get the current year
|
43
|
+
current_year = datetime.today().year
|
44
|
+
|
45
|
+
# Convert the input string to a datetime object (assuming the current year first)
|
46
|
+
date_obj = datetime.strptime(f"{date_str} {current_year}", "%A %d %B %Y")
|
47
|
+
|
48
|
+
# If the date has already passed this year, use next year
|
49
|
+
if date_obj < datetime.today():
|
50
|
+
date_obj = datetime.strptime(
|
51
|
+
f"{date_str} {current_year + 1}", "%A %d %B %Y"
|
52
|
+
)
|
53
|
+
|
54
|
+
return date_obj.strftime(date_format) # Return in YYYY-MM-DD format
|
55
|
+
|
56
|
+
for panel in panels:
|
57
|
+
bin_type_tag = panel.find("h4") # Extracts bin type
|
58
|
+
date_text = panel.find_all("td") # Extracts collection date
|
59
|
+
|
60
|
+
date_text = date_text[1]
|
61
|
+
|
62
|
+
if bin_type_tag and date_text:
|
63
|
+
bin_type = bin_type_tag.text.strip()
|
64
|
+
try:
|
65
|
+
collection_date = date_text.text.strip().split(":")[1]
|
66
|
+
except IndexError:
|
67
|
+
continue
|
68
|
+
|
69
|
+
bin_type = (
|
70
|
+
(" ".join(bin_type.splitlines())).replace(" ", " ")
|
71
|
+
).lstrip()
|
72
|
+
collection_date = (
|
73
|
+
(" ".join(collection_date.splitlines())).replace(" ", " ")
|
74
|
+
).lstrip()
|
75
|
+
|
76
|
+
dict_data = {
|
77
|
+
"type": bin_type,
|
78
|
+
"collectionDate": get_full_date(collection_date),
|
79
|
+
}
|
80
|
+
bindata["bins"].append(dict_data)
|
81
|
+
|
82
|
+
bindata["bins"].sort(
|
83
|
+
key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
|
84
|
+
)
|
85
|
+
|
86
|
+
return bindata
|
@@ -113,7 +113,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
113
113
|
bin_data["bins"].append(dict_data)
|
114
114
|
|
115
115
|
bin_data["bins"].sort(
|
116
|
-
key=lambda x: datetime.strptime(x.get("collectionDate"),
|
116
|
+
key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
|
117
117
|
)
|
118
118
|
|
119
119
|
except Exception as e:
|
@@ -104,7 +104,8 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
104
104
|
bin_types = soup.find_all("ul", class_="binCollectionTimesList")
|
105
105
|
|
106
106
|
for bin_collection_dates in bin_types:
|
107
|
-
|
107
|
+
|
108
|
+
bin_collection_list = bin_collection_dates.find_all("li")
|
108
109
|
|
109
110
|
if bin_collection_list:
|
110
111
|
collection_dates = [
|