uk_bin_collection 0.151.0__py3-none-any.whl → 0.152.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- uk_bin_collection/tests/input.json +45 -8
- uk_bin_collection/uk_bin_collection/councils/AngusCouncil.py +149 -0
- uk_bin_collection/uk_bin_collection/councils/BarkingDagenham.py +11 -2
- uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py +11 -2
- uk_bin_collection/uk_bin_collection/councils/BroadlandDistrictCouncil.py +21 -6
- uk_bin_collection/uk_bin_collection/councils/ChichesterDistrictCouncil.py +105 -53
- uk_bin_collection/uk_bin_collection/councils/FermanaghOmaghDistrictCouncil.py +102 -0
- uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py +61 -39
- uk_bin_collection/uk_bin_collection/councils/SloughBoroughCouncil.py +140 -0
- uk_bin_collection/uk_bin_collection/councils/TewkesburyBoroughCouncil.py +40 -0
- {uk_bin_collection-0.151.0.dist-info → uk_bin_collection-0.152.0.dist-info}/METADATA +1 -1
- {uk_bin_collection-0.151.0.dist-info → uk_bin_collection-0.152.0.dist-info}/RECORD +15 -11
- {uk_bin_collection-0.151.0.dist-info → uk_bin_collection-0.152.0.dist-info}/LICENSE +0 -0
- {uk_bin_collection-0.151.0.dist-info → uk_bin_collection-0.152.0.dist-info}/WHEEL +0 -0
- {uk_bin_collection-0.151.0.dist-info → uk_bin_collection-0.152.0.dist-info}/entry_points.txt +0 -0
@@ -28,6 +28,16 @@
|
|
28
28
|
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN.",
|
29
29
|
"LAD24CD": "E07000032"
|
30
30
|
},
|
31
|
+
"AngusCouncil": {
|
32
|
+
"uprn": "117053733",
|
33
|
+
"skip_get_url": true,
|
34
|
+
"postcode": "DD7 7LE",
|
35
|
+
"url": "https://www.angus.gov.uk/bins_litter_and_recycling/bin_collection_days",
|
36
|
+
"web_driver": "http://selenium:4444",
|
37
|
+
"wiki_name": "Angus",
|
38
|
+
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN. Requires Selenium",
|
39
|
+
"LAD24CD": "S12000041"
|
40
|
+
},
|
31
41
|
"AntrimAndNewtonabbeyCouncil": {
|
32
42
|
"LAD24CD": "N09000001",
|
33
43
|
"url": "https://antrimandnewtownabbey.gov.uk/residents/bins-recycling/bins-schedule/?Id=643",
|
@@ -117,13 +127,13 @@
|
|
117
127
|
"LAD24CD": "E07000200"
|
118
128
|
},
|
119
129
|
"BarkingDagenham": {
|
120
|
-
"house_number": "19
|
130
|
+
"house_number": "19",
|
121
131
|
"postcode": "RM6 6XH",
|
122
132
|
"skip_get_url": true,
|
123
133
|
"web_driver": "http://selenium:4444",
|
124
134
|
"url": "https://www.lbbd.gov.uk/rubbish-recycling/household-bin-collection/check-your-bin-collection-days",
|
125
135
|
"wiki_name": "Barking and Dagenham",
|
126
|
-
"wiki_note": "Use
|
136
|
+
"wiki_note": "Use house number and postcode. Requires Selenium.",
|
127
137
|
"LAD24CD": "E09000002"
|
128
138
|
},
|
129
139
|
"BarnetCouncil": {
|
@@ -311,13 +321,13 @@
|
|
311
321
|
"LAD24CD": "E09000005"
|
312
322
|
},
|
313
323
|
"BrightonandHoveCityCouncil": {
|
314
|
-
"house_number": "44
|
324
|
+
"house_number": "44",
|
315
325
|
"postcode": "BN1 8NE",
|
316
326
|
"skip_get_url": true,
|
317
327
|
"url": "https://cityclean.brighton-hove.gov.uk/link/collections",
|
318
328
|
"web_driver": "http://selenium:4444",
|
319
329
|
"wiki_name": "Brighton and Hove",
|
320
|
-
"wiki_note": "Use
|
330
|
+
"wiki_note": "Use house number and postcode. Requires Selenium",
|
321
331
|
"LAD24CD": "E06000043"
|
322
332
|
},
|
323
333
|
"BristolCityCouncil": {
|
@@ -330,12 +340,12 @@
|
|
330
340
|
},
|
331
341
|
"BroadlandDistrictCouncil": {
|
332
342
|
"skip_get_url": true,
|
333
|
-
"house_number": "1
|
343
|
+
"house_number": "1",
|
334
344
|
"postcode": "NR10 3FD",
|
335
345
|
"url": "https://area.southnorfolkandbroadland.gov.uk/FindAddress",
|
336
346
|
"web_driver": "http://selenium:4444",
|
337
347
|
"wiki_name": "Broadland",
|
338
|
-
"wiki_note": "Use
|
348
|
+
"wiki_note": "Use house number and postcode. Requires Selenium.",
|
339
349
|
"LAD24CD": "E07000144"
|
340
350
|
},
|
341
351
|
"BromleyBoroughCouncil": {
|
@@ -521,7 +531,7 @@
|
|
521
531
|
"LAD24CD": "E07000034"
|
522
532
|
},
|
523
533
|
"ChichesterDistrictCouncil": {
|
524
|
-
"house_number": "7
|
534
|
+
"house_number": "7",
|
525
535
|
"postcode": "RH14 0JT",
|
526
536
|
"skip_get_url": true,
|
527
537
|
"url": "https://www.chichester.gov.uk/checkyourbinday",
|
@@ -901,6 +911,15 @@
|
|
901
911
|
"wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
|
902
912
|
"LAD24CD": "E07000010"
|
903
913
|
},
|
914
|
+
"FermanaghOmaghDistrictCouncil": {
|
915
|
+
"house_number": "20",
|
916
|
+
"postcode": "BT74 6DQ",
|
917
|
+
"skip_get_url": true,
|
918
|
+
"url": "https://www.fermanaghomagh.com/services/environment-and-waste/waste-collection-calendar/",
|
919
|
+
"wiki_name": "Fermanagh and Omagh",
|
920
|
+
"wiki_note": "Pass the house number and postcode in their respective parameters.",
|
921
|
+
"LAD24CD": "N09000006"
|
922
|
+
},
|
904
923
|
"FifeCouncil": {
|
905
924
|
"uprn": "320203521",
|
906
925
|
"url": "https://www.fife.gov.uk",
|
@@ -1199,6 +1218,7 @@
|
|
1199
1218
|
"LAD24CD": "E07000120",
|
1200
1219
|
"uprn": "100010448773",
|
1201
1220
|
"url": "https://iapp.itouchvision.com/iappcollectionday/collection-day/?uuid=FEBA68993831481FD81B2E605364D00A8DC017A4",
|
1221
|
+
"skip_get_url": true,
|
1202
1222
|
"web_driver": "http://selenium:4444",
|
1203
1223
|
"wiki_name": "Hyndburn",
|
1204
1224
|
"wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search). This parser requires a Selenium webdriver."
|
@@ -1622,7 +1642,7 @@
|
|
1622
1642
|
"NorthEastDerbyshireDistrictCouncil": {
|
1623
1643
|
"postcode": "S42 5RB",
|
1624
1644
|
"skip_get_url": true,
|
1625
|
-
"uprn": "
|
1645
|
+
"uprn": "010034492222",
|
1626
1646
|
"url": "https://myselfservice.ne-derbyshire.gov.uk/service/Check_your_Bin_Day",
|
1627
1647
|
"web_driver": "http://selenium:4444",
|
1628
1648
|
"wiki_name": "North East Derbyshire",
|
@@ -2013,6 +2033,15 @@
|
|
2013
2033
|
"wiki_note": "Follow the instructions [here](https://bins.shropshire.gov.uk/) until you get the page showing your bin collection dates, then copy the URL and replace the URL in the command.",
|
2014
2034
|
"LAD24CD": "E06000051"
|
2015
2035
|
},
|
2036
|
+
"SloughBoroughCouncil": {
|
2037
|
+
"postcode": "SL2 2EW",
|
2038
|
+
"skip_get_url": true,
|
2039
|
+
"url": "https://www.slough.gov.uk/bin-collections",
|
2040
|
+
"web_driver": "http://selenium:4444",
|
2041
|
+
"wiki_name": "Slough",
|
2042
|
+
"wiki_note": "Pass the UPRN and postcode in their respective parameters. This parser requires a Selenium webdriver.",
|
2043
|
+
"LAD24CD": "E06000039"
|
2044
|
+
},
|
2016
2045
|
"SolihullCouncil": {
|
2017
2046
|
"url": "https://digital.solihull.gov.uk/BinCollectionCalendar/Calendar.aspx?UPRN=100071005444",
|
2018
2047
|
"wiki_command_url_override": "https://digital.solihull.gov.uk/BinCollectionCalendar/Calendar.aspx?UPRN=XXXXXXXX",
|
@@ -2332,6 +2361,14 @@
|
|
2332
2361
|
"wiki_note": "Provide your UPRN. Find it using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
|
2333
2362
|
"LAD24CD": "E06000020"
|
2334
2363
|
},
|
2364
|
+
"TewkesburyBoroughCouncil": {
|
2365
|
+
"skip_get_url": true,
|
2366
|
+
"uprn": "10067626314",
|
2367
|
+
"url": "https://tewkesbury.gov.uk/services/waste-and-recycling/",
|
2368
|
+
"wiki_name": "Tewkesbury",
|
2369
|
+
"wiki_note": "Provide your UPRN. Find it using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
|
2370
|
+
"LAD24CD": "E07000083"
|
2371
|
+
},
|
2335
2372
|
"TendringDistrictCouncil": {
|
2336
2373
|
"postcode": "CO15 4EU",
|
2337
2374
|
"skip_get_url": true,
|
@@ -0,0 +1,149 @@
|
|
1
|
+
import time
|
2
|
+
import re
|
3
|
+
from datetime import datetime
|
4
|
+
|
5
|
+
from bs4 import BeautifulSoup
|
6
|
+
from selenium.webdriver.common.by import By
|
7
|
+
from selenium.webdriver.common.keys import Keys
|
8
|
+
from selenium.webdriver.support import expected_conditions as EC
|
9
|
+
from selenium.webdriver.support.ui import Select, WebDriverWait
|
10
|
+
|
11
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
12
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
13
|
+
|
14
|
+
|
15
|
+
class CouncilClass(AbstractGetBinDataClass):
|
16
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
17
|
+
driver = None
|
18
|
+
try:
|
19
|
+
user_postcode = kwargs.get("postcode")
|
20
|
+
if not user_postcode:
|
21
|
+
raise ValueError("No postcode provided.")
|
22
|
+
check_postcode(user_postcode)
|
23
|
+
|
24
|
+
user_uprn = kwargs.get("uprn")
|
25
|
+
check_uprn(user_uprn)
|
26
|
+
|
27
|
+
headless = kwargs.get("headless")
|
28
|
+
web_driver = kwargs.get("web_driver")
|
29
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
30
|
+
page = "https://www.angus.gov.uk/bins_litter_and_recycling/bin_collection_days"
|
31
|
+
|
32
|
+
driver.get(page)
|
33
|
+
|
34
|
+
wait = WebDriverWait(driver, 10)
|
35
|
+
accept_cookies_button = wait.until(
|
36
|
+
EC.element_to_be_clickable((By.ID, "ccc-recommended-settings"))
|
37
|
+
)
|
38
|
+
accept_cookies_button.click()
|
39
|
+
|
40
|
+
find_your_collection_button = wait.until(
|
41
|
+
EC.element_to_be_clickable(
|
42
|
+
(By.XPATH, "/html/body/div[2]/div[2]/div/div/section/div[2]/div/article/div/div/p[2]/a")
|
43
|
+
)
|
44
|
+
)
|
45
|
+
find_your_collection_button.click()
|
46
|
+
|
47
|
+
iframe = wait.until(EC.presence_of_element_located((By.ID, "fillform-frame-1")))
|
48
|
+
driver.switch_to.frame(iframe)
|
49
|
+
|
50
|
+
postcode_input = wait.until(EC.presence_of_element_located((By.ID, "searchString")))
|
51
|
+
postcode_input.send_keys(user_postcode + Keys.TAB + Keys.ENTER)
|
52
|
+
|
53
|
+
time.sleep(15)
|
54
|
+
|
55
|
+
select_elem = wait.until(EC.presence_of_element_located((By.ID, "customerAddress")))
|
56
|
+
WebDriverWait(driver, 10).until(
|
57
|
+
lambda d: len(select_elem.find_elements(By.TAG_NAME, "option")) > 1
|
58
|
+
)
|
59
|
+
dropdown = Select(select_elem)
|
60
|
+
dropdown.select_by_value(user_uprn)
|
61
|
+
|
62
|
+
time.sleep(10)
|
63
|
+
|
64
|
+
wait.until(
|
65
|
+
EC.presence_of_element_located(
|
66
|
+
(By.CSS_SELECTOR, "span.fieldInput.content.html.non-input"))
|
67
|
+
)
|
68
|
+
|
69
|
+
soup = BeautifulSoup(driver.page_source, "html.parser")
|
70
|
+
bin_data = {"bins": []}
|
71
|
+
current_date = datetime.now()
|
72
|
+
current_formatted_date = None
|
73
|
+
|
74
|
+
spans = soup.select("span.fieldInput.content.html.non-input")
|
75
|
+
print(f"Found {len(spans)} bin info spans.")
|
76
|
+
|
77
|
+
for i, span in enumerate(spans):
|
78
|
+
try:
|
79
|
+
# Look for any non-empty <u> tag recursively
|
80
|
+
date_tag = next(
|
81
|
+
(u for u in span.find_all("u") if u and u.text.strip()),
|
82
|
+
None
|
83
|
+
)
|
84
|
+
bin_type_tag = span.find("b")
|
85
|
+
|
86
|
+
if date_tag:
|
87
|
+
raw_date = date_tag.text.strip().replace(",", "")
|
88
|
+
full_date_str = f"{raw_date} {current_date.year}"
|
89
|
+
full_date_str = re.sub(r"\s+", " ", full_date_str)
|
90
|
+
|
91
|
+
try:
|
92
|
+
parsed_date = datetime.strptime(full_date_str, "%A %d %B %Y")
|
93
|
+
if parsed_date.date() < current_date.date():
|
94
|
+
parsed_date = parsed_date.replace(year=current_date.year + 1)
|
95
|
+
current_formatted_date = parsed_date.strftime("%d/%m/%Y")
|
96
|
+
print(f"[{i}] Parsed date: {current_formatted_date}")
|
97
|
+
except ValueError as ve:
|
98
|
+
print(f"[{i}] Could not parse date: '{full_date_str}' - {ve}")
|
99
|
+
continue
|
100
|
+
else:
|
101
|
+
print(f"[{i}] No date tag found, using last valid date: {current_formatted_date}")
|
102
|
+
|
103
|
+
if not current_formatted_date:
|
104
|
+
print(f"[{i}] No current date to associate bin type with — skipping.")
|
105
|
+
continue
|
106
|
+
|
107
|
+
if not bin_type_tag or not bin_type_tag.text.strip():
|
108
|
+
print(f"[{i}] No bin type found — skipping.")
|
109
|
+
continue
|
110
|
+
|
111
|
+
bin_type = bin_type_tag.text.strip()
|
112
|
+
|
113
|
+
# Optional seasonal override
|
114
|
+
try:
|
115
|
+
overrides_dict = get_seasonal_overrides()
|
116
|
+
if current_formatted_date in overrides_dict:
|
117
|
+
current_formatted_date = overrides_dict[current_formatted_date]
|
118
|
+
except Exception:
|
119
|
+
pass
|
120
|
+
|
121
|
+
print(f"[{i}] Found bin: {bin_type} on {current_formatted_date}")
|
122
|
+
|
123
|
+
bin_data["bins"].append({
|
124
|
+
"type": bin_type,
|
125
|
+
"collectionDate": current_formatted_date
|
126
|
+
})
|
127
|
+
|
128
|
+
except Exception as inner_e:
|
129
|
+
print(f"[{i}] Skipping span due to error: {inner_e}")
|
130
|
+
continue
|
131
|
+
|
132
|
+
except Exception as inner_e:
|
133
|
+
print(f"Skipping span due to error: {inner_e}")
|
134
|
+
continue
|
135
|
+
|
136
|
+
if not bin_data["bins"]:
|
137
|
+
raise ValueError("No bin data found.")
|
138
|
+
|
139
|
+
print(bin_data)
|
140
|
+
|
141
|
+
return bin_data
|
142
|
+
|
143
|
+
except Exception as e:
|
144
|
+
print(f"An error occurred: {e}")
|
145
|
+
raise
|
146
|
+
|
147
|
+
finally:
|
148
|
+
if driver:
|
149
|
+
driver.quit()
|
@@ -84,10 +84,19 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
84
84
|
EC.element_to_be_clickable((By.ID, "address")),
|
85
85
|
message="Address dropdown not found",
|
86
86
|
)
|
87
|
+
|
87
88
|
dropdown = Select(address_select)
|
88
89
|
|
89
|
-
|
90
|
-
|
90
|
+
found = False
|
91
|
+
for option in dropdown.options:
|
92
|
+
if user_paon in option.text:
|
93
|
+
option.click()
|
94
|
+
found = True
|
95
|
+
print("Address selected successfully")
|
96
|
+
break
|
97
|
+
|
98
|
+
if not found:
|
99
|
+
raise Exception(f"No matching address containing '{user_paon}' found.")
|
91
100
|
|
92
101
|
driver.switch_to.active_element.send_keys(Keys.TAB + Keys.ENTER)
|
93
102
|
print("Pressed ENTER on Next button")
|
@@ -63,8 +63,16 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
63
63
|
|
64
64
|
# Create a 'Select' for it, then select the first address in the list
|
65
65
|
# (Index 0 is "Make a selection from the list")
|
66
|
-
|
67
|
-
|
66
|
+
options = parent_element.find_elements(By.TAG_NAME, "option")
|
67
|
+
found = False
|
68
|
+
for option in options:
|
69
|
+
if user_paon in option.text:
|
70
|
+
option.click()
|
71
|
+
found = True
|
72
|
+
break
|
73
|
+
|
74
|
+
if not found:
|
75
|
+
raise Exception(f"Address containing '{user_paon}' not found in dropdown options")
|
68
76
|
|
69
77
|
submit_btn = wait.until(
|
70
78
|
EC.presence_of_element_located(
|
@@ -125,6 +133,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
125
133
|
break
|
126
134
|
dict_data = {"type": bin_type, "collectionDate": bin_date}
|
127
135
|
data["bins"].append(dict_data)
|
136
|
+
print(data)
|
128
137
|
except Exception as e:
|
129
138
|
# Here you can log the exception if needed
|
130
139
|
print(f"An error occurred: {e}")
|
@@ -83,15 +83,30 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
83
83
|
)
|
84
84
|
print("Found address dropdown")
|
85
85
|
|
86
|
-
# Create a Select object for the dropdown
|
87
86
|
dropdown_select = Select(address_dropdown)
|
88
87
|
|
89
|
-
|
90
|
-
print(f"Looking for address: {user_paon}")
|
88
|
+
print(f"Looking for address containing: {user_paon}")
|
91
89
|
|
92
|
-
|
93
|
-
|
94
|
-
|
90
|
+
found = False
|
91
|
+
user_paon_clean = user_paon.lower().strip()
|
92
|
+
|
93
|
+
for option in dropdown_select.options:
|
94
|
+
option_text_clean = option.text.lower().strip()
|
95
|
+
|
96
|
+
if (
|
97
|
+
option_text_clean == user_paon_clean # Exact match if full address given
|
98
|
+
or option_text_clean.startswith(f"{user_paon_clean} ") # Startswith match if just a number
|
99
|
+
):
|
100
|
+
option.click()
|
101
|
+
found = True
|
102
|
+
print(f"Selected address: {option.text.strip()}")
|
103
|
+
break
|
104
|
+
|
105
|
+
if not found:
|
106
|
+
all_options = [opt.text for opt in dropdown_select.options]
|
107
|
+
raise Exception(
|
108
|
+
f"Could not find a matching address for '{user_paon}'. Available options: {all_options}"
|
109
|
+
)
|
95
110
|
|
96
111
|
print("Looking for submit button after address selection...")
|
97
112
|
submit_btn = wait.until(
|
@@ -1,110 +1,162 @@
|
|
1
1
|
import time
|
2
2
|
from datetime import datetime
|
3
3
|
|
4
|
-
from selenium.webdriver.support.ui import Select
|
5
4
|
from bs4 import BeautifulSoup
|
6
5
|
from selenium.webdriver.common.by import By
|
7
|
-
from selenium.webdriver.support import expected_conditions as EC
|
8
|
-
from selenium.webdriver.support.ui import Select
|
9
|
-
from selenium.webdriver.support.wait import WebDriverWait
|
10
6
|
from selenium.webdriver.common.keys import Keys
|
7
|
+
from selenium.webdriver.support.ui import WebDriverWait, Select
|
8
|
+
from selenium.webdriver.support import expected_conditions as EC
|
9
|
+
from selenium.common.exceptions import StaleElementReferenceException, TimeoutException
|
11
10
|
|
12
11
|
from uk_bin_collection.uk_bin_collection.common import *
|
13
12
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
14
13
|
|
14
|
+
date_format = "%d/%m/%Y"
|
15
15
|
|
16
|
-
# import the wonderful Beautiful Soup and the URL grabber
|
17
16
|
class CouncilClass(AbstractGetBinDataClass):
|
18
|
-
"""
|
19
|
-
Concrete classes have to implement all abstract operations of the
|
20
|
-
base class. They can also override some operations with a default
|
21
|
-
implementation.
|
22
|
-
"""
|
23
|
-
|
24
17
|
def parse_data(self, page: str, **kwargs) -> dict:
|
25
18
|
driver = None
|
26
19
|
try:
|
27
|
-
# Make a BS4 object
|
28
|
-
|
29
20
|
page = "https://www.chichester.gov.uk/checkyourbinday"
|
30
21
|
|
31
22
|
user_postcode = kwargs.get("postcode")
|
32
|
-
|
23
|
+
house_number = kwargs.get("paon")
|
33
24
|
web_driver = kwargs.get("web_driver")
|
34
25
|
headless = kwargs.get("headless")
|
35
|
-
house_number = kwargs.get("paon")
|
36
26
|
|
37
27
|
driver = create_webdriver(web_driver, headless, None, __name__)
|
38
28
|
driver.get(page)
|
39
29
|
|
40
30
|
wait = WebDriverWait(driver, 60)
|
41
31
|
|
42
|
-
|
32
|
+
input_postcode = wait.until(
|
43
33
|
EC.visibility_of_element_located(
|
44
34
|
(By.ID, "WASTECOLLECTIONCALENDARV5_CALENDAR_ADDRESSLOOKUPPOSTCODE")
|
45
35
|
)
|
46
36
|
)
|
37
|
+
input_postcode.send_keys(user_postcode)
|
47
38
|
|
48
|
-
|
49
|
-
|
50
|
-
inputElement_postcodesearch_btn = wait.until(
|
51
|
-
EC.visibility_of_element_located(
|
52
|
-
(By.ID, "WASTECOLLECTIONCALENDARV5_CALENDAR_ADDRESSLOOKUPSEARCH")
|
53
|
-
)
|
54
|
-
)
|
55
|
-
inputElement_postcodesearch_btn.send_keys(Keys.ENTER)
|
56
|
-
|
57
|
-
inputElement_select_address = wait.until(
|
39
|
+
search_button = wait.until(
|
58
40
|
EC.element_to_be_clickable(
|
59
|
-
(By.ID, "
|
41
|
+
(By.ID, "WASTECOLLECTIONCALENDARV5_CALENDAR_ADDRESSLOOKUPSEARCH")
|
60
42
|
)
|
61
43
|
)
|
62
|
-
|
63
|
-
By.ID, "WASTECOLLECTIONCALENDARV5_CALENDAR_ADDRESSLOOKUPADDRESS"
|
64
|
-
)
|
44
|
+
search_button.send_keys(Keys.ENTER)
|
65
45
|
|
66
|
-
|
67
|
-
dropdown = Select(dropdown_element)
|
46
|
+
self.smart_select_address(driver, house_number)
|
68
47
|
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
results = wait.until(
|
73
|
-
EC.element_to_be_clickable(
|
48
|
+
wait.until(
|
49
|
+
EC.presence_of_element_located(
|
74
50
|
(By.CLASS_NAME, "bin-collection-dates-container")
|
75
51
|
)
|
76
52
|
)
|
77
53
|
|
78
54
|
soup = BeautifulSoup(driver.page_source, features="html.parser")
|
79
|
-
soup.
|
55
|
+
table = soup.find("table", class_="defaultgeneral bin-collection-dates")
|
56
|
+
rows = table.find_all("tr") if table else []
|
80
57
|
|
81
|
-
# Extract data from the table
|
82
58
|
bin_collection_data = []
|
83
|
-
rows = soup.find(
|
84
|
-
"table", class_="defaultgeneral bin-collection-dates"
|
85
|
-
).find_all("tr")
|
86
59
|
for row in rows:
|
87
60
|
cells = row.find_all("td")
|
88
61
|
if cells:
|
89
62
|
date_str = cells[0].text.strip()
|
90
63
|
bin_type = cells[1].text.strip()
|
91
|
-
# Convert date string to the required format DD/MM/YYYY
|
92
64
|
date_obj = datetime.strptime(date_str, "%d %B %Y")
|
93
|
-
|
94
|
-
bin_collection_data.append(
|
95
|
-
|
96
|
-
|
65
|
+
formatted_date = date_obj.strftime(date_format)
|
66
|
+
bin_collection_data.append({
|
67
|
+
"collectionDate": formatted_date,
|
68
|
+
"type": bin_type
|
69
|
+
})
|
97
70
|
|
98
|
-
|
99
|
-
|
71
|
+
print(bin_collection_data)
|
72
|
+
|
73
|
+
return {"bins": bin_collection_data}
|
100
74
|
|
101
75
|
except Exception as e:
|
102
|
-
# Here you can log the exception if needed
|
103
76
|
print(f"An error occurred: {e}")
|
104
|
-
# Optionally, re-raise the exception if you want it to propagate
|
105
77
|
raise
|
106
78
|
finally:
|
107
|
-
# This block ensures that the driver is closed regardless of an exception
|
108
79
|
if driver:
|
109
80
|
driver.quit()
|
110
|
-
|
81
|
+
|
82
|
+
def smart_select_address(self, driver, house_number: str):
|
83
|
+
dropdown_id = "WASTECOLLECTIONCALENDARV5_CALENDAR_ADDRESSLOOKUPADDRESS"
|
84
|
+
|
85
|
+
print("Waiting for address dropdown...")
|
86
|
+
|
87
|
+
def dropdown_has_addresses(d):
|
88
|
+
try:
|
89
|
+
dropdown_el = d.find_element(By.ID, dropdown_id)
|
90
|
+
select = Select(dropdown_el)
|
91
|
+
return len(select.options) > 1
|
92
|
+
except StaleElementReferenceException:
|
93
|
+
return False
|
94
|
+
|
95
|
+
WebDriverWait(driver, 30).until(dropdown_has_addresses)
|
96
|
+
|
97
|
+
dropdown_el = driver.find_element(By.ID, dropdown_id)
|
98
|
+
dropdown = Select(dropdown_el)
|
99
|
+
|
100
|
+
print("Address dropdown options:")
|
101
|
+
for opt in dropdown.options:
|
102
|
+
print(f"- {opt.text.strip()}")
|
103
|
+
|
104
|
+
user_input_clean = house_number.lower().strip()
|
105
|
+
found = False
|
106
|
+
|
107
|
+
for option in dropdown.options:
|
108
|
+
option_text_clean = option.text.lower().strip()
|
109
|
+
print(f"Comparing: {repr(option_text_clean)} == {repr(user_input_clean)}")
|
110
|
+
|
111
|
+
if (
|
112
|
+
option_text_clean == user_input_clean
|
113
|
+
or option_text_clean.startswith(f"{user_input_clean},")
|
114
|
+
):
|
115
|
+
try:
|
116
|
+
option.click()
|
117
|
+
found = True
|
118
|
+
print(f"Strict match clicked: {option.text.strip()}")
|
119
|
+
break
|
120
|
+
except StaleElementReferenceException:
|
121
|
+
print("Stale during click, retrying...")
|
122
|
+
dropdown_el = driver.find_element(By.ID, dropdown_id)
|
123
|
+
dropdown = Select(dropdown_el)
|
124
|
+
for fresh_option in dropdown.options:
|
125
|
+
if fresh_option.text.lower().strip() == option_text_clean:
|
126
|
+
fresh_option.click()
|
127
|
+
found = True
|
128
|
+
print(f"Strict match clicked after refresh: {fresh_option.text.strip()}")
|
129
|
+
break
|
130
|
+
|
131
|
+
if found:
|
132
|
+
break
|
133
|
+
|
134
|
+
if not found:
|
135
|
+
print("No strict match found, trying fuzzy match...")
|
136
|
+
for option in dropdown.options:
|
137
|
+
option_text_clean = option.text.lower().strip()
|
138
|
+
if user_input_clean in option_text_clean:
|
139
|
+
try:
|
140
|
+
option.click()
|
141
|
+
found = True
|
142
|
+
print(f"Fuzzy match clicked: {option.text.strip()}")
|
143
|
+
break
|
144
|
+
except StaleElementReferenceException:
|
145
|
+
print("Stale during fuzzy click, retrying...")
|
146
|
+
dropdown_el = driver.find_element(By.ID, dropdown_id)
|
147
|
+
dropdown = Select(dropdown_el)
|
148
|
+
for fresh_option in dropdown.options:
|
149
|
+
if fresh_option.text.lower().strip() == option_text_clean:
|
150
|
+
fresh_option.click()
|
151
|
+
found = True
|
152
|
+
print(f"Fuzzy match clicked after refresh: {fresh_option.text.strip()}")
|
153
|
+
break
|
154
|
+
|
155
|
+
if found:
|
156
|
+
break
|
157
|
+
|
158
|
+
if not found:
|
159
|
+
all_opts = [opt.text.strip() for opt in dropdown.options]
|
160
|
+
raise Exception(
|
161
|
+
f"Could not find address '{house_number}' in options: {all_opts}"
|
162
|
+
)
|
@@ -0,0 +1,102 @@
|
|
1
|
+
import difflib
|
2
|
+
from datetime import date, datetime
|
3
|
+
|
4
|
+
import requests
|
5
|
+
from bs4 import BeautifulSoup
|
6
|
+
|
7
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
8
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
9
|
+
|
10
|
+
|
11
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
12
|
+
class CouncilClass(AbstractGetBinDataClass):
|
13
|
+
"""
|
14
|
+
Concrete classes have to implement all abstract operations of the
|
15
|
+
base class. They can also override some operations with a default
|
16
|
+
implementation.
|
17
|
+
"""
|
18
|
+
|
19
|
+
base_url = "https://fermanaghomagh.isl-fusion.com/"
|
20
|
+
|
21
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
22
|
+
"""
|
23
|
+
This function will make a request to the search endpoint with the postcode, extract the
|
24
|
+
house numbers from the responses, then retrieve the ID of the entry with the house number that matches,
|
25
|
+
to then retrieve the bin schedule.
|
26
|
+
|
27
|
+
The API here is a weird combination of HTML in json responses.
|
28
|
+
"""
|
29
|
+
postcode = kwargs.get("postcode")
|
30
|
+
paon = kwargs.get("paon")
|
31
|
+
|
32
|
+
if not postcode:
|
33
|
+
raise ValueError("Must provide a postcode")
|
34
|
+
|
35
|
+
if not paon:
|
36
|
+
raise ValueError("Must provide a house number")
|
37
|
+
|
38
|
+
search_url = f"{self.base_url}/address/{postcode}"
|
39
|
+
|
40
|
+
requests.packages.urllib3.disable_warnings()
|
41
|
+
s = requests.Session()
|
42
|
+
response = s.get(search_url)
|
43
|
+
response.raise_for_status()
|
44
|
+
|
45
|
+
address_data = response.json()
|
46
|
+
|
47
|
+
address_list = address_data["html"]
|
48
|
+
|
49
|
+
soup = BeautifulSoup(address_list, features="html.parser")
|
50
|
+
|
51
|
+
address_by_id = {}
|
52
|
+
|
53
|
+
for li in soup.find_all("li"):
|
54
|
+
link = li.find_all("a")[0]
|
55
|
+
address_id = link.attrs["href"]
|
56
|
+
address = link.text
|
57
|
+
|
58
|
+
address_by_id[address_id] = address
|
59
|
+
|
60
|
+
addresses = list(address_by_id.values())
|
61
|
+
|
62
|
+
common = difflib.SequenceMatcher(
|
63
|
+
a=addresses[0], b=addresses[1]
|
64
|
+
).find_longest_match()
|
65
|
+
extra_bit = addresses[0][common.a : common.a + common.size]
|
66
|
+
|
67
|
+
ids_by_paon = {
|
68
|
+
a.replace(extra_bit, ""): a_id.replace("/view/", "").replace("/", "")
|
69
|
+
for a_id, a in address_by_id.items()
|
70
|
+
}
|
71
|
+
|
72
|
+
property_id = ids_by_paon.get(paon)
|
73
|
+
if not property_id:
|
74
|
+
raise ValueError(
|
75
|
+
f"Invalid house number, valid values are {', '.join(ids_by_paon.keys())}"
|
76
|
+
)
|
77
|
+
|
78
|
+
today = date.today()
|
79
|
+
calendar_url = (
|
80
|
+
f"{self.base_url}/calendar/{property_id}/{today.strftime('%Y-%m-%d')}"
|
81
|
+
)
|
82
|
+
response = s.get(calendar_url)
|
83
|
+
response.raise_for_status()
|
84
|
+
calendar_data = response.json()
|
85
|
+
next_collections = calendar_data["nextCollections"]
|
86
|
+
|
87
|
+
collections = list(next_collections["collections"].values())
|
88
|
+
|
89
|
+
data = {"bins": []}
|
90
|
+
|
91
|
+
for collection in collections:
|
92
|
+
collection_date = datetime.strptime(collection["date"], "%Y-%m-%d")
|
93
|
+
bins = [c["name"] for c in collection["collections"].values()]
|
94
|
+
|
95
|
+
for bin in bins:
|
96
|
+
data["bins"].append(
|
97
|
+
{
|
98
|
+
"type": bin,
|
99
|
+
"collectionDate": collection_date.strftime(date_format),
|
100
|
+
}
|
101
|
+
)
|
102
|
+
return data
|
@@ -1,4 +1,5 @@
|
|
1
1
|
from datetime import datetime
|
2
|
+
from time import sleep
|
2
3
|
|
3
4
|
from bs4 import BeautifulSoup
|
4
5
|
from selenium.webdriver.common.by import By
|
@@ -9,8 +10,6 @@ from selenium.webdriver.support.wait import WebDriverWait
|
|
9
10
|
from uk_bin_collection.uk_bin_collection.common import *
|
10
11
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
11
12
|
|
12
|
-
# import the wonderful Beautiful Soup and the URL grabber
|
13
|
-
|
14
13
|
|
15
14
|
class CouncilClass(AbstractGetBinDataClass):
|
16
15
|
"""
|
@@ -34,82 +33,105 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
34
33
|
headless = kwargs.get("headless")
|
35
34
|
check_uprn(user_uprn)
|
36
35
|
check_postcode(user_postcode)
|
37
|
-
|
36
|
+
|
38
37
|
driver = create_webdriver(web_driver, headless, None, __name__)
|
39
38
|
driver.get(page)
|
40
39
|
|
41
|
-
# If you bang in the house number (or property name) and postcode in the box it should find your property
|
42
|
-
|
43
40
|
iframe_presense = WebDriverWait(driver, 30).until(
|
44
41
|
EC.presence_of_element_located((By.ID, "fillform-frame-1"))
|
45
42
|
)
|
46
43
|
|
47
44
|
driver.switch_to.frame(iframe_presense)
|
48
45
|
wait = WebDriverWait(driver, 60)
|
46
|
+
|
49
47
|
inputElement_postcodesearch = wait.until(
|
50
48
|
EC.element_to_be_clickable((By.NAME, "postcode_search"))
|
51
49
|
)
|
52
|
-
|
53
50
|
inputElement_postcodesearch.send_keys(str(user_postcode))
|
54
51
|
|
55
|
-
# Wait for the 'Select your property' dropdown to appear and select the first result
|
56
52
|
dropdown = wait.until(EC.element_to_be_clickable((By.NAME, "selAddress")))
|
57
|
-
|
58
53
|
dropdown_options = wait.until(
|
59
54
|
EC.presence_of_element_located((By.CLASS_NAME, "lookup-option"))
|
60
55
|
)
|
61
56
|
|
62
|
-
# Create a 'Select' for it, then select the first address in the list
|
63
|
-
# (Index 0 is "Make a selection from the list")
|
64
57
|
drop_down_values = Select(dropdown)
|
65
58
|
option_element = wait.until(
|
66
59
|
EC.presence_of_element_located(
|
67
60
|
(By.CSS_SELECTOR, f'option.lookup-option[value="{str(user_uprn)}"]')
|
68
61
|
)
|
69
62
|
)
|
70
|
-
|
71
63
|
drop_down_values.select_by_value(str(user_uprn))
|
72
64
|
|
73
|
-
# Wait for the 'View more' link to appear, then click it to get the full set of dates
|
74
65
|
h3_element = wait.until(
|
75
66
|
EC.presence_of_element_located(
|
76
67
|
(By.XPATH, "//th[contains(text(), 'Waste Collection')]")
|
77
68
|
)
|
78
69
|
)
|
79
70
|
|
71
|
+
sleep(10)
|
72
|
+
|
80
73
|
soup = BeautifulSoup(driver.page_source, features="html.parser")
|
74
|
+
print("Parsing HTML content...")
|
75
|
+
|
76
|
+
collection_rows = soup.find_all("tr")
|
77
|
+
|
78
|
+
for row in collection_rows:
|
79
|
+
cells = row.find_all("td")
|
80
|
+
if len(cells) == 3: # Date, Image, Bin Type
|
81
|
+
# Extract date carefully
|
82
|
+
date_labels = cells[0].find_all("label")
|
83
|
+
collection_date = None
|
84
|
+
for label in date_labels:
|
85
|
+
label_text = label.get_text().strip()
|
86
|
+
if contains_date(label_text):
|
87
|
+
collection_date = label_text
|
88
|
+
break
|
89
|
+
|
90
|
+
# Extract bin type
|
91
|
+
bin_label = cells[2].find("label")
|
92
|
+
bin_types = bin_label.get_text().strip() if bin_label else None
|
93
|
+
|
94
|
+
if collection_date and bin_types:
|
95
|
+
print(f"Found collection: {collection_date} - {bin_types}")
|
96
|
+
|
97
|
+
# Handle combined collections
|
98
|
+
if "&" in bin_types:
|
99
|
+
if "Burgundy" in bin_types:
|
100
|
+
data["bins"].append(
|
101
|
+
{
|
102
|
+
"type": "Burgundy Bin",
|
103
|
+
"collectionDate": datetime.strptime(
|
104
|
+
collection_date, "%d/%m/%Y"
|
105
|
+
).strftime(date_format),
|
106
|
+
}
|
107
|
+
)
|
108
|
+
if "Green" in bin_types:
|
109
|
+
data["bins"].append(
|
110
|
+
{
|
111
|
+
"type": "Green Bin",
|
112
|
+
"collectionDate": datetime.strptime(
|
113
|
+
collection_date, "%d/%m/%Y"
|
114
|
+
).strftime(date_format),
|
115
|
+
}
|
116
|
+
)
|
117
|
+
else:
|
118
|
+
if "Black" in bin_types:
|
119
|
+
data["bins"].append(
|
120
|
+
{
|
121
|
+
"type": "Black Bin",
|
122
|
+
"collectionDate": datetime.strptime(
|
123
|
+
collection_date, "%d/%m/%Y"
|
124
|
+
).strftime(date_format),
|
125
|
+
}
|
126
|
+
)
|
127
|
+
|
128
|
+
print(f"Found {len(data['bins'])} collections")
|
129
|
+
print(f"Final data: {data}")
|
81
130
|
|
82
|
-
target_h3 = soup.find("h3", string="Collection Details")
|
83
|
-
tables_after_h3 = target_h3.parent.parent.find_next("table")
|
84
|
-
|
85
|
-
table_rows = tables_after_h3.find_all("tr")
|
86
|
-
for row in table_rows:
|
87
|
-
rowdata = row.find_all("td")
|
88
|
-
if len(rowdata) == 3:
|
89
|
-
labels = rowdata[0].find_all("label")
|
90
|
-
# Strip the day (i.e., Monday) out of the collection date string for parsing
|
91
|
-
if len(labels) >= 2:
|
92
|
-
date_label = labels[1]
|
93
|
-
datestring = date_label.text.strip()
|
94
|
-
|
95
|
-
# Add the bin type and collection date to the 'data' dictionary
|
96
|
-
data["bins"].append(
|
97
|
-
{
|
98
|
-
"type": rowdata[2].text.strip(),
|
99
|
-
"collectionDate": datetime.strptime(
|
100
|
-
datestring, "%d/%m/%Y"
|
101
|
-
).strftime(
|
102
|
-
date_format
|
103
|
-
), # Format the date as needed
|
104
|
-
}
|
105
|
-
)
|
106
131
|
except Exception as e:
|
107
|
-
# Here you can log the exception if needed
|
108
132
|
print(f"An error occurred: {e}")
|
109
|
-
# Optionally, re-raise the exception if you want it to propagate
|
110
133
|
raise
|
111
134
|
finally:
|
112
|
-
# This block ensures that the driver is closed regardless of an exception
|
113
135
|
if driver:
|
114
136
|
driver.quit()
|
115
137
|
return data
|
@@ -0,0 +1,140 @@
|
|
1
|
+
import time
|
2
|
+
import re
|
3
|
+
import requests
|
4
|
+
from datetime import datetime
|
5
|
+
from bs4 import BeautifulSoup
|
6
|
+
from selenium.webdriver.common.by import By
|
7
|
+
from selenium.webdriver.common.keys import Keys
|
8
|
+
from selenium.webdriver.support import expected_conditions as EC
|
9
|
+
from selenium.webdriver.support.ui import WebDriverWait
|
10
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
11
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
12
|
+
|
13
|
+
def get_street_from_postcode(postcode: str, api_key: str) -> str:
|
14
|
+
url = "https://maps.googleapis.com/maps/api/geocode/json"
|
15
|
+
params = {"address": postcode, "key": api_key}
|
16
|
+
response = requests.get(url, params=params)
|
17
|
+
data = response.json()
|
18
|
+
|
19
|
+
if data["status"] != "OK":
|
20
|
+
raise ValueError(f"API error: {data['status']}")
|
21
|
+
|
22
|
+
for component in data["results"][0]["address_components"]:
|
23
|
+
if "route" in component["types"]:
|
24
|
+
return component["long_name"]
|
25
|
+
|
26
|
+
raise ValueError("No street (route) found in the response.")
|
27
|
+
|
28
|
+
class CouncilClass(AbstractGetBinDataClass):
|
29
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
30
|
+
driver = None
|
31
|
+
bin_data = {"bins": []}
|
32
|
+
try:
|
33
|
+
user_postcode = kwargs.get("postcode")
|
34
|
+
if not user_postcode:
|
35
|
+
raise ValueError("No postcode provided.")
|
36
|
+
check_postcode(user_postcode)
|
37
|
+
|
38
|
+
headless = kwargs.get("headless")
|
39
|
+
web_driver = kwargs.get("web_driver")
|
40
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
41
|
+
page = "https://www.slough.gov.uk/bin-collections"
|
42
|
+
driver.get(page)
|
43
|
+
|
44
|
+
# Accept cookies
|
45
|
+
WebDriverWait(driver, 10).until(
|
46
|
+
EC.element_to_be_clickable((By.ID, "ccc-recommended-settings"))
|
47
|
+
).click()
|
48
|
+
|
49
|
+
# Enter the street name into the address search
|
50
|
+
address_input = WebDriverWait(driver, 10).until(
|
51
|
+
EC.presence_of_element_located((By.ID, "keyword_directory25"))
|
52
|
+
)
|
53
|
+
user_address = get_street_from_postcode(user_postcode, "AIzaSyBDLULT7EIlNtHerswPtfmL15Tt3Oc0bV8")
|
54
|
+
address_input.send_keys(user_address + Keys.ENTER)
|
55
|
+
|
56
|
+
# Wait for address results to load
|
57
|
+
WebDriverWait(driver, 10).until(
|
58
|
+
EC.presence_of_all_elements_located((By.CSS_SELECTOR, "span.list__link-text"))
|
59
|
+
)
|
60
|
+
span_elements = driver.find_elements(By.CSS_SELECTOR, "span.list__link-text")
|
61
|
+
|
62
|
+
for span in span_elements:
|
63
|
+
if user_address.lower() in span.text.lower():
|
64
|
+
span.click()
|
65
|
+
break
|
66
|
+
else:
|
67
|
+
raise Exception(f"No link found containing address: {user_address}")
|
68
|
+
|
69
|
+
# Wait for address detail page
|
70
|
+
WebDriverWait(driver, 10).until(
|
71
|
+
EC.presence_of_element_located((By.CSS_SELECTOR, "section.site-content"))
|
72
|
+
)
|
73
|
+
soup = BeautifulSoup(driver.page_source, "html.parser")
|
74
|
+
|
75
|
+
# Extract each bin link and type
|
76
|
+
for heading in soup.select("dt.definition__heading"):
|
77
|
+
heading_text = heading.get_text(strip=True)
|
78
|
+
if "bin day details" in heading_text.lower():
|
79
|
+
bin_type = heading_text.split()[0].capitalize() + " bin"
|
80
|
+
dd = heading.find_next_sibling("dd")
|
81
|
+
link = dd.find("a", href=True)
|
82
|
+
|
83
|
+
if link:
|
84
|
+
bin_url = link["href"]
|
85
|
+
if not bin_url.startswith("http"):
|
86
|
+
bin_url = "https://www.slough.gov.uk" + bin_url
|
87
|
+
|
88
|
+
# Visit the child page
|
89
|
+
print(f"Navigating to {bin_url}")
|
90
|
+
driver.get(bin_url)
|
91
|
+
WebDriverWait(driver, 10).until(
|
92
|
+
EC.presence_of_element_located((By.CSS_SELECTOR, "div.page-content"))
|
93
|
+
)
|
94
|
+
child_soup = BeautifulSoup(driver.page_source, "html.parser")
|
95
|
+
|
96
|
+
editor_div = child_soup.find("div", class_="editor")
|
97
|
+
if not editor_div:
|
98
|
+
print("No editor div found on bin detail page.")
|
99
|
+
continue
|
100
|
+
|
101
|
+
ul = editor_div.find("ul")
|
102
|
+
if not ul:
|
103
|
+
print("No <ul> with dates found in editor div.")
|
104
|
+
continue
|
105
|
+
|
106
|
+
for li in ul.find_all("li"):
|
107
|
+
raw_text = li.get_text(strip=True).replace(".", "")
|
108
|
+
|
109
|
+
if "no collection" in raw_text.lower() or "no collections" in raw_text.lower():
|
110
|
+
print(f"Ignoring non-collection note: {raw_text}")
|
111
|
+
continue
|
112
|
+
|
113
|
+
raw_date = raw_text
|
114
|
+
|
115
|
+
try:
|
116
|
+
parsed_date = datetime.strptime(raw_date, "%d %B %Y")
|
117
|
+
except ValueError:
|
118
|
+
raw_date_cleaned = raw_date.split("(")[0].strip()
|
119
|
+
try:
|
120
|
+
parsed_date = datetime.strptime(raw_date_cleaned, "%d %B %Y")
|
121
|
+
except Exception:
|
122
|
+
print(f"Could not parse date: {raw_text}")
|
123
|
+
continue
|
124
|
+
|
125
|
+
formatted_date = parsed_date.strftime("%d/%m/%Y")
|
126
|
+
contains_date(formatted_date)
|
127
|
+
bin_data["bins"].append({
|
128
|
+
"type": bin_type,
|
129
|
+
"collectionDate": formatted_date
|
130
|
+
})
|
131
|
+
|
132
|
+
print(f"Type: {bin_type}, Date: {formatted_date}")
|
133
|
+
|
134
|
+
except Exception as e:
|
135
|
+
print(f"An error occurred: {e}")
|
136
|
+
raise
|
137
|
+
finally:
|
138
|
+
if driver:
|
139
|
+
driver.quit()
|
140
|
+
return bin_data
|
@@ -0,0 +1,40 @@
|
|
1
|
+
import json
|
2
|
+
import requests
|
3
|
+
from datetime import datetime
|
4
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
5
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
6
|
+
|
7
|
+
|
8
|
+
class CouncilClass(AbstractGetBinDataClass):
|
9
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
10
|
+
user_uprn = kwargs.get("uprn")
|
11
|
+
check_uprn(user_uprn)
|
12
|
+
|
13
|
+
url = f"https://api-2.tewkesbury.gov.uk/incab/rounds/{user_uprn}/next-collection"
|
14
|
+
response = requests.get(url)
|
15
|
+
response.raise_for_status()
|
16
|
+
|
17
|
+
json_data = response.json()
|
18
|
+
|
19
|
+
data = {"bins": []}
|
20
|
+
|
21
|
+
if json_data.get("status") == "OK" and "body" in json_data:
|
22
|
+
for entry in json_data["body"]:
|
23
|
+
bin_type = entry.get("collectionType")
|
24
|
+
date_str = entry.get("NextCollection")
|
25
|
+
|
26
|
+
if bin_type and date_str:
|
27
|
+
try:
|
28
|
+
collection_date = datetime.strptime(date_str, "%Y-%m-%d")
|
29
|
+
data["bins"].append({
|
30
|
+
"type": bin_type,
|
31
|
+
"collectionDate": collection_date.strftime(date_format)
|
32
|
+
})
|
33
|
+
except ValueError:
|
34
|
+
continue
|
35
|
+
|
36
|
+
# Sort by date
|
37
|
+
data["bins"].sort(key=lambda x: x["collectionDate"])
|
38
|
+
|
39
|
+
print(json.dumps(data, indent=2))
|
40
|
+
return data
|
@@ -7,7 +7,7 @@ uk_bin_collection/tests/council_feature_input_parity.py,sha256=DO6Mk4ImYgM5ZCZ-c
|
|
7
7
|
uk_bin_collection/tests/features/environment.py,sha256=VQZjJdJI_kZn08M0j5cUgvKT4k3iTw8icJge1DGOkoA,127
|
8
8
|
uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=SJK-Vc737hrf03tssxxbeg_JIvAH-ddB8f6gU1LTbuQ,251
|
9
9
|
uk_bin_collection/tests/generate_map_test_results.py,sha256=CKnGK2ZgiSXomRGkomX90DitgMP-X7wkHhyKORDcL2E,1144
|
10
|
-
uk_bin_collection/tests/input.json,sha256=
|
10
|
+
uk_bin_collection/tests/input.json,sha256=pwAeJYV_DJ4zMkXDYolTi5e-v9g_0gd-AMv_VCeJsyE,133191
|
11
11
|
uk_bin_collection/tests/output.schema,sha256=ZwKQBwYyTDEM4G2hJwfLUVM-5v1vKRvRK9W9SS1sd18,1086
|
12
12
|
uk_bin_collection/tests/step_defs/step_helpers/file_handler.py,sha256=Ygzi4V0S1MIHqbdstUlIqtRIwnynvhu4UtpweJ6-5N8,1474
|
13
13
|
uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=VZ0a81sioJULD7syAYHjvK_-nT_Rd36tUyzPetSA0gk,3475
|
@@ -21,6 +21,7 @@ uk_bin_collection/uk_bin_collection/councils/AberdeenCityCouncil.py,sha256=Je8Vw
|
|
21
21
|
uk_bin_collection/uk_bin_collection/councils/AberdeenshireCouncil.py,sha256=aO1CSdyqa8oAD0fB79y1Q9bikAWCP_JFa7CsyTa2j9s,1655
|
22
22
|
uk_bin_collection/uk_bin_collection/councils/AdurAndWorthingCouncils.py,sha256=ppbrmm-MzB1wOulK--CU_0j4P-djNf3ozMhHnmQFqLo,1511
|
23
23
|
uk_bin_collection/uk_bin_collection/councils/AmberValleyBoroughCouncil.py,sha256=mTeluIIEcuxLxhfDQ95A1fp8RM6AkJT5tRGZPUbYGdk,1853
|
24
|
+
uk_bin_collection/uk_bin_collection/councils/AngusCouncil.py,sha256=YlhAnxkRAAvrwbUvleNKUuLROcwMTps2eMHElpuctm4,5894
|
24
25
|
uk_bin_collection/uk_bin_collection/councils/AntrimAndNewtonabbeyCouncil.py,sha256=Hp5pteaC5RjL5ZqPZ564S9WQ6ZTKLMO6Dl_fxip2TUc,1653
|
25
26
|
uk_bin_collection/uk_bin_collection/councils/ArdsAndNorthDownCouncil.py,sha256=iMBldxNErgi-ok1o6xpqdNgMvR6qapaNqoTWDTqMeGo,3824
|
26
27
|
uk_bin_collection/uk_bin_collection/councils/ArgyllandButeCouncil.py,sha256=NBeXjzv0bOblkS5xjSy5D0DdW9vtx_TGMLVP0FP1JqA,5073
|
@@ -31,7 +32,7 @@ uk_bin_collection/uk_bin_collection/councils/AshfordBoroughCouncil.py,sha256=kuL
|
|
31
32
|
uk_bin_collection/uk_bin_collection/councils/AylesburyValeCouncil.py,sha256=LouqjspEMt1TkOGqWHs2zkxwOETIy3n7p64uKIlAgUg,2401
|
32
33
|
uk_bin_collection/uk_bin_collection/councils/BCPCouncil.py,sha256=W7QBx6Mgso8RYosuXsaYo3GGNAu-tiyBSmuYxr1JSOU,1707
|
33
34
|
uk_bin_collection/uk_bin_collection/councils/BaberghDistrictCouncil.py,sha256=1eXdST58xFRMdYl8AGNG_EwyQeLa31WSWUe882hQ2ec,6329
|
34
|
-
uk_bin_collection/uk_bin_collection/councils/BarkingDagenham.py,sha256=
|
35
|
+
uk_bin_collection/uk_bin_collection/councils/BarkingDagenham.py,sha256=fgbaD8jsUBJrnUWKbcjSdJHDVRAh9KRUsG6JwB31NYA,6263
|
35
36
|
uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py,sha256=Sd4-pbv0QZsR7soxvXYqsfdOUIqZqS6notyoZthG77s,9182
|
36
37
|
uk_bin_collection/uk_bin_collection/councils/BarnsleyMBCouncil.py,sha256=hqzVKEqwYmjcQjCYBletcCt9_pE96qQ3kn7eDroJeNk,4764
|
37
38
|
uk_bin_collection/uk_bin_collection/councils/BasildonCouncil.py,sha256=NymPmq5pud0PJ8ePcc2r1SKED4EHQ0EY2l71O-Metxc,3313
|
@@ -53,9 +54,9 @@ uk_bin_collection/uk_bin_collection/councils/BradfordMDC.py,sha256=BEWS2c62cOsf2
|
|
53
54
|
uk_bin_collection/uk_bin_collection/councils/BraintreeDistrictCouncil.py,sha256=2vYHilpI8mSwC2Ykdr1gxYAN3excDWqF6AwtGbkwbTw,2441
|
54
55
|
uk_bin_collection/uk_bin_collection/councils/BrecklandCouncil.py,sha256=PX6A_pDvaN109aSNWmEhm88GFKfkClIkmbwGURWvsks,1744
|
55
56
|
uk_bin_collection/uk_bin_collection/councils/BrentCouncil.py,sha256=BsP7V0vezteX0WAxcxqMf3g6ro-J78W6hubefALRMyg,5222
|
56
|
-
uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py,sha256=
|
57
|
+
uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py,sha256=trTCVrSwQ8EuKe30V1vShkkCSw5fEG1kg0DimnsZqdM,6133
|
57
58
|
uk_bin_collection/uk_bin_collection/councils/BristolCityCouncil.py,sha256=nQeRBKrDcZE2m_EzjUBr9dJ5tcUdGcUuA5FcnLkbLr4,5575
|
58
|
-
uk_bin_collection/uk_bin_collection/councils/BroadlandDistrictCouncil.py,sha256=
|
59
|
+
uk_bin_collection/uk_bin_collection/councils/BroadlandDistrictCouncil.py,sha256=YhzP8zar_oSSkBOA3mdMAehnMTrcTBmGO0RfC4UBzvM,8236
|
59
60
|
uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py,sha256=dii85JLmYU1uMidCEsWVo3stTcq_QqyC65DxG8u1UmE,4302
|
60
61
|
uk_bin_collection/uk_bin_collection/councils/BromsgroveDistrictCouncil.py,sha256=PUfxP8j5Oh9wFHkdjbrJzQli9UzMHZzwrZ2hkThrvhI,1781
|
61
62
|
uk_bin_collection/uk_bin_collection/councils/BroxbourneCouncil.py,sha256=mcRtkFc9g3YNN17OQfhzYJtNYeWrZPP_e7m7goEhz5I,3012
|
@@ -78,7 +79,7 @@ uk_bin_collection/uk_bin_collection/councils/CherwellDistrictCouncil.py,sha256=V
|
|
78
79
|
uk_bin_collection/uk_bin_collection/councils/CheshireEastCouncil.py,sha256=I7Dj8LzG-Q4yrJ99jLRIwKwW5WQ9he8UksvF_YPzTxI,1681
|
79
80
|
uk_bin_collection/uk_bin_collection/councils/CheshireWestAndChesterCouncil.py,sha256=5mKZf22NgdyBY-SqV0c2q8b8IJobkoZrsfGEVUcxUyM,3544
|
80
81
|
uk_bin_collection/uk_bin_collection/councils/ChesterfieldBoroughCouncil.py,sha256=mZiM8Ugm_OP0JkC5pLaQmi4i79mAp4SNNrcIdsREjHw,7198
|
81
|
-
uk_bin_collection/uk_bin_collection/councils/ChichesterDistrictCouncil.py,sha256=
|
82
|
+
uk_bin_collection/uk_bin_collection/councils/ChichesterDistrictCouncil.py,sha256=bvtqXSZN64jSND0NxwzJ-WA9H49FwARQMGS3PlUSiUg,6311
|
82
83
|
uk_bin_collection/uk_bin_collection/councils/ChorleyCouncil.py,sha256=M7HjuUaFq8aSnOf_9m1QS4MmPPMmPhF3mLHSrfDPtV0,5194
|
83
84
|
uk_bin_collection/uk_bin_collection/councils/ColchesterCityCouncil.py,sha256=Mny-q2rQkWe2Tj1gINwEM1L4AkqQl1EDMAaKY0-deD4,3968
|
84
85
|
uk_bin_collection/uk_bin_collection/councils/ConwyCountyBorough.py,sha256=QcFHTzW7Q1jYyHWq6thpA6fhO8w38eCOpUPk9EfjGIA,1291
|
@@ -124,6 +125,7 @@ uk_bin_collection/uk_bin_collection/councils/ExeterCityCouncil.py,sha256=FPNyBuQ
|
|
124
125
|
uk_bin_collection/uk_bin_collection/councils/FalkirkCouncil.py,sha256=C3OA9PEhBsCYPzwsSdqVi_SbF8uiB186i2XfHWKd3VI,1694
|
125
126
|
uk_bin_collection/uk_bin_collection/councils/FarehamBoroughCouncil.py,sha256=25QxeN5q3ad1Wwexs2d-B7ooH0ru6pOUx58413FOTY4,2352
|
126
127
|
uk_bin_collection/uk_bin_collection/councils/FenlandDistrictCouncil.py,sha256=sFrnKzIE2tIcz0YrC6A9HcevzgNdf6E6_HLGMWDKtGw,2513
|
128
|
+
uk_bin_collection/uk_bin_collection/councils/FermanaghOmaghDistrictCouncil.py,sha256=om9bdOv3_n16DMNX3-ndRwBEAlddhY1BB8z6doXrDfo,3317
|
127
129
|
uk_bin_collection/uk_bin_collection/councils/FifeCouncil.py,sha256=eP_NnHtBLyflRUko9ubi_nxUPb7qg9SbaaSxqWZxNEs,2157
|
128
130
|
uk_bin_collection/uk_bin_collection/councils/FlintshireCountyCouncil.py,sha256=RvPHhGbzP3mcjgWe2rIQux43UuDH7XofJGIKs7wJRe0,2060
|
129
131
|
uk_bin_collection/uk_bin_collection/councils/FolkstoneandHytheDistrictCouncil.py,sha256=yKgZhua-2hjMihHshhncXVUBagbTOQBnNbKzdIZkWjw,3114
|
@@ -207,7 +209,7 @@ uk_bin_collection/uk_bin_collection/councils/NewhamCouncil.py,sha256=-UTgDseUtAG
|
|
207
209
|
uk_bin_collection/uk_bin_collection/councils/NewportCityCouncil.py,sha256=dAcl5P97bttl2xCPvxof1a18kmqOrMmiElgtn3Ej7zs,8480
|
208
210
|
uk_bin_collection/uk_bin_collection/councils/NorthAyrshireCouncil.py,sha256=o8zv40Wt19d51mrN5lsgLMCKMokMPmI1cMHBNT5yAho,1976
|
209
211
|
uk_bin_collection/uk_bin_collection/councils/NorthDevonCountyCouncil.py,sha256=tgJKIvu7nnCAHu_HImfG5SQABD6ygKFqrZU-ZoC6ObY,6260
|
210
|
-
uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py,sha256=
|
212
|
+
uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py,sha256=BfNpYcjG3z0Yz8OYN6NkfzvZ5k1FI-80D-rv211kPPU,5449
|
211
213
|
uk_bin_collection/uk_bin_collection/councils/NorthEastLincs.py,sha256=fYf438VZIaOaqPSwdTTWVjFTdrI0jGfFsxVzOc-QdkA,1817
|
212
214
|
uk_bin_collection/uk_bin_collection/councils/NorthHertfordshireDistrictCouncil.py,sha256=dFgvZqVKEVEP0zSPeh2s9xIWSCGbhYHpXn2U6Nk0HXM,2847
|
213
215
|
uk_bin_collection/uk_bin_collection/councils/NorthKestevenDistrictCouncil.py,sha256=vYOCerJXr9LTP6F2wm4vpYNYbQaWNZ6yfHEQ33N_hTw,1681
|
@@ -254,6 +256,7 @@ uk_bin_collection/uk_bin_collection/councils/SeftonCouncil.py,sha256=XUEz2li0oHr
|
|
254
256
|
uk_bin_collection/uk_bin_collection/councils/SevenoaksDistrictCouncil.py,sha256=qqrrRaSVm9CYAtm0rB2ZnyH_nLwaReuacoUxZpo597k,4260
|
255
257
|
uk_bin_collection/uk_bin_collection/councils/SheffieldCityCouncil.py,sha256=9g9AeiackoWyej9EVlKUzywzAtMuBVD0f93ZryAUha8,2016
|
256
258
|
uk_bin_collection/uk_bin_collection/councils/ShropshireCouncil.py,sha256=6OIEhJmv-zLJiEo8WaJePA7JuhNRGkh2WohFLhzN8Kk,1477
|
259
|
+
uk_bin_collection/uk_bin_collection/councils/SloughBoroughCouncil.py,sha256=B6ksHUgcPns9fRUrPpyTkGGAwqn1DH0rQfS3Ma6Gzrs,5987
|
257
260
|
uk_bin_collection/uk_bin_collection/councils/SolihullCouncil.py,sha256=gbTHjbdV46evGfLfF8rxVMQIgNZD-XPHgZeuyje7kGY,1609
|
258
261
|
uk_bin_collection/uk_bin_collection/councils/SomersetCouncil.py,sha256=CZJnkCGn4-yH31HH5_ix-8V2_vsjGuKYxgzAPGZdSAw,8480
|
259
262
|
uk_bin_collection/uk_bin_collection/councils/SouthAyrshireCouncil.py,sha256=03eapeXwxneKI4ccKPSHoviIbhmV1m90I-0WQ_s3KsY,2722
|
@@ -294,6 +297,7 @@ uk_bin_collection/uk_bin_collection/councils/TeignbridgeCouncil.py,sha256=-NowMN
|
|
294
297
|
uk_bin_collection/uk_bin_collection/councils/TelfordAndWrekinCouncil.py,sha256=p1ZS5R4EGxbEWlRBrkGXgKwE_lkyBT-R60yKFFhVObc,1844
|
295
298
|
uk_bin_collection/uk_bin_collection/councils/TendringDistrictCouncil.py,sha256=1_CkpWPTfRUEP5YJ9R4_dJRLtb-O9i83hfWJc1shw_c,4283
|
296
299
|
uk_bin_collection/uk_bin_collection/councils/TestValleyBoroughCouncil.py,sha256=Dtfkyrwt795W7gqFJxVGRR8t3R5WMNQZwTWJckLpZWE,8480
|
300
|
+
uk_bin_collection/uk_bin_collection/councils/TewkesburyBoroughCouncil.py,sha256=cSULCQBddk4tfaJZHv7mrNwM4sZkOYJpmlQXlvrZvPk,1396
|
297
301
|
uk_bin_collection/uk_bin_collection/councils/ThanetDistrictCouncil.py,sha256=Cxrf0tUryDL-wFclPH5yovVt8i7Sc7g-ZFrU9_wg6KY,2717
|
298
302
|
uk_bin_collection/uk_bin_collection/councils/ThreeRiversDistrictCouncil.py,sha256=RHt3e9oeKzwxjjY-M8aC0nk-ZXhHIoyC81JzxkPVxsE,5531
|
299
303
|
uk_bin_collection/uk_bin_collection/councils/ThurrockCouncil.py,sha256=vAZMm6mcsdEcOkP15xwxWy9gdXpmLYQFH7qRifurNoY,2935
|
@@ -340,8 +344,8 @@ uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py,sha256=I2kBYMlsD4bId
|
|
340
344
|
uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=QD4v4xpsEE0QheR_fGaNOIRMc2FatcUfKkkhAhseyVU,1159
|
341
345
|
uk_bin_collection/uk_bin_collection/create_new_council.py,sha256=m-IhmWmeWQlFsTZC4OxuFvtw5ZtB8EAJHxJTH4O59lQ,1536
|
342
346
|
uk_bin_collection/uk_bin_collection/get_bin_data.py,sha256=YvmHfZqanwrJ8ToGch34x-L-7yPe31nB_x77_Mgl_vo,4545
|
343
|
-
uk_bin_collection-0.
|
344
|
-
uk_bin_collection-0.
|
345
|
-
uk_bin_collection-0.
|
346
|
-
uk_bin_collection-0.
|
347
|
-
uk_bin_collection-0.
|
347
|
+
uk_bin_collection-0.152.0.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
|
348
|
+
uk_bin_collection-0.152.0.dist-info/METADATA,sha256=2gnhvjy_Hu9moRYLw5s6ogGMDh1dYkitouNtc5DOOM4,20914
|
349
|
+
uk_bin_collection-0.152.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
350
|
+
uk_bin_collection-0.152.0.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
|
351
|
+
uk_bin_collection-0.152.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
{uk_bin_collection-0.151.0.dist-info → uk_bin_collection-0.152.0.dist-info}/entry_points.txt
RENAMED
File without changes
|