uk_bin_collection 0.102.0__py3-none-any.whl → 0.104.0__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- uk_bin_collection/tests/input.json +38 -1
- uk_bin_collection/uk_bin_collection/councils/AberdeenshireCouncil.py +52 -0
- uk_bin_collection/uk_bin_collection/councils/BasildonCouncil.py +42 -39
- uk_bin_collection/uk_bin_collection/councils/BelfastCityCouncil.py +13 -8
- uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py +24 -21
- uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/CanterburyCityCouncil.py +54 -0
- uk_bin_collection/uk_bin_collection/councils/CheshireEastCouncil.py +25 -10
- uk_bin_collection/uk_bin_collection/councils/CornwallCouncil.py +21 -20
- uk_bin_collection/uk_bin_collection/councils/EnfieldCouncil.py +16 -18
- uk_bin_collection/uk_bin_collection/councils/GedlingBoroughCouncil.py +10 -4
- uk_bin_collection/uk_bin_collection/councils/IslingtonCouncil.py +6 -4
- uk_bin_collection/uk_bin_collection/councils/LutonBoroughCouncil.py +81 -0
- uk_bin_collection/uk_bin_collection/councils/MoleValleyDistrictCouncil.py +37 -20
- uk_bin_collection/uk_bin_collection/councils/NorthTynesideCouncil.py +11 -9
- uk_bin_collection/uk_bin_collection/councils/RochfordCouncil.py +1 -2
- uk_bin_collection/uk_bin_collection/councils/RotherhamCouncil.py +8 -6
- uk_bin_collection/uk_bin_collection/councils/SwindonBoroughCouncil.py +56 -0
- uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py +21 -11
- uk_bin_collection/uk_bin_collection/councils/WestOxfordshireDistrictCouncil.py +113 -0
- uk_bin_collection/uk_bin_collection/councils/WokinghamBoroughCouncil.py +1 -1
- {uk_bin_collection-0.102.0.dist-info → uk_bin_collection-0.104.0.dist-info}/METADATA +1 -1
- {uk_bin_collection-0.102.0.dist-info → uk_bin_collection-0.104.0.dist-info}/RECORD +26 -21
- {uk_bin_collection-0.102.0.dist-info → uk_bin_collection-0.104.0.dist-info}/LICENSE +0 -0
- {uk_bin_collection-0.102.0.dist-info → uk_bin_collection-0.104.0.dist-info}/WHEEL +0 -0
- {uk_bin_collection-0.102.0.dist-info → uk_bin_collection-0.104.0.dist-info}/entry_points.txt +0 -0
@@ -1,4 +1,11 @@
|
|
1
1
|
{
|
2
|
+
"AberdeenshireCouncil": {
|
3
|
+
"url": "https://online.aberdeenshire.gov.uk",
|
4
|
+
"wiki_command_url_override": "https://online.aberdeenshire.gov.uk",
|
5
|
+
"uprn": "151176430",
|
6
|
+
"wiki_name": "Aberdeenshire Council",
|
7
|
+
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
|
8
|
+
},
|
2
9
|
"AdurAndWorthingCouncils": {
|
3
10
|
"url": "https://www.adur-worthing.gov.uk/bin-day/?brlu-selected-address=100061878829",
|
4
11
|
"wiki_command_url_override": "https://www.adur-worthing.gov.uk/bin-day/?brlu-selected-address=XXXXXXXX",
|
@@ -201,6 +208,13 @@
|
|
201
208
|
"wiki_name": "Cannock Chase District Council",
|
202
209
|
"wiki_note": "To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)"
|
203
210
|
},
|
211
|
+
"CanterburyCityCouncil": {
|
212
|
+
"url": "https://www.canterbury.gov.uk",
|
213
|
+
"wiki_command_url_override": "https://www.canterbury.gov.uk",
|
214
|
+
"uprn": "10094583181",
|
215
|
+
"wiki_name": "Canterbury City Council",
|
216
|
+
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
|
217
|
+
},
|
204
218
|
"CardiffCouncil": {
|
205
219
|
"skip_get_url": true,
|
206
220
|
"uprn": "100100112419",
|
@@ -367,7 +381,7 @@
|
|
367
381
|
},
|
368
382
|
"EastDevonDC": {
|
369
383
|
"url": "https://eastdevon.gov.uk/recycling-and-waste/recycling-waste-information/when-is-my-bin-collected/future-collections-calendar/?UPRN=010090909915",
|
370
|
-
"wiki_command_url_override": "https://eastdevon.gov.uk/recycling-waste/recycling-
|
384
|
+
"wiki_command_url_override": "https://eastdevon.gov.uk/recycling-and-waste/recycling-waste-information/when-is-my-bin-collected/future-collections-calendar/?UPRN=XXXXXXXX",
|
371
385
|
"wiki_name": "East Devon District Council",
|
372
386
|
"wiki_note": "Replace XXXXXXXX with UPRN."
|
373
387
|
},
|
@@ -681,6 +695,13 @@
|
|
681
695
|
"wiki_name": "London Borough Redbridge",
|
682
696
|
"wiki_note": "Follow the instructions [here](https://my.redbridge.gov.uk/RecycleRefuse) until you get the page listing your \"Address\" then copy the entire address text and use that in the house number field."
|
683
697
|
},
|
698
|
+
"LutonBoroughCouncil": {
|
699
|
+
"url": "https://myforms.luton.gov.uk",
|
700
|
+
"wiki_command_url_override": "https://myforms.luton.gov.uk",
|
701
|
+
"uprn": "100080155778",
|
702
|
+
"wiki_name": "Luton Borough Council",
|
703
|
+
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
|
704
|
+
},
|
684
705
|
"MaldonDistrictCouncil": {
|
685
706
|
"skip_get_url": true,
|
686
707
|
"uprn": "100090557253",
|
@@ -1189,6 +1210,13 @@
|
|
1189
1210
|
"url": "https://www1.swansea.gov.uk/recyclingsearch/",
|
1190
1211
|
"wiki_name": "SwanseaCouncil"
|
1191
1212
|
},
|
1213
|
+
"SwindonBoroughCouncil": {
|
1214
|
+
"url": "https://www.swindon.gov.uk",
|
1215
|
+
"wiki_command_url_override": "https://www.swindon.gov.uk",
|
1216
|
+
"uprn": "10022793351",
|
1217
|
+
"wiki_name": "Swindon Borough Council",
|
1218
|
+
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
|
1219
|
+
},
|
1192
1220
|
"TamesideMBCouncil": {
|
1193
1221
|
"skip_get_url": true,
|
1194
1222
|
"uprn": "100012835362",
|
@@ -1369,6 +1397,15 @@
|
|
1369
1397
|
"url": "https://www.northampton.gov.uk/info/200084/bins-waste-and-recycling/1602/check-your-collection-day",
|
1370
1398
|
"wiki_name": "West Northamptonshire Council"
|
1371
1399
|
},
|
1400
|
+
"WestOxfordshireDistrictCouncil": {
|
1401
|
+
"house_number": "24",
|
1402
|
+
"postcode": "OX28 1YA",
|
1403
|
+
"skip_get_url": true,
|
1404
|
+
"url": "https://community.westoxon.gov.uk/s/waste-collection-enquiry",
|
1405
|
+
"web_driver": "http://selenium:4444",
|
1406
|
+
"wiki_name": "West Oxfordshire District Council",
|
1407
|
+
"wiki_note": "Pass the full address in the house number and postcode in"
|
1408
|
+
},
|
1372
1409
|
"WestSuffolkCouncil": {
|
1373
1410
|
"postcode": "IP28 6DR",
|
1374
1411
|
"skip_get_url": true,
|
@@ -0,0 +1,52 @@
|
|
1
|
+
import requests
|
2
|
+
from bs4 import BeautifulSoup
|
3
|
+
|
4
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
5
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
6
|
+
|
7
|
+
|
8
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
9
|
+
class CouncilClass(AbstractGetBinDataClass):
|
10
|
+
"""
|
11
|
+
Concrete classes have to implement all abstract operations of the
|
12
|
+
base class. They can also override some operations with a default
|
13
|
+
implementation.
|
14
|
+
"""
|
15
|
+
|
16
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
17
|
+
|
18
|
+
user_uprn = kwargs.get("uprn")
|
19
|
+
check_uprn(user_uprn)
|
20
|
+
bindata = {"bins": []}
|
21
|
+
|
22
|
+
URI = f"https://online.aberdeenshire.gov.uk/Apps/Waste-Collections/Routes/Route/{user_uprn}"
|
23
|
+
|
24
|
+
# Make the GET request
|
25
|
+
response = requests.get(URI)
|
26
|
+
|
27
|
+
soup = BeautifulSoup(response.content, features="html.parser")
|
28
|
+
soup.prettify()
|
29
|
+
|
30
|
+
for collection in soup.find("table").find("tbody").find_all("tr"):
|
31
|
+
th = collection.find("th")
|
32
|
+
if th:
|
33
|
+
continue
|
34
|
+
td = collection.find_all("td")
|
35
|
+
collection_date = datetime.strptime(
|
36
|
+
td[0].text,
|
37
|
+
"%d/%m/%Y %A",
|
38
|
+
)
|
39
|
+
bin_type = td[1].text.split(" and ")
|
40
|
+
|
41
|
+
for bin in bin_type:
|
42
|
+
dict_data = {
|
43
|
+
"type": bin,
|
44
|
+
"collectionDate": collection_date.strftime(date_format),
|
45
|
+
}
|
46
|
+
bindata["bins"].append(dict_data)
|
47
|
+
|
48
|
+
bindata["bins"].sort(
|
49
|
+
key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
|
50
|
+
)
|
51
|
+
|
52
|
+
return bindata
|
@@ -1,13 +1,13 @@
|
|
1
|
-
|
1
|
+
import requests
|
2
|
+
import json
|
3
|
+
from datetime import datetime
|
4
|
+
from uk_bin_collection.uk_bin_collection.common import check_uprn, date_format as DATE_FORMAT
|
2
5
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
3
6
|
|
4
7
|
|
5
|
-
# import the wonderful Beautiful Soup and the URL grabber
|
6
8
|
class CouncilClass(AbstractGetBinDataClass):
|
7
9
|
"""
|
8
|
-
Concrete
|
9
|
-
base class. They can also override some operations with a default
|
10
|
-
implementation.
|
10
|
+
Concrete class that implements the abstract bin data fetching and parsing logic.
|
11
11
|
"""
|
12
12
|
|
13
13
|
def parse_data(self, page: str, **kwargs) -> dict:
|
@@ -18,64 +18,67 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
18
18
|
check_uprn(uprn)
|
19
19
|
|
20
20
|
payload = {
|
21
|
-
# Add your payload details here (replace this with the actual payload structure if required)
|
22
21
|
"uprn": uprn
|
23
22
|
}
|
24
23
|
|
25
|
-
|
26
|
-
headers = {
|
27
|
-
"Content-Type": "application/json"
|
28
|
-
}
|
24
|
+
headers = {"Content-Type": "application/json"}
|
29
25
|
|
30
26
|
response = requests.post(url_base, data=json.dumps(payload), headers=headers)
|
31
27
|
|
32
|
-
# Ensure the request was successful
|
33
28
|
if response.status_code == 200:
|
34
29
|
data = response.json()
|
35
30
|
|
36
31
|
# Initialize an empty list to store the bin collection details
|
37
|
-
|
38
32
|
bins = []
|
39
33
|
|
40
34
|
# Function to add collection details to bins list
|
41
35
|
def add_collection(service_name, collection_data):
|
42
|
-
bins.append(
|
43
|
-
|
44
|
-
|
45
|
-
|
36
|
+
bins.append(
|
37
|
+
{
|
38
|
+
"type": service_name,
|
39
|
+
"collectionDate": collection_data.get("current_collection_date"),
|
40
|
+
}
|
41
|
+
)
|
46
42
|
|
47
|
-
|
48
|
-
|
43
|
+
available_services = data.get("refuse", {}).get("available_services", {})
|
44
|
+
|
45
|
+
date_format = "%d-%m-%Y" # Define the desired date format
|
49
46
|
|
50
47
|
for service_name, service_data in available_services.items():
|
51
|
-
#
|
48
|
+
# Handle the different cases of service data
|
52
49
|
match service_data["container"]:
|
53
50
|
case "Green Wheelie Bin":
|
54
|
-
subscription_status =
|
55
|
-
|
51
|
+
subscription_status = (
|
52
|
+
service_data["subscription"]["active"]
|
53
|
+
if service_data.get("subscription")
|
54
|
+
else False
|
55
|
+
)
|
56
|
+
type_descr = f"Green Wheelie Bin ({'Active' if subscription_status else 'Expired'})"
|
56
57
|
case "N/A":
|
57
|
-
type_descr = service_data
|
58
|
+
type_descr = service_data.get("name", "Unknown Service")
|
58
59
|
case _:
|
59
|
-
type_descr = service_data
|
60
|
-
|
60
|
+
type_descr = service_data.get("container", "Unknown Container")
|
61
61
|
|
62
62
|
date_str = service_data.get("current_collection_date")
|
63
|
-
#
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
"
|
72
|
-
|
63
|
+
if date_str: # Ensure the date string exists
|
64
|
+
try:
|
65
|
+
# Parse and format the date string
|
66
|
+
date_obj = datetime.strptime(date_str, "%Y-%m-%d")
|
67
|
+
formatted_date = date_obj.strftime(DATE_FORMAT)
|
68
|
+
except ValueError:
|
69
|
+
formatted_date = "Invalid Date"
|
70
|
+
else:
|
71
|
+
formatted_date = "No Collection Date"
|
72
|
+
|
73
|
+
bins.append(
|
74
|
+
{
|
75
|
+
"type": type_descr, # Use service name from the data
|
76
|
+
"collectionDate": formatted_date,
|
77
|
+
}
|
78
|
+
)
|
73
79
|
|
74
80
|
else:
|
75
81
|
print(f"Failed to fetch data. Status code: {response.status_code}")
|
82
|
+
return {}
|
76
83
|
|
77
|
-
|
78
|
-
"bins": bins
|
79
|
-
}
|
80
|
-
|
81
|
-
return data
|
84
|
+
return {"bins": bins}
|
@@ -9,7 +9,6 @@ from uk_bin_collection.uk_bin_collection.common import *
|
|
9
9
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
10
10
|
|
11
11
|
|
12
|
-
|
13
12
|
# import the wonderful Beautiful Soup and the URL grabber
|
14
13
|
class CouncilClass(AbstractGetBinDataClass):
|
15
14
|
"""
|
@@ -34,7 +33,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
34
33
|
|
35
34
|
session = requests.Session()
|
36
35
|
session.headers.update(headers)
|
37
|
-
|
36
|
+
|
38
37
|
user_uprn = kwargs.get("uprn")
|
39
38
|
user_postcode = kwargs.get("postcode")
|
40
39
|
URL = "https://online.belfastcity.gov.uk/find-bin-collection-day/Default.aspx"
|
@@ -47,14 +46,16 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
47
46
|
"__EVENTTARGET": "",
|
48
47
|
"__EVENTARGUMENT": "",
|
49
48
|
"__VIEWSTATE": self.get_session_variable(soup, "__VIEWSTATE"),
|
50
|
-
"__VIEWSTATEGENERATOR": self.get_session_variable(
|
49
|
+
"__VIEWSTATEGENERATOR": self.get_session_variable(
|
50
|
+
soup, "__VIEWSTATEGENERATOR"
|
51
|
+
),
|
51
52
|
"__SCROLLPOSITIONX": "0",
|
52
53
|
"__SCROLLPOSITIONY": "0",
|
53
54
|
"__EVENTVALIDATION": self.get_session_variable(soup, "__EVENTVALIDATION"),
|
54
55
|
"ctl00$MainContent$searchBy_radio": "P",
|
55
56
|
"ctl00$MainContent$Street_textbox": "",
|
56
57
|
"ctl00$MainContent$Postcode_textbox": user_postcode,
|
57
|
-
"ctl00$MainContent$AddressLookup_button": "Find address"
|
58
|
+
"ctl00$MainContent$AddressLookup_button": "Find address",
|
58
59
|
}
|
59
60
|
|
60
61
|
# Build intermediate ASP.NET variables for uprn Select address
|
@@ -65,7 +66,9 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
65
66
|
"__EVENTTARGET": "",
|
66
67
|
"__EVENTARGUMENT": "",
|
67
68
|
"__VIEWSTATE": self.get_session_variable(soup, "__VIEWSTATE"),
|
68
|
-
"__VIEWSTATEGENERATOR": self.get_session_variable(
|
69
|
+
"__VIEWSTATEGENERATOR": self.get_session_variable(
|
70
|
+
soup, "__VIEWSTATEGENERATOR"
|
71
|
+
),
|
69
72
|
"__SCROLLPOSITIONX": "0",
|
70
73
|
"__SCROLLPOSITIONY": "0",
|
71
74
|
"__EVENTVALIDATION": self.get_session_variable(soup, "__EVENTVALIDATION"),
|
@@ -73,14 +76,14 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
73
76
|
"ctl00$MainContent$Street_textbox": "",
|
74
77
|
"ctl00$MainContent$Postcode_textbox": user_postcode,
|
75
78
|
"ctl00$MainContent$lstAddresses": user_uprn,
|
76
|
-
"ctl00$MainContent$SelectAddress_button": "Select address"
|
79
|
+
"ctl00$MainContent$SelectAddress_button": "Select address",
|
77
80
|
}
|
78
81
|
|
79
82
|
# Actual http call to get Bins Data
|
80
83
|
response = session.post(URL, data=form_data)
|
81
84
|
response.raise_for_status()
|
82
85
|
soup = BeautifulSoup(response.text, "html.parser")
|
83
|
-
|
86
|
+
|
84
87
|
# Find Bins table and data
|
85
88
|
table = soup.find("div", {"id": "binsGrid"})
|
86
89
|
if table:
|
@@ -91,7 +94,9 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
91
94
|
collection_type = columns[0].get_text(strip=True)
|
92
95
|
collection_date_raw = columns[3].get_text(strip=True)
|
93
96
|
# if the month number is a single digit there are 2 spaces, stripping all spaces to make it consistent
|
94
|
-
collection_date = datetime.strptime(
|
97
|
+
collection_date = datetime.strptime(
|
98
|
+
collection_date_raw.replace(" ", ""), "%a%b%d%Y"
|
99
|
+
)
|
95
100
|
bin_entry = {
|
96
101
|
"type": collection_type,
|
97
102
|
"collectionDate": collection_date.strftime(date_format),
|
@@ -45,17 +45,13 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
45
45
|
)
|
46
46
|
inputElement_postcodesearch.send_keys(user_postcode)
|
47
47
|
|
48
|
-
|
49
|
-
|
50
48
|
find_address_btn = wait.until(
|
51
49
|
EC.element_to_be_clickable((By.XPATH, '//*[@id="sub"]'))
|
52
50
|
)
|
53
51
|
find_address_btn.click()
|
54
52
|
|
55
53
|
dropdown_options = wait.until(
|
56
|
-
EC.presence_of_element_located(
|
57
|
-
(By.XPATH, '//*[@id="address"]')
|
58
|
-
)
|
54
|
+
EC.presence_of_element_located((By.XPATH, '//*[@id="address"]'))
|
59
55
|
)
|
60
56
|
time.sleep(2)
|
61
57
|
dropdown_options.click()
|
@@ -71,11 +67,8 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
71
67
|
# Click the element
|
72
68
|
address.click()
|
73
69
|
|
74
|
-
|
75
70
|
submit_address = wait.until(
|
76
|
-
EC.presence_of_element_located(
|
77
|
-
(By.XPATH, '//*[@id="go"]')
|
78
|
-
)
|
71
|
+
EC.presence_of_element_located((By.XPATH, '//*[@id="go"]'))
|
79
72
|
)
|
80
73
|
time.sleep(2)
|
81
74
|
submit_address.click()
|
@@ -83,13 +76,11 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
83
76
|
results_found = wait.until(
|
84
77
|
EC.element_to_be_clickable(
|
85
78
|
(By.XPATH, '//h1[contains(text(), "Your bin days")]')
|
86
|
-
)
|
87
79
|
)
|
80
|
+
)
|
88
81
|
|
89
82
|
final_page = wait.until(
|
90
|
-
EC.presence_of_element_located(
|
91
|
-
(By.CLASS_NAME, "waste__collections")
|
92
|
-
)
|
83
|
+
EC.presence_of_element_located((By.CLASS_NAME, "waste__collections"))
|
93
84
|
)
|
94
85
|
|
95
86
|
soup = BeautifulSoup(driver.page_source, features="html.parser")
|
@@ -103,29 +94,41 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
103
94
|
# Loop through each bin field
|
104
95
|
for bin_section in bin_sections:
|
105
96
|
# Extract the bin type (e.g., "Brown Caddy", "Green Wheelie Bin", etc.)
|
106
|
-
bin_type = bin_section.get_text(strip=True).split("\n")[
|
97
|
+
bin_type = bin_section.get_text(strip=True).split("\n")[
|
98
|
+
0
|
99
|
+
] # The first part is the bin type
|
107
100
|
|
108
101
|
# Find the next sibling <dl> tag that contains the next collection information
|
109
102
|
summary_list = bin_section.find_next("dl", class_="govuk-summary-list")
|
110
103
|
|
111
104
|
if summary_list:
|
112
105
|
# Now, instead of finding by class, we'll search by text within the dt element
|
113
|
-
next_collection_dt = summary_list.find(
|
106
|
+
next_collection_dt = summary_list.find(
|
107
|
+
"dt", string=lambda text: "Next collection" in text
|
108
|
+
)
|
114
109
|
|
115
110
|
if next_collection_dt:
|
116
111
|
# Find the sibling <dd> tag for the collection date
|
117
|
-
next_collection = next_collection_dt.find_next_sibling(
|
112
|
+
next_collection = next_collection_dt.find_next_sibling(
|
113
|
+
"dd"
|
114
|
+
).get_text(strip=True)
|
118
115
|
|
119
116
|
if next_collection:
|
120
117
|
try:
|
121
118
|
# Parse the next collection date (assuming the format is like "Tuesday 15 October 2024")
|
122
|
-
parsed_date = datetime.strptime(
|
119
|
+
parsed_date = datetime.strptime(
|
120
|
+
next_collection, "%A %d %B %Y"
|
121
|
+
)
|
123
122
|
|
124
123
|
# Add the bin information to the data dictionary
|
125
|
-
data["bins"].append(
|
126
|
-
|
127
|
-
|
128
|
-
|
124
|
+
data["bins"].append(
|
125
|
+
{
|
126
|
+
"type": bin_type,
|
127
|
+
"collectionDate": parsed_date.strftime(
|
128
|
+
date_format
|
129
|
+
),
|
130
|
+
}
|
131
|
+
)
|
129
132
|
except ValueError as e:
|
130
133
|
print(f"Error parsing date for {bin_type}: {e}")
|
131
134
|
else:
|
@@ -82,7 +82,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
82
82
|
bin_type = " ".join(words).capitalize()
|
83
83
|
date_list = item.find_all("p")
|
84
84
|
for d in date_list:
|
85
|
-
clean_date_str = re.sub(r
|
85
|
+
clean_date_str = re.sub(r"[^A-Za-z0-9 ]+", "", d.text.strip())
|
86
86
|
next_collection = datetime.strptime(clean_date_str, "%A %d %B %Y")
|
87
87
|
collections.append((bin_type, next_collection))
|
88
88
|
|
@@ -0,0 +1,54 @@
|
|
1
|
+
import time
|
2
|
+
|
3
|
+
import requests
|
4
|
+
|
5
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
6
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
7
|
+
|
8
|
+
|
9
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
10
|
+
class CouncilClass(AbstractGetBinDataClass):
|
11
|
+
"""
|
12
|
+
Concrete classes have to implement all abstract operations of the
|
13
|
+
base class. They can also override some operations with a default
|
14
|
+
implementation.
|
15
|
+
"""
|
16
|
+
|
17
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
18
|
+
|
19
|
+
user_uprn = kwargs.get("uprn")
|
20
|
+
check_uprn(user_uprn)
|
21
|
+
bindata = {"bins": []}
|
22
|
+
|
23
|
+
data = {"uprn": user_uprn, "usrn": "1"}
|
24
|
+
|
25
|
+
URI = (
|
26
|
+
"https://zbr7r13ke2.execute-api.eu-west-2.amazonaws.com/Beta/get-bin-dates"
|
27
|
+
)
|
28
|
+
|
29
|
+
# Make the GET request
|
30
|
+
response = requests.post(URI, json=data)
|
31
|
+
response.raise_for_status()
|
32
|
+
|
33
|
+
# Parse the JSON response
|
34
|
+
bin_collection = json.loads(response.json()["dates"])
|
35
|
+
collections = {
|
36
|
+
"General": bin_collection["blackBinDay"],
|
37
|
+
"Recycling": bin_collection["recyclingBinDay"],
|
38
|
+
"Food": bin_collection["foodBinDay"],
|
39
|
+
"Garden": bin_collection["gardenBinDay"],
|
40
|
+
}
|
41
|
+
# Loop through each collection in bin_collection
|
42
|
+
for collection in collections:
|
43
|
+
print(collection)
|
44
|
+
|
45
|
+
if len(collections[collection]) <= 0:
|
46
|
+
continue
|
47
|
+
for date in collections[collection]:
|
48
|
+
date = (
|
49
|
+
datetime.strptime(date, "%Y-%m-%dT%H:%M:%S").strftime("%d/%m/%Y"),
|
50
|
+
)
|
51
|
+
dict_data = {"type": collection, "collectionDate": date[0]}
|
52
|
+
bindata["bins"].append(dict_data)
|
53
|
+
|
54
|
+
return bindata
|
@@ -1,26 +1,41 @@
|
|
1
|
-
from
|
1
|
+
from typing import Dict, Any, Optional
|
2
|
+
from bs4 import BeautifulSoup, Tag, NavigableString
|
2
3
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
3
4
|
|
4
|
-
|
5
|
+
"""
|
6
|
+
This module provides bin collection data for Cheshire East Council.
|
7
|
+
"""
|
8
|
+
|
9
|
+
|
5
10
|
class CouncilClass(AbstractGetBinDataClass):
|
6
|
-
|
11
|
+
"""
|
12
|
+
A class to fetch and parse bin collection data for Cheshire East Council.
|
13
|
+
"""
|
14
|
+
|
15
|
+
def parse_data(self, page: Any, **kwargs: Any) -> Dict[str, Any]:
|
7
16
|
soup = BeautifulSoup(page.text, features="html.parser")
|
8
17
|
|
9
|
-
bin_data_dict = {"bins": []}
|
18
|
+
bin_data_dict: Dict[str, Any] = {"bins": []}
|
10
19
|
|
11
|
-
table = soup.find(
|
12
|
-
|
20
|
+
table: Optional[Tag | NavigableString] = soup.find(
|
21
|
+
"table", {"class": "job-details"}
|
22
|
+
)
|
23
|
+
if isinstance(table, Tag): # Ensure we only proceed if 'table' is a Tag
|
13
24
|
rows = table.find_all("tr", {"class": "data-row"})
|
14
25
|
|
15
26
|
for row in rows:
|
16
27
|
cells = row.find_all(
|
17
|
-
"td",
|
28
|
+
"td",
|
29
|
+
{
|
30
|
+
"class": lambda L: isinstance(L, str)
|
31
|
+
and L.startswith("visible-cell")
|
32
|
+
}, # Explicitly check if L is a string
|
18
33
|
)
|
19
|
-
labels = cells[0].find_all("label") if cells else []
|
34
|
+
labels: list[Tag] = cells[0].find_all("label") if cells else []
|
20
35
|
|
21
36
|
if len(labels) >= 3:
|
22
|
-
bin_type = labels[2].get_text(strip=True)
|
23
|
-
collection_date = labels[1].get_text(strip=True)
|
37
|
+
bin_type: str = labels[2].get_text(strip=True)
|
38
|
+
collection_date: str = labels[1].get_text(strip=True)
|
24
39
|
|
25
40
|
bin_data_dict["bins"].append(
|
26
41
|
{
|
@@ -4,7 +4,6 @@ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataC
|
|
4
4
|
from dateutil.relativedelta import relativedelta
|
5
5
|
|
6
6
|
|
7
|
-
|
8
7
|
# import the wonderful Beautiful Soup and the URL grabber
|
9
8
|
class CouncilClass(AbstractGetBinDataClass):
|
10
9
|
"""
|
@@ -23,37 +22,39 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
23
22
|
check_uprn(user_uprn)
|
24
23
|
|
25
24
|
headers = {
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
25
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
26
|
+
"Accept-Language": "en-GB,en;q=0.9",
|
27
|
+
"Cache-Control": "no-cache",
|
28
|
+
"Connection": "keep-alive",
|
29
|
+
"Pragma": "no-cache",
|
30
|
+
"Sec-Fetch-Dest": "document",
|
31
|
+
"Sec-Fetch-Mode": "navigate",
|
32
|
+
"Sec-Fetch-Site": "none",
|
33
|
+
"Sec-Fetch-User": "?1",
|
34
|
+
"Upgrade-Insecure-Requests": "1",
|
35
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.143 Safari/537.36",
|
36
|
+
"sec-ch-ua": '"Opera GX";v="111", "Chromium";v="125", "Not.A/Brand";v="24"',
|
37
|
+
"sec-ch-ua-mobile": "?0",
|
38
|
+
"sec-ch-ua-platform": '"Windows"',
|
40
39
|
}
|
41
40
|
params = {
|
42
|
-
|
41
|
+
"uprn": f"{user_uprn}",
|
43
42
|
# 'uprn': f'100040128734',
|
44
43
|
}
|
45
44
|
response = requests.get(
|
46
|
-
|
45
|
+
"https://www.cornwall.gov.uk/umbraco/surface/waste/MyCollectionDays",
|
47
46
|
params=params,
|
48
|
-
headers=headers
|
47
|
+
headers=headers,
|
49
48
|
)
|
50
49
|
|
51
50
|
soup = BeautifulSoup(response.text, features="html.parser")
|
52
51
|
soup.prettify()
|
53
52
|
|
54
|
-
for item in soup.find_all(
|
53
|
+
for item in soup.find_all("div", class_="collection text-center service"):
|
55
54
|
bin_type = item.contents[1].text + " bin"
|
56
|
-
collection_date = datetime.strptime(item.contents[5].text, "%d %b").replace(
|
55
|
+
collection_date = datetime.strptime(item.contents[5].text, "%d %b").replace(
|
56
|
+
year=curr_date.year
|
57
|
+
)
|
57
58
|
if curr_date.month == 12 and collection_date.month == 1:
|
58
59
|
collection_date = collection_date + relativedelta(years=1)
|
59
60
|
collections.append((bin_type, collection_date))
|