uk_bin_collection 0.102.0__py3-none-any.whl → 0.104.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- uk_bin_collection/tests/input.json +38 -1
- uk_bin_collection/uk_bin_collection/councils/AberdeenshireCouncil.py +52 -0
- uk_bin_collection/uk_bin_collection/councils/BasildonCouncil.py +42 -39
- uk_bin_collection/uk_bin_collection/councils/BelfastCityCouncil.py +13 -8
- uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py +24 -21
- uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/CanterburyCityCouncil.py +54 -0
- uk_bin_collection/uk_bin_collection/councils/CheshireEastCouncil.py +25 -10
- uk_bin_collection/uk_bin_collection/councils/CornwallCouncil.py +21 -20
- uk_bin_collection/uk_bin_collection/councils/EnfieldCouncil.py +16 -18
- uk_bin_collection/uk_bin_collection/councils/GedlingBoroughCouncil.py +10 -4
- uk_bin_collection/uk_bin_collection/councils/IslingtonCouncil.py +6 -4
- uk_bin_collection/uk_bin_collection/councils/LutonBoroughCouncil.py +81 -0
- uk_bin_collection/uk_bin_collection/councils/MoleValleyDistrictCouncil.py +37 -20
- uk_bin_collection/uk_bin_collection/councils/NorthTynesideCouncil.py +11 -9
- uk_bin_collection/uk_bin_collection/councils/RochfordCouncil.py +1 -2
- uk_bin_collection/uk_bin_collection/councils/RotherhamCouncil.py +8 -6
- uk_bin_collection/uk_bin_collection/councils/SwindonBoroughCouncil.py +56 -0
- uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py +21 -11
- uk_bin_collection/uk_bin_collection/councils/WestOxfordshireDistrictCouncil.py +113 -0
- uk_bin_collection/uk_bin_collection/councils/WokinghamBoroughCouncil.py +1 -1
- {uk_bin_collection-0.102.0.dist-info → uk_bin_collection-0.104.0.dist-info}/METADATA +1 -1
- {uk_bin_collection-0.102.0.dist-info → uk_bin_collection-0.104.0.dist-info}/RECORD +26 -21
- {uk_bin_collection-0.102.0.dist-info → uk_bin_collection-0.104.0.dist-info}/LICENSE +0 -0
- {uk_bin_collection-0.102.0.dist-info → uk_bin_collection-0.104.0.dist-info}/WHEEL +0 -0
- {uk_bin_collection-0.102.0.dist-info → uk_bin_collection-0.104.0.dist-info}/entry_points.txt +0 -0
@@ -1,4 +1,11 @@
|
|
1
1
|
{
|
2
|
+
"AberdeenshireCouncil": {
|
3
|
+
"url": "https://online.aberdeenshire.gov.uk",
|
4
|
+
"wiki_command_url_override": "https://online.aberdeenshire.gov.uk",
|
5
|
+
"uprn": "151176430",
|
6
|
+
"wiki_name": "Aberdeenshire Council",
|
7
|
+
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
|
8
|
+
},
|
2
9
|
"AdurAndWorthingCouncils": {
|
3
10
|
"url": "https://www.adur-worthing.gov.uk/bin-day/?brlu-selected-address=100061878829",
|
4
11
|
"wiki_command_url_override": "https://www.adur-worthing.gov.uk/bin-day/?brlu-selected-address=XXXXXXXX",
|
@@ -201,6 +208,13 @@
|
|
201
208
|
"wiki_name": "Cannock Chase District Council",
|
202
209
|
"wiki_note": "To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)"
|
203
210
|
},
|
211
|
+
"CanterburyCityCouncil": {
|
212
|
+
"url": "https://www.canterbury.gov.uk",
|
213
|
+
"wiki_command_url_override": "https://www.canterbury.gov.uk",
|
214
|
+
"uprn": "10094583181",
|
215
|
+
"wiki_name": "Canterbury City Council",
|
216
|
+
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
|
217
|
+
},
|
204
218
|
"CardiffCouncil": {
|
205
219
|
"skip_get_url": true,
|
206
220
|
"uprn": "100100112419",
|
@@ -367,7 +381,7 @@
|
|
367
381
|
},
|
368
382
|
"EastDevonDC": {
|
369
383
|
"url": "https://eastdevon.gov.uk/recycling-and-waste/recycling-waste-information/when-is-my-bin-collected/future-collections-calendar/?UPRN=010090909915",
|
370
|
-
"wiki_command_url_override": "https://eastdevon.gov.uk/recycling-waste/recycling-
|
384
|
+
"wiki_command_url_override": "https://eastdevon.gov.uk/recycling-and-waste/recycling-waste-information/when-is-my-bin-collected/future-collections-calendar/?UPRN=XXXXXXXX",
|
371
385
|
"wiki_name": "East Devon District Council",
|
372
386
|
"wiki_note": "Replace XXXXXXXX with UPRN."
|
373
387
|
},
|
@@ -681,6 +695,13 @@
|
|
681
695
|
"wiki_name": "London Borough Redbridge",
|
682
696
|
"wiki_note": "Follow the instructions [here](https://my.redbridge.gov.uk/RecycleRefuse) until you get the page listing your \"Address\" then copy the entire address text and use that in the house number field."
|
683
697
|
},
|
698
|
+
"LutonBoroughCouncil": {
|
699
|
+
"url": "https://myforms.luton.gov.uk",
|
700
|
+
"wiki_command_url_override": "https://myforms.luton.gov.uk",
|
701
|
+
"uprn": "100080155778",
|
702
|
+
"wiki_name": "Luton Borough Council",
|
703
|
+
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
|
704
|
+
},
|
684
705
|
"MaldonDistrictCouncil": {
|
685
706
|
"skip_get_url": true,
|
686
707
|
"uprn": "100090557253",
|
@@ -1189,6 +1210,13 @@
|
|
1189
1210
|
"url": "https://www1.swansea.gov.uk/recyclingsearch/",
|
1190
1211
|
"wiki_name": "SwanseaCouncil"
|
1191
1212
|
},
|
1213
|
+
"SwindonBoroughCouncil": {
|
1214
|
+
"url": "https://www.swindon.gov.uk",
|
1215
|
+
"wiki_command_url_override": "https://www.swindon.gov.uk",
|
1216
|
+
"uprn": "10022793351",
|
1217
|
+
"wiki_name": "Swindon Borough Council",
|
1218
|
+
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
|
1219
|
+
},
|
1192
1220
|
"TamesideMBCouncil": {
|
1193
1221
|
"skip_get_url": true,
|
1194
1222
|
"uprn": "100012835362",
|
@@ -1369,6 +1397,15 @@
|
|
1369
1397
|
"url": "https://www.northampton.gov.uk/info/200084/bins-waste-and-recycling/1602/check-your-collection-day",
|
1370
1398
|
"wiki_name": "West Northamptonshire Council"
|
1371
1399
|
},
|
1400
|
+
"WestOxfordshireDistrictCouncil": {
|
1401
|
+
"house_number": "24",
|
1402
|
+
"postcode": "OX28 1YA",
|
1403
|
+
"skip_get_url": true,
|
1404
|
+
"url": "https://community.westoxon.gov.uk/s/waste-collection-enquiry",
|
1405
|
+
"web_driver": "http://selenium:4444",
|
1406
|
+
"wiki_name": "West Oxfordshire District Council",
|
1407
|
+
"wiki_note": "Pass the full address in the house number and postcode in"
|
1408
|
+
},
|
1372
1409
|
"WestSuffolkCouncil": {
|
1373
1410
|
"postcode": "IP28 6DR",
|
1374
1411
|
"skip_get_url": true,
|
@@ -0,0 +1,52 @@
|
|
1
|
+
import requests
|
2
|
+
from bs4 import BeautifulSoup
|
3
|
+
|
4
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
5
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
6
|
+
|
7
|
+
|
8
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
9
|
+
class CouncilClass(AbstractGetBinDataClass):
|
10
|
+
"""
|
11
|
+
Concrete classes have to implement all abstract operations of the
|
12
|
+
base class. They can also override some operations with a default
|
13
|
+
implementation.
|
14
|
+
"""
|
15
|
+
|
16
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
17
|
+
|
18
|
+
user_uprn = kwargs.get("uprn")
|
19
|
+
check_uprn(user_uprn)
|
20
|
+
bindata = {"bins": []}
|
21
|
+
|
22
|
+
URI = f"https://online.aberdeenshire.gov.uk/Apps/Waste-Collections/Routes/Route/{user_uprn}"
|
23
|
+
|
24
|
+
# Make the GET request
|
25
|
+
response = requests.get(URI)
|
26
|
+
|
27
|
+
soup = BeautifulSoup(response.content, features="html.parser")
|
28
|
+
soup.prettify()
|
29
|
+
|
30
|
+
for collection in soup.find("table").find("tbody").find_all("tr"):
|
31
|
+
th = collection.find("th")
|
32
|
+
if th:
|
33
|
+
continue
|
34
|
+
td = collection.find_all("td")
|
35
|
+
collection_date = datetime.strptime(
|
36
|
+
td[0].text,
|
37
|
+
"%d/%m/%Y %A",
|
38
|
+
)
|
39
|
+
bin_type = td[1].text.split(" and ")
|
40
|
+
|
41
|
+
for bin in bin_type:
|
42
|
+
dict_data = {
|
43
|
+
"type": bin,
|
44
|
+
"collectionDate": collection_date.strftime(date_format),
|
45
|
+
}
|
46
|
+
bindata["bins"].append(dict_data)
|
47
|
+
|
48
|
+
bindata["bins"].sort(
|
49
|
+
key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
|
50
|
+
)
|
51
|
+
|
52
|
+
return bindata
|
@@ -1,13 +1,13 @@
|
|
1
|
-
|
1
|
+
import requests
|
2
|
+
import json
|
3
|
+
from datetime import datetime
|
4
|
+
from uk_bin_collection.uk_bin_collection.common import check_uprn, date_format as DATE_FORMAT
|
2
5
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
3
6
|
|
4
7
|
|
5
|
-
# import the wonderful Beautiful Soup and the URL grabber
|
6
8
|
class CouncilClass(AbstractGetBinDataClass):
|
7
9
|
"""
|
8
|
-
Concrete
|
9
|
-
base class. They can also override some operations with a default
|
10
|
-
implementation.
|
10
|
+
Concrete class that implements the abstract bin data fetching and parsing logic.
|
11
11
|
"""
|
12
12
|
|
13
13
|
def parse_data(self, page: str, **kwargs) -> dict:
|
@@ -18,64 +18,67 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
18
18
|
check_uprn(uprn)
|
19
19
|
|
20
20
|
payload = {
|
21
|
-
# Add your payload details here (replace this with the actual payload structure if required)
|
22
21
|
"uprn": uprn
|
23
22
|
}
|
24
23
|
|
25
|
-
|
26
|
-
headers = {
|
27
|
-
"Content-Type": "application/json"
|
28
|
-
}
|
24
|
+
headers = {"Content-Type": "application/json"}
|
29
25
|
|
30
26
|
response = requests.post(url_base, data=json.dumps(payload), headers=headers)
|
31
27
|
|
32
|
-
# Ensure the request was successful
|
33
28
|
if response.status_code == 200:
|
34
29
|
data = response.json()
|
35
30
|
|
36
31
|
# Initialize an empty list to store the bin collection details
|
37
|
-
|
38
32
|
bins = []
|
39
33
|
|
40
34
|
# Function to add collection details to bins list
|
41
35
|
def add_collection(service_name, collection_data):
|
42
|
-
bins.append(
|
43
|
-
|
44
|
-
|
45
|
-
|
36
|
+
bins.append(
|
37
|
+
{
|
38
|
+
"type": service_name,
|
39
|
+
"collectionDate": collection_data.get("current_collection_date"),
|
40
|
+
}
|
41
|
+
)
|
46
42
|
|
47
|
-
|
48
|
-
|
43
|
+
available_services = data.get("refuse", {}).get("available_services", {})
|
44
|
+
|
45
|
+
date_format = "%d-%m-%Y" # Define the desired date format
|
49
46
|
|
50
47
|
for service_name, service_data in available_services.items():
|
51
|
-
#
|
48
|
+
# Handle the different cases of service data
|
52
49
|
match service_data["container"]:
|
53
50
|
case "Green Wheelie Bin":
|
54
|
-
subscription_status =
|
55
|
-
|
51
|
+
subscription_status = (
|
52
|
+
service_data["subscription"]["active"]
|
53
|
+
if service_data.get("subscription")
|
54
|
+
else False
|
55
|
+
)
|
56
|
+
type_descr = f"Green Wheelie Bin ({'Active' if subscription_status else 'Expired'})"
|
56
57
|
case "N/A":
|
57
|
-
type_descr = service_data
|
58
|
+
type_descr = service_data.get("name", "Unknown Service")
|
58
59
|
case _:
|
59
|
-
type_descr = service_data
|
60
|
-
|
60
|
+
type_descr = service_data.get("container", "Unknown Container")
|
61
61
|
|
62
62
|
date_str = service_data.get("current_collection_date")
|
63
|
-
#
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
"
|
72
|
-
|
63
|
+
if date_str: # Ensure the date string exists
|
64
|
+
try:
|
65
|
+
# Parse and format the date string
|
66
|
+
date_obj = datetime.strptime(date_str, "%Y-%m-%d")
|
67
|
+
formatted_date = date_obj.strftime(DATE_FORMAT)
|
68
|
+
except ValueError:
|
69
|
+
formatted_date = "Invalid Date"
|
70
|
+
else:
|
71
|
+
formatted_date = "No Collection Date"
|
72
|
+
|
73
|
+
bins.append(
|
74
|
+
{
|
75
|
+
"type": type_descr, # Use service name from the data
|
76
|
+
"collectionDate": formatted_date,
|
77
|
+
}
|
78
|
+
)
|
73
79
|
|
74
80
|
else:
|
75
81
|
print(f"Failed to fetch data. Status code: {response.status_code}")
|
82
|
+
return {}
|
76
83
|
|
77
|
-
|
78
|
-
"bins": bins
|
79
|
-
}
|
80
|
-
|
81
|
-
return data
|
84
|
+
return {"bins": bins}
|
@@ -9,7 +9,6 @@ from uk_bin_collection.uk_bin_collection.common import *
|
|
9
9
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
10
10
|
|
11
11
|
|
12
|
-
|
13
12
|
# import the wonderful Beautiful Soup and the URL grabber
|
14
13
|
class CouncilClass(AbstractGetBinDataClass):
|
15
14
|
"""
|
@@ -34,7 +33,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
34
33
|
|
35
34
|
session = requests.Session()
|
36
35
|
session.headers.update(headers)
|
37
|
-
|
36
|
+
|
38
37
|
user_uprn = kwargs.get("uprn")
|
39
38
|
user_postcode = kwargs.get("postcode")
|
40
39
|
URL = "https://online.belfastcity.gov.uk/find-bin-collection-day/Default.aspx"
|
@@ -47,14 +46,16 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
47
46
|
"__EVENTTARGET": "",
|
48
47
|
"__EVENTARGUMENT": "",
|
49
48
|
"__VIEWSTATE": self.get_session_variable(soup, "__VIEWSTATE"),
|
50
|
-
"__VIEWSTATEGENERATOR": self.get_session_variable(
|
49
|
+
"__VIEWSTATEGENERATOR": self.get_session_variable(
|
50
|
+
soup, "__VIEWSTATEGENERATOR"
|
51
|
+
),
|
51
52
|
"__SCROLLPOSITIONX": "0",
|
52
53
|
"__SCROLLPOSITIONY": "0",
|
53
54
|
"__EVENTVALIDATION": self.get_session_variable(soup, "__EVENTVALIDATION"),
|
54
55
|
"ctl00$MainContent$searchBy_radio": "P",
|
55
56
|
"ctl00$MainContent$Street_textbox": "",
|
56
57
|
"ctl00$MainContent$Postcode_textbox": user_postcode,
|
57
|
-
"ctl00$MainContent$AddressLookup_button": "Find address"
|
58
|
+
"ctl00$MainContent$AddressLookup_button": "Find address",
|
58
59
|
}
|
59
60
|
|
60
61
|
# Build intermediate ASP.NET variables for uprn Select address
|
@@ -65,7 +66,9 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
65
66
|
"__EVENTTARGET": "",
|
66
67
|
"__EVENTARGUMENT": "",
|
67
68
|
"__VIEWSTATE": self.get_session_variable(soup, "__VIEWSTATE"),
|
68
|
-
"__VIEWSTATEGENERATOR": self.get_session_variable(
|
69
|
+
"__VIEWSTATEGENERATOR": self.get_session_variable(
|
70
|
+
soup, "__VIEWSTATEGENERATOR"
|
71
|
+
),
|
69
72
|
"__SCROLLPOSITIONX": "0",
|
70
73
|
"__SCROLLPOSITIONY": "0",
|
71
74
|
"__EVENTVALIDATION": self.get_session_variable(soup, "__EVENTVALIDATION"),
|
@@ -73,14 +76,14 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
73
76
|
"ctl00$MainContent$Street_textbox": "",
|
74
77
|
"ctl00$MainContent$Postcode_textbox": user_postcode,
|
75
78
|
"ctl00$MainContent$lstAddresses": user_uprn,
|
76
|
-
"ctl00$MainContent$SelectAddress_button": "Select address"
|
79
|
+
"ctl00$MainContent$SelectAddress_button": "Select address",
|
77
80
|
}
|
78
81
|
|
79
82
|
# Actual http call to get Bins Data
|
80
83
|
response = session.post(URL, data=form_data)
|
81
84
|
response.raise_for_status()
|
82
85
|
soup = BeautifulSoup(response.text, "html.parser")
|
83
|
-
|
86
|
+
|
84
87
|
# Find Bins table and data
|
85
88
|
table = soup.find("div", {"id": "binsGrid"})
|
86
89
|
if table:
|
@@ -91,7 +94,9 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
91
94
|
collection_type = columns[0].get_text(strip=True)
|
92
95
|
collection_date_raw = columns[3].get_text(strip=True)
|
93
96
|
# if the month number is a single digit there are 2 spaces, stripping all spaces to make it consistent
|
94
|
-
collection_date = datetime.strptime(
|
97
|
+
collection_date = datetime.strptime(
|
98
|
+
collection_date_raw.replace(" ", ""), "%a%b%d%Y"
|
99
|
+
)
|
95
100
|
bin_entry = {
|
96
101
|
"type": collection_type,
|
97
102
|
"collectionDate": collection_date.strftime(date_format),
|
@@ -45,17 +45,13 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
45
45
|
)
|
46
46
|
inputElement_postcodesearch.send_keys(user_postcode)
|
47
47
|
|
48
|
-
|
49
|
-
|
50
48
|
find_address_btn = wait.until(
|
51
49
|
EC.element_to_be_clickable((By.XPATH, '//*[@id="sub"]'))
|
52
50
|
)
|
53
51
|
find_address_btn.click()
|
54
52
|
|
55
53
|
dropdown_options = wait.until(
|
56
|
-
EC.presence_of_element_located(
|
57
|
-
(By.XPATH, '//*[@id="address"]')
|
58
|
-
)
|
54
|
+
EC.presence_of_element_located((By.XPATH, '//*[@id="address"]'))
|
59
55
|
)
|
60
56
|
time.sleep(2)
|
61
57
|
dropdown_options.click()
|
@@ -71,11 +67,8 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
71
67
|
# Click the element
|
72
68
|
address.click()
|
73
69
|
|
74
|
-
|
75
70
|
submit_address = wait.until(
|
76
|
-
EC.presence_of_element_located(
|
77
|
-
(By.XPATH, '//*[@id="go"]')
|
78
|
-
)
|
71
|
+
EC.presence_of_element_located((By.XPATH, '//*[@id="go"]'))
|
79
72
|
)
|
80
73
|
time.sleep(2)
|
81
74
|
submit_address.click()
|
@@ -83,13 +76,11 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
83
76
|
results_found = wait.until(
|
84
77
|
EC.element_to_be_clickable(
|
85
78
|
(By.XPATH, '//h1[contains(text(), "Your bin days")]')
|
86
|
-
)
|
87
79
|
)
|
80
|
+
)
|
88
81
|
|
89
82
|
final_page = wait.until(
|
90
|
-
EC.presence_of_element_located(
|
91
|
-
(By.CLASS_NAME, "waste__collections")
|
92
|
-
)
|
83
|
+
EC.presence_of_element_located((By.CLASS_NAME, "waste__collections"))
|
93
84
|
)
|
94
85
|
|
95
86
|
soup = BeautifulSoup(driver.page_source, features="html.parser")
|
@@ -103,29 +94,41 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
103
94
|
# Loop through each bin field
|
104
95
|
for bin_section in bin_sections:
|
105
96
|
# Extract the bin type (e.g., "Brown Caddy", "Green Wheelie Bin", etc.)
|
106
|
-
bin_type = bin_section.get_text(strip=True).split("\n")[
|
97
|
+
bin_type = bin_section.get_text(strip=True).split("\n")[
|
98
|
+
0
|
99
|
+
] # The first part is the bin type
|
107
100
|
|
108
101
|
# Find the next sibling <dl> tag that contains the next collection information
|
109
102
|
summary_list = bin_section.find_next("dl", class_="govuk-summary-list")
|
110
103
|
|
111
104
|
if summary_list:
|
112
105
|
# Now, instead of finding by class, we'll search by text within the dt element
|
113
|
-
next_collection_dt = summary_list.find(
|
106
|
+
next_collection_dt = summary_list.find(
|
107
|
+
"dt", string=lambda text: "Next collection" in text
|
108
|
+
)
|
114
109
|
|
115
110
|
if next_collection_dt:
|
116
111
|
# Find the sibling <dd> tag for the collection date
|
117
|
-
next_collection = next_collection_dt.find_next_sibling(
|
112
|
+
next_collection = next_collection_dt.find_next_sibling(
|
113
|
+
"dd"
|
114
|
+
).get_text(strip=True)
|
118
115
|
|
119
116
|
if next_collection:
|
120
117
|
try:
|
121
118
|
# Parse the next collection date (assuming the format is like "Tuesday 15 October 2024")
|
122
|
-
parsed_date = datetime.strptime(
|
119
|
+
parsed_date = datetime.strptime(
|
120
|
+
next_collection, "%A %d %B %Y"
|
121
|
+
)
|
123
122
|
|
124
123
|
# Add the bin information to the data dictionary
|
125
|
-
data["bins"].append(
|
126
|
-
|
127
|
-
|
128
|
-
|
124
|
+
data["bins"].append(
|
125
|
+
{
|
126
|
+
"type": bin_type,
|
127
|
+
"collectionDate": parsed_date.strftime(
|
128
|
+
date_format
|
129
|
+
),
|
130
|
+
}
|
131
|
+
)
|
129
132
|
except ValueError as e:
|
130
133
|
print(f"Error parsing date for {bin_type}: {e}")
|
131
134
|
else:
|
@@ -82,7 +82,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
82
82
|
bin_type = " ".join(words).capitalize()
|
83
83
|
date_list = item.find_all("p")
|
84
84
|
for d in date_list:
|
85
|
-
clean_date_str = re.sub(r
|
85
|
+
clean_date_str = re.sub(r"[^A-Za-z0-9 ]+", "", d.text.strip())
|
86
86
|
next_collection = datetime.strptime(clean_date_str, "%A %d %B %Y")
|
87
87
|
collections.append((bin_type, next_collection))
|
88
88
|
|
@@ -0,0 +1,54 @@
|
|
1
|
+
import time
|
2
|
+
|
3
|
+
import requests
|
4
|
+
|
5
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
6
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
7
|
+
|
8
|
+
|
9
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
10
|
+
class CouncilClass(AbstractGetBinDataClass):
|
11
|
+
"""
|
12
|
+
Concrete classes have to implement all abstract operations of the
|
13
|
+
base class. They can also override some operations with a default
|
14
|
+
implementation.
|
15
|
+
"""
|
16
|
+
|
17
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
18
|
+
|
19
|
+
user_uprn = kwargs.get("uprn")
|
20
|
+
check_uprn(user_uprn)
|
21
|
+
bindata = {"bins": []}
|
22
|
+
|
23
|
+
data = {"uprn": user_uprn, "usrn": "1"}
|
24
|
+
|
25
|
+
URI = (
|
26
|
+
"https://zbr7r13ke2.execute-api.eu-west-2.amazonaws.com/Beta/get-bin-dates"
|
27
|
+
)
|
28
|
+
|
29
|
+
# Make the GET request
|
30
|
+
response = requests.post(URI, json=data)
|
31
|
+
response.raise_for_status()
|
32
|
+
|
33
|
+
# Parse the JSON response
|
34
|
+
bin_collection = json.loads(response.json()["dates"])
|
35
|
+
collections = {
|
36
|
+
"General": bin_collection["blackBinDay"],
|
37
|
+
"Recycling": bin_collection["recyclingBinDay"],
|
38
|
+
"Food": bin_collection["foodBinDay"],
|
39
|
+
"Garden": bin_collection["gardenBinDay"],
|
40
|
+
}
|
41
|
+
# Loop through each collection in bin_collection
|
42
|
+
for collection in collections:
|
43
|
+
print(collection)
|
44
|
+
|
45
|
+
if len(collections[collection]) <= 0:
|
46
|
+
continue
|
47
|
+
for date in collections[collection]:
|
48
|
+
date = (
|
49
|
+
datetime.strptime(date, "%Y-%m-%dT%H:%M:%S").strftime("%d/%m/%Y"),
|
50
|
+
)
|
51
|
+
dict_data = {"type": collection, "collectionDate": date[0]}
|
52
|
+
bindata["bins"].append(dict_data)
|
53
|
+
|
54
|
+
return bindata
|
@@ -1,26 +1,41 @@
|
|
1
|
-
from
|
1
|
+
from typing import Dict, Any, Optional
|
2
|
+
from bs4 import BeautifulSoup, Tag, NavigableString
|
2
3
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
3
4
|
|
4
|
-
|
5
|
+
"""
|
6
|
+
This module provides bin collection data for Cheshire East Council.
|
7
|
+
"""
|
8
|
+
|
9
|
+
|
5
10
|
class CouncilClass(AbstractGetBinDataClass):
|
6
|
-
|
11
|
+
"""
|
12
|
+
A class to fetch and parse bin collection data for Cheshire East Council.
|
13
|
+
"""
|
14
|
+
|
15
|
+
def parse_data(self, page: Any, **kwargs: Any) -> Dict[str, Any]:
|
7
16
|
soup = BeautifulSoup(page.text, features="html.parser")
|
8
17
|
|
9
|
-
bin_data_dict = {"bins": []}
|
18
|
+
bin_data_dict: Dict[str, Any] = {"bins": []}
|
10
19
|
|
11
|
-
table = soup.find(
|
12
|
-
|
20
|
+
table: Optional[Tag | NavigableString] = soup.find(
|
21
|
+
"table", {"class": "job-details"}
|
22
|
+
)
|
23
|
+
if isinstance(table, Tag): # Ensure we only proceed if 'table' is a Tag
|
13
24
|
rows = table.find_all("tr", {"class": "data-row"})
|
14
25
|
|
15
26
|
for row in rows:
|
16
27
|
cells = row.find_all(
|
17
|
-
"td",
|
28
|
+
"td",
|
29
|
+
{
|
30
|
+
"class": lambda L: isinstance(L, str)
|
31
|
+
and L.startswith("visible-cell")
|
32
|
+
}, # Explicitly check if L is a string
|
18
33
|
)
|
19
|
-
labels = cells[0].find_all("label") if cells else []
|
34
|
+
labels: list[Tag] = cells[0].find_all("label") if cells else []
|
20
35
|
|
21
36
|
if len(labels) >= 3:
|
22
|
-
bin_type = labels[2].get_text(strip=True)
|
23
|
-
collection_date = labels[1].get_text(strip=True)
|
37
|
+
bin_type: str = labels[2].get_text(strip=True)
|
38
|
+
collection_date: str = labels[1].get_text(strip=True)
|
24
39
|
|
25
40
|
bin_data_dict["bins"].append(
|
26
41
|
{
|
@@ -4,7 +4,6 @@ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataC
|
|
4
4
|
from dateutil.relativedelta import relativedelta
|
5
5
|
|
6
6
|
|
7
|
-
|
8
7
|
# import the wonderful Beautiful Soup and the URL grabber
|
9
8
|
class CouncilClass(AbstractGetBinDataClass):
|
10
9
|
"""
|
@@ -23,37 +22,39 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
23
22
|
check_uprn(user_uprn)
|
24
23
|
|
25
24
|
headers = {
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
25
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
26
|
+
"Accept-Language": "en-GB,en;q=0.9",
|
27
|
+
"Cache-Control": "no-cache",
|
28
|
+
"Connection": "keep-alive",
|
29
|
+
"Pragma": "no-cache",
|
30
|
+
"Sec-Fetch-Dest": "document",
|
31
|
+
"Sec-Fetch-Mode": "navigate",
|
32
|
+
"Sec-Fetch-Site": "none",
|
33
|
+
"Sec-Fetch-User": "?1",
|
34
|
+
"Upgrade-Insecure-Requests": "1",
|
35
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.143 Safari/537.36",
|
36
|
+
"sec-ch-ua": '"Opera GX";v="111", "Chromium";v="125", "Not.A/Brand";v="24"',
|
37
|
+
"sec-ch-ua-mobile": "?0",
|
38
|
+
"sec-ch-ua-platform": '"Windows"',
|
40
39
|
}
|
41
40
|
params = {
|
42
|
-
|
41
|
+
"uprn": f"{user_uprn}",
|
43
42
|
# 'uprn': f'100040128734',
|
44
43
|
}
|
45
44
|
response = requests.get(
|
46
|
-
|
45
|
+
"https://www.cornwall.gov.uk/umbraco/surface/waste/MyCollectionDays",
|
47
46
|
params=params,
|
48
|
-
headers=headers
|
47
|
+
headers=headers,
|
49
48
|
)
|
50
49
|
|
51
50
|
soup = BeautifulSoup(response.text, features="html.parser")
|
52
51
|
soup.prettify()
|
53
52
|
|
54
|
-
for item in soup.find_all(
|
53
|
+
for item in soup.find_all("div", class_="collection text-center service"):
|
55
54
|
bin_type = item.contents[1].text + " bin"
|
56
|
-
collection_date = datetime.strptime(item.contents[5].text, "%d %b").replace(
|
55
|
+
collection_date = datetime.strptime(item.contents[5].text, "%d %b").replace(
|
56
|
+
year=curr_date.year
|
57
|
+
)
|
57
58
|
if curr_date.month == 12 and collection_date.month == 1:
|
58
59
|
collection_date = collection_date + relativedelta(years=1)
|
59
60
|
collections.append((bin_type, collection_date))
|