uk_bin_collection 0.108.2__py3-none-any.whl → 0.109.1__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- uk_bin_collection/tests/input.json +29 -4
- uk_bin_collection/tests/step_defs/test_validate_council.py +1 -0
- uk_bin_collection/uk_bin_collection/councils/BrecklandCouncil.py +55 -0
- uk_bin_collection/uk_bin_collection/councils/ChesterfieldBoroughCouncil.py +175 -0
- uk_bin_collection/uk_bin_collection/councils/CotswoldDistrictCouncil.py +120 -0
- uk_bin_collection/uk_bin_collection/councils/StHelensBC.py +107 -42
- uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py +1 -2
- {uk_bin_collection-0.108.2.dist-info → uk_bin_collection-0.109.1.dist-info}/METADATA +1 -4
- {uk_bin_collection-0.108.2.dist-info → uk_bin_collection-0.109.1.dist-info}/RECORD +12 -9
- {uk_bin_collection-0.108.2.dist-info → uk_bin_collection-0.109.1.dist-info}/LICENSE +0 -0
- {uk_bin_collection-0.108.2.dist-info → uk_bin_collection-0.109.1.dist-info}/WHEEL +0 -0
- {uk_bin_collection-0.108.2.dist-info → uk_bin_collection-0.109.1.dist-info}/entry_points.txt +0 -0
@@ -159,6 +159,13 @@
|
|
159
159
|
"wiki_name": "Bradford MDC",
|
160
160
|
"wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search). Post code isn't parsed by this script, but you can pass it in double quotes."
|
161
161
|
},
|
162
|
+
"BrecklandCouncil": {
|
163
|
+
"url": "https://www.breckland.gov.uk",
|
164
|
+
"wiki_command_url_override": "https://www.breckland.gov.uk",
|
165
|
+
"uprn": "100091495479",
|
166
|
+
"wiki_name": "Breckland Council",
|
167
|
+
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
|
168
|
+
},
|
162
169
|
"BrightonandHoveCityCouncil": {
|
163
170
|
"house_number": "44 Carden Avenue, Brighton, BN1 8NE",
|
164
171
|
"postcode": "BN1 8NE",
|
@@ -272,6 +279,12 @@
|
|
272
279
|
"web_driver": "http://selenium:4444",
|
273
280
|
"wiki_name": "Cheshire West and Chester Council"
|
274
281
|
},
|
282
|
+
"ChesterfieldBoroughCouncil": {
|
283
|
+
"uprn": "74008234",
|
284
|
+
"skip_get_url": true,
|
285
|
+
"url": "https://www.cheshirewestandchester.gov.uk/residents/waste-and-recycling/your-bin-collection/collection-day",
|
286
|
+
"wiki_name": "Chesterfield Borough Council"
|
287
|
+
},
|
275
288
|
"ChichesterDistrictCouncil": {
|
276
289
|
"house_number": "7, Plaistow Road, Kirdford, Billingshurst, West Sussex",
|
277
290
|
"postcode": "RH14 0JT",
|
@@ -314,6 +327,15 @@
|
|
314
327
|
"wiki_name": "Cornwall Council",
|
315
328
|
"wiki_note": "Use https://uprn.uk/ to find your UPRN."
|
316
329
|
},
|
330
|
+
"CotswoldDistrictCouncil": {
|
331
|
+
"house_number": "19",
|
332
|
+
"postcode": "GL56 0GB",
|
333
|
+
"skip_get_url": true,
|
334
|
+
"url": "https://community.cotswold.gov.uk/s/waste-collection-enquiry",
|
335
|
+
"web_driver": "http://selenium:4444",
|
336
|
+
"wiki_name": "Cotswold District Council",
|
337
|
+
"wiki_note": "Pass the full address in the house number and postcode in"
|
338
|
+
},
|
317
339
|
"CoventryCityCouncil": {
|
318
340
|
"url": "https://www.coventry.gov.uk/directory-record/56384/abberton-way-",
|
319
341
|
"wiki_command_url_override": "https://www.coventry.gov.uk/directory_record/XXXXXX/XXXXXX",
|
@@ -837,8 +859,8 @@
|
|
837
859
|
"wiki_name": "New Forest Council"
|
838
860
|
},
|
839
861
|
"NewarkAndSherwoodDC": {
|
840
|
-
"url": "http://app.newark-sherwooddc.gov.uk/bincollection/calendar?pid=200004258529",
|
841
|
-
"wiki_command_url_override": "http://app.newark-sherwooddc.gov.uk/bincollection/calendar?pid=XXXXXXXX",
|
862
|
+
"url": "http://app.newark-sherwooddc.gov.uk/bincollection/calendar?pid=200004258529&nc=1",
|
863
|
+
"wiki_command_url_override": "http://app.newark-sherwooddc.gov.uk/bincollection/calendar?pid=XXXXXXXX&nc=1",
|
842
864
|
"wiki_name": "Newark and Sherwood District Council",
|
843
865
|
"wiki_note": "Replace XXXXXXXX with UPRN."
|
844
866
|
},
|
@@ -1200,10 +1222,13 @@
|
|
1200
1222
|
"wiki_name": "St Albans City and District Council"
|
1201
1223
|
},
|
1202
1224
|
"StHelensBC": {
|
1225
|
+
"house_number": "15",
|
1226
|
+
"postcode": "L34 2GA",
|
1203
1227
|
"skip_get_url": true,
|
1204
|
-
"uprn": "39081672",
|
1205
1228
|
"url": "https://www.sthelens.gov.uk/",
|
1206
|
-
"
|
1229
|
+
"web_driver": "http://selenium:4444",
|
1230
|
+
"wiki_name": "St Helens Borough Council",
|
1231
|
+
"wiki_note": "Pass the house name/number in the house number parameter, wrapped in double quotes"
|
1207
1232
|
},
|
1208
1233
|
"StaffordBoroughCouncil": {
|
1209
1234
|
"uprn": "100032203010",
|
@@ -0,0 +1,55 @@
|
|
1
|
+
import time
|
2
|
+
|
3
|
+
import requests
|
4
|
+
|
5
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
6
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
7
|
+
|
8
|
+
|
9
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
10
|
+
class CouncilClass(AbstractGetBinDataClass):
|
11
|
+
"""
|
12
|
+
Concrete classes have to implement all abstract operations of the
|
13
|
+
base class. They can also override some operations with a default
|
14
|
+
implementation.
|
15
|
+
"""
|
16
|
+
|
17
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
18
|
+
|
19
|
+
user_uprn = kwargs.get("uprn")
|
20
|
+
check_uprn(user_uprn)
|
21
|
+
bindata = {"bins": []}
|
22
|
+
|
23
|
+
URI = "https://www.breckland.gov.uk/apiserver/ajaxlibrary"
|
24
|
+
|
25
|
+
data = {
|
26
|
+
"id": "1730410741649",
|
27
|
+
"jsonrpc": "2.0",
|
28
|
+
"method": "Breckland.Whitespace.JointWasteAPI.GetBinCollectionsByUprn",
|
29
|
+
"params": {"uprn": user_uprn, "environment": "live"},
|
30
|
+
}
|
31
|
+
# Make the GET request
|
32
|
+
response = requests.post(URI, json=data)
|
33
|
+
|
34
|
+
# Parse the JSON response
|
35
|
+
bin_collection = response.json()
|
36
|
+
|
37
|
+
# Loop through each collection in bin_collection
|
38
|
+
for collection in bin_collection["result"]:
|
39
|
+
bin_type = collection.get("collectiontype")
|
40
|
+
collection_date = collection.get("nextcollection")
|
41
|
+
|
42
|
+
dict_data = {
|
43
|
+
"type": bin_type,
|
44
|
+
"collectionDate": datetime.strptime(
|
45
|
+
collection_date,
|
46
|
+
"%d/%m/%Y %H:%M:%S",
|
47
|
+
).strftime("%d/%m/%Y"),
|
48
|
+
}
|
49
|
+
bindata["bins"].append(dict_data)
|
50
|
+
|
51
|
+
bindata["bins"].sort(
|
52
|
+
key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
|
53
|
+
)
|
54
|
+
|
55
|
+
return bindata
|
@@ -0,0 +1,175 @@
|
|
1
|
+
import json
|
2
|
+
import logging
|
3
|
+
import re
|
4
|
+
from datetime import datetime, timedelta
|
5
|
+
|
6
|
+
import requests
|
7
|
+
from bs4 import BeautifulSoup
|
8
|
+
from uk_bin_collection.uk_bin_collection.common import check_uprn, date_format
|
9
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
10
|
+
import urllib3
|
11
|
+
|
12
|
+
|
13
|
+
# Suppress only the single warning from urllib3 needed.
|
14
|
+
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
15
|
+
|
16
|
+
_LOGGER = logging.getLogger(__name__)
|
17
|
+
|
18
|
+
class CouncilClass(AbstractGetBinDataClass):
|
19
|
+
"""
|
20
|
+
Implementation for Chesterfield Borough Council waste collection data retrieval.
|
21
|
+
"""
|
22
|
+
|
23
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
24
|
+
"""
|
25
|
+
Fetch and parse waste collection data for Chesterfield Borough Council.
|
26
|
+
|
27
|
+
Args:
|
28
|
+
page (str): Not used in this implementation.
|
29
|
+
**kwargs: Should contain 'uprn' key.
|
30
|
+
|
31
|
+
Returns:
|
32
|
+
dict: Parsed bin collection data.
|
33
|
+
"""
|
34
|
+
# Get and check UPRN
|
35
|
+
user_uprn = kwargs.get("uprn")
|
36
|
+
check_uprn(user_uprn)
|
37
|
+
bindata = {"bins": []}
|
38
|
+
|
39
|
+
# Define API URLs
|
40
|
+
API_URLS = {
|
41
|
+
"session": "https://www.chesterfield.gov.uk/bins-and-recycling/bin-collections/check-bin-collections.aspx",
|
42
|
+
"fwuid": "https://myaccount.chesterfield.gov.uk/anonymous/c/cbc_VE_CollectionDaysLO.app?aura.format=JSON&aura.formatAdapter=LIGHTNING_OUT",
|
43
|
+
"search": "https://myaccount.chesterfield.gov.uk/anonymous/aura?r=2&aura.ApexAction.execute=1",
|
44
|
+
}
|
45
|
+
|
46
|
+
HEADERS = {
|
47
|
+
"User-Agent": "Mozilla/5.0",
|
48
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
49
|
+
}
|
50
|
+
|
51
|
+
# Initialize session
|
52
|
+
session = requests.Session()
|
53
|
+
|
54
|
+
try:
|
55
|
+
# Step 1: Get session
|
56
|
+
session.get(API_URLS["session"], headers=HEADERS, verify=False)
|
57
|
+
|
58
|
+
# Step 2: Get fwuid
|
59
|
+
fwuid_response = session.get(API_URLS["fwuid"], headers=HEADERS, verify=False)
|
60
|
+
fwuid_data = fwuid_response.json()
|
61
|
+
fwuid = fwuid_data.get("auraConfig", {}).get("context", {}).get("fwuid")
|
62
|
+
|
63
|
+
if not fwuid:
|
64
|
+
_LOGGER.error("Failed to retrieve fwuid from the response.")
|
65
|
+
return bindata
|
66
|
+
|
67
|
+
# Step 3: Prepare payload for UPRN search
|
68
|
+
payload = {
|
69
|
+
"message": json.dumps({
|
70
|
+
"actions": [{
|
71
|
+
"id": "4;a",
|
72
|
+
"descriptor": "aura://ApexActionController/ACTION$execute",
|
73
|
+
"callingDescriptor": "UNKNOWN",
|
74
|
+
"params": {
|
75
|
+
"namespace": "",
|
76
|
+
"classname": "CBC_VE_CollectionDays",
|
77
|
+
"method": "getServicesByUPRN",
|
78
|
+
"params": {
|
79
|
+
"propertyUprn": user_uprn,
|
80
|
+
"executedFrom": "Main Website"
|
81
|
+
},
|
82
|
+
"cacheable": False,
|
83
|
+
"isContinuation": False
|
84
|
+
}
|
85
|
+
}]
|
86
|
+
}),
|
87
|
+
"aura.context": json.dumps({
|
88
|
+
"mode": "PROD",
|
89
|
+
"fwuid": fwuid,
|
90
|
+
"app": "c:cbc_VE_CollectionDaysLO",
|
91
|
+
"loaded": {
|
92
|
+
"APPLICATION@markup://c:cbc_VE_CollectionDaysLO": "pqeNg7kPWCbx1pO8sIjdLA"
|
93
|
+
},
|
94
|
+
"dn": [],
|
95
|
+
"globals": {},
|
96
|
+
"uad": True
|
97
|
+
}),
|
98
|
+
"aura.pageURI": "/bins-and-recycling/bin-collections/check-bin-collections.aspx",
|
99
|
+
"aura.token": "null",
|
100
|
+
}
|
101
|
+
|
102
|
+
# Step 4: Make POST request to fetch collection data
|
103
|
+
search_response = session.post(
|
104
|
+
API_URLS["search"],
|
105
|
+
data=payload,
|
106
|
+
headers=HEADERS,
|
107
|
+
verify=False
|
108
|
+
)
|
109
|
+
search_data = search_response.json()
|
110
|
+
|
111
|
+
# Step 5: Extract service units
|
112
|
+
service_units = search_data.get("actions", [])[0].get("returnValue", {}).get("returnValue", {}).get("serviceUnits", [])
|
113
|
+
|
114
|
+
if not service_units:
|
115
|
+
_LOGGER.warning("No service units found for the given UPRN.")
|
116
|
+
return bindata
|
117
|
+
|
118
|
+
# Initialize dictionary to store bin dates
|
119
|
+
bin_schedule = {}
|
120
|
+
|
121
|
+
# Define icon mapping
|
122
|
+
ICON_MAP = {
|
123
|
+
"DOMESTIC REFUSE": "mdi:trash-can",
|
124
|
+
"DOMESTIC RECYCLING": "mdi:recycle",
|
125
|
+
"DOMESTIC ORGANIC": "mdi:leaf",
|
126
|
+
}
|
127
|
+
|
128
|
+
# Define regex pattern to capture day and date (e.g., Tue 5 Nov)
|
129
|
+
date_pattern = re.compile(r"\b\w{3} \d{1,2} \w{3}\b")
|
130
|
+
|
131
|
+
current_year = datetime.now().year
|
132
|
+
|
133
|
+
# Process each service unit
|
134
|
+
for item in service_units:
|
135
|
+
try:
|
136
|
+
waste_type = item["serviceTasks"][0]["taskTypeName"]
|
137
|
+
waste_type = str(waste_type).replace("Collect ", "").upper()
|
138
|
+
except (IndexError, KeyError):
|
139
|
+
_LOGGER.debug("Skipping a service unit due to missing data.")
|
140
|
+
continue
|
141
|
+
|
142
|
+
# Extract the next scheduled date
|
143
|
+
try:
|
144
|
+
dt_zulu = item["serviceTasks"][0]["serviceTaskSchedules"][0]["nextInstance"]["currentScheduledDate"]
|
145
|
+
dt_utc = datetime.strptime(dt_zulu, "%Y-%m-%dT%H:%M:%S.%f%z")
|
146
|
+
dt_local = dt_utc.astimezone(None)
|
147
|
+
collection_date = dt_local.date()
|
148
|
+
except (IndexError, KeyError, ValueError) as e:
|
149
|
+
_LOGGER.warning(f"Failed to parse date for {waste_type}: {e}")
|
150
|
+
continue
|
151
|
+
|
152
|
+
# Append to bin_schedule
|
153
|
+
bin_schedule[waste_type] = collection_date.strftime(date_format)
|
154
|
+
|
155
|
+
# Convert bin_schedule to the expected format
|
156
|
+
for bin_type, collection_date in bin_schedule.items():
|
157
|
+
dict_data = {
|
158
|
+
"type": bin_type,
|
159
|
+
"collectionDate": collection_date,
|
160
|
+
}
|
161
|
+
bindata["bins"].append(dict_data)
|
162
|
+
|
163
|
+
# Sort the bins by collection date
|
164
|
+
bindata["bins"].sort(
|
165
|
+
key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
|
166
|
+
)
|
167
|
+
|
168
|
+
except requests.RequestException as e:
|
169
|
+
_LOGGER.error(f"Network error occurred: {e}")
|
170
|
+
except json.JSONDecodeError as e:
|
171
|
+
_LOGGER.error(f"JSON decoding failed: {e}")
|
172
|
+
except Exception as e:
|
173
|
+
_LOGGER.error(f"An unexpected error occurred: {e}")
|
174
|
+
|
175
|
+
return bindata
|
@@ -0,0 +1,120 @@
|
|
1
|
+
import time
|
2
|
+
from datetime import datetime
|
3
|
+
|
4
|
+
from bs4 import BeautifulSoup
|
5
|
+
from selenium.webdriver.common.by import By
|
6
|
+
from selenium.webdriver.common.keys import Keys
|
7
|
+
from selenium.webdriver.support import expected_conditions as EC
|
8
|
+
from selenium.webdriver.support.ui import Select
|
9
|
+
from selenium.webdriver.support.wait import WebDriverWait
|
10
|
+
|
11
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
12
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
13
|
+
|
14
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
15
|
+
|
16
|
+
|
17
|
+
class CouncilClass(AbstractGetBinDataClass):
|
18
|
+
"""
|
19
|
+
Concrete classes have to implement all abstract operations of the
|
20
|
+
base class. They can also override some operations with a default
|
21
|
+
implementation.
|
22
|
+
"""
|
23
|
+
|
24
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
25
|
+
driver = None
|
26
|
+
try:
|
27
|
+
page = "https://community.cotswold.gov.uk/s/waste-collection-enquiry"
|
28
|
+
|
29
|
+
data = {"bins": []}
|
30
|
+
|
31
|
+
house_number = kwargs.get("paon")
|
32
|
+
postcode = kwargs.get("postcode")
|
33
|
+
full_address = f"{house_number}, {postcode}"
|
34
|
+
web_driver = kwargs.get("web_driver")
|
35
|
+
headless = kwargs.get("headless")
|
36
|
+
|
37
|
+
# Create Selenium webdriver
|
38
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
39
|
+
driver.get(page)
|
40
|
+
|
41
|
+
# If you bang in the house number (or property name) and postcode in the box it should find your property
|
42
|
+
wait = WebDriverWait(driver, 60)
|
43
|
+
address_entry_field = wait.until(
|
44
|
+
EC.presence_of_element_located(
|
45
|
+
(By.XPATH, '//*[@id="combobox-input-19"]')
|
46
|
+
)
|
47
|
+
)
|
48
|
+
|
49
|
+
address_entry_field.send_keys(str(full_address))
|
50
|
+
|
51
|
+
address_entry_field = wait.until(
|
52
|
+
EC.element_to_be_clickable((By.XPATH, '//*[@id="combobox-input-19"]'))
|
53
|
+
)
|
54
|
+
address_entry_field.click()
|
55
|
+
address_entry_field.send_keys(Keys.BACKSPACE)
|
56
|
+
address_entry_field.send_keys(str(full_address[len(full_address) - 1]))
|
57
|
+
|
58
|
+
first_found_address = wait.until(
|
59
|
+
EC.element_to_be_clickable(
|
60
|
+
(By.XPATH, '//*[@id="dropdown-element-19"]/ul')
|
61
|
+
)
|
62
|
+
)
|
63
|
+
|
64
|
+
first_found_address.click()
|
65
|
+
# Wait for the 'Select your property' dropdown to appear and select the first result
|
66
|
+
next_btn = wait.until(
|
67
|
+
EC.element_to_be_clickable((By.XPATH, "//lightning-button/button"))
|
68
|
+
)
|
69
|
+
next_btn.click()
|
70
|
+
bin_data = wait.until(
|
71
|
+
EC.presence_of_element_located(
|
72
|
+
(By.XPATH, "//span[contains(text(), 'Container')]")
|
73
|
+
)
|
74
|
+
)
|
75
|
+
|
76
|
+
soup = BeautifulSoup(driver.page_source, features="html.parser")
|
77
|
+
|
78
|
+
rows = soup.find_all("tr", class_="slds-hint-parent")
|
79
|
+
current_year = datetime.now().year
|
80
|
+
|
81
|
+
for row in rows:
|
82
|
+
columns = row.find_all("td")
|
83
|
+
if columns:
|
84
|
+
container_type = row.find("th").text.strip()
|
85
|
+
if columns[0].get_text() == "Today":
|
86
|
+
collection_day = datetime.now().strftime("%a, %d %B")
|
87
|
+
elif columns[0].get_text() == "Tomorrow":
|
88
|
+
collection_day = (datetime.now() + timedelta(days=1)).strftime(
|
89
|
+
"%a, %d %B"
|
90
|
+
)
|
91
|
+
else:
|
92
|
+
collection_day = re.sub(
|
93
|
+
r"[^a-zA-Z0-9,\s]", "", columns[0].get_text()
|
94
|
+
).strip()
|
95
|
+
|
96
|
+
# Parse the date from the string
|
97
|
+
parsed_date = datetime.strptime(collection_day, "%a, %d %B")
|
98
|
+
if parsed_date < datetime(
|
99
|
+
parsed_date.year, parsed_date.month, parsed_date.day
|
100
|
+
):
|
101
|
+
parsed_date = parsed_date.replace(year=current_year + 1)
|
102
|
+
else:
|
103
|
+
parsed_date = parsed_date.replace(year=current_year)
|
104
|
+
# Format the date as %d/%m/%Y
|
105
|
+
formatted_date = parsed_date.strftime("%d/%m/%Y")
|
106
|
+
|
107
|
+
# Add the bin type and collection date to the 'data' dictionary
|
108
|
+
data["bins"].append(
|
109
|
+
{"type": container_type, "collectionDate": formatted_date}
|
110
|
+
)
|
111
|
+
except Exception as e:
|
112
|
+
# Here you can log the exception if needed
|
113
|
+
print(f"An error occurred: {e}")
|
114
|
+
# Optionally, re-raise the exception if you want it to propagate
|
115
|
+
raise
|
116
|
+
finally:
|
117
|
+
# This block ensures that the driver is closed regardless of an exception
|
118
|
+
if driver:
|
119
|
+
driver.quit()
|
120
|
+
return data
|
@@ -1,4 +1,8 @@
|
|
1
1
|
from bs4 import BeautifulSoup
|
2
|
+
from selenium.webdriver.common.by import By
|
3
|
+
from selenium.webdriver.support import expected_conditions as EC
|
4
|
+
from selenium.webdriver.support.ui import Select
|
5
|
+
from selenium.webdriver.support.wait import WebDriverWait
|
2
6
|
|
3
7
|
from uk_bin_collection.uk_bin_collection.common import *
|
4
8
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
@@ -8,50 +12,111 @@ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataC
|
|
8
12
|
class CouncilClass(AbstractGetBinDataClass):
|
9
13
|
"""
|
10
14
|
Concrete classes have to implement all abstract operations of the
|
11
|
-
|
12
|
-
|
15
|
+
base class. They can also override some operations with a default
|
16
|
+
implementation.
|
13
17
|
"""
|
14
18
|
|
15
19
|
def parse_data(self, page: str, **kwargs) -> dict:
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
s = requests.Session()
|
26
|
-
page = s.get(url)
|
27
|
-
|
28
|
-
# Make a BS4 object
|
29
|
-
soup = BeautifulSoup(
|
30
|
-
re.sub("<div([^>]+)>", "", page.text).replace("</div>", ""),
|
31
|
-
features="html.parser",
|
32
|
-
)
|
33
|
-
soup.prettify()
|
34
|
-
|
35
|
-
data = {"bins": []}
|
36
|
-
collection_rows = (
|
37
|
-
soup.find("table", {"class": "multitable"}).find("tbody").find_all("tr")
|
38
|
-
)
|
39
|
-
|
40
|
-
for collection_row in collection_rows:
|
41
|
-
# Get bin collection type
|
42
|
-
bin_type = collection_row.find("th")
|
43
|
-
if bin_type:
|
44
|
-
bin_type = bin_type.get_text(strip=True)
|
45
|
-
# Get bin collection dates
|
46
|
-
for bin_date in collection_row.find_all("td"):
|
47
|
-
if bin_date.get_text(strip=True) != "Dates not allocated":
|
48
|
-
collection_date = datetime.strptime(
|
49
|
-
bin_date.get_text(strip=True), "%a %d %b %Y"
|
50
|
-
)
|
51
|
-
dict_data = {
|
52
|
-
"type": bin_type,
|
53
|
-
"collectionDate": collection_date.strftime(date_format),
|
54
|
-
}
|
55
|
-
data["bins"].append(dict_data)
|
20
|
+
driver = None
|
21
|
+
try:
|
22
|
+
data = {"bins": []}
|
23
|
+
user_paon = kwargs.get("paon")
|
24
|
+
user_postcode = kwargs.get("postcode")
|
25
|
+
web_driver = kwargs.get("web_driver")
|
26
|
+
headless = kwargs.get("headless")
|
27
|
+
check_paon(user_paon)
|
28
|
+
check_postcode(user_postcode)
|
56
29
|
|
30
|
+
# Create Selenium webdriver
|
31
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
32
|
+
driver.get(
|
33
|
+
"https://www.sthelens.gov.uk/article/3473/Check-your-collection-dates"
|
34
|
+
)
|
35
|
+
|
36
|
+
"""
|
37
|
+
accept_button = WebDriverWait(driver, timeout=30).until(
|
38
|
+
EC.element_to_be_clickable((By.ID, "ccc-notify-accept"))
|
39
|
+
)
|
40
|
+
accept_button.click()
|
41
|
+
"""
|
42
|
+
|
43
|
+
# Wait for the postcode field to appear then populate it
|
44
|
+
inputElement_postcode = WebDriverWait(driver, 30).until(
|
45
|
+
EC.presence_of_element_located(
|
46
|
+
(By.ID, "RESIDENTCOLLECTIONDATES_PAGE1_POSTCODE")
|
47
|
+
)
|
48
|
+
)
|
49
|
+
inputElement_postcode.send_keys(user_postcode)
|
50
|
+
|
51
|
+
# Click search button
|
52
|
+
findAddress = WebDriverWait(driver, 10).until(
|
53
|
+
EC.presence_of_element_located(
|
54
|
+
(By.ID, "RESIDENTCOLLECTIONDATES_PAGE1_FINDADDRESS_NEXT")
|
55
|
+
)
|
56
|
+
)
|
57
|
+
findAddress.click()
|
58
|
+
|
59
|
+
WebDriverWait(driver, timeout=30).until(
|
60
|
+
EC.element_to_be_clickable(
|
61
|
+
(By.ID, "RESIDENTCOLLECTIONDATES_PAGE1_ADDRESS_chosen")
|
62
|
+
)
|
63
|
+
).click()
|
64
|
+
|
65
|
+
WebDriverWait(driver, 10).until(
|
66
|
+
EC.element_to_be_clickable(
|
67
|
+
(
|
68
|
+
By.XPATH,
|
69
|
+
f"//ul[@id='RESIDENTCOLLECTIONDATES_PAGE1_ADDRESS-chosen-search-results']/li[starts-with(text(), '{user_paon}')]",
|
70
|
+
)
|
71
|
+
)
|
72
|
+
).click()
|
73
|
+
|
74
|
+
WebDriverWait(driver, timeout=30).until(
|
75
|
+
EC.element_to_be_clickable(
|
76
|
+
(By.ID, "RESIDENTCOLLECTIONDATES_PAGE1_ADDRESSNEXT_NEXT")
|
77
|
+
)
|
78
|
+
).click()
|
79
|
+
|
80
|
+
# Wait for the collections table to appear
|
81
|
+
WebDriverWait(driver, 10).until(
|
82
|
+
EC.presence_of_element_located(
|
83
|
+
(By.ID, "RESIDENTCOLLECTIONDATES__FIELDS_OUTER")
|
84
|
+
)
|
85
|
+
)
|
86
|
+
|
87
|
+
soup = BeautifulSoup(driver.page_source, features="html.parser")
|
88
|
+
|
89
|
+
# Get the month rows first
|
90
|
+
current_month = ""
|
91
|
+
for row in soup.find_all("tr"):
|
92
|
+
# Check if the row is a month header (contains 'th' tag)
|
93
|
+
if row.find("th"):
|
94
|
+
current_month = row.find("th").get_text(strip=True)
|
95
|
+
else:
|
96
|
+
# Extract the date, day, and waste types
|
97
|
+
columns = row.find_all("td")
|
98
|
+
if len(columns) >= 4:
|
99
|
+
day = columns[0].get_text(strip=True)
|
100
|
+
date = day + " " + current_month
|
101
|
+
waste_types = columns[3].get_text(strip=True)
|
102
|
+
|
103
|
+
for type in waste_types.split(" & "):
|
104
|
+
dict_data = {
|
105
|
+
"type": type,
|
106
|
+
"collectionDate": datetime.strptime(
|
107
|
+
date,
|
108
|
+
"%d %B %Y",
|
109
|
+
).strftime("%d/%m/%Y"),
|
110
|
+
}
|
111
|
+
data["bins"].append(dict_data)
|
112
|
+
|
113
|
+
except Exception as e:
|
114
|
+
# Here you can log the exception if needed
|
115
|
+
print(f"An error occurred: {e}")
|
116
|
+
# Optionally, re-raise the exception if you want it to propagate
|
117
|
+
raise
|
118
|
+
finally:
|
119
|
+
# This block ensures that the driver is closed regardless of an exception
|
120
|
+
if driver:
|
121
|
+
driver.quit()
|
57
122
|
return data
|
@@ -2,7 +2,6 @@ from bs4 import BeautifulSoup
|
|
2
2
|
from uk_bin_collection.uk_bin_collection.common import *
|
3
3
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
4
4
|
|
5
|
-
|
6
5
|
# import the wonderful Beautiful Soup and the URL grabber
|
7
6
|
class CouncilClass(AbstractGetBinDataClass):
|
8
7
|
"""
|
@@ -25,7 +24,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
25
24
|
dict_data = {
|
26
25
|
"type": bin_type,
|
27
26
|
"collectionDate": datetime.strptime(
|
28
|
-
bin_collection.get_text(strip=True),
|
27
|
+
bin_collection.get_text(strip=True), date_format
|
29
28
|
),
|
30
29
|
}
|
31
30
|
data["bins"].append(dict_data)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: uk_bin_collection
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.109.1
|
4
4
|
Summary: Python Lib to collect UK Bin Data
|
5
5
|
Author: Robert Bradley
|
6
6
|
Author-email: robbrad182@gmail.com
|
@@ -12,9 +12,6 @@ Requires-Dist: bs4
|
|
12
12
|
Requires-Dist: holidays
|
13
13
|
Requires-Dist: lxml
|
14
14
|
Requires-Dist: pandas
|
15
|
-
Requires-Dist: pytest-asyncio (>=0.24.0,<0.25.0)
|
16
|
-
Requires-Dist: pytest-freezer (>=0.4.8,<0.5.0)
|
17
|
-
Requires-Dist: pytest-homeassistant-custom-component (==0.13.175)
|
18
15
|
Requires-Dist: python-dateutil
|
19
16
|
Requires-Dist: requests
|
20
17
|
Requires-Dist: selenium
|
@@ -2,10 +2,10 @@ uk_bin_collection/README.rst,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
|
|
2
2
|
uk_bin_collection/tests/council_feature_input_parity.py,sha256=DO6Mk4ImYgM5ZCZ-cutwz5RoYYWZRLYx2tr6zIs_9Rc,3843
|
3
3
|
uk_bin_collection/tests/features/environment.py,sha256=VQZjJdJI_kZn08M0j5cUgvKT4k3iTw8icJge1DGOkoA,127
|
4
4
|
uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=SJK-Vc737hrf03tssxxbeg_JIvAH-ddB8f6gU1LTbuQ,251
|
5
|
-
uk_bin_collection/tests/input.json,sha256=
|
5
|
+
uk_bin_collection/tests/input.json,sha256=mQ5rDFi-l9QHP6AvgNnxADOGtgdybTT24hOoEh7MkRI,78144
|
6
6
|
uk_bin_collection/tests/output.schema,sha256=ZwKQBwYyTDEM4G2hJwfLUVM-5v1vKRvRK9W9SS1sd18,1086
|
7
7
|
uk_bin_collection/tests/step_defs/step_helpers/file_handler.py,sha256=Ygzi4V0S1MIHqbdstUlIqtRIwnynvhu4UtpweJ6-5N8,1474
|
8
|
-
uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=
|
8
|
+
uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=VZ0a81sioJULD7syAYHjvK_-nT_Rd36tUyzPetSA0gk,3475
|
9
9
|
uk_bin_collection/tests/test_collect_data.py,sha256=xit4lopMGXb5g2faCK9VrOjHl81QDV9sZAu7vBdr2Uw,2253
|
10
10
|
uk_bin_collection/tests/test_common_functions.py,sha256=WRm7AYI9qaDqW0dNROTFh-KZRdNFhhYHvjrFZUN4IPs,14084
|
11
11
|
uk_bin_collection/tests/test_conftest.py,sha256=GWqP6-fCv5A2VUYiyXqUO-Dm0hlUOwbFgfi6CkdGqvQ,1061
|
@@ -34,6 +34,7 @@ uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py,sha256=jHbCK8sL
|
|
34
34
|
uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py,sha256=WI68r8jB0IHPUT4CgmZMtng899AAMFTxkyTdPg9yLF8,4117
|
35
35
|
uk_bin_collection/uk_bin_collection/councils/BracknellForestCouncil.py,sha256=Llo1rULaAZ8rChVYZqXFFLo7CN6vbT0ULUJD6ActouY,9015
|
36
36
|
uk_bin_collection/uk_bin_collection/councils/BradfordMDC.py,sha256=VFrdcqKpHPw8v77Ll9QzBz_4carUfC1XYnxqUvDihkA,4275
|
37
|
+
uk_bin_collection/uk_bin_collection/councils/BrecklandCouncil.py,sha256=PX6A_pDvaN109aSNWmEhm88GFKfkClIkmbwGURWvsks,1744
|
37
38
|
uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py,sha256=k6qt4cds-Ejd97Z-__pw2BYvGVbFdc9SUfF73PPrTNA,5823
|
38
39
|
uk_bin_collection/uk_bin_collection/councils/BristolCityCouncil.py,sha256=kJmmDJz_kQ45DHmG7ocrUpNJonEn0kuXYEDQyZaf9ks,5576
|
39
40
|
uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py,sha256=_bAFykZWZkEVUB-QKeVLfWO8plG6nRgn71QF2BUN2rk,4329
|
@@ -49,11 +50,13 @@ uk_bin_collection/uk_bin_collection/councils/CharnwoodBoroughCouncil.py,sha256=t
|
|
49
50
|
uk_bin_collection/uk_bin_collection/councils/ChelmsfordCityCouncil.py,sha256=EB88D0MNJwuDZ2GX1ENc5maGYx17mnHTCtNl6s-v11E,5090
|
50
51
|
uk_bin_collection/uk_bin_collection/councils/CheshireEastCouncil.py,sha256=aMqT5sy1Z1gklFO5Xl893OgeBmpf19OwpizWEKWQ3hg,1680
|
51
52
|
uk_bin_collection/uk_bin_collection/councils/CheshireWestAndChesterCouncil.py,sha256=xsJcx-Dcds0ZcX2vZ-xHVkCg-faQRvbhrJzRDY6Lguw,4779
|
53
|
+
uk_bin_collection/uk_bin_collection/councils/ChesterfieldBoroughCouncil.py,sha256=KtMasJBLed5qw15uzZ9bZ2ndy-ofCyu3MwCya6i9oig,6784
|
52
54
|
uk_bin_collection/uk_bin_collection/councils/ChichesterDistrictCouncil.py,sha256=HxrLcJves7ZsE8FbooymeecTUmScY4R7Oi71vwCePPo,4118
|
53
55
|
uk_bin_collection/uk_bin_collection/councils/ChorleyCouncil.py,sha256=M7HjuUaFq8aSnOf_9m1QS4MmPPMmPhF3mLHSrfDPtV0,5194
|
54
56
|
uk_bin_collection/uk_bin_collection/councils/ColchesterCityCouncil.py,sha256=Mny-q2rQkWe2Tj1gINwEM1L4AkqQl1EDMAaKY0-deD4,3968
|
55
57
|
uk_bin_collection/uk_bin_collection/councils/ConwyCountyBorough.py,sha256=el75qv2QyfWZBU09tJLvD8vLQZ9pCg73u1NBFs6ybo8,1034
|
56
58
|
uk_bin_collection/uk_bin_collection/councils/CornwallCouncil.py,sha256=WZiz50svwyZgO8QKUCLy7hfFuy2HmAx5h-TG3yAweRA,2836
|
59
|
+
uk_bin_collection/uk_bin_collection/councils/CotswoldDistrictCouncil.py,sha256=K_HVkAJHEs-i3PRdoBqWVtWUl3BNed6mRLGaqvtSskg,4896
|
57
60
|
uk_bin_collection/uk_bin_collection/councils/CoventryCityCouncil.py,sha256=kfAvA2e4MlO0W9YT70U_mW9gxVPrmr0BOGzV99Tw2Bg,2012
|
58
61
|
uk_bin_collection/uk_bin_collection/councils/CrawleyBoroughCouncil.py,sha256=_BEKZAjlS5Ad5DjyxqAEFSLn8F-KYox0zmn4BXaAD6A,2367
|
59
62
|
uk_bin_collection/uk_bin_collection/councils/CroydonCouncil.py,sha256=Vxh5ICoaXTAvx0nDOq_95XQ4He9sQKcLdI5keV2uxM4,11384
|
@@ -180,7 +183,7 @@ uk_bin_collection/uk_bin_collection/councils/SouthRibbleCouncil.py,sha256=Odexbe
|
|
180
183
|
uk_bin_collection/uk_bin_collection/councils/SouthTynesideCouncil.py,sha256=dxXGrJfg_fn2IPTBgq6Duwy0WY8GYLafMuisaCjOnbs,3426
|
181
184
|
uk_bin_collection/uk_bin_collection/councils/SouthwarkCouncil.py,sha256=Z6JIbUt3yr4oG60n1At4AjPIGrs7Qzn_sDNY-TsS62E,4882
|
182
185
|
uk_bin_collection/uk_bin_collection/councils/StAlbansCityAndDistrictCouncil.py,sha256=mPZz6Za6kTSkrfHnj0OfwtnpRYR1dKvxbuFEKnWsiL8,1451
|
183
|
-
uk_bin_collection/uk_bin_collection/councils/StHelensBC.py,sha256=
|
186
|
+
uk_bin_collection/uk_bin_collection/councils/StHelensBC.py,sha256=X9dvnQTNn7QUO8gv1A587e1aDI92TWN4iNLATTn3H3w,4777
|
184
187
|
uk_bin_collection/uk_bin_collection/councils/StaffordBoroughCouncil.py,sha256=9Qj4HJI7Dbiqb2mVSG2UtkBe27Y7wvQ5SYFTwGzJ5g0,2292
|
185
188
|
uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py,sha256=_N8Cg26EbTaKp0RsWvQuELVcZDHbT2BlD2LW8qhkS_Q,4361
|
186
189
|
uk_bin_collection/uk_bin_collection/councils/StockportBoroughCouncil.py,sha256=v0HmioNVRoU1-9OnLJl2V3M5pVR1aVu1BgOLHFR1Sf4,1429
|
@@ -232,11 +235,11 @@ uk_bin_collection/uk_bin_collection/councils/WorcesterCityCouncil.py,sha256=dKHB
|
|
232
235
|
uk_bin_collection/uk_bin_collection/councils/WychavonDistrictCouncil.py,sha256=GnNNMe33YMlK6S7rjM3c4BQkBnXelS0DKl2x5V4fb2w,5775
|
233
236
|
uk_bin_collection/uk_bin_collection/councils/WyreCouncil.py,sha256=zDDa7n4K_zm5PgDL08A26gD9yOOsOhuexI3x2seaBF4,3511
|
234
237
|
uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py,sha256=I2kBYMlsD4bIdsvmoSzBjJAvTTi6yPfJa8xjJx1ys2w,1490
|
235
|
-
uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=
|
238
|
+
uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=EQWRhZ2pEejlvm0fPyOTsOHKvUZmPnxEYO_OWRGKTjs,1158
|
236
239
|
uk_bin_collection/uk_bin_collection/create_new_council.py,sha256=m-IhmWmeWQlFsTZC4OxuFvtw5ZtB8EAJHxJTH4O59lQ,1536
|
237
240
|
uk_bin_collection/uk_bin_collection/get_bin_data.py,sha256=YvmHfZqanwrJ8ToGch34x-L-7yPe31nB_x77_Mgl_vo,4545
|
238
|
-
uk_bin_collection-0.
|
239
|
-
uk_bin_collection-0.
|
240
|
-
uk_bin_collection-0.
|
241
|
-
uk_bin_collection-0.
|
242
|
-
uk_bin_collection-0.
|
241
|
+
uk_bin_collection-0.109.1.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
|
242
|
+
uk_bin_collection-0.109.1.dist-info/METADATA,sha256=k1L8njxhEISbOUBLjK44nNQIvFv5F0DHDdLD-s3Vq_0,17574
|
243
|
+
uk_bin_collection-0.109.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
244
|
+
uk_bin_collection-0.109.1.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
|
245
|
+
uk_bin_collection-0.109.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
{uk_bin_collection-0.108.2.dist-info → uk_bin_collection-0.109.1.dist-info}/entry_points.txt
RENAMED
File without changes
|