uk_bin_collection 0.74.0__py3-none-any.whl → 0.74.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- uk_bin_collection/tests/input.json +3 -2
- uk_bin_collection/uk_bin_collection/councils/CheshireWestAndChesterCouncil.py +54 -79
- {uk_bin_collection-0.74.0.dist-info → uk_bin_collection-0.74.1.dist-info}/METADATA +1 -1
- {uk_bin_collection-0.74.0.dist-info → uk_bin_collection-0.74.1.dist-info}/RECORD +7 -7
- {uk_bin_collection-0.74.0.dist-info → uk_bin_collection-0.74.1.dist-info}/LICENSE +0 -0
- {uk_bin_collection-0.74.0.dist-info → uk_bin_collection-0.74.1.dist-info}/WHEEL +0 -0
- {uk_bin_collection-0.74.0.dist-info → uk_bin_collection-0.74.1.dist-info}/entry_points.txt +0 -0
|
@@ -207,8 +207,9 @@
|
|
|
207
207
|
"wiki_note": "Both the UPRN and a one-line address are passed in the URL, which needs to be wrapped in double quotes. The one-line address is made up of the house number, street name and postcode.\nUse the form [here](https://online.cheshireeast.gov.uk/mycollectionday/) to find them, then take the first line and post code and replace all spaces with `%20`."
|
|
208
208
|
},
|
|
209
209
|
"CheshireWestAndChesterCouncil": {
|
|
210
|
-
"house_number": "
|
|
211
|
-
"postcode": "
|
|
210
|
+
"house_number": "Hill View House",
|
|
211
|
+
"postcode": "CH3 9ER",
|
|
212
|
+
"uprn": "100012346655",
|
|
212
213
|
"skip_get_url": true,
|
|
213
214
|
"url": "https://www.cheshirewestandchester.gov.uk/residents/waste-and-recycling/your-bin-collection/collection-day",
|
|
214
215
|
"web_driver": "http://selenium:4444",
|
|
@@ -1,29 +1,23 @@
|
|
|
1
1
|
import time
|
|
2
|
-
|
|
2
|
+
import logging
|
|
3
3
|
from bs4 import BeautifulSoup
|
|
4
4
|
from selenium.webdriver.common.by import By
|
|
5
5
|
from selenium.webdriver.support import expected_conditions as EC
|
|
6
6
|
from selenium.webdriver.support.ui import Select
|
|
7
7
|
from selenium.webdriver.support.wait import WebDriverWait
|
|
8
|
-
|
|
9
8
|
from uk_bin_collection.uk_bin_collection.common import *
|
|
10
9
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
|
11
10
|
|
|
11
|
+
# Set up logging
|
|
12
|
+
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
12
13
|
|
|
13
|
-
# import the wonderful Beautiful Soup and the URL grabber
|
|
14
14
|
class CouncilClass(AbstractGetBinDataClass):
|
|
15
|
-
"""
|
|
16
|
-
Concrete classes have to implement all abstract operations of the
|
|
17
|
-
base class. They can also override some operations with a default
|
|
18
|
-
implementation.
|
|
19
|
-
"""
|
|
20
|
-
|
|
21
15
|
def parse_data(self, page: str, **kwargs) -> dict:
|
|
22
16
|
driver = None
|
|
23
17
|
try:
|
|
24
18
|
data = {"bins": []}
|
|
25
19
|
collections = []
|
|
26
|
-
|
|
20
|
+
user_uprn = kwargs.get("uprn")
|
|
27
21
|
user_paon = kwargs.get("paon")
|
|
28
22
|
user_postcode = kwargs.get("postcode")
|
|
29
23
|
web_driver = kwargs.get("web_driver")
|
|
@@ -33,77 +27,59 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
|
33
27
|
|
|
34
28
|
# Create Selenium webdriver
|
|
35
29
|
driver = create_webdriver(web_driver, headless)
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
EC.
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
)
|
|
52
|
-
find_collection_button.click()
|
|
53
|
-
|
|
54
|
-
banner_close_button = WebDriverWait(driver, timeout=30).until(
|
|
55
|
-
EC.presence_of_element_located((By.ID, "close-cookie-message"))
|
|
56
|
-
)
|
|
57
|
-
banner_close_button.click()
|
|
58
|
-
|
|
59
|
-
time.sleep(5)
|
|
60
|
-
|
|
61
|
-
frame = driver.find_element(
|
|
62
|
-
By.XPATH, "/html/body/div[4]/section/div/div[2]/div[2]/div/iframe"
|
|
63
|
-
)
|
|
64
|
-
driver.switch_to.frame(frame)
|
|
65
|
-
|
|
66
|
-
# Wait for the postcode field to appear then populate it
|
|
67
|
-
inputElement_postcode = WebDriverWait(driver, 30).until(
|
|
68
|
-
EC.presence_of_element_located((By.NAME, "postcode_search"))
|
|
69
|
-
)
|
|
70
|
-
inputElement_postcode.send_keys(user_postcode)
|
|
71
|
-
|
|
72
|
-
address_box_text = WebDriverWait(driver, 30).until(
|
|
73
|
-
EC.presence_of_element_located((By.ID, "label_Choose_Address"))
|
|
74
|
-
)
|
|
75
|
-
address_box_text.click()
|
|
76
|
-
time.sleep(2)
|
|
77
|
-
|
|
78
|
-
address_selection_menu = Select(
|
|
79
|
-
driver.find_element(By.ID, "Choose_Address")
|
|
80
|
-
)
|
|
81
|
-
for idx, addr_option in enumerate(address_selection_menu.options):
|
|
82
|
-
option_name = addr_option.text[0 : len(user_paon)]
|
|
83
|
-
if option_name == user_paon:
|
|
84
|
-
selected_address = addr_option
|
|
85
|
-
break
|
|
86
|
-
address_selection_menu.select_by_visible_text(selected_address.text)
|
|
87
|
-
|
|
88
|
-
WebDriverWait(driver, 30).until(
|
|
89
|
-
EC.presence_of_element_located(
|
|
90
|
-
(By.XPATH, '//*[@id="bin-schedule-content"]/div/h3')
|
|
91
|
-
)
|
|
92
|
-
)
|
|
30
|
+
if headless:
|
|
31
|
+
driver.set_window_size(1920, 1080)
|
|
32
|
+
|
|
33
|
+
driver.get("https://www.cheshirewestandchester.gov.uk/residents/waste-and-recycling/your-bin-collection/collection-day")
|
|
34
|
+
wait = WebDriverWait(driver, 60)
|
|
35
|
+
|
|
36
|
+
def click_element(by, value):
|
|
37
|
+
element = wait.until(EC.element_to_be_clickable((by, value)))
|
|
38
|
+
driver.execute_script("arguments[0].scrollIntoView();", element)
|
|
39
|
+
element.click()
|
|
40
|
+
|
|
41
|
+
logging.info("Accepting cookies")
|
|
42
|
+
click_element(By.ID, "ccc-close")
|
|
43
|
+
|
|
44
|
+
logging.info("Finding collection day")
|
|
45
|
+
click_element(By.LINK_TEXT, "Find your collection day")
|
|
93
46
|
|
|
47
|
+
logging.info("Switching to iframe")
|
|
48
|
+
iframe_presence = wait.until(EC.presence_of_element_located((By.ID, "fillform-frame-1")))
|
|
49
|
+
driver.switch_to.frame(iframe_presence)
|
|
50
|
+
|
|
51
|
+
logging.info("Entering postcode")
|
|
52
|
+
input_element_postcode = wait.until(EC.presence_of_element_located((By.XPATH, '//input[@id="postcode_search"]')))
|
|
53
|
+
input_element_postcode.send_keys(user_postcode)
|
|
54
|
+
|
|
55
|
+
pcsearch_btn = wait.until(EC.element_to_be_clickable((By.XPATH, "//input[@id='postcode_search']")))
|
|
56
|
+
click_element(By.XPATH, "//input[@id='postcode_search']")
|
|
57
|
+
|
|
58
|
+
logging.info("Selecting address")
|
|
59
|
+
dropdown = wait.until(EC.element_to_be_clickable((By.ID, "Choose_Address")))
|
|
60
|
+
dropdown_options = wait.until(EC.presence_of_element_located((By.CLASS_NAME, "lookup-option")))
|
|
61
|
+
drop_down_values = Select(dropdown)
|
|
62
|
+
option_element = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, f'option.lookup-option[value="{str(user_uprn)}"]')))
|
|
63
|
+
driver.execute_script("arguments[0].scrollIntoView();", option_element)
|
|
64
|
+
drop_down_values.select_by_value(str(user_uprn))
|
|
65
|
+
|
|
66
|
+
logging.info("Waiting for bin schedule")
|
|
67
|
+
wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'bin-schedule-content-bin-card')))
|
|
68
|
+
|
|
69
|
+
logging.info("Extracting bin collection data")
|
|
94
70
|
soup = BeautifulSoup(driver.page_source, features="html.parser")
|
|
95
|
-
soup.
|
|
71
|
+
bin_cards = soup.find_all("div", {"class": "bin-schedule-content-bin-card"})
|
|
72
|
+
collections = []
|
|
96
73
|
|
|
97
|
-
# Get collections
|
|
98
|
-
bin_cards = soup.find_all("div", {"class": "bin-schedule-content-info"})
|
|
99
74
|
for card in bin_cards:
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
)
|
|
75
|
+
bin_info = card.find("div", {"class": "bin-schedule-content-info"})
|
|
76
|
+
bin_name = bin_info.find_all("p")[0].text.strip() + " bin"
|
|
77
|
+
bin_date_str = bin_info.find_all("p")[1].text.split(":")[1].strip()
|
|
78
|
+
bin_date = datetime.strptime(bin_date_str, "%A, %B %d, %Y")
|
|
104
79
|
collections.append((bin_name, bin_date))
|
|
105
80
|
|
|
106
81
|
ordered_data = sorted(collections, key=lambda x: x[1])
|
|
82
|
+
|
|
107
83
|
for item in ordered_data:
|
|
108
84
|
dict_data = {
|
|
109
85
|
"type": item[0].capitalize(),
|
|
@@ -111,15 +87,14 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
|
111
87
|
}
|
|
112
88
|
data["bins"].append(dict_data)
|
|
113
89
|
|
|
90
|
+
logging.info("Data extraction complete")
|
|
91
|
+
return data
|
|
92
|
+
|
|
114
93
|
except Exception as e:
|
|
115
|
-
|
|
116
|
-
print(f"An error occurred: {e}")
|
|
117
|
-
# Optionally, re-raise the exception if you want it to propagate
|
|
94
|
+
logging.error(f"An error occurred: {e}")
|
|
118
95
|
raise
|
|
119
96
|
|
|
120
97
|
finally:
|
|
121
|
-
# This block ensures that the driver is closed regardless of an exception
|
|
122
98
|
if driver:
|
|
123
99
|
driver.quit()
|
|
124
100
|
|
|
125
|
-
return data
|
|
@@ -2,7 +2,7 @@ uk_bin_collection/README.rst,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
|
|
|
2
2
|
uk_bin_collection/tests/council_feature_input_parity.py,sha256=qS3JuQD__NomFiVvA0_xHL_kWRuvvRhLj_M30xUzcbg,3072
|
|
3
3
|
uk_bin_collection/tests/features/environment.py,sha256=VQZjJdJI_kZn08M0j5cUgvKT4k3iTw8icJge1DGOkoA,127
|
|
4
4
|
uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=XIYnckDPBwBd7jTX2CN7Nv4mhUQsiJYOnYzvXVGfpdk,16246
|
|
5
|
-
uk_bin_collection/tests/input.json,sha256=
|
|
5
|
+
uk_bin_collection/tests/input.json,sha256=lbJZCvGd0kvIvI-0dD7Q9clMnkk_zX2UWbp8j2Ji1ow,53474
|
|
6
6
|
uk_bin_collection/tests/output.schema,sha256=ZwKQBwYyTDEM4G2hJwfLUVM-5v1vKRvRK9W9SS1sd18,1086
|
|
7
7
|
uk_bin_collection/tests/step_defs/step_helpers/file_handler.py,sha256=Ygzi4V0S1MIHqbdstUlIqtRIwnynvhu4UtpweJ6-5N8,1474
|
|
8
8
|
uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=Pg3Z7c3zrebx1w-yIZ9xTJ3E-okVndlNIAJL-F46HKU,2962
|
|
@@ -38,7 +38,7 @@ uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py,sha25
|
|
|
38
38
|
uk_bin_collection/uk_bin_collection/councils/CharnwoodBoroughCouncil.py,sha256=tXfzMetN6wxahuGGRp2mIyCCDSL4F2aG61HhUxw6COQ,2172
|
|
39
39
|
uk_bin_collection/uk_bin_collection/councils/ChelmsfordCityCouncil.py,sha256=pEMQKNNTExYEqAyEn9ZMplwLVMly8R6tJeN5zNS_vpY,5074
|
|
40
40
|
uk_bin_collection/uk_bin_collection/councils/CheshireEastCouncil.py,sha256=pyCxi_UZQkRnuHnwynLFHiux-__UIgAqhcpI3c2mkfA,1132
|
|
41
|
-
uk_bin_collection/uk_bin_collection/councils/CheshireWestAndChesterCouncil.py,sha256=
|
|
41
|
+
uk_bin_collection/uk_bin_collection/councils/CheshireWestAndChesterCouncil.py,sha256=GE5l66hB6qnhEbewKvioW9YOp8a90Xnnzu9_6Gl2n5A,4432
|
|
42
42
|
uk_bin_collection/uk_bin_collection/councils/ChorleyCouncil.py,sha256=A6HSR7bapz0X03sMzu0BXIke9wlsCpWhIddDXkfVFZc,5183
|
|
43
43
|
uk_bin_collection/uk_bin_collection/councils/ConwyCountyBorough.py,sha256=el75qv2QyfWZBU09tJLvD8vLQZ9pCg73u1NBFs6ybo8,1034
|
|
44
44
|
uk_bin_collection/uk_bin_collection/councils/CrawleyBoroughCouncil.py,sha256=rZIRiXp0rOkc20M9DpgxuhbykACip7LNjpn6ytDsnwk,2361
|
|
@@ -164,8 +164,8 @@ uk_bin_collection/uk_bin_collection/councils/WyreCouncil.py,sha256=zDDa7n4K_zm5P
|
|
|
164
164
|
uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py,sha256=I2kBYMlsD4bIdsvmoSzBjJAvTTi6yPfJa8xjJx1ys2w,1490
|
|
165
165
|
uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=4s9ODGPAwPqwXc8SrTX5Wlfmizs3_58iXUtHc4Ir86o,1162
|
|
166
166
|
uk_bin_collection/uk_bin_collection/get_bin_data.py,sha256=9qppF2oPkhmOoK8-ZkRIU1M6vhBh-yUCWAZEEd07iLk,5414
|
|
167
|
-
uk_bin_collection-0.74.
|
|
168
|
-
uk_bin_collection-0.74.
|
|
169
|
-
uk_bin_collection-0.74.
|
|
170
|
-
uk_bin_collection-0.74.
|
|
171
|
-
uk_bin_collection-0.74.
|
|
167
|
+
uk_bin_collection-0.74.1.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
|
|
168
|
+
uk_bin_collection-0.74.1.dist-info/METADATA,sha256=mZbP454WX8TthYdJSeCodIRKf62cZaU_CWjOhVTl6cI,12594
|
|
169
|
+
uk_bin_collection-0.74.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
170
|
+
uk_bin_collection-0.74.1.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
|
|
171
|
+
uk_bin_collection-0.74.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|