uk_bin_collection 0.98.4__py3-none-any.whl → 0.98.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- uk_bin_collection/tests/input.json +2 -1
- uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py +24 -7
- uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py +4 -0
- uk_bin_collection/uk_bin_collection/councils/SwaleBoroughCouncil.py +63 -25
- uk_bin_collection/uk_bin_collection/councils/WestBerkshireCouncil.py +45 -24
- {uk_bin_collection-0.98.4.dist-info → uk_bin_collection-0.98.5.dist-info}/METADATA +1 -1
- {uk_bin_collection-0.98.4.dist-info → uk_bin_collection-0.98.5.dist-info}/RECORD +10 -10
- {uk_bin_collection-0.98.4.dist-info → uk_bin_collection-0.98.5.dist-info}/LICENSE +0 -0
- {uk_bin_collection-0.98.4.dist-info → uk_bin_collection-0.98.5.dist-info}/WHEEL +0 -0
- {uk_bin_collection-0.98.4.dist-info → uk_bin_collection-0.98.5.dist-info}/entry_points.txt +0 -0
@@ -1128,7 +1128,8 @@
|
|
1128
1128
|
"SwaleBoroughCouncil": {
|
1129
1129
|
"postcode": "ME12 2NQ",
|
1130
1130
|
"skip_get_url": true,
|
1131
|
-
"
|
1131
|
+
"house_number": "81",
|
1132
|
+
"web_driver": "http://selenium:4444",
|
1132
1133
|
"url": "https://swale.gov.uk/bins-littering-and-the-environment/bins/collection-days",
|
1133
1134
|
"wiki_name": "Swale Borough Council"
|
1134
1135
|
},
|
@@ -74,13 +74,30 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
74
74
|
|
75
75
|
driver.get(page)
|
76
76
|
|
77
|
+
wait = WebDriverWait(driver, 10)
|
78
|
+
accept_cookies_button = wait.until(
|
79
|
+
EC.element_to_be_clickable(
|
80
|
+
(
|
81
|
+
By.XPATH,
|
82
|
+
"//button[contains(text(), 'Accept additional cookies')]",
|
83
|
+
)
|
84
|
+
)
|
85
|
+
)
|
86
|
+
accept_cookies_button.click()
|
87
|
+
|
77
88
|
# Wait for the element to be clickable
|
78
|
-
|
89
|
+
wait = WebDriverWait(driver, 10)
|
90
|
+
find_your_collection_button = wait.until(
|
79
91
|
EC.element_to_be_clickable(
|
80
|
-
(By.
|
92
|
+
(By.LINK_TEXT, "Find your household collection day")
|
81
93
|
)
|
82
94
|
)
|
83
95
|
|
96
|
+
# Scroll to the element (in case something is blocking it)
|
97
|
+
driver.execute_script(
|
98
|
+
"arguments[0].scrollIntoView();", find_your_collection_button
|
99
|
+
)
|
100
|
+
|
84
101
|
# Click the element
|
85
102
|
find_your_collection_button.click()
|
86
103
|
|
@@ -107,12 +124,12 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
107
124
|
|
108
125
|
postcode_input.send_keys(user_postcode)
|
109
126
|
|
110
|
-
find_address_button = WebDriverWait(driver,
|
111
|
-
EC.
|
112
|
-
(By.CSS_SELECTOR, '[value="Find address"]')
|
113
|
-
)
|
127
|
+
find_address_button = WebDriverWait(driver, 30).until(
|
128
|
+
EC.element_to_be_clickable((By.CSS_SELECTOR, '[value="Find address"]'))
|
114
129
|
)
|
115
|
-
|
130
|
+
driver.execute_script("arguments[0].scrollIntoView();", find_address_button)
|
131
|
+
driver.execute_script("arguments[0].click();", find_address_button)
|
132
|
+
# find_address_button.click()
|
116
133
|
|
117
134
|
time.sleep(15)
|
118
135
|
# Wait for address box to be visible
|
@@ -80,6 +80,10 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
80
80
|
)
|
81
81
|
)
|
82
82
|
search_btn.send_keys(Keys.ENTER)
|
83
|
+
WebDriverWait(driver, 10).until(
|
84
|
+
EC.presence_of_element_located((By.ID, "collectionTabs"))
|
85
|
+
)
|
86
|
+
|
83
87
|
soup = BeautifulSoup(driver.page_source, features="html.parser")
|
84
88
|
|
85
89
|
# Find all tab panels within the collectionTabs
|
@@ -1,9 +1,11 @@
|
|
1
|
-
import requests
|
2
1
|
from bs4 import BeautifulSoup
|
2
|
+
from selenium.webdriver.common.by import By
|
3
|
+
from selenium.webdriver.support import expected_conditions as EC
|
4
|
+
from selenium.webdriver.support.wait import WebDriverWait
|
5
|
+
|
3
6
|
from uk_bin_collection.uk_bin_collection.common import *
|
4
7
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
5
8
|
|
6
|
-
|
7
9
|
# import the wonderful Beautiful Soup and the URL grabber
|
8
10
|
|
9
11
|
|
@@ -17,36 +19,72 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
17
19
|
def parse_data(self, page: str, **kwargs) -> dict:
|
18
20
|
# Get postcode and UPRN from kwargs
|
19
21
|
user_postcode = kwargs.get("postcode")
|
20
|
-
|
22
|
+
user_paon = kwargs.get("paon")
|
23
|
+
web_driver = kwargs.get("web_driver")
|
24
|
+
headless = kwargs.get("headless")
|
21
25
|
check_postcode(user_postcode)
|
22
|
-
|
26
|
+
check_paon(user_paon)
|
23
27
|
|
24
28
|
# Build URL to parse
|
25
|
-
council_url =
|
29
|
+
council_url = "https://swale.gov.uk/bins-littering-and-the-environment/bins/my-collection-day"
|
30
|
+
|
31
|
+
# Create Selenium webdriver
|
32
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
33
|
+
driver.get(council_url)
|
34
|
+
|
35
|
+
# Wait for the postcode field to appear then populate it
|
36
|
+
try:
|
37
|
+
inputElement_postcode = WebDriverWait(driver, 10).until(
|
38
|
+
EC.presence_of_element_located((By.ID, "q462406_q1"))
|
39
|
+
)
|
40
|
+
inputElement_postcode.send_keys(user_postcode)
|
41
|
+
except Exception:
|
42
|
+
print("Page failed to load. Probably due to Cloudflare robot check!")
|
43
|
+
|
44
|
+
# Click search button
|
45
|
+
findAddress = WebDriverWait(driver, 10).until(
|
46
|
+
EC.presence_of_element_located((By.ID, "form_email_462397_submit"))
|
47
|
+
)
|
48
|
+
driver.execute_script("arguments[0].click();", findAddress)
|
49
|
+
|
50
|
+
# Wait for the 'Select address' dropdown to appear and select option matching the house name/number
|
51
|
+
WebDriverWait(driver, 10).until(
|
52
|
+
EC.element_to_be_clickable(
|
53
|
+
(
|
54
|
+
By.XPATH,
|
55
|
+
"//select[@id='SBCYBDAddressList']//option[contains(., '"
|
56
|
+
+ user_paon
|
57
|
+
+ "')]",
|
58
|
+
)
|
59
|
+
)
|
60
|
+
).click()
|
61
|
+
|
62
|
+
# Click search button
|
63
|
+
getBins = WebDriverWait(driver, 10).until(
|
64
|
+
EC.presence_of_element_located((By.ID, "form_email_462397_submit"))
|
65
|
+
)
|
66
|
+
driver.execute_script("arguments[0].click();", getBins)
|
67
|
+
|
68
|
+
BinTable = WebDriverWait(driver, 30).until(
|
69
|
+
EC.presence_of_element_located((By.ID, "SBC-YBD-Main"))
|
70
|
+
)
|
26
71
|
|
27
|
-
|
28
|
-
|
29
|
-
response = requests.get(council_url, verify=False)
|
30
|
-
if response.status_code == 200:
|
31
|
-
soup = BeautifulSoup(response.text, features="html.parser")
|
32
|
-
soup.prettify()
|
33
|
-
else:
|
34
|
-
raise ConnectionAbortedError("Could not parse council website.")
|
72
|
+
soup = BeautifulSoup(driver.page_source, features="html.parser")
|
73
|
+
soup.prettify()
|
35
74
|
|
36
75
|
data = {"bins": []}
|
37
76
|
|
38
77
|
# Get the collection bullet points on the page and parse them
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
data["bins"].append(dict_data)
|
78
|
+
nextCollections = soup.find("div", {"id": "nextCollections"})
|
79
|
+
for c in nextCollections:
|
80
|
+
collection = c.find_all("strong")
|
81
|
+
for bin in collection:
|
82
|
+
split = (bin.text).split(" on ")
|
83
|
+
bin_type = split[0]
|
84
|
+
bin_date = datetime.strptime(split[1], "%A %d %b %Y").strftime(
|
85
|
+
"%d/%m/%Y"
|
86
|
+
)
|
87
|
+
dict_data = {"type": bin_type, "collectionDate": bin_date}
|
88
|
+
data["bins"].append(dict_data)
|
51
89
|
|
52
90
|
return data
|
@@ -74,30 +74,51 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
74
74
|
soup = BeautifulSoup(driver.page_source, features="html.parser")
|
75
75
|
soup.prettify()
|
76
76
|
|
77
|
-
|
78
|
-
" "
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
.
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
77
|
+
rubbish_div = soup.find(
|
78
|
+
"div", {"id": "FINDYOURBINDAYS_RUBBISHDATE_OUTERDIV"}
|
79
|
+
)
|
80
|
+
try:
|
81
|
+
rubbish_date = rubbish_div.find_all("div")[2]
|
82
|
+
rubbish_date = datetime.strptime(
|
83
|
+
rubbish_date.text,
|
84
|
+
"%A %d %B",
|
85
|
+
).replace(year=datetime.now().year)
|
86
|
+
except:
|
87
|
+
rubbish_date = rubbish_div.find_all("div")[3]
|
88
|
+
rubbish_date = datetime.strptime(
|
89
|
+
rubbish_date.text,
|
90
|
+
"%A %d %B",
|
91
|
+
).replace(year=datetime.now().year)
|
92
|
+
recycling_div = soup.find(
|
93
|
+
"div", {"id": "FINDYOURBINDAYS_RECYCLINGDATE_OUTERDIV"}
|
94
|
+
)
|
95
|
+
try:
|
96
|
+
recycling_date = recycling_div.find_all("div")[2]
|
97
|
+
recycling_date = datetime.strptime(
|
98
|
+
recycling_date.text,
|
99
|
+
"%A %d %B",
|
100
|
+
).replace(year=datetime.now().year)
|
101
|
+
except:
|
102
|
+
rubbish_date = recycling_div.find_all("div")[3]
|
103
|
+
rubbish_date = datetime.strptime(
|
104
|
+
rubbish_date.text,
|
105
|
+
"%A %d %B",
|
106
|
+
).replace(year=datetime.now().year)
|
107
|
+
food_div = soup.find(
|
108
|
+
"div", {"id": "FINDYOURBINDAYS_RECYCLINGDATE_OUTERDIV"}
|
109
|
+
)
|
110
|
+
try:
|
111
|
+
food_date = food_div.find_all("div")[2]
|
112
|
+
food_date = datetime.strptime(
|
113
|
+
food_date.text,
|
114
|
+
"%A %d %B",
|
115
|
+
).replace(year=datetime.now().year)
|
116
|
+
except:
|
117
|
+
food_date = food_div.find_all("div")[3]
|
118
|
+
food_date = datetime.strptime(
|
119
|
+
food_date.text,
|
120
|
+
"%A %d %B",
|
121
|
+
).replace(year=datetime.now().year)
|
101
122
|
|
102
123
|
if datetime.now().month == 12 and rubbish_date.month == 1:
|
103
124
|
rubbish_date = rubbish_date + relativedelta(years=1)
|
@@ -2,7 +2,7 @@ uk_bin_collection/README.rst,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
|
|
2
2
|
uk_bin_collection/tests/council_feature_input_parity.py,sha256=DO6Mk4ImYgM5ZCZ-cutwz5RoYYWZRLYx2tr6zIs_9Rc,3843
|
3
3
|
uk_bin_collection/tests/features/environment.py,sha256=VQZjJdJI_kZn08M0j5cUgvKT4k3iTw8icJge1DGOkoA,127
|
4
4
|
uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=SJK-Vc737hrf03tssxxbeg_JIvAH-ddB8f6gU1LTbuQ,251
|
5
|
-
uk_bin_collection/tests/input.json,sha256=
|
5
|
+
uk_bin_collection/tests/input.json,sha256=gkqIfeMzQQC76jc6J4DPRn1ZCp6AVdPU_7ra3gb7pyc,68114
|
6
6
|
uk_bin_collection/tests/output.schema,sha256=ZwKQBwYyTDEM4G2hJwfLUVM-5v1vKRvRK9W9SS1sd18,1086
|
7
7
|
uk_bin_collection/tests/step_defs/step_helpers/file_handler.py,sha256=Ygzi4V0S1MIHqbdstUlIqtRIwnynvhu4UtpweJ6-5N8,1474
|
8
8
|
uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=LrOSt_loA1Mw3vTqaO2LpaDMu7rYJy6k5Kr-EOBln7s,3424
|
@@ -17,7 +17,7 @@ uk_bin_collection/uk_bin_collection/councils/ArmaghBanbridgeCraigavonCouncil.py,
|
|
17
17
|
uk_bin_collection/uk_bin_collection/councils/ArunCouncil.py,sha256=yfhthv9nuogP19VOZ3TYQrq51qqjiCZcSel4sXhiKjs,4012
|
18
18
|
uk_bin_collection/uk_bin_collection/councils/AylesburyValeCouncil.py,sha256=LouqjspEMt1TkOGqWHs2zkxwOETIy3n7p64uKIlAgUg,2401
|
19
19
|
uk_bin_collection/uk_bin_collection/councils/BCPCouncil.py,sha256=W7QBx6Mgso8RYosuXsaYo3GGNAu-tiyBSmuYxr1JSOU,1707
|
20
|
-
uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py,sha256=
|
20
|
+
uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py,sha256=Sd4-pbv0QZsR7soxvXYqsfdOUIqZqS6notyoZthG77s,9182
|
21
21
|
uk_bin_collection/uk_bin_collection/councils/BarnsleyMBCouncil.py,sha256=MgF_7XyIcIoNzFR0OJsjBkLCZKgWxBrV6nTcutMxO1Q,4244
|
22
22
|
uk_bin_collection/uk_bin_collection/councils/BasildonCouncil.py,sha256=SBvAa0GZM3V7ygK8ARawbHAPH6R_303U30RH8WYPi5Q,3020
|
23
23
|
uk_bin_collection/uk_bin_collection/councils/BasingstokeCouncil.py,sha256=VPWGljnH4C3q8qs5ZmCtqjNjgWQvviALzjk00q3EZeQ,2632
|
@@ -81,7 +81,7 @@ uk_bin_collection/uk_bin_collection/councils/GedlingBoroughCouncil.py,sha256=Iss
|
|
81
81
|
uk_bin_collection/uk_bin_collection/councils/GlasgowCityCouncil.py,sha256=i7BympEhCm7D9yR0p5_QQICtWvNcDYNJIWB19SA0g2k,2303
|
82
82
|
uk_bin_collection/uk_bin_collection/councils/GloucesterCityCouncil.py,sha256=8Wjvmdvg5blHVrREaEnhhWZaWhYVP4v_KdDVPLIUxaU,4889
|
83
83
|
uk_bin_collection/uk_bin_collection/councils/GuildfordCouncil.py,sha256=9pVrmQhZcK2AD8gX8mNvP--L4L9KaY6L3B822VX6fec,5695
|
84
|
-
uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py,sha256=
|
84
|
+
uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py,sha256=gq_CPqi6qM2oNiHhKKF1lZC86fyKL4lPhh_DN9pJZ04,5971
|
85
85
|
uk_bin_collection/uk_bin_collection/councils/HaringeyCouncil.py,sha256=t_6AkAu4wrv8Q0WlDhWh_82I0djl5tk531Pzs-SjWzg,2647
|
86
86
|
uk_bin_collection/uk_bin_collection/councils/HarrogateBoroughCouncil.py,sha256=_g3fP5Nq-OUjgNrfRf4UEyFKzq0x8QK-4enh5RP1efA,2050
|
87
87
|
uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py,sha256=oqF8M0lcT3KsrG6W6I6JJX07E6Sc_-_sr7MybfIMab8,4626
|
@@ -170,7 +170,7 @@ uk_bin_collection/uk_bin_collection/councils/StokeOnTrentCityCouncil.py,sha256=K
|
|
170
170
|
uk_bin_collection/uk_bin_collection/councils/StratfordUponAvonCouncil.py,sha256=DMTAcXT_lay8Cl1hBbzf_LN7-GwTDGxT3Ug9QJkaF9Y,3936
|
171
171
|
uk_bin_collection/uk_bin_collection/councils/StroudDistrictCouncil.py,sha256=9bYWppi7ViLGHL4VEg--nFn28MLYJYbiEntull1uZxU,3561
|
172
172
|
uk_bin_collection/uk_bin_collection/councils/SunderlandCityCouncil.py,sha256=4DnKyyu56_AwuchD6_oL1dvpDStMvkkxQtYN79rUKOs,3825
|
173
|
-
uk_bin_collection/uk_bin_collection/councils/SwaleBoroughCouncil.py,sha256=
|
173
|
+
uk_bin_collection/uk_bin_collection/councils/SwaleBoroughCouncil.py,sha256=ak0zqBJ6UAS8_t5zKGwigMxPMdlQuYXRsGCQJCNERJs,3415
|
174
174
|
uk_bin_collection/uk_bin_collection/councils/SwanseaCouncil.py,sha256=nmVPoPhnFgVi--vczX2i4Sf3bqM5RWJuwfhioRUr5XE,2303
|
175
175
|
uk_bin_collection/uk_bin_collection/councils/TamesideMBCouncil.py,sha256=k2TAAZG7n2S1BWVyxbE_-4-lZuzhOimCNz4yimUCOGk,1995
|
176
176
|
uk_bin_collection/uk_bin_collection/councils/TandridgeDistrictCouncil.py,sha256=KLVvM2NNq_DQylVe5dwO2l7qPahLHg08jJGLCv1MBQ4,2324
|
@@ -191,7 +191,7 @@ uk_bin_collection/uk_bin_collection/councils/WarwickDistrictCouncil.py,sha256=3W
|
|
191
191
|
uk_bin_collection/uk_bin_collection/councils/WaverleyBoroughCouncil.py,sha256=tp9l7vdgSGRzNNG0pDfnNuFj4D2bpRJUJmAiTJ6bM0g,4662
|
192
192
|
uk_bin_collection/uk_bin_collection/councils/WealdenDistrictCouncil.py,sha256=SvSSaLkx7iJjzypAwKkaJwegXkSsIQtUOS2V605kz1A,3368
|
193
193
|
uk_bin_collection/uk_bin_collection/councils/WelhatCouncil.py,sha256=ikUft37dYNJghfe-_6Fskiq1JihqpLmLNj38QkKSUUA,2316
|
194
|
-
uk_bin_collection/uk_bin_collection/councils/WestBerkshireCouncil.py,sha256=
|
194
|
+
uk_bin_collection/uk_bin_collection/councils/WestBerkshireCouncil.py,sha256=XhTimZAPNgcuFgNp5mQjkR8mC4LRqUEUCy6e6plHspM,6004
|
195
195
|
uk_bin_collection/uk_bin_collection/councils/WestLindseyDistrictCouncil.py,sha256=JFWUy4w0CKulGq16PfbRDKAdQEbokVEuabwlZYigdEU,4606
|
196
196
|
uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py,sha256=dq0jimtARvRkZiGbVFrXXZgY-BODtz3uYZ5UKn0bf64,4114
|
197
197
|
uk_bin_collection/uk_bin_collection/councils/WestMorlandAndFurness.py,sha256=jbqV3460rn9D0yTBGWjpSe1IvWWcdGur5pzgj-hJcQ4,2513
|
@@ -210,8 +210,8 @@ uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py,sha256=I2kBYMlsD4bId
|
|
210
210
|
uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=4s9ODGPAwPqwXc8SrTX5Wlfmizs3_58iXUtHc4Ir86o,1162
|
211
211
|
uk_bin_collection/uk_bin_collection/create_new_council.py,sha256=m-IhmWmeWQlFsTZC4OxuFvtw5ZtB8EAJHxJTH4O59lQ,1536
|
212
212
|
uk_bin_collection/uk_bin_collection/get_bin_data.py,sha256=YvmHfZqanwrJ8ToGch34x-L-7yPe31nB_x77_Mgl_vo,4545
|
213
|
-
uk_bin_collection-0.98.
|
214
|
-
uk_bin_collection-0.98.
|
215
|
-
uk_bin_collection-0.98.
|
216
|
-
uk_bin_collection-0.98.
|
217
|
-
uk_bin_collection-0.98.
|
213
|
+
uk_bin_collection-0.98.5.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
|
214
|
+
uk_bin_collection-0.98.5.dist-info/METADATA,sha256=hbuONB_eNHQmV4S23Zggy6yYeSXl5wI-YQrWkYlld58,16843
|
215
|
+
uk_bin_collection-0.98.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
216
|
+
uk_bin_collection-0.98.5.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
|
217
|
+
uk_bin_collection-0.98.5.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|