uk_bin_collection 0.98.4__py3-none-any.whl → 0.98.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1128,7 +1128,8 @@
1128
1128
  "SwaleBoroughCouncil": {
1129
1129
  "postcode": "ME12 2NQ",
1130
1130
  "skip_get_url": true,
1131
- "uprn": "100061081168",
1131
+ "house_number": "81",
1132
+ "web_driver": "http://selenium:4444",
1132
1133
  "url": "https://swale.gov.uk/bins-littering-and-the-environment/bins/collection-days",
1133
1134
  "wiki_name": "Swale Borough Council"
1134
1135
  },
@@ -74,13 +74,30 @@ class CouncilClass(AbstractGetBinDataClass):
74
74
 
75
75
  driver.get(page)
76
76
 
77
+ wait = WebDriverWait(driver, 10)
78
+ accept_cookies_button = wait.until(
79
+ EC.element_to_be_clickable(
80
+ (
81
+ By.XPATH,
82
+ "//button[contains(text(), 'Accept additional cookies')]",
83
+ )
84
+ )
85
+ )
86
+ accept_cookies_button.click()
87
+
77
88
  # Wait for the element to be clickable
78
- find_your_collection_button = WebDriverWait(driver, 10).until(
89
+ wait = WebDriverWait(driver, 10)
90
+ find_your_collection_button = wait.until(
79
91
  EC.element_to_be_clickable(
80
- (By.XPATH, '//a[contains(text(), "Find your household collection day")]')
92
+ (By.LINK_TEXT, "Find your household collection day")
81
93
  )
82
94
  )
83
95
 
96
+ # Scroll to the element (in case something is blocking it)
97
+ driver.execute_script(
98
+ "arguments[0].scrollIntoView();", find_your_collection_button
99
+ )
100
+
84
101
  # Click the element
85
102
  find_your_collection_button.click()
86
103
 
@@ -107,12 +124,12 @@ class CouncilClass(AbstractGetBinDataClass):
107
124
 
108
125
  postcode_input.send_keys(user_postcode)
109
126
 
110
- find_address_button = WebDriverWait(driver, 10).until(
111
- EC.presence_of_element_located(
112
- (By.CSS_SELECTOR, '[value="Find address"]')
113
- )
127
+ find_address_button = WebDriverWait(driver, 30).until(
128
+ EC.element_to_be_clickable((By.CSS_SELECTOR, '[value="Find address"]'))
114
129
  )
115
- find_address_button.click()
130
+ driver.execute_script("arguments[0].scrollIntoView();", find_address_button)
131
+ driver.execute_script("arguments[0].click();", find_address_button)
132
+ # find_address_button.click()
116
133
 
117
134
  time.sleep(15)
118
135
  # Wait for address box to be visible
@@ -80,6 +80,10 @@ class CouncilClass(AbstractGetBinDataClass):
80
80
  )
81
81
  )
82
82
  search_btn.send_keys(Keys.ENTER)
83
+ WebDriverWait(driver, 10).until(
84
+ EC.presence_of_element_located((By.ID, "collectionTabs"))
85
+ )
86
+
83
87
  soup = BeautifulSoup(driver.page_source, features="html.parser")
84
88
 
85
89
  # Find all tab panels within the collectionTabs
@@ -1,9 +1,11 @@
1
- import requests
2
1
  from bs4 import BeautifulSoup
2
+ from selenium.webdriver.common.by import By
3
+ from selenium.webdriver.support import expected_conditions as EC
4
+ from selenium.webdriver.support.wait import WebDriverWait
5
+
3
6
  from uk_bin_collection.uk_bin_collection.common import *
4
7
  from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
5
8
 
6
-
7
9
  # import the wonderful Beautiful Soup and the URL grabber
8
10
 
9
11
 
@@ -17,36 +19,72 @@ class CouncilClass(AbstractGetBinDataClass):
17
19
  def parse_data(self, page: str, **kwargs) -> dict:
18
20
  # Get postcode and UPRN from kwargs
19
21
  user_postcode = kwargs.get("postcode")
20
- user_uprn = kwargs.get("uprn")
22
+ user_paon = kwargs.get("paon")
23
+ web_driver = kwargs.get("web_driver")
24
+ headless = kwargs.get("headless")
21
25
  check_postcode(user_postcode)
22
- check_uprn(user_uprn)
26
+ check_paon(user_paon)
23
27
 
24
28
  # Build URL to parse
25
- council_url = f"https://swale.gov.uk/bins-littering-and-the-environment/bins/collection-days?postcode={user_postcode.replace(' ', '+')}&addresses={user_uprn}&address-submit="
29
+ council_url = "https://swale.gov.uk/bins-littering-and-the-environment/bins/my-collection-day"
30
+
31
+ # Create Selenium webdriver
32
+ driver = create_webdriver(web_driver, headless, None, __name__)
33
+ driver.get(council_url)
34
+
35
+ # Wait for the postcode field to appear then populate it
36
+ try:
37
+ inputElement_postcode = WebDriverWait(driver, 10).until(
38
+ EC.presence_of_element_located((By.ID, "q462406_q1"))
39
+ )
40
+ inputElement_postcode.send_keys(user_postcode)
41
+ except Exception:
42
+ print("Page failed to load. Probably due to Cloudflare robot check!")
43
+
44
+ # Click search button
45
+ findAddress = WebDriverWait(driver, 10).until(
46
+ EC.presence_of_element_located((By.ID, "form_email_462397_submit"))
47
+ )
48
+ driver.execute_script("arguments[0].click();", findAddress)
49
+
50
+ # Wait for the 'Select address' dropdown to appear and select option matching the house name/number
51
+ WebDriverWait(driver, 10).until(
52
+ EC.element_to_be_clickable(
53
+ (
54
+ By.XPATH,
55
+ "//select[@id='SBCYBDAddressList']//option[contains(., '"
56
+ + user_paon
57
+ + "')]",
58
+ )
59
+ )
60
+ ).click()
61
+
62
+ # Click search button
63
+ getBins = WebDriverWait(driver, 10).until(
64
+ EC.presence_of_element_located((By.ID, "form_email_462397_submit"))
65
+ )
66
+ driver.execute_script("arguments[0].click();", getBins)
67
+
68
+ BinTable = WebDriverWait(driver, 30).until(
69
+ EC.presence_of_element_located((By.ID, "SBC-YBD-Main"))
70
+ )
26
71
 
27
- # Parse URL and read if connection successful
28
- requests.packages.urllib3.disable_warnings()
29
- response = requests.get(council_url, verify=False)
30
- if response.status_code == 200:
31
- soup = BeautifulSoup(response.text, features="html.parser")
32
- soup.prettify()
33
- else:
34
- raise ConnectionAbortedError("Could not parse council website.")
72
+ soup = BeautifulSoup(driver.page_source, features="html.parser")
73
+ soup.prettify()
35
74
 
36
75
  data = {"bins": []}
37
76
 
38
77
  # Get the collection bullet points on the page and parse them
39
- form_area = soup.find("form", {"class": "integration bin-lookup"})
40
- collections = [
41
- item.text.strip().split(",") for item in form_area.find_all("li")
42
- ]
43
- for c in collections:
44
- bin_type = c[0].strip()
45
- # temp_date = c[2].strip() + " " + str(datetime.now().year)
46
- bin_date = datetime.strptime(
47
- c[2].strip() + " " + str(datetime.now().year), "%d %B %Y"
48
- ).strftime(date_format)
49
- dict_data = {"type": bin_type, "collectionDate": bin_date}
50
- data["bins"].append(dict_data)
78
+ nextCollections = soup.find("div", {"id": "nextCollections"})
79
+ for c in nextCollections:
80
+ collection = c.find_all("strong")
81
+ for bin in collection:
82
+ split = (bin.text).split(" on ")
83
+ bin_type = split[0]
84
+ bin_date = datetime.strptime(split[1], "%A %d %b %Y").strftime(
85
+ "%d/%m/%Y"
86
+ )
87
+ dict_data = {"type": bin_type, "collectionDate": bin_date}
88
+ data["bins"].append(dict_data)
51
89
 
52
90
  return data
@@ -74,30 +74,51 @@ class CouncilClass(AbstractGetBinDataClass):
74
74
  soup = BeautifulSoup(driver.page_source, features="html.parser")
75
75
  soup.prettify()
76
76
 
77
- rubbish_date = datetime.strptime(
78
- " ".join(
79
- soup.find("div", {"id": "FINDYOURBINDAYS_RUBBISHDATE_OUTERDIV"})
80
- .get_text(strip=True)
81
- .split()[6:8]
82
- ),
83
- "%d %B",
84
- ).replace(year=datetime.now().year)
85
- recycling_date = datetime.strptime(
86
- " ".join(
87
- soup.find("div", {"id": "FINDYOURBINDAYS_RECYCLINGDATE_OUTERDIV"})
88
- .get_text(strip=True)
89
- .split()[6:8]
90
- ),
91
- "%d %B",
92
- ).replace(year=datetime.now().year)
93
- food_date = datetime.strptime(
94
- " ".join(
95
- soup.find("div", {"id": "FINDYOURBINDAYS_FOODWASTEDATE_OUTERDIV"})
96
- .get_text(strip=True)
97
- .split()[8:10]
98
- ),
99
- "%d %B",
100
- ).replace(year=datetime.now().year)
77
+ rubbish_div = soup.find(
78
+ "div", {"id": "FINDYOURBINDAYS_RUBBISHDATE_OUTERDIV"}
79
+ )
80
+ try:
81
+ rubbish_date = rubbish_div.find_all("div")[2]
82
+ rubbish_date = datetime.strptime(
83
+ rubbish_date.text,
84
+ "%A %d %B",
85
+ ).replace(year=datetime.now().year)
86
+ except:
87
+ rubbish_date = rubbish_div.find_all("div")[3]
88
+ rubbish_date = datetime.strptime(
89
+ rubbish_date.text,
90
+ "%A %d %B",
91
+ ).replace(year=datetime.now().year)
92
+ recycling_div = soup.find(
93
+ "div", {"id": "FINDYOURBINDAYS_RECYCLINGDATE_OUTERDIV"}
94
+ )
95
+ try:
96
+ recycling_date = recycling_div.find_all("div")[2]
97
+ recycling_date = datetime.strptime(
98
+ recycling_date.text,
99
+ "%A %d %B",
100
+ ).replace(year=datetime.now().year)
101
+ except:
102
+ rubbish_date = recycling_div.find_all("div")[3]
103
+ rubbish_date = datetime.strptime(
104
+ rubbish_date.text,
105
+ "%A %d %B",
106
+ ).replace(year=datetime.now().year)
107
+ food_div = soup.find(
108
+ "div", {"id": "FINDYOURBINDAYS_RECYCLINGDATE_OUTERDIV"}
109
+ )
110
+ try:
111
+ food_date = food_div.find_all("div")[2]
112
+ food_date = datetime.strptime(
113
+ food_date.text,
114
+ "%A %d %B",
115
+ ).replace(year=datetime.now().year)
116
+ except:
117
+ food_date = food_div.find_all("div")[3]
118
+ food_date = datetime.strptime(
119
+ food_date.text,
120
+ "%A %d %B",
121
+ ).replace(year=datetime.now().year)
101
122
 
102
123
  if datetime.now().month == 12 and rubbish_date.month == 1:
103
124
  rubbish_date = rubbish_date + relativedelta(years=1)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: uk_bin_collection
3
- Version: 0.98.4
3
+ Version: 0.98.5
4
4
  Summary: Python Lib to collect UK Bin Data
5
5
  Author: Robert Bradley
6
6
  Author-email: robbrad182@gmail.com
@@ -2,7 +2,7 @@ uk_bin_collection/README.rst,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
2
2
  uk_bin_collection/tests/council_feature_input_parity.py,sha256=DO6Mk4ImYgM5ZCZ-cutwz5RoYYWZRLYx2tr6zIs_9Rc,3843
3
3
  uk_bin_collection/tests/features/environment.py,sha256=VQZjJdJI_kZn08M0j5cUgvKT4k3iTw8icJge1DGOkoA,127
4
4
  uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=SJK-Vc737hrf03tssxxbeg_JIvAH-ddB8f6gU1LTbuQ,251
5
- uk_bin_collection/tests/input.json,sha256=R0NULatIVLIsX13cKIDBD9ukIbCJ3nw_Gg3iC4nudvE,68070
5
+ uk_bin_collection/tests/input.json,sha256=gkqIfeMzQQC76jc6J4DPRn1ZCp6AVdPU_7ra3gb7pyc,68114
6
6
  uk_bin_collection/tests/output.schema,sha256=ZwKQBwYyTDEM4G2hJwfLUVM-5v1vKRvRK9W9SS1sd18,1086
7
7
  uk_bin_collection/tests/step_defs/step_helpers/file_handler.py,sha256=Ygzi4V0S1MIHqbdstUlIqtRIwnynvhu4UtpweJ6-5N8,1474
8
8
  uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=LrOSt_loA1Mw3vTqaO2LpaDMu7rYJy6k5Kr-EOBln7s,3424
@@ -17,7 +17,7 @@ uk_bin_collection/uk_bin_collection/councils/ArmaghBanbridgeCraigavonCouncil.py,
17
17
  uk_bin_collection/uk_bin_collection/councils/ArunCouncil.py,sha256=yfhthv9nuogP19VOZ3TYQrq51qqjiCZcSel4sXhiKjs,4012
18
18
  uk_bin_collection/uk_bin_collection/councils/AylesburyValeCouncil.py,sha256=LouqjspEMt1TkOGqWHs2zkxwOETIy3n7p64uKIlAgUg,2401
19
19
  uk_bin_collection/uk_bin_collection/councils/BCPCouncil.py,sha256=W7QBx6Mgso8RYosuXsaYo3GGNAu-tiyBSmuYxr1JSOU,1707
20
- uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py,sha256=NccMd0830aiNRQ8SH6mc4r5Hlugfey5-PyUd24QLo4s,8478
20
+ uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py,sha256=Sd4-pbv0QZsR7soxvXYqsfdOUIqZqS6notyoZthG77s,9182
21
21
  uk_bin_collection/uk_bin_collection/councils/BarnsleyMBCouncil.py,sha256=MgF_7XyIcIoNzFR0OJsjBkLCZKgWxBrV6nTcutMxO1Q,4244
22
22
  uk_bin_collection/uk_bin_collection/councils/BasildonCouncil.py,sha256=SBvAa0GZM3V7ygK8ARawbHAPH6R_303U30RH8WYPi5Q,3020
23
23
  uk_bin_collection/uk_bin_collection/councils/BasingstokeCouncil.py,sha256=VPWGljnH4C3q8qs5ZmCtqjNjgWQvviALzjk00q3EZeQ,2632
@@ -81,7 +81,7 @@ uk_bin_collection/uk_bin_collection/councils/GedlingBoroughCouncil.py,sha256=Iss
81
81
  uk_bin_collection/uk_bin_collection/councils/GlasgowCityCouncil.py,sha256=i7BympEhCm7D9yR0p5_QQICtWvNcDYNJIWB19SA0g2k,2303
82
82
  uk_bin_collection/uk_bin_collection/councils/GloucesterCityCouncil.py,sha256=8Wjvmdvg5blHVrREaEnhhWZaWhYVP4v_KdDVPLIUxaU,4889
83
83
  uk_bin_collection/uk_bin_collection/councils/GuildfordCouncil.py,sha256=9pVrmQhZcK2AD8gX8mNvP--L4L9KaY6L3B822VX6fec,5695
84
- uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py,sha256=r8cmtWhMJg-XG63ZHxidKKW7i4yQNrZSSMSCkBwrqjI,5837
84
+ uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py,sha256=gq_CPqi6qM2oNiHhKKF1lZC86fyKL4lPhh_DN9pJZ04,5971
85
85
  uk_bin_collection/uk_bin_collection/councils/HaringeyCouncil.py,sha256=t_6AkAu4wrv8Q0WlDhWh_82I0djl5tk531Pzs-SjWzg,2647
86
86
  uk_bin_collection/uk_bin_collection/councils/HarrogateBoroughCouncil.py,sha256=_g3fP5Nq-OUjgNrfRf4UEyFKzq0x8QK-4enh5RP1efA,2050
87
87
  uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py,sha256=oqF8M0lcT3KsrG6W6I6JJX07E6Sc_-_sr7MybfIMab8,4626
@@ -170,7 +170,7 @@ uk_bin_collection/uk_bin_collection/councils/StokeOnTrentCityCouncil.py,sha256=K
170
170
  uk_bin_collection/uk_bin_collection/councils/StratfordUponAvonCouncil.py,sha256=DMTAcXT_lay8Cl1hBbzf_LN7-GwTDGxT3Ug9QJkaF9Y,3936
171
171
  uk_bin_collection/uk_bin_collection/councils/StroudDistrictCouncil.py,sha256=9bYWppi7ViLGHL4VEg--nFn28MLYJYbiEntull1uZxU,3561
172
172
  uk_bin_collection/uk_bin_collection/councils/SunderlandCityCouncil.py,sha256=4DnKyyu56_AwuchD6_oL1dvpDStMvkkxQtYN79rUKOs,3825
173
- uk_bin_collection/uk_bin_collection/councils/SwaleBoroughCouncil.py,sha256=CCAjclhkCxwf_MkdrIif7MZpLfV5SPy7hyM7BE0TRPc,2048
173
+ uk_bin_collection/uk_bin_collection/councils/SwaleBoroughCouncil.py,sha256=ak0zqBJ6UAS8_t5zKGwigMxPMdlQuYXRsGCQJCNERJs,3415
174
174
  uk_bin_collection/uk_bin_collection/councils/SwanseaCouncil.py,sha256=nmVPoPhnFgVi--vczX2i4Sf3bqM5RWJuwfhioRUr5XE,2303
175
175
  uk_bin_collection/uk_bin_collection/councils/TamesideMBCouncil.py,sha256=k2TAAZG7n2S1BWVyxbE_-4-lZuzhOimCNz4yimUCOGk,1995
176
176
  uk_bin_collection/uk_bin_collection/councils/TandridgeDistrictCouncil.py,sha256=KLVvM2NNq_DQylVe5dwO2l7qPahLHg08jJGLCv1MBQ4,2324
@@ -191,7 +191,7 @@ uk_bin_collection/uk_bin_collection/councils/WarwickDistrictCouncil.py,sha256=3W
191
191
  uk_bin_collection/uk_bin_collection/councils/WaverleyBoroughCouncil.py,sha256=tp9l7vdgSGRzNNG0pDfnNuFj4D2bpRJUJmAiTJ6bM0g,4662
192
192
  uk_bin_collection/uk_bin_collection/councils/WealdenDistrictCouncil.py,sha256=SvSSaLkx7iJjzypAwKkaJwegXkSsIQtUOS2V605kz1A,3368
193
193
  uk_bin_collection/uk_bin_collection/councils/WelhatCouncil.py,sha256=ikUft37dYNJghfe-_6Fskiq1JihqpLmLNj38QkKSUUA,2316
194
- uk_bin_collection/uk_bin_collection/councils/WestBerkshireCouncil.py,sha256=r90AIUdPgo85VuuvN_NeCDUy3NEJXdO4Ntt93yKo6qI,5110
194
+ uk_bin_collection/uk_bin_collection/councils/WestBerkshireCouncil.py,sha256=XhTimZAPNgcuFgNp5mQjkR8mC4LRqUEUCy6e6plHspM,6004
195
195
  uk_bin_collection/uk_bin_collection/councils/WestLindseyDistrictCouncil.py,sha256=JFWUy4w0CKulGq16PfbRDKAdQEbokVEuabwlZYigdEU,4606
196
196
  uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py,sha256=dq0jimtARvRkZiGbVFrXXZgY-BODtz3uYZ5UKn0bf64,4114
197
197
  uk_bin_collection/uk_bin_collection/councils/WestMorlandAndFurness.py,sha256=jbqV3460rn9D0yTBGWjpSe1IvWWcdGur5pzgj-hJcQ4,2513
@@ -210,8 +210,8 @@ uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py,sha256=I2kBYMlsD4bId
210
210
  uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=4s9ODGPAwPqwXc8SrTX5Wlfmizs3_58iXUtHc4Ir86o,1162
211
211
  uk_bin_collection/uk_bin_collection/create_new_council.py,sha256=m-IhmWmeWQlFsTZC4OxuFvtw5ZtB8EAJHxJTH4O59lQ,1536
212
212
  uk_bin_collection/uk_bin_collection/get_bin_data.py,sha256=YvmHfZqanwrJ8ToGch34x-L-7yPe31nB_x77_Mgl_vo,4545
213
- uk_bin_collection-0.98.4.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
214
- uk_bin_collection-0.98.4.dist-info/METADATA,sha256=YKK2go_0HJQg2QyV6u0dz0ALkinbMCvzNteuuslvn3I,16843
215
- uk_bin_collection-0.98.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
216
- uk_bin_collection-0.98.4.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
217
- uk_bin_collection-0.98.4.dist-info/RECORD,,
213
+ uk_bin_collection-0.98.5.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
214
+ uk_bin_collection-0.98.5.dist-info/METADATA,sha256=hbuONB_eNHQmV4S23Zggy6yYeSXl5wI-YQrWkYlld58,16843
215
+ uk_bin_collection-0.98.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
216
+ uk_bin_collection-0.98.5.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
217
+ uk_bin_collection-0.98.5.dist-info/RECORD,,