uk_bin_collection 0.134.1__py3-none-any.whl → 0.135.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. uk_bin_collection/tests/check_selenium_url_in_input.json.py +209 -0
  2. uk_bin_collection/tests/input.json +55 -5
  3. uk_bin_collection/uk_bin_collection/councils/AmberValleyBoroughCouncil.py +60 -0
  4. uk_bin_collection/uk_bin_collection/councils/BolsoverCouncil.py +298 -0
  5. uk_bin_collection/uk_bin_collection/councils/CherwellDistrictCouncil.py +75 -0
  6. uk_bin_collection/uk_bin_collection/councils/ConwyCountyBorough.py +11 -3
  7. uk_bin_collection/uk_bin_collection/councils/CotswoldDistrictCouncil.py +3 -5
  8. uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py +54 -50
  9. uk_bin_collection/uk_bin_collection/councils/EpsomandEwellBoroughCouncil.py +86 -0
  10. uk_bin_collection/uk_bin_collection/councils/GloucesterCityCouncil.py +1 -1
  11. uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py +2 -1
  12. uk_bin_collection/uk_bin_collection/councils/MiddlesbroughCouncil.py +100 -0
  13. uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py +2 -0
  14. uk_bin_collection/uk_bin_collection/councils/NorthYorkshire.py +17 -15
  15. uk_bin_collection/uk_bin_collection/councils/RedcarandClevelandCouncil.py +108 -0
  16. uk_bin_collection/uk_bin_collection/councils/RunnymedeBoroughCouncil.py +54 -0
  17. uk_bin_collection/uk_bin_collection/councils/SunderlandCityCouncil.py +21 -15
  18. uk_bin_collection/uk_bin_collection/councils/TendringDistrictCouncil.py +1 -1
  19. uk_bin_collection/uk_bin_collection/councils/TorridgeDistrictCouncil.py +1 -35
  20. {uk_bin_collection-0.134.1.dist-info → uk_bin_collection-0.135.2.dist-info}/METADATA +1 -1
  21. {uk_bin_collection-0.134.1.dist-info → uk_bin_collection-0.135.2.dist-info}/RECORD +24 -16
  22. {uk_bin_collection-0.134.1.dist-info → uk_bin_collection-0.135.2.dist-info}/LICENSE +0 -0
  23. {uk_bin_collection-0.134.1.dist-info → uk_bin_collection-0.135.2.dist-info}/WHEEL +0 -0
  24. {uk_bin_collection-0.134.1.dist-info → uk_bin_collection-0.135.2.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,298 @@
1
+ import time
2
+
3
+ import requests
4
+
5
+ from uk_bin_collection.uk_bin_collection.common import *
6
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
7
+
8
+
9
+ # import the wonderful Beautiful Soup and the URL grabber
10
+ class CouncilClass(AbstractGetBinDataClass):
11
+ """
12
+ Concrete classes have to implement all abstract operations of the
13
+ base class. They can also override some operations with a default
14
+ implementation.
15
+ """
16
+
17
+ def parse_data(self, page: str, **kwargs) -> dict:
18
+
19
+ user_uprn = kwargs.get("uprn")
20
+ check_uprn(user_uprn)
21
+ bindata = {"bins": []}
22
+
23
+ SESSION_URL = "https://selfservice.bolsover.gov.uk/authapi/isauthenticated?uri=https%253A%252F%252Fselfservice.bolsover.gov.uk%252Fservice%252FCheck_your_Bin_Day&hostname=selfservice.bolsover.gov.uk&withCredentials=true"
24
+
25
+ API_URL = "https://selfservice.bolsover.gov.uk/apibroker/runLookup"
26
+
27
+ data = {
28
+ "formValues": {"Bin Collection": {"uprnLoggedIn": {"value": user_uprn}}},
29
+ }
30
+ headers = {
31
+ "Content-Type": "application/json",
32
+ "Accept": "application/json",
33
+ "User-Agent": "Mozilla/5.0",
34
+ "X-Requested-With": "XMLHttpRequest",
35
+ "Referer": "https://selfservice.bolsover.gov.uk/fillform/?iframe_id=fillform-frame-1&db_id=",
36
+ }
37
+ s = requests.session()
38
+ r = s.get(SESSION_URL)
39
+ r.raise_for_status()
40
+ session_data = r.json()
41
+ sid = session_data["auth-session"]
42
+ params = {
43
+ "id": "6023d37e037c3",
44
+ "repeat_against": "",
45
+ "noRetry": "true",
46
+ "getOnlyTokens": "undefined",
47
+ "log_id": "",
48
+ "app_name": "AF-Renderer::Self",
49
+ # unix_timestamp
50
+ "_": str(int(time.time() * 1000)),
51
+ "sid": sid,
52
+ }
53
+
54
+ r = s.post(API_URL, json=data, headers=headers, params=params)
55
+ r.raise_for_status()
56
+
57
+ data = r.json()
58
+ rows_data = data["integration"]["transformed"]["rows_data"]["0"]
59
+ if not isinstance(rows_data, dict):
60
+ raise ValueError("Invalid data returned from API")
61
+
62
+ # print(rows_data)
63
+
64
+ route = rows_data["Route"]
65
+
66
+ # print(route)
67
+
68
+ def get_route_number(route):
69
+ if route[:2] == "Mo":
70
+ return 0
71
+ elif route[:2] == "Tu":
72
+ return 1
73
+ elif route[:2] == "We":
74
+ return 2
75
+ elif route[:2] == "Th":
76
+ return 3
77
+ elif route[:2] == "Fr":
78
+ return 4
79
+ else:
80
+ return None # Default case if none of the conditions match
81
+
82
+ dayOfCollectionAsNumber = get_route_number(route)
83
+ # print(dayOfCollectionAsNumber)
84
+
85
+ def calculate_collection_date(
86
+ dayOfCollectionAsNumber,
87
+ currentDayAsNumber,
88
+ today,
89
+ dayDiffPlus,
90
+ dayDiffMinus,
91
+ ):
92
+ if dayOfCollectionAsNumber == currentDayAsNumber:
93
+ return today
94
+ elif dayOfCollectionAsNumber > currentDayAsNumber:
95
+ return today + timedelta(days=dayDiffPlus)
96
+ else:
97
+ return today + timedelta(days=dayDiffMinus)
98
+
99
+ # Example usage
100
+ today = datetime.today() # Current date
101
+ currentDayAsNumber = today.weekday()
102
+ dayDiffPlus = dayOfCollectionAsNumber - currentDayAsNumber
103
+ dayDiffMinus = dayOfCollectionAsNumber - currentDayAsNumber + 7
104
+
105
+ week1 = calculate_collection_date(
106
+ dayOfCollectionAsNumber,
107
+ currentDayAsNumber,
108
+ today,
109
+ dayDiffPlus,
110
+ dayDiffMinus,
111
+ )
112
+ week2 = week1 + timedelta(days=7)
113
+ week3 = week2 + timedelta(days=7)
114
+ week4 = week3 + timedelta(days=7)
115
+
116
+ # print(week1.strftime(date_format))
117
+ # print(week2.strftime(date_format))
118
+ # print(week3.strftime(date_format))
119
+ # print(week4.strftime(date_format))
120
+
121
+ greenSusStart = datetime.strptime("2024-11-08", "%Y-%m-%d")
122
+ greenSusEnd = datetime.strptime("2025-03-18", "%Y-%m-%d")
123
+
124
+ def is_within_green_sus(dtDay0, greenSusStart, greenSusEnd):
125
+ return "Yes" if greenSusStart <= dtDay0 < greenSusEnd else "No"
126
+
127
+ week1InSus = is_within_green_sus(week1, greenSusStart, greenSusEnd)
128
+ week2InSus = is_within_green_sus(week2, greenSusStart, greenSusEnd)
129
+ week3InSus = is_within_green_sus(week3, greenSusStart, greenSusEnd)
130
+ week4InSus = is_within_green_sus(week4, greenSusStart, greenSusEnd)
131
+
132
+ # print(week1InSus)
133
+ # print(week2InSus)
134
+ # print(week3InSus)
135
+ # print(week4InSus)
136
+
137
+ WeekBlack = rows_data["WeekBlack"]
138
+ WeekBandG = rows_data["WeekBandG"]
139
+
140
+ if WeekBlack == "1":
141
+ WeekBandG = ""
142
+ if WeekBandG == "1":
143
+ WeekBlack = ""
144
+
145
+ def determine_bin_collection_week1(
146
+ txtBlack, txtBurgGreen, dtDay0, today, week1InSus
147
+ ):
148
+ # Check for empty values
149
+ if txtBlack == "" and txtBurgGreen == "":
150
+ return ""
151
+
152
+ # Black Bin Collection
153
+ if txtBlack == "1" and dtDay0 >= today:
154
+ return "Black Bin"
155
+
156
+ # Burgundy Bin Collection
157
+ if txtBurgGreen == "1" and dtDay0 > today:
158
+ if week1InSus == "Yes":
159
+ return "Burgundy Bin"
160
+ elif week1InSus == "No":
161
+ return "Burgundy Bin & Green Bin"
162
+
163
+ # Default cases based on week1InSus
164
+ if txtBlack == "" and dtDay0 >= today:
165
+ if week1InSus == "Yes":
166
+ return "Burgundy Bin"
167
+ elif week1InSus == "No":
168
+ return "Burgundy Bin & Green Bin"
169
+
170
+ return "" # Default empty case
171
+
172
+ def determine_bin_collection_week2(
173
+ txtBlack, txtBurgGreen, dtDay7, today, week2InSus
174
+ ):
175
+ # Check for empty values
176
+ if txtBlack == "" and txtBurgGreen == "":
177
+ return ""
178
+
179
+ # Black Bin Collection
180
+ if txtBlack == "" and dtDay7 >= today:
181
+ return "Black Bin"
182
+
183
+ # Burgundy Bin Collection (week2InSus check)
184
+ if txtBurgGreen == "1" and dtDay7 > today:
185
+ if week2InSus == "Yes":
186
+ return "Burgundy Bin"
187
+ elif week2InSus == "No":
188
+ return "Burgundy Bin & Green Bin"
189
+
190
+ # Burgundy Bin Collection for txtBlack = '1'
191
+ if txtBlack == "1" and dtDay7 >= today:
192
+ if week2InSus == "Yes":
193
+ return "Burgundy Bin"
194
+ elif week2InSus == "No":
195
+ return "Burgundy Bin & Green Bin"
196
+
197
+ return "" # Default empty case
198
+
199
+ def determine_bin_collection_week3(
200
+ txtBlack, txtBurgGreen, dtDay14, today, week3InSus
201
+ ):
202
+ # Check for empty values
203
+ if txtBlack == "" and txtBurgGreen == "":
204
+ return ""
205
+
206
+ # Black Bin Collection
207
+ if txtBlack == "1" and dtDay14 >= today:
208
+ return "Black Bin"
209
+
210
+ # Burgundy Bin Collection (week3InSus check)
211
+ if txtBurgGreen == "1" and dtDay14 > today:
212
+ if week3InSus == "Yes":
213
+ return "Burgundy Bin"
214
+ elif week3InSus == "No":
215
+ return "Burgundy Bin & Green Bin"
216
+
217
+ # Burgundy Bin Collection for txtBlack = ''
218
+ if txtBlack == "" and dtDay14 >= today:
219
+ if week3InSus == "Yes":
220
+ return "Burgundy Bin"
221
+ elif week3InSus == "No":
222
+ return "Burgundy Bin & Green Bin"
223
+
224
+ return "" # Default empty case
225
+
226
+ def determine_bin_collection_week4(
227
+ txtBlack, txtBurgGreen, dtDay21, today, week4InSus
228
+ ):
229
+ # Check for empty values
230
+ if txtBlack == "" and txtBurgGreen == "":
231
+ return ""
232
+
233
+ # Black Bin Collection
234
+ if txtBlack == "" and dtDay21 >= today:
235
+ return "Black Bin"
236
+
237
+ # Burgundy Bin Collection (week4InSus check)
238
+ if txtBurgGreen == "1" and dtDay21 > today:
239
+ if week4InSus == "Yes":
240
+ return "Burgundy Bin"
241
+ elif week4InSus == "No":
242
+ return "Burgundy Bin & Green Bin"
243
+
244
+ # Burgundy Bin Collection for txtBlack = '1'
245
+ if txtBlack == "1" and dtDay21 >= today:
246
+ if week4InSus == "Yes":
247
+ return "Burgundy Bin"
248
+ elif week4InSus == "No":
249
+ return "Burgundy Bin & Green Bin"
250
+
251
+ return "" # Default empty case
252
+
253
+ week1Text = determine_bin_collection_week1(
254
+ WeekBlack, WeekBandG, week1, today, week1InSus
255
+ )
256
+ week2Text = determine_bin_collection_week2(
257
+ WeekBlack, WeekBandG, week2, today, week2InSus
258
+ )
259
+ week3Text = determine_bin_collection_week3(
260
+ WeekBlack, WeekBandG, week3, today, week3InSus
261
+ )
262
+ week4Text = determine_bin_collection_week4(
263
+ WeekBlack, WeekBandG, week4, today, week4InSus
264
+ )
265
+
266
+ # print(week1Text)
267
+ # print(week2Text)
268
+ # print(week3Text)
269
+ # print(week4Text)
270
+
271
+ week_data = [
272
+ (week1Text, week1),
273
+ (week2Text, week2),
274
+ (week3Text, week3),
275
+ (week4Text, week4),
276
+ ]
277
+
278
+ # print(week_data)
279
+
280
+ # Iterate through the array
281
+ for week_text, week_date in week_data:
282
+ # Check if '&' exists and split
283
+ if "&" in week_text:
284
+ split_texts = [text.strip() for text in week_text.split("&")]
285
+ for text in split_texts:
286
+ dict_data = {
287
+ "type": text,
288
+ "collectionDate": week_date.strftime(date_format),
289
+ }
290
+ bindata["bins"].append(dict_data)
291
+ else:
292
+ dict_data = {
293
+ "type": week_text,
294
+ "collectionDate": week_date.strftime(date_format),
295
+ }
296
+ bindata["bins"].append(dict_data)
297
+
298
+ return bindata
@@ -0,0 +1,75 @@
1
+ from datetime import datetime, timedelta
2
+
3
+ import requests
4
+ from bs4 import BeautifulSoup
5
+
6
+ from uk_bin_collection.uk_bin_collection.common import *
7
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
8
+
9
+
10
+ # import the wonderful Beautiful Soup and the URL grabber
11
+ class CouncilClass(AbstractGetBinDataClass):
12
+ """
13
+ Concrete classes have to implement all abstract operations of the
14
+ base class. They can also override some operations with a default
15
+ implementation.
16
+ """
17
+
18
+ def parse_data(self, page: str, **kwargs) -> dict:
19
+
20
+ user_uprn = kwargs.get("uprn")
21
+ check_uprn(user_uprn)
22
+ bindata = {"bins": []}
23
+
24
+ URI = f"https://www.cherwell.gov.uk/homepage/129/bin-collection-search?uprn={user_uprn}"
25
+
26
+ # Make the GET request
27
+ response = requests.get(URI)
28
+
29
+ soup = BeautifulSoup(response.text, "html.parser")
30
+
31
+ def get_full_date(date_str):
32
+ # Get the current year
33
+ current_year = datetime.today().year
34
+
35
+ date_str = remove_ordinal_indicator_from_date_string(date_str)
36
+
37
+ # Convert the input string to a datetime object (assuming the current year first)
38
+ date_obj = datetime.strptime(f"{date_str} {current_year}", "%d %B %Y")
39
+
40
+ # If the date has already passed this year, use next year
41
+ if date_obj < datetime.today():
42
+ date_obj = datetime.strptime(
43
+ f"{date_str} {current_year + 1}", "%d %B %Y"
44
+ )
45
+
46
+ return date_obj.strftime(date_format) # Return in YYYY-MM-DD format
47
+
48
+ # print(soup)
49
+
50
+ div = soup.find("div", class_="bin-collection-results__tasks")
51
+
52
+ for item in div.find_all("li", class_="list__item"):
53
+ # Extract bin type
54
+ bin_type_tag = item.find("h3", class_="bin-collection-tasks__heading")
55
+ bin_type = (
56
+ "".join(bin_type_tag.find_all(text=True, recursive=False)).strip()
57
+ if bin_type_tag
58
+ else "Unknown Bin"
59
+ )
60
+
61
+ # Extract collection date
62
+ date_tag = item.find("p", class_="bin-collection-tasks__date")
63
+ collection_date = date_tag.text.strip() if date_tag else "Unknown Date"
64
+
65
+ dict_data = {
66
+ "type": bin_type,
67
+ "collectionDate": get_full_date(collection_date),
68
+ }
69
+ bindata["bins"].append(dict_data)
70
+
71
+ bindata["bins"].sort(
72
+ key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
73
+ )
74
+
75
+ return bindata
@@ -1,12 +1,20 @@
1
+ from datetime import datetime
2
+
3
+ import requests
1
4
  from bs4 import BeautifulSoup
2
- from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
5
+
3
6
  from uk_bin_collection.uk_bin_collection.common import *
4
- from datetime import datetime
7
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
5
8
 
6
9
 
7
10
  class CouncilClass(AbstractGetBinDataClass):
8
11
  def parse_data(self, page: str, **kwargs) -> dict:
9
- soup = BeautifulSoup(page.text, features="html.parser")
12
+ user_uprn = kwargs.get("uprn")
13
+ check_uprn(user_uprn)
14
+ uri = f"https://www.conwy.gov.uk/Contensis-Forms/erf/collection-result-soap-xmas2025.asp?ilangid=1&uprn={user_uprn}"
15
+
16
+ response = requests.get(uri)
17
+ soup = BeautifulSoup(response.content, features="html.parser")
10
18
  data = {"bins": []}
11
19
 
12
20
  for bin_section in soup.select('div[class*="containererf"]'):
@@ -41,15 +41,13 @@ class CouncilClass(AbstractGetBinDataClass):
41
41
  # If you bang in the house number (or property name) and postcode in the box it should find your property
42
42
  wait = WebDriverWait(driver, 60)
43
43
  address_entry_field = wait.until(
44
- EC.presence_of_element_located(
45
- (By.XPATH, '//*[@id="combobox-input-20"]')
46
- )
44
+ EC.element_to_be_clickable((By.XPATH, '//*[@id="combobox-input-22"]'))
47
45
  )
48
46
 
49
47
  address_entry_field.send_keys(str(full_address))
50
48
 
51
49
  address_entry_field = wait.until(
52
- EC.element_to_be_clickable((By.XPATH, '//*[@id="combobox-input-20"]'))
50
+ EC.element_to_be_clickable((By.XPATH, '//*[@id="combobox-input-22"]'))
53
51
  )
54
52
  address_entry_field.click()
55
53
  address_entry_field.send_keys(Keys.BACKSPACE)
@@ -57,7 +55,7 @@ class CouncilClass(AbstractGetBinDataClass):
57
55
 
58
56
  first_found_address = wait.until(
59
57
  EC.element_to_be_clickable(
60
- (By.XPATH, '//*[@id="dropdown-element-20"]/ul')
58
+ (By.XPATH, '//*[@id="dropdown-element-22"]/ul')
61
59
  )
62
60
  )
63
61
 
@@ -1,8 +1,5 @@
1
+ import requests
1
2
  from bs4 import BeautifulSoup
2
- from selenium.webdriver.common.by import By
3
- from selenium.webdriver.support import expected_conditions as EC
4
- from selenium.webdriver.support.ui import Select
5
- from selenium.webdriver.support.wait import WebDriverWait
6
3
 
7
4
  from uk_bin_collection.uk_bin_collection.common import *
8
5
  from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
@@ -19,63 +16,70 @@ class CouncilClass(AbstractGetBinDataClass):
19
16
  def parse_data(self, page: str, **kwargs) -> dict:
20
17
  driver = None
21
18
  try:
22
- page = "https://selfserve.derbyshiredales.gov.uk/renderform.aspx?t=103&k=9644C066D2168A4C21BCDA351DA2642526359DFF"
19
+ uri = "https://selfserve.derbyshiredales.gov.uk/renderform.aspx?t=103&k=9644C066D2168A4C21BCDA351DA2642526359DFF"
23
20
 
24
- data = {"bins": []}
21
+ bindata = {"bins": []}
25
22
 
26
23
  user_uprn = kwargs.get("uprn")
27
24
  user_postcode = kwargs.get("postcode")
28
- web_driver = kwargs.get("web_driver")
29
- headless = kwargs.get("headless")
30
25
  check_uprn(user_uprn)
31
26
  check_postcode(user_postcode)
32
27
 
33
- # Create Selenium webdriver
34
- driver = create_webdriver(web_driver, headless, None, __name__)
35
- driver.get(page)
28
+ # Start a session
29
+ session = requests.Session()
36
30
 
37
- # Populate postcode field
38
- inputElement_postcode = driver.find_element(
39
- By.ID,
40
- "ctl00_ContentPlaceHolder1_FF2924TB",
41
- )
42
- inputElement_postcode.send_keys(user_postcode)
43
-
44
- # Click search button
45
- driver.find_element(
46
- By.ID,
47
- "ctl00_ContentPlaceHolder1_FF2924BTN",
48
- ).click()
49
-
50
- # Wait for the 'Select address' dropdown to appear and select option matching UPRN
51
- dropdown = WebDriverWait(driver, 10).until(
52
- EC.presence_of_element_located(
53
- (By.ID, "ctl00_ContentPlaceHolder1_FF2924DDL")
54
- )
55
- )
56
- # Create a 'Select' for it, then select the matching URPN option
57
- dropdownSelect = Select(dropdown)
58
- dropdownSelect.select_by_value("U" + user_uprn)
59
-
60
- # Wait for the submit button to appear, then click it to get the collection dates
61
- submit = WebDriverWait(driver, 10).until(
62
- EC.presence_of_element_located(
63
- (By.ID, "ctl00_ContentPlaceHolder1_btnSubmit")
64
- )
65
- )
66
- submit.click()
31
+ response = session.get(uri)
67
32
 
68
- soup = BeautifulSoup(driver.page_source, features="html.parser")
33
+ soup = BeautifulSoup(response.content, features="html.parser")
69
34
 
70
- bin_rows = (
71
- soup.find("div", id="ctl00_ContentPlaceHolder1_pnlConfirmation")
72
- .find("div", {"class": "row"})
73
- .find_all("div", {"class": "row"})
74
- )
35
+ # Function to extract hidden input values
36
+ def get_hidden_value(soup, name):
37
+ element = soup.find("input", {"name": name})
38
+ return element["value"] if element else None
39
+
40
+ # Extract the required values
41
+ data = {
42
+ "__RequestVerificationToken": get_hidden_value(
43
+ soup, "__RequestVerificationToken"
44
+ ),
45
+ "FormGuid": get_hidden_value(soup, "FormGuid"),
46
+ "ObjectTemplateID": get_hidden_value(soup, "ObjectTemplateID"),
47
+ "Trigger": "submit",
48
+ "CurrentSectionID": get_hidden_value(soup, "CurrentSectionID"),
49
+ "TriggerCtl": "",
50
+ "FF2924": "U" + user_uprn,
51
+ "FF2924lbltxt": "Collection address",
52
+ "FF2924-text": user_postcode,
53
+ }
54
+
55
+ # Print extracted data
56
+ # print("Extracted Data:", data)
57
+
58
+ # Step 2: Submit the extracted data via a POST request
59
+ headers = {
60
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
61
+ "Referer": uri,
62
+ "Content-Type": "application/x-www-form-urlencoded",
63
+ }
64
+
65
+ URI = "https://selfserve.derbyshiredales.gov.uk/renderform/Form"
66
+
67
+ # Make the POST request
68
+ post_response = session.post(URI, data=data, headers=headers)
69
+
70
+ soup = BeautifulSoup(post_response.content, features="html.parser")
71
+
72
+ # print(soup)
73
+
74
+ bin_rows = soup.find("div", {"class": "ss_confPanel"})
75
+
76
+ bin_rows = bin_rows.find_all("div", {"class": "row"})
75
77
  if bin_rows:
76
78
  for bin_row in bin_rows:
77
79
  bin_data = bin_row.find_all("div")
78
80
  if bin_data and bin_data[0] and bin_data[1]:
81
+ if bin_data[0].get_text(strip=True) == "Your Collections":
82
+ continue
79
83
  collection_date = datetime.strptime(
80
84
  bin_data[0].get_text(strip=True), "%A%d %B, %Y"
81
85
  )
@@ -83,9 +87,9 @@ class CouncilClass(AbstractGetBinDataClass):
83
87
  "type": bin_data[1].get_text(strip=True),
84
88
  "collectionDate": collection_date.strftime(date_format),
85
89
  }
86
- data["bins"].append(dict_data)
90
+ bindata["bins"].append(dict_data)
87
91
 
88
- data["bins"].sort(
92
+ bindata["bins"].sort(
89
93
  key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
90
94
  )
91
95
  except Exception as e:
@@ -97,4 +101,4 @@ class CouncilClass(AbstractGetBinDataClass):
97
101
  # This block ensures that the driver is closed regardless of an exception
98
102
  if driver:
99
103
  driver.quit()
100
- return data
104
+ return bindata
@@ -0,0 +1,86 @@
1
+ from datetime import datetime, timedelta
2
+
3
+ import requests
4
+ from bs4 import BeautifulSoup
5
+
6
+ from uk_bin_collection.uk_bin_collection.common import *
7
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
8
+
9
+
10
+ # import the wonderful Beautiful Soup and the URL grabber
11
+ class CouncilClass(AbstractGetBinDataClass):
12
+ """
13
+ Concrete classes have to implement all abstract operations of the
14
+ base class. They can also override some operations with a default
15
+ implementation.
16
+ """
17
+
18
+ def parse_data(self, page: str, **kwargs) -> dict:
19
+
20
+ user_uprn = kwargs.get("uprn")
21
+ check_uprn(user_uprn)
22
+ bindata = {"bins": []}
23
+
24
+ URI = f"https://maps.epsom-ewell.gov.uk/myeebc.aspx?action=SetAddress&UniqueId={user_uprn}"
25
+
26
+ # Make the GET request
27
+ response = requests.get(URI)
28
+
29
+ soup = BeautifulSoup(response.text, "html.parser")
30
+
31
+ # print(soup)
32
+
33
+ div = soup.find_all("div", class_="atPanelContent atAlt1 atLast")
34
+
35
+ # print(div[1])
36
+
37
+ panels = div[1].find_all("div", class_="atPanelData")
38
+
39
+ # print(panels)
40
+
41
+ def get_full_date(date_str):
42
+ # Get the current year
43
+ current_year = datetime.today().year
44
+
45
+ # Convert the input string to a datetime object (assuming the current year first)
46
+ date_obj = datetime.strptime(f"{date_str} {current_year}", "%A %d %B %Y")
47
+
48
+ # If the date has already passed this year, use next year
49
+ if date_obj < datetime.today():
50
+ date_obj = datetime.strptime(
51
+ f"{date_str} {current_year + 1}", "%A %d %B %Y"
52
+ )
53
+
54
+ return date_obj.strftime(date_format) # Return in YYYY-MM-DD format
55
+
56
+ for panel in panels:
57
+ bin_type_tag = panel.find("h4") # Extracts bin type
58
+ date_text = panel.find_all("td") # Extracts collection date
59
+
60
+ date_text = date_text[1]
61
+
62
+ if bin_type_tag and date_text:
63
+ bin_type = bin_type_tag.text.strip()
64
+ try:
65
+ collection_date = date_text.text.strip().split(":")[1]
66
+ except IndexError:
67
+ continue
68
+
69
+ bin_type = (
70
+ (" ".join(bin_type.splitlines())).replace(" ", " ")
71
+ ).lstrip()
72
+ collection_date = (
73
+ (" ".join(collection_date.splitlines())).replace(" ", " ")
74
+ ).lstrip()
75
+
76
+ dict_data = {
77
+ "type": bin_type,
78
+ "collectionDate": get_full_date(collection_date),
79
+ }
80
+ bindata["bins"].append(dict_data)
81
+
82
+ bindata["bins"].sort(
83
+ key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
84
+ )
85
+
86
+ return bindata
@@ -113,7 +113,7 @@ class CouncilClass(AbstractGetBinDataClass):
113
113
  bin_data["bins"].append(dict_data)
114
114
 
115
115
  bin_data["bins"].sort(
116
- key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
116
+ key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
117
117
  )
118
118
 
119
119
  except Exception as e:
@@ -104,7 +104,8 @@ class CouncilClass(AbstractGetBinDataClass):
104
104
  bin_types = soup.find_all("ul", class_="binCollectionTimesList")
105
105
 
106
106
  for bin_collection_dates in bin_types:
107
- bin_collection_list = bin_collection_dates.find_all("li", class_="")
107
+
108
+ bin_collection_list = bin_collection_dates.find_all("li")
108
109
 
109
110
  if bin_collection_list:
110
111
  collection_dates = [