uk_bin_collection 0.119.0__py3-none-any.whl → 0.121.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. uk_bin_collection/tests/input.json +116 -9
  2. uk_bin_collection/uk_bin_collection/councils/BaberghDistrictCouncil.py +3 -1
  3. uk_bin_collection/uk_bin_collection/councils/CheltenhamBoroughCouncil.py +102 -0
  4. uk_bin_collection/uk_bin_collection/councils/CotswoldDistrictCouncil.py +3 -3
  5. uk_bin_collection/uk_bin_collection/councils/CumberlandAllerdaleCouncil.py +93 -0
  6. uk_bin_collection/uk_bin_collection/councils/EastAyrshireCouncil.py +11 -8
  7. uk_bin_collection/uk_bin_collection/councils/EnvironmentFirst.py +14 -0
  8. uk_bin_collection/uk_bin_collection/councils/FolkstoneandHytheDistrictCouncil.py +81 -0
  9. uk_bin_collection/uk_bin_collection/councils/HackneyCouncil.py +85 -0
  10. uk_bin_collection/uk_bin_collection/councils/HartlepoolBoroughCouncil.py +83 -0
  11. uk_bin_collection/uk_bin_collection/councils/KingsLynnandWestNorfolkBC.py +59 -0
  12. uk_bin_collection/uk_bin_collection/councils/LondonBoroughHavering.py +75 -0
  13. uk_bin_collection/uk_bin_collection/councils/LondonBoroughLewisham.py +132 -0
  14. uk_bin_collection/uk_bin_collection/councils/MidSuffolkDistrictCouncil.py +3 -1
  15. uk_bin_collection/uk_bin_collection/councils/MorayCouncil.py +65 -0
  16. uk_bin_collection/uk_bin_collection/councils/NewcastleUnderLymeCouncil.py +66 -0
  17. uk_bin_collection/uk_bin_collection/councils/NorthHertfordshireDistrictCouncil.py +93 -0
  18. uk_bin_collection/uk_bin_collection/councils/RoyalBoroughofGreenwich.py +113 -0
  19. uk_bin_collection/uk_bin_collection/councils/SandwellBoroughCouncil.py +87 -0
  20. uk_bin_collection/uk_bin_collection/councils/ThurrockCouncil.py +93 -0
  21. uk_bin_collection/uk_bin_collection/councils/WestNorthamptonshireCouncil.py +12 -10
  22. uk_bin_collection/uk_bin_collection/councils/WyreForestDistrictCouncil.py +65 -0
  23. {uk_bin_collection-0.119.0.dist-info → uk_bin_collection-0.121.0.dist-info}/METADATA +1 -1
  24. {uk_bin_collection-0.119.0.dist-info → uk_bin_collection-0.121.0.dist-info}/RECORD +27 -12
  25. {uk_bin_collection-0.119.0.dist-info → uk_bin_collection-0.121.0.dist-info}/LICENSE +0 -0
  26. {uk_bin_collection-0.119.0.dist-info → uk_bin_collection-0.121.0.dist-info}/WHEEL +0 -0
  27. {uk_bin_collection-0.119.0.dist-info → uk_bin_collection-0.121.0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,85 @@
1
+ import time
2
+
3
+ import requests
4
+
5
+ from uk_bin_collection.uk_bin_collection.common import *
6
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
7
+
8
+
9
+ # import the wonderful Beautiful Soup and the URL grabber
10
+ class CouncilClass(AbstractGetBinDataClass):
11
+ """
12
+ Concrete classes have to implement all abstract operations of the
13
+ base class. They can also override some operations with a default
14
+ implementation.
15
+ """
16
+
17
+ def parse_data(self, page: str, **kwargs) -> dict:
18
+
19
+ user_paon = kwargs.get("paon")
20
+ user_postcode = kwargs.get("postcode")
21
+ check_postcode(user_postcode)
22
+ check_paon(user_paon)
23
+ bindata = {"bins": []}
24
+
25
+ URI = "https://waste-api-hackney-live.ieg4.net/f806d91c-e133-43a6-ba9a-c0ae4f4cccf6/property/opensearch"
26
+
27
+ data = {
28
+ "Postcode": user_postcode,
29
+ }
30
+ headers = {"Content-Type": "application/json"}
31
+
32
+ # Make the GET request
33
+ response = requests.post(URI, json=data, headers=headers)
34
+
35
+ addresses = response.json()
36
+
37
+ for address in addresses["addressSummaries"]:
38
+ summary = address["summary"]
39
+ if user_paon in summary:
40
+ systemId = address["systemId"]
41
+ if systemId:
42
+ URI = f"https://waste-api-hackney-live.ieg4.net/f806d91c-e133-43a6-ba9a-c0ae4f4cccf6/alloywastepages/getproperty/{systemId}"
43
+
44
+ response = requests.get(URI)
45
+
46
+ address = response.json()
47
+
48
+ binIDs = address["providerSpecificFields"][
49
+ "attributes_wasteContainersAssignableWasteContainers"
50
+ ]
51
+ for binID in binIDs.split(","):
52
+ URI = f"https://waste-api-hackney-live.ieg4.net/f806d91c-e133-43a6-ba9a-c0ae4f4cccf6/alloywastepages/getbin/{binID}"
53
+ response = requests.get(URI)
54
+ getBin = response.json()
55
+
56
+ bin_type = getBin["subTitle"]
57
+
58
+ URI = f"https://waste-api-hackney-live.ieg4.net/f806d91c-e133-43a6-ba9a-c0ae4f4cccf6/alloywastepages/getcollection/{binID}"
59
+ response = requests.get(URI)
60
+ getcollection = response.json()
61
+
62
+ collectionID = getcollection["scheduleCodeWorkflowID"]
63
+
64
+ URI = f"https://waste-api-hackney-live.ieg4.net/f806d91c-e133-43a6-ba9a-c0ae4f4cccf6/alloywastepages/getworkflow/{collectionID}"
65
+ response = requests.get(URI)
66
+ collection_dates = response.json()
67
+
68
+ dates = collection_dates["trigger"]["dates"]
69
+
70
+ for date in dates:
71
+ parsed_datetime = datetime.strptime(
72
+ date, "%Y-%m-%dT%H:%M:%SZ"
73
+ ).strftime(date_format)
74
+
75
+ dict_data = {
76
+ "type": bin_type.strip(),
77
+ "collectionDate": parsed_datetime,
78
+ }
79
+ bindata["bins"].append(dict_data)
80
+
81
+ bindata["bins"].sort(
82
+ key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
83
+ )
84
+
85
+ return bindata
@@ -0,0 +1,83 @@
1
+ import time
2
+
3
+ import requests
4
+ from bs4 import BeautifulSoup
5
+
6
+ from uk_bin_collection.uk_bin_collection.common import *
7
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
8
+
9
+
10
+ # import the wonderful Beautiful Soup and the URL grabber
11
+ class CouncilClass(AbstractGetBinDataClass):
12
+ """
13
+ Concrete classes have to implement all abstract operations of the
14
+ base class. They can also override some operations with a default
15
+ implementation.
16
+ """
17
+
18
+ def parse_data(self, page: str, **kwargs) -> dict:
19
+
20
+ user_uprn = kwargs.get("uprn")
21
+ check_uprn(user_uprn)
22
+ bindata = {"bins": []}
23
+
24
+ SESSION_URL = "https://online.hartlepool.gov.uk/authapi/isauthenticated?uri=https%253A%252F%252Fonline.hartlepool.gov.uk%252Fservice%252FRefuse_and_recycling___check_bin_day&hostname=online.hartlepool.gov.uk&withCredentials=true"
25
+
26
+ API_URL = "https://online.hartlepool.gov.uk/apibroker/runLookup"
27
+
28
+ headers = {
29
+ "Content-Type": "application/json",
30
+ "Accept": "application/json",
31
+ "User-Agent": "Mozilla/5.0",
32
+ "X-Requested-With": "XMLHttpRequest",
33
+ "Referer": "https://online.hartlepool.gov.uk/fillform/?iframe_id=fillform-frame-1&db_id=",
34
+ }
35
+ s = requests.session()
36
+ r = s.get(SESSION_URL)
37
+ r.raise_for_status()
38
+ session_data = r.json()
39
+ sid = session_data["auth-session"]
40
+ params = {
41
+ "id": "5ec67e019ffdd",
42
+ "repeat_against": "",
43
+ "noRetry": "true",
44
+ "getOnlyTokens": "undefined",
45
+ "log_id": "",
46
+ "app_name": "AF-Renderer::Self",
47
+ # unix_timestamp
48
+ "_": str(int(time.time() * 1000)),
49
+ "sid": sid,
50
+ }
51
+
52
+ data = {
53
+ "formValues": {
54
+ "Section 1": {
55
+ "collectionLocationUPRN": {
56
+ "value": user_uprn,
57
+ },
58
+ },
59
+ },
60
+ }
61
+
62
+ r = s.post(API_URL, json=data, headers=headers, params=params)
63
+ r.raise_for_status()
64
+
65
+ data = r.json()
66
+ rows_data = data["integration"]["transformed"]["rows_data"]["0"]
67
+ if not isinstance(rows_data, dict):
68
+ raise ValueError("Invalid data returned from API")
69
+
70
+ soup = BeautifulSoup(rows_data["HTMLCollectionDatesText"], "html.parser")
71
+
72
+ # Find all div elements containing the bin schedule
73
+ for div in soup.find_all("div"):
74
+ # Extract bin type and date from the span tag
75
+ text = div.find("span").text.strip()
76
+ bin_type, date = text.split(" ", 1)
77
+ dict_data = {
78
+ "type": bin_type,
79
+ "collectionDate": date,
80
+ }
81
+ bindata["bins"].append(dict_data)
82
+
83
+ return bindata
@@ -0,0 +1,59 @@
1
+ import requests
2
+ from bs4 import BeautifulSoup
3
+
4
+ from uk_bin_collection.uk_bin_collection.common import *
5
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
6
+
7
+
8
+ # import the wonderful Beautiful Soup and the URL grabber
9
+ class CouncilClass(AbstractGetBinDataClass):
10
+ """
11
+ Concrete classes have to implement all abstract operations of the
12
+ base class. They can also override some operations with a default
13
+ implementation.
14
+ """
15
+
16
+ def parse_data(self, page: str, **kwargs) -> dict:
17
+
18
+ user_uprn = kwargs.get("uprn")
19
+ check_uprn(user_uprn)
20
+ user_uprn = user_uprn.zfill(12)
21
+ bindata = {"bins": []}
22
+
23
+ URI = "https://www.west-norfolk.gov.uk/info/20174/bins_and_recycling_collection_dates"
24
+
25
+ headers = {"Cookie": f"bcklwn_uprn={user_uprn}"}
26
+
27
+ # Make the GET request
28
+ response = requests.get(URI, headers=headers)
29
+
30
+ soup = BeautifulSoup(response.content, features="html.parser")
31
+ soup.prettify()
32
+
33
+ # Find all bin_date_container divs
34
+ bin_date_containers = soup.find_all("div", class_="bin_date_container")
35
+
36
+ # Loop through each bin_date_container
37
+ for container in bin_date_containers:
38
+ # Extract the collection date
39
+ date = (
40
+ container.find("h3", class_="collectiondate").text.strip().rstrip(":")
41
+ )
42
+
43
+ # Extract the bin type from the alt attribute of the img tag
44
+ bin_type = container.find("img")["alt"]
45
+
46
+ dict_data = {
47
+ "type": bin_type,
48
+ "collectionDate": datetime.strptime(
49
+ date,
50
+ "%A %d %B %Y",
51
+ ).strftime("%d/%m/%Y"),
52
+ }
53
+ bindata["bins"].append(dict_data)
54
+
55
+ bindata["bins"].sort(
56
+ key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
57
+ )
58
+
59
+ return bindata
@@ -0,0 +1,75 @@
1
+ import time
2
+
3
+ import requests
4
+ from bs4 import BeautifulSoup
5
+
6
+ from uk_bin_collection.uk_bin_collection.common import *
7
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
8
+
9
+
10
+ # import the wonderful Beautiful Soup and the URL grabber
11
+ class CouncilClass(AbstractGetBinDataClass):
12
+ """
13
+ Concrete classes have to implement all abstract operations of the
14
+ base class. They can also override some operations with a default
15
+ implementation.
16
+ """
17
+
18
+ def parse_data(self, page: str, **kwargs) -> dict:
19
+
20
+ user_uprn = kwargs.get("uprn")
21
+ check_uprn(user_uprn)
22
+ bindata = {"bins": []}
23
+
24
+ URI = "https://lbhapiprod.azure-api.net"
25
+ endpoint = f"{URI}/whitespace/GetCollectionByUprnAndDate"
26
+ subscription_key = "2ea6a75f9ea34bb58d299a0c9f84e72e"
27
+
28
+ # Get today's date in 'YYYY-MM-DD' format
29
+ collection_date = datetime.now().strftime("%Y-%m-%d")
30
+
31
+ # Define the request headers
32
+ headers = {
33
+ "Content-Type": "application/json",
34
+ "Ocp-Apim-Subscription-Key": subscription_key,
35
+ }
36
+
37
+ # Define the request body
38
+ data = {
39
+ "getCollectionByUprnAndDate": {
40
+ "getCollectionByUprnAndDateInput": {
41
+ "uprn": user_uprn,
42
+ "nextCollectionFromDate": collection_date,
43
+ }
44
+ }
45
+ }
46
+ # Make the POST request
47
+ response = requests.post(endpoint, headers=headers, data=json.dumps(data))
48
+ response.raise_for_status() # Raise an exception for HTTP errors
49
+
50
+ # Parse the JSON response
51
+ response_data = response.json()
52
+
53
+ collections = (
54
+ response_data.get("getCollectionByUprnAndDateResponse", {})
55
+ .get("getCollectionByUprnAndDateResult", {})
56
+ .get("Collections", [])
57
+ )
58
+
59
+ for collection in collections:
60
+ bin_type = collection["service"]
61
+ collection_date = collection["date"]
62
+
63
+ dict_data = {
64
+ "type": bin_type,
65
+ "collectionDate": datetime.strptime(
66
+ collection_date,
67
+ "%d/%m/%Y %H:%M:%S",
68
+ ).strftime(date_format),
69
+ }
70
+ bindata["bins"].append(dict_data)
71
+ bindata["bins"].sort(
72
+ key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
73
+ )
74
+
75
+ return bindata
@@ -0,0 +1,132 @@
1
+ import time
2
+
3
+ from bs4 import BeautifulSoup
4
+ from selenium.webdriver.common.by import By
5
+ from selenium.webdriver.support import expected_conditions as EC
6
+ from selenium.webdriver.support.ui import Select, WebDriverWait
7
+
8
+ from uk_bin_collection.uk_bin_collection.common import *
9
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
10
+
11
+
12
+ # import the wonderful Beautiful Soup and the URL grabber
13
+ class CouncilClass(AbstractGetBinDataClass):
14
+ """
15
+ Concrete classes have to implement all abstract operations of the
16
+ base class. They can also override some operations with a default
17
+ implementation.
18
+ """
19
+
20
+ def parse_data(self, page: str, **kwargs) -> dict:
21
+
22
+ user_uprn = kwargs.get("uprn")
23
+ user_postcode = kwargs.get("postcode")
24
+ web_driver = kwargs.get("web_driver")
25
+ headless = kwargs.get("headless")
26
+ check_uprn(user_uprn)
27
+ bindata = {"bins": []}
28
+
29
+ # Initialize the WebDriver (Chrome in this case)
30
+ driver = create_webdriver(
31
+ web_driver,
32
+ headless,
33
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
34
+ __name__,
35
+ )
36
+
37
+ # Step 1: Navigate to the form page
38
+ driver.get(
39
+ "https://lewisham.gov.uk/myservices/recycling-and-rubbish/your-bins/collection"
40
+ )
41
+
42
+ try:
43
+ cookie_accept_button = WebDriverWait(driver, 5).until(
44
+ EC.element_to_be_clickable(
45
+ (By.ID, "CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll")
46
+ )
47
+ )
48
+ cookie_accept_button.click()
49
+ except Exception:
50
+ print("No cookie consent banner found or already dismissed.")
51
+
52
+ # Wait for the form to load
53
+ WebDriverWait(driver, 10).until(
54
+ EC.presence_of_element_located((By.CLASS_NAME, "address-finder"))
55
+ )
56
+
57
+ # Step 2: Locate the input field for the postcode
58
+ postcode_input = driver.find_element(By.CLASS_NAME, "js-address-finder-input")
59
+
60
+ # Enter the postcode
61
+ postcode_input.send_keys(user_postcode) # Replace with your desired postcode
62
+ time.sleep(1) # Optional: Wait for the UI to react
63
+
64
+ # Step 4: Click the "Find address" button with retry logic
65
+ find_button = WebDriverWait(driver, 10).until(
66
+ EC.element_to_be_clickable(
67
+ (By.CLASS_NAME, "js-address-finder-step-address")
68
+ )
69
+ )
70
+ find_button.click()
71
+
72
+ # Wait for the address selector to appear and options to load
73
+ WebDriverWait(driver, 10).until(
74
+ lambda d: len(
75
+ d.find_element(By.ID, "address-selector").find_elements(
76
+ By.TAG_NAME, "option"
77
+ )
78
+ )
79
+ > 1
80
+ )
81
+
82
+ # Select the dropdown and print available options
83
+ address_selector = driver.find_element(By.ID, "address-selector")
84
+
85
+ # Use Select class to interact with the dropdown
86
+ select = Select(address_selector)
87
+ if len(select.options) > 1:
88
+ select.select_by_value(user_uprn)
89
+ else:
90
+ print("No additional addresses available to select")
91
+
92
+ # Wait until the URL contains the expected substring
93
+ WebDriverWait(driver, 10).until(
94
+ EC.url_contains("/find-your-collection-day-result")
95
+ )
96
+
97
+ # Parse the HTML
98
+ soup = BeautifulSoup(driver.page_source, "html.parser")
99
+
100
+ # Extract the main container
101
+ collection_result = soup.find("div", class_="js-find-collection-result")
102
+
103
+ # Extract each collection type and its frequency/day
104
+ for strong_tag in collection_result.find_all("strong"):
105
+ bin_type = strong_tag.text.strip() # e.g., "Food waste"
106
+ # Extract day from the sibling text
107
+ schedule_text = (
108
+ strong_tag.next_sibling.next_sibling.next_sibling.text.strip()
109
+ .split("on\n")[-1]
110
+ .replace("\n", "")
111
+ .replace("\t", "")
112
+ )
113
+ day = schedule_text.strip().split(".")[0]
114
+
115
+ # Extract the next collection date
116
+ if "Your next collection date is" in schedule_text:
117
+ start_index = schedule_text.index("Your next collection date is") + len(
118
+ "Your next collection date is"
119
+ )
120
+ next_collection_date = (
121
+ schedule_text[start_index:].strip().split("\n")[0].strip()
122
+ )
123
+ else:
124
+ next_collection_date = get_next_day_of_week(day, date_format)
125
+
126
+ dict_data = {
127
+ "type": bin_type,
128
+ "collectionDate": next_collection_date,
129
+ }
130
+ bindata["bins"].append(dict_data)
131
+
132
+ return bindata
@@ -24,6 +24,7 @@ class CouncilClass(AbstractGetBinDataClass):
24
24
 
25
25
  collection_day = kwargs.get("paon")
26
26
  garden_collection_week = kwargs.get("postcode")
27
+ garden_collection_day = kwargs.get("uprn")
27
28
  bindata = {"bins": []}
28
29
 
29
30
  days_of_week = [
@@ -42,6 +43,7 @@ class CouncilClass(AbstractGetBinDataClass):
42
43
  recyclingstartDate = datetime(2024, 11, 4)
43
44
 
44
45
  offset_days = days_of_week.index(collection_day)
46
+ offset_days_garden = days_of_week.index(garden_collection_day)
45
47
  if garden_collection_week:
46
48
  garden_collection = garden_week.index(garden_collection_week)
47
49
 
@@ -155,7 +157,7 @@ class CouncilClass(AbstractGetBinDataClass):
155
157
 
156
158
  collection_date = (
157
159
  datetime.strptime(gardenDate, "%d/%m/%Y")
158
- + timedelta(days=offset_days)
160
+ + timedelta(days=offset_days_garden)
159
161
  ).strftime("%d/%m/%Y")
160
162
 
161
163
  garden_holiday = next(
@@ -0,0 +1,65 @@
1
+ import requests
2
+ from bs4 import BeautifulSoup
3
+
4
+ from uk_bin_collection.uk_bin_collection.common import *
5
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
6
+
7
+
8
+ # import the wonderful Beautiful Soup and the URL grabber
9
+ class CouncilClass(AbstractGetBinDataClass):
10
+ """
11
+ Concrete classes have to implement all abstract operations of the
12
+ base class. They can also override some operations with a default
13
+ implementation.
14
+ """
15
+
16
+ def parse_data(self, page: str, **kwargs) -> dict:
17
+
18
+ user_uprn = kwargs.get("uprn")
19
+ bindata = {"bins": []}
20
+
21
+ user_uprn = user_uprn.zfill(8)
22
+
23
+ year = datetime.today().year
24
+ response = requests.get(
25
+ f"https://bindayfinder.moray.gov.uk/cal_{year}_view.php",
26
+ params={"id": user_uprn},
27
+ )
28
+ if response.status_code != 200:
29
+ # fall back to known good calendar URL
30
+ response = requests.get(
31
+ "https://bindayfinder.moray.gov.uk/cal_2024_view.php",
32
+ params={"id": user_uprn},
33
+ )
34
+ soup = BeautifulSoup(response.text, "html.parser")
35
+
36
+ bin_types = {
37
+ "G": "Green",
38
+ "B": "Brown",
39
+ "P": "Purple",
40
+ "C": "Blue",
41
+ "O": "Orange",
42
+ }
43
+
44
+ for month_container in soup.findAll("div", class_="month-container"):
45
+ for div in month_container.findAll("div"):
46
+ if "month-header" in div["class"]:
47
+ month = div.text
48
+ elif div["class"] and div["class"][0] in ["B", "GPOC", "GBPOC"]:
49
+ bins = div["class"][0]
50
+ dom = int(div.text)
51
+ for i in bins:
52
+ dict_data = {
53
+ "type": bin_types.get(i),
54
+ "collectionDate": datetime.strptime(
55
+ f"{dom} {month} {year}",
56
+ "%d %B %Y",
57
+ ).strftime("%d/%m/%Y"),
58
+ }
59
+ bindata["bins"].append(dict_data)
60
+
61
+ bindata["bins"].sort(
62
+ key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
63
+ )
64
+
65
+ return bindata
@@ -0,0 +1,66 @@
1
+ import requests
2
+ from bs4 import BeautifulSoup
3
+ from dateutil.relativedelta import relativedelta
4
+
5
+ from uk_bin_collection.uk_bin_collection.common import *
6
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
7
+
8
+
9
+ # import the wonderful Beautiful Soup and the URL grabber
10
+ class CouncilClass(AbstractGetBinDataClass):
11
+ """
12
+ Concrete classes have to implement all abstract operations of the
13
+ base class. They can also override some operations with a default
14
+ implementation.
15
+ """
16
+
17
+ def parse_data(self, page: str, **kwargs) -> dict:
18
+
19
+ user_uprn = kwargs.get("uprn")
20
+ check_uprn(user_uprn)
21
+ bindata = {"bins": []}
22
+
23
+ URI = f"https://www.newcastle-staffs.gov.uk/homepage/97/check-your-bin-day?uprn={user_uprn}"
24
+
25
+ # Make the GET request
26
+ response = requests.get(URI)
27
+ response.raise_for_status()
28
+ soup = BeautifulSoup(response.text, features="html.parser")
29
+ soup.prettify()
30
+
31
+ # Find the table
32
+ table = soup.find("table", {"class": "data-table"})
33
+
34
+ if table:
35
+ rows = table.find("tbody").find_all("tr")
36
+ for row in rows:
37
+ date = datetime.strptime(
38
+ (
39
+ row.find_all("td")[0]
40
+ .get_text(strip=True)
41
+ .replace("Date:", "")
42
+ .strip()
43
+ ),
44
+ "%A %d %B",
45
+ ).replace(year=datetime.now().year)
46
+ if datetime.now().month > 10 and date.month < 3:
47
+ date = date + relativedelta(years=1)
48
+ bin_types = (
49
+ row.find_all("td")[1]
50
+ .text.replace("Collection Type:", "")
51
+ .splitlines()
52
+ )
53
+ for bin_type in bin_types:
54
+ bin_type = bin_type.strip()
55
+ if bin_type:
56
+ dict_data = {
57
+ "type": bin_type.strip(),
58
+ "collectionDate": date.strftime("%d/%m/%Y"),
59
+ }
60
+ bindata["bins"].append(dict_data)
61
+
62
+ bindata["bins"].sort(
63
+ key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
64
+ )
65
+
66
+ return bindata
@@ -0,0 +1,93 @@
1
+ import requests
2
+ from bs4 import BeautifulSoup
3
+
4
+ from uk_bin_collection.uk_bin_collection.common import *
5
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
6
+
7
+
8
+ # import the wonderful Beautiful Soup and the URL grabber
9
+ class CouncilClass(AbstractGetBinDataClass):
10
+ """
11
+ Concrete classes have to implement all abstract operations of the
12
+ base class. They can also override some operations with a default
13
+ implementation.
14
+ """
15
+
16
+ def parse_data(self, page: str, **kwargs) -> dict:
17
+
18
+ user_postcode = kwargs.get("postcode")
19
+ user_paon = kwargs.get("paon")
20
+ check_postcode(user_postcode)
21
+ check_paon(user_paon)
22
+ bindata = {"bins": []}
23
+
24
+ URI = "https://uhtn-wrp.whitespacews.com/"
25
+
26
+ session = requests.Session()
27
+
28
+ # get link from first page as has some kind of unique hash
29
+ r = session.get(
30
+ URI,
31
+ )
32
+ r.raise_for_status()
33
+ soup = BeautifulSoup(r.text, features="html.parser")
34
+
35
+ alink = soup.find("a", text="Find my bin collection day")
36
+
37
+ if alink is None:
38
+ raise Exception("Initial page did not load correctly")
39
+
40
+ # greplace 'seq' query string to skip next step
41
+ nextpageurl = alink["href"].replace("seq=1", "seq=2")
42
+
43
+ data = {
44
+ "address_name_number": user_paon,
45
+ "address_postcode": user_postcode,
46
+ }
47
+
48
+ # get list of addresses
49
+ r = session.post(nextpageurl, data)
50
+ r.raise_for_status()
51
+
52
+ soup = BeautifulSoup(r.text, features="html.parser")
53
+
54
+ # get first address (if you don't enter enough argument values this won't find the right address)
55
+ alink = soup.find("div", id="property_list").find("a")
56
+
57
+ if alink is None:
58
+ raise Exception("Address not found")
59
+
60
+ nextpageurl = URI + alink["href"]
61
+
62
+ # get collection page
63
+ r = session.get(
64
+ nextpageurl,
65
+ )
66
+ r.raise_for_status()
67
+ soup = BeautifulSoup(r.text, features="html.parser")
68
+
69
+ if soup.find("span", id="waste-hint"):
70
+ raise Exception("No scheduled services at this address")
71
+
72
+ u1s = soup.find("section", id="scheduled-collections").find_all("u1")
73
+
74
+ for u1 in u1s:
75
+ lis = u1.find_all("li", recursive=False)
76
+
77
+ date = lis[1].text.replace("\n", "")
78
+ bin_type = lis[2].text.replace("\n", "")
79
+
80
+ dict_data = {
81
+ "type": bin_type,
82
+ "collectionDate": datetime.strptime(
83
+ date,
84
+ "%d/%m/%Y",
85
+ ).strftime(date_format),
86
+ }
87
+ bindata["bins"].append(dict_data)
88
+
89
+ bindata["bins"].sort(
90
+ key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
91
+ )
92
+
93
+ return bindata