uk_bin_collection 0.97.1__py3-none-any.whl → 0.98.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- uk_bin_collection/tests/input.json +36 -1
- uk_bin_collection/uk_bin_collection/councils/ArmaghBanbridgeCraigavonCouncil.py +72 -0
- uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py +11 -2
- uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py +66 -66
- uk_bin_collection/uk_bin_collection/councils/LichfieldDistrictCouncil.py +60 -0
- uk_bin_collection/uk_bin_collection/councils/WalsallCouncil.py +56 -0
- uk_bin_collection/uk_bin_collection/councils/WestMorlandAndFurness.py +65 -0
- uk_bin_collection/uk_bin_collection/councils/WirralCouncil.py +64 -0
- {uk_bin_collection-0.97.1.dist-info → uk_bin_collection-0.98.1.dist-info}/METADATA +1 -1
- {uk_bin_collection-0.97.1.dist-info → uk_bin_collection-0.98.1.dist-info}/RECORD +13 -8
- {uk_bin_collection-0.97.1.dist-info → uk_bin_collection-0.98.1.dist-info}/LICENSE +0 -0
- {uk_bin_collection-0.97.1.dist-info → uk_bin_collection-0.98.1.dist-info}/WHEEL +0 -0
- {uk_bin_collection-0.97.1.dist-info → uk_bin_collection-0.98.1.dist-info}/entry_points.txt +0 -0
@@ -5,6 +5,13 @@
|
|
5
5
|
"wiki_name": "Adur and Worthing Councils",
|
6
6
|
"wiki_note": "Replace XXXXXXXX with UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
|
7
7
|
},
|
8
|
+
"ArmaghBanbridgeCraigavonCouncil": {
|
9
|
+
"url": "https://www.armaghbanbridgecraigavon.gov.uk/",
|
10
|
+
"wiki_command_url_override": "https://www.armaghbanbridgecraigavon.gov.uk/",
|
11
|
+
"uprn": "185625284",
|
12
|
+
"wiki_name": "Armagh Banbridge Craigavon Council",
|
13
|
+
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
|
14
|
+
},
|
8
15
|
"ArunCouncil": {
|
9
16
|
"house_number": "1",
|
10
17
|
"postcode": "BN16 4DA",
|
@@ -42,7 +49,7 @@
|
|
42
49
|
"wiki_name": "Barnsley Metropolitan Borough Council",
|
43
50
|
"wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search)."
|
44
51
|
},
|
45
|
-
"BasildonCouncil": {
|
52
|
+
"BasildonCouncil": {
|
46
53
|
"skip_get_url": true,
|
47
54
|
"uprn": "10013350430",
|
48
55
|
"url": "https://basildonportal.azurewebsites.net/api/getPropertyRefuseInformation",
|
@@ -606,6 +613,13 @@
|
|
606
613
|
"web_driver": "http://selenium:4444",
|
607
614
|
"wiki_name": "Leeds City Council"
|
608
615
|
},
|
616
|
+
"LichfieldDistrictCouncil": {
|
617
|
+
"url": "https://www.lichfielddc.gov.uk",
|
618
|
+
"wiki_command_url_override": "https://www.lichfielddc.gov.uk",
|
619
|
+
"uprn": "100031694085",
|
620
|
+
"wiki_name": "Lichfield District Council",
|
621
|
+
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
|
622
|
+
},
|
609
623
|
"LisburnCastlereaghCityCouncil": {
|
610
624
|
"house_number": "97",
|
611
625
|
"postcode": "BT28 1JN",
|
@@ -1216,6 +1230,13 @@
|
|
1216
1230
|
"wiki_name": "Wakefield City Council",
|
1217
1231
|
"wiki_note": "Follow the instructions [here](https://www.wakefield.gov.uk/where-i-live/) until you get the page that includes a \"Bin Collections\" section then copy the URL and replace the URL in the command."
|
1218
1232
|
},
|
1233
|
+
"WalsallCouncil": {
|
1234
|
+
"url": "https://cag.walsall.gov.uk/",
|
1235
|
+
"wiki_command_url_override": "https://cag.walsall.gov.uk/",
|
1236
|
+
"uprn": "100071080513",
|
1237
|
+
"wiki_name": "Walsall Council",
|
1238
|
+
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
|
1239
|
+
},
|
1219
1240
|
"WalthamForest": {
|
1220
1241
|
"house_number": "17 Chingford Road, Walthamstow",
|
1221
1242
|
"postcode": "E17 4PW",
|
@@ -1278,6 +1299,13 @@
|
|
1278
1299
|
"wiki_name": "West Lothian Council",
|
1279
1300
|
"wiki_note": "Pass the house name/number in the house number parameter, wrapped in double quotes"
|
1280
1301
|
},
|
1302
|
+
"WestMorlandAndFurness": {
|
1303
|
+
"url": "https://www.westmorlandandfurness.gov.uk/",
|
1304
|
+
"wiki_command_url_override": "https://www.westmorlandandfurness.gov.uk/",
|
1305
|
+
"uprn": "100110353478",
|
1306
|
+
"wiki_name": "West Morland And Furness Council",
|
1307
|
+
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
|
1308
|
+
},
|
1281
1309
|
"WestNorthamptonshireCouncil": {
|
1282
1310
|
"postcode": "NN3 2JB",
|
1283
1311
|
"skip_get_url": true,
|
@@ -1322,6 +1350,13 @@
|
|
1322
1350
|
"url": "https://forms.rbwm.gov.uk/bincollections?uprn=",
|
1323
1351
|
"wiki_name": "Windsor and Maidenhead Council"
|
1324
1352
|
},
|
1353
|
+
"WirralCouncil": {
|
1354
|
+
"url": "https://www.wirral.gov.uk",
|
1355
|
+
"wiki_command_url_override": "https://www.wirral.gov.uk",
|
1356
|
+
"uprn": "Vernon Avenue,Seacombe",
|
1357
|
+
"wiki_name": "Wirral Council",
|
1358
|
+
"wiki_note": "Please use the UPRN field to enter your street name and suburb separated by a comma, for example 'Vernon Avenue,Seacombe'"
|
1359
|
+
},
|
1325
1360
|
"WokingBoroughCouncil": {
|
1326
1361
|
"house_number": "2",
|
1327
1362
|
"postcode": "GU214JY",
|
@@ -0,0 +1,72 @@
|
|
1
|
+
import requests
|
2
|
+
from bs4 import BeautifulSoup
|
3
|
+
|
4
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
5
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
6
|
+
|
7
|
+
|
8
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
9
|
+
class CouncilClass(AbstractGetBinDataClass):
|
10
|
+
"""
|
11
|
+
Concrete classes have to implement all abstract operations of the
|
12
|
+
base class. They can also override some operations with a default
|
13
|
+
implementation.
|
14
|
+
"""
|
15
|
+
|
16
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
17
|
+
|
18
|
+
user_uprn = kwargs.get("uprn")
|
19
|
+
check_uprn(user_uprn)
|
20
|
+
bindata = {"bins": []}
|
21
|
+
|
22
|
+
# Function to extract bin collection information
|
23
|
+
def extract_bin_schedule(soup, heading_class):
|
24
|
+
collections = []
|
25
|
+
|
26
|
+
# Find the relevant section based on the heading class
|
27
|
+
section_heading = soup.find("div", class_=heading_class)
|
28
|
+
if section_heading:
|
29
|
+
# Find all the bin collection dates in that section
|
30
|
+
collection_dates = section_heading.find_next(
|
31
|
+
"div", class_="col-sm-12 col-md-9"
|
32
|
+
).find_all("h4")
|
33
|
+
for date in collection_dates:
|
34
|
+
# Clean and add the date to the list
|
35
|
+
collections.append(date.get_text(strip=True))
|
36
|
+
|
37
|
+
return collections
|
38
|
+
|
39
|
+
# URL for bin collection schedule
|
40
|
+
url = f"https://www.armaghbanbridgecraigavon.gov.uk/resident/binday-result/?address={user_uprn}"
|
41
|
+
|
42
|
+
# Send a GET request to fetch the page content
|
43
|
+
response = requests.get(url)
|
44
|
+
|
45
|
+
# Check if the request was successful
|
46
|
+
if response.status_code == 200:
|
47
|
+
# Parse the page content using BeautifulSoup
|
48
|
+
soup = BeautifulSoup(response.text, "html.parser")
|
49
|
+
|
50
|
+
# Extract bin collection schedules by their sections
|
51
|
+
domestic_collections = extract_bin_schedule(soup, "heading bg-black")
|
52
|
+
for collection in domestic_collections:
|
53
|
+
bindata["bins"].append(
|
54
|
+
{"collectionDate": collection, "type": "Domestic"}
|
55
|
+
)
|
56
|
+
recycling_collections = extract_bin_schedule(soup, "heading bg-green")
|
57
|
+
for collection in recycling_collections:
|
58
|
+
bindata["bins"].append(
|
59
|
+
{"collectionDate": collection, "type": "Recycling"}
|
60
|
+
)
|
61
|
+
garden_collections = extract_bin_schedule(soup, "heading bg-brown")
|
62
|
+
for collection in garden_collections:
|
63
|
+
bindata["bins"].append({"collectionDate": collection, "type": "Garden"})
|
64
|
+
|
65
|
+
else:
|
66
|
+
print(f"Failed to retrieve data. Status code: {response.status_code}")
|
67
|
+
|
68
|
+
bindata["bins"].sort(
|
69
|
+
key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
|
70
|
+
)
|
71
|
+
|
72
|
+
return bindata
|
@@ -70,10 +70,19 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
70
70
|
headless = kwargs.get("headless")
|
71
71
|
web_driver = kwargs.get("web_driver")
|
72
72
|
driver = create_webdriver(web_driver, headless, None, __name__)
|
73
|
-
page = "https://
|
73
|
+
page = "https://www.barnet.gov.uk/recycling-and-waste/bin-collections/find-your-bin-collection-day"
|
74
|
+
|
74
75
|
driver.get(page)
|
75
76
|
|
76
|
-
|
77
|
+
# Wait for the element to be clickable
|
78
|
+
find_your_collection_button = WebDriverWait(driver, 10).until(
|
79
|
+
EC.element_to_be_clickable(
|
80
|
+
(By.XPATH, '//a[contains(text(), "Find your household collection day")]')
|
81
|
+
)
|
82
|
+
)
|
83
|
+
|
84
|
+
# Click the element
|
85
|
+
find_your_collection_button.click()
|
77
86
|
|
78
87
|
try:
|
79
88
|
accept_cookies = WebDriverWait(driver, timeout=10).until(
|
@@ -24,7 +24,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
24
24
|
def parse_data(self, page: str, **kwargs) -> dict:
|
25
25
|
driver = None
|
26
26
|
try:
|
27
|
-
page = "https://
|
27
|
+
page = "https://waste.bexley.gov.uk/waste"
|
28
28
|
|
29
29
|
data = {"bins": []}
|
30
30
|
|
@@ -38,102 +38,102 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
38
38
|
driver = create_webdriver(web_driver, headless, None, __name__)
|
39
39
|
driver.get(page)
|
40
40
|
|
41
|
-
|
42
|
-
|
43
|
-
iframe_presense = WebDriverWait(driver, 30).until(
|
44
|
-
EC.presence_of_element_located((By.ID, "fillform-frame-1"))
|
45
|
-
)
|
46
|
-
|
47
|
-
driver.switch_to.frame(iframe_presense)
|
48
|
-
wait = WebDriverWait(driver, 60)
|
49
|
-
start_btn = wait.until(
|
50
|
-
EC.element_to_be_clickable(
|
51
|
-
(By.XPATH, "//button/span[contains(text(), 'Next')]")
|
52
|
-
)
|
53
|
-
)
|
54
|
-
|
55
|
-
start_btn.click()
|
41
|
+
wait = WebDriverWait(driver, 10)
|
56
42
|
|
57
43
|
inputElement_postcodesearch = wait.until(
|
58
|
-
EC.element_to_be_clickable((By.ID, "
|
44
|
+
EC.element_to_be_clickable((By.ID, "pc"))
|
59
45
|
)
|
60
46
|
inputElement_postcodesearch.send_keys(user_postcode)
|
61
47
|
|
48
|
+
|
49
|
+
|
62
50
|
find_address_btn = wait.until(
|
63
|
-
EC.element_to_be_clickable((By.XPATH, '//*[@id="
|
51
|
+
EC.element_to_be_clickable((By.XPATH, '//*[@id="sub"]'))
|
64
52
|
)
|
65
53
|
find_address_btn.click()
|
66
54
|
|
67
55
|
dropdown_options = wait.until(
|
68
56
|
EC.presence_of_element_located(
|
69
|
-
(By.XPATH, '//*[@id="
|
57
|
+
(By.XPATH, '//*[@id="address"]')
|
70
58
|
)
|
71
59
|
)
|
72
60
|
time.sleep(2)
|
73
61
|
dropdown_options.click()
|
74
62
|
time.sleep(1)
|
75
|
-
|
63
|
+
|
64
|
+
# Wait for the element to be clickable
|
65
|
+
address = WebDriverWait(driver, 10).until(
|
66
|
+
EC.element_to_be_clickable(
|
67
|
+
(By.XPATH, f'//li[contains(text(), "{user_paon}")]')
|
68
|
+
)
|
69
|
+
)
|
70
|
+
|
71
|
+
# Click the element
|
72
|
+
address.click()
|
73
|
+
|
74
|
+
|
75
|
+
submit_address = wait.until(
|
76
76
|
EC.presence_of_element_located(
|
77
|
-
(By.XPATH, '//*[@id="
|
77
|
+
(By.XPATH, '//*[@id="go"]')
|
78
78
|
)
|
79
79
|
)
|
80
|
-
time.sleep(
|
81
|
-
|
82
|
-
dropdown_input.send_keys(Keys.ENTER)
|
80
|
+
time.sleep(2)
|
81
|
+
submit_address.click()
|
83
82
|
|
84
83
|
results_found = wait.until(
|
85
|
-
EC.presence_of_element_located((By.CLASS_NAME, "found-content"))
|
86
|
-
)
|
87
|
-
finish_btn = wait.until(
|
88
84
|
EC.element_to_be_clickable(
|
89
|
-
(By.XPATH,
|
85
|
+
(By.XPATH, '//h1[contains(text(), "Your bin days")]')
|
86
|
+
)
|
90
87
|
)
|
91
|
-
|
92
|
-
finish_btn.click()
|
88
|
+
|
93
89
|
final_page = wait.until(
|
94
90
|
EC.presence_of_element_located(
|
95
|
-
(By.CLASS_NAME, "
|
91
|
+
(By.CLASS_NAME, "waste__collections")
|
96
92
|
)
|
97
93
|
)
|
98
94
|
|
99
95
|
soup = BeautifulSoup(driver.page_source, features="html.parser")
|
100
96
|
|
101
|
-
|
102
|
-
# Define your XPath
|
103
|
-
|
104
|
-
for bin in bin_fields:
|
105
|
-
# Extract h3 text from the current element
|
106
|
-
h3_text = (
|
107
|
-
bin.find("h3", class_="container-name").get_text(strip=True)
|
108
|
-
if bin.find("h3", class_="container-name")
|
109
|
-
else None
|
110
|
-
)
|
111
|
-
|
112
|
-
date_text = (
|
113
|
-
bin.find("p", class_="container-status").get_text(strip=True)
|
114
|
-
if bin.find("p", class_="container-status")
|
115
|
-
else None
|
116
|
-
)
|
97
|
+
# Find all waste services
|
117
98
|
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
99
|
+
# Initialize the data dictionary
|
100
|
+
data = {"bins": []}
|
101
|
+
bin_sections = soup.find_all("h3", class_="waste-service-name")
|
102
|
+
|
103
|
+
# Loop through each bin field
|
104
|
+
for bin_section in bin_sections:
|
105
|
+
# Extract the bin type (e.g., "Brown Caddy", "Green Wheelie Bin", etc.)
|
106
|
+
bin_type = bin_section.get_text(strip=True).split("\n")[0] # The first part is the bin type
|
107
|
+
|
108
|
+
# Find the next sibling <dl> tag that contains the next collection information
|
109
|
+
summary_list = bin_section.find_next("dl", class_="govuk-summary-list")
|
110
|
+
|
111
|
+
if summary_list:
|
112
|
+
# Now, instead of finding by class, we'll search by text within the dt element
|
113
|
+
next_collection_dt = summary_list.find("dt", string=lambda text: "Next collection" in text)
|
114
|
+
|
115
|
+
if next_collection_dt:
|
116
|
+
# Find the sibling <dd> tag for the collection date
|
117
|
+
next_collection = next_collection_dt.find_next_sibling("dd").get_text(strip=True)
|
118
|
+
|
119
|
+
if next_collection:
|
120
|
+
try:
|
121
|
+
# Parse the next collection date (assuming the format is like "Tuesday 15 October 2024")
|
122
|
+
parsed_date = datetime.strptime(next_collection, "%A %d %B %Y")
|
123
|
+
|
124
|
+
# Add the bin information to the data dictionary
|
125
|
+
data["bins"].append({
|
126
|
+
"type": bin_type,
|
127
|
+
"collectionDate": parsed_date.strftime(date_format),
|
128
|
+
})
|
129
|
+
except ValueError as e:
|
130
|
+
print(f"Error parsing date for {bin_type}: {e}")
|
131
|
+
else:
|
132
|
+
print(f"No next collection date found for {bin_type}")
|
133
|
+
else:
|
134
|
+
print(f"No 'Next collection' text found for {bin_type}")
|
135
|
+
else:
|
136
|
+
print(f"No summary list found for {bin_type}")
|
137
137
|
|
138
138
|
except Exception as e:
|
139
139
|
# Here you can log the exception if needed
|
@@ -0,0 +1,60 @@
|
|
1
|
+
import re
|
2
|
+
|
3
|
+
import requests
|
4
|
+
from bs4 import BeautifulSoup
|
5
|
+
|
6
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
7
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
8
|
+
|
9
|
+
|
10
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
11
|
+
class CouncilClass(AbstractGetBinDataClass):
|
12
|
+
"""
|
13
|
+
Concrete classes have to implement all abstract operations of the
|
14
|
+
base class. They can also override some operations with a default
|
15
|
+
implementation.
|
16
|
+
"""
|
17
|
+
|
18
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
19
|
+
|
20
|
+
user_uprn = kwargs.get("uprn")
|
21
|
+
check_uprn(user_uprn)
|
22
|
+
bindata = {"bins": []}
|
23
|
+
|
24
|
+
def solve(s):
|
25
|
+
return re.sub(r"(\d)(st|nd|rd|th)", r"\1", s)
|
26
|
+
|
27
|
+
URI = f"https://www.lichfielddc.gov.uk/homepage/6/bin-collection-dates?uprn={user_uprn}"
|
28
|
+
|
29
|
+
# Make the GET request
|
30
|
+
response = requests.get(URI)
|
31
|
+
|
32
|
+
soup = BeautifulSoup(response.text, "html.parser")
|
33
|
+
|
34
|
+
bins = soup.find_all("h3", class_="bin-collection-tasks__heading")
|
35
|
+
dates = soup.find_all("p", class_="bin-collection-tasks__date")
|
36
|
+
|
37
|
+
current_year = datetime.now().year
|
38
|
+
current_month = datetime.now().month
|
39
|
+
|
40
|
+
for i in range(len(dates)):
|
41
|
+
bint = " ".join(bins[i].text.split()[2:4])
|
42
|
+
date = dates[i].text
|
43
|
+
|
44
|
+
date = datetime.strptime(
|
45
|
+
solve(date),
|
46
|
+
"%d %B",
|
47
|
+
)
|
48
|
+
|
49
|
+
if (current_month > 10) and (date.month < 3):
|
50
|
+
date = date.replace(year=(current_year + 1))
|
51
|
+
else:
|
52
|
+
date = date.replace(year=current_year)
|
53
|
+
|
54
|
+
dict_data = {
|
55
|
+
"type": bint,
|
56
|
+
"collectionDate": date.strftime("%d/%m/%Y"),
|
57
|
+
}
|
58
|
+
bindata["bins"].append(dict_data)
|
59
|
+
|
60
|
+
return bindata
|
@@ -0,0 +1,56 @@
|
|
1
|
+
import requests
|
2
|
+
from bs4 import BeautifulSoup
|
3
|
+
|
4
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
5
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
6
|
+
|
7
|
+
|
8
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
9
|
+
class CouncilClass(AbstractGetBinDataClass):
|
10
|
+
"""
|
11
|
+
Concrete classes have to implement all abstract operations of the
|
12
|
+
base class. They can also override some operations with a default
|
13
|
+
implementation.
|
14
|
+
"""
|
15
|
+
|
16
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
17
|
+
|
18
|
+
user_uprn = kwargs.get("uprn")
|
19
|
+
check_uprn(user_uprn)
|
20
|
+
bindata = {"bins": []}
|
21
|
+
|
22
|
+
URI = f"https://cag.walsall.gov.uk/BinCollections/GetBins?uprn={user_uprn}"
|
23
|
+
|
24
|
+
headers = {
|
25
|
+
"user-agent": "Mozilla/5.0",
|
26
|
+
}
|
27
|
+
|
28
|
+
response = requests.get(URI, headers=headers)
|
29
|
+
|
30
|
+
soup = BeautifulSoup(response.text, "html.parser")
|
31
|
+
# Extract links to collection shedule pages and iterate through the pages
|
32
|
+
schedule_links = soup.findAll("a", {"class": "nav-link"}, href=True)
|
33
|
+
for item in schedule_links:
|
34
|
+
if "roundname" in item["href"]:
|
35
|
+
# get bin colour
|
36
|
+
bincolour = item["href"].split("=")[-1].split("%")[0].upper()
|
37
|
+
binURL = "https://cag.walsall.gov.uk" + item["href"]
|
38
|
+
r = requests.get(binURL, headers=headers)
|
39
|
+
soup = BeautifulSoup(r.text, "html.parser")
|
40
|
+
table = soup.findAll("tr")
|
41
|
+
for tr in table:
|
42
|
+
td = tr.findAll("td")
|
43
|
+
if td:
|
44
|
+
dict_data = {
|
45
|
+
"type": bincolour,
|
46
|
+
"collectionDate": datetime.strptime(
|
47
|
+
td[1].text.strip(), "%d/%m/%Y"
|
48
|
+
).strftime("%d/%m/%Y"),
|
49
|
+
}
|
50
|
+
bindata["bins"].append(dict_data)
|
51
|
+
|
52
|
+
bindata["bins"].sort(
|
53
|
+
key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
|
54
|
+
)
|
55
|
+
|
56
|
+
return bindata
|
@@ -0,0 +1,65 @@
|
|
1
|
+
import requests
|
2
|
+
from bs4 import BeautifulSoup
|
3
|
+
|
4
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
5
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
6
|
+
|
7
|
+
|
8
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
9
|
+
class CouncilClass(AbstractGetBinDataClass):
|
10
|
+
"""
|
11
|
+
Concrete classes have to implement all abstract operations of the
|
12
|
+
base class. They can also override some operations with a default
|
13
|
+
implementation.
|
14
|
+
"""
|
15
|
+
|
16
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
17
|
+
|
18
|
+
user_uprn = kwargs.get("uprn")
|
19
|
+
check_uprn(user_uprn)
|
20
|
+
bindata = {"bins": []}
|
21
|
+
|
22
|
+
URI = f"https://www.westmorlandandfurness.gov.uk/bins-recycling-and-street-cleaning/waste-collection-schedule/view/{user_uprn}"
|
23
|
+
|
24
|
+
headers = {
|
25
|
+
"user-agent": "Mozilla/5.0",
|
26
|
+
}
|
27
|
+
|
28
|
+
current_year = datetime.now().year
|
29
|
+
current_month = datetime.now().month
|
30
|
+
|
31
|
+
response = requests.get(URI)
|
32
|
+
|
33
|
+
soup = BeautifulSoup(response.text, "html.parser")
|
34
|
+
# Extract links to collection shedule pages and iterate through the pages
|
35
|
+
schedule = soup.findAll("div", {"class": "waste-collection__month"})
|
36
|
+
for month in schedule:
|
37
|
+
collectionmonth = datetime.strptime(month.find("h3").text, "%B")
|
38
|
+
collectionmonth = collectionmonth.month
|
39
|
+
collectiondays = month.findAll("li", {"class": "waste-collection__day"})
|
40
|
+
for collectionday in collectiondays:
|
41
|
+
day = collectionday.find(
|
42
|
+
"span", {"class": "waste-collection__day--day"}
|
43
|
+
).text.strip()
|
44
|
+
collectiondate = datetime.strptime(day, "%d")
|
45
|
+
collectiondate = collectiondate.replace(month=collectionmonth)
|
46
|
+
bintype = collectionday.find(
|
47
|
+
"span", {"class": "waste-collection__day--colour"}
|
48
|
+
).text
|
49
|
+
|
50
|
+
if (current_month > 9) and (collectiondate.month < 4):
|
51
|
+
collectiondate = collectiondate.replace(year=(current_year + 1))
|
52
|
+
else:
|
53
|
+
collectiondate = collectiondate.replace(year=current_year)
|
54
|
+
|
55
|
+
dict_data = {
|
56
|
+
"type": bintype,
|
57
|
+
"collectionDate": collectiondate.strftime("%d/%m/%Y"),
|
58
|
+
}
|
59
|
+
bindata["bins"].append(dict_data)
|
60
|
+
|
61
|
+
bindata["bins"].sort(
|
62
|
+
key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
|
63
|
+
)
|
64
|
+
|
65
|
+
return bindata
|
@@ -0,0 +1,64 @@
|
|
1
|
+
import requests
|
2
|
+
from bs4 import BeautifulSoup
|
3
|
+
|
4
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
5
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
6
|
+
|
7
|
+
|
8
|
+
# import the wonderful Beautiful Soup and the URL grabber
|
9
|
+
class CouncilClass(AbstractGetBinDataClass):
|
10
|
+
"""
|
11
|
+
Concrete classes have to implement all abstract operations of the
|
12
|
+
base class. They can also override some operations with a default
|
13
|
+
implementation.
|
14
|
+
"""
|
15
|
+
|
16
|
+
def parse_data(self, page: str, **kwargs) -> dict:
|
17
|
+
|
18
|
+
user_uprn = kwargs.get("uprn")
|
19
|
+
bindata = {"bins": []}
|
20
|
+
|
21
|
+
street = user_uprn.split(",")[0]
|
22
|
+
suburb = user_uprn.split(",")[1]
|
23
|
+
|
24
|
+
if not street:
|
25
|
+
print("Street name could not be parsed"),
|
26
|
+
return
|
27
|
+
if not suburb:
|
28
|
+
print("Street name could not be parsed"),
|
29
|
+
return
|
30
|
+
|
31
|
+
wastes = {
|
32
|
+
"Non recycleable (green bin)",
|
33
|
+
"Garden waste (brown bin)",
|
34
|
+
"Paper and packaging (grey bin)",
|
35
|
+
}
|
36
|
+
DATE_REGEX = "^([0-9]{1,2} [A-Za-z]+ [0-9]{4})"
|
37
|
+
|
38
|
+
s = requests.Session()
|
39
|
+
# Loop through waste types
|
40
|
+
for waste in wastes:
|
41
|
+
r = s.get(
|
42
|
+
f"https://ww3.wirral.gov.uk//recycling/detailContentDru7.asp?s={street}&t={suburb}&c={waste}"
|
43
|
+
)
|
44
|
+
# extract dates
|
45
|
+
soup = BeautifulSoup(r.text, "html.parser")
|
46
|
+
dates = soup.findAll("li")
|
47
|
+
if len(dates) != 0:
|
48
|
+
for item in dates:
|
49
|
+
match = re.match(DATE_REGEX, item.text)
|
50
|
+
if match:
|
51
|
+
dict_data = {
|
52
|
+
"type": waste,
|
53
|
+
"collectionDate": datetime.strptime(
|
54
|
+
match.group(1),
|
55
|
+
"%d %B %Y",
|
56
|
+
).strftime("%d/%m/%Y"),
|
57
|
+
}
|
58
|
+
bindata["bins"].append(dict_data)
|
59
|
+
|
60
|
+
bindata["bins"].sort(
|
61
|
+
key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
|
62
|
+
)
|
63
|
+
|
64
|
+
return bindata
|
@@ -2,7 +2,7 @@ uk_bin_collection/README.rst,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
|
|
2
2
|
uk_bin_collection/tests/council_feature_input_parity.py,sha256=DO6Mk4ImYgM5ZCZ-cutwz5RoYYWZRLYx2tr6zIs_9Rc,3843
|
3
3
|
uk_bin_collection/tests/features/environment.py,sha256=VQZjJdJI_kZn08M0j5cUgvKT4k3iTw8icJge1DGOkoA,127
|
4
4
|
uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=SJK-Vc737hrf03tssxxbeg_JIvAH-ddB8f6gU1LTbuQ,251
|
5
|
-
uk_bin_collection/tests/input.json,sha256=
|
5
|
+
uk_bin_collection/tests/input.json,sha256=Lb2LZ6Q1LvG-yaG4KCjPIiKdLyK9IE6N_4vb7wwdExw,68074
|
6
6
|
uk_bin_collection/tests/output.schema,sha256=ZwKQBwYyTDEM4G2hJwfLUVM-5v1vKRvRK9W9SS1sd18,1086
|
7
7
|
uk_bin_collection/tests/step_defs/step_helpers/file_handler.py,sha256=Ygzi4V0S1MIHqbdstUlIqtRIwnynvhu4UtpweJ6-5N8,1474
|
8
8
|
uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=LrOSt_loA1Mw3vTqaO2LpaDMu7rYJy6k5Kr-EOBln7s,3424
|
@@ -13,10 +13,11 @@ uk_bin_collection/tests/test_get_data.py,sha256=tQGyuTR8pZvnwzVsDqFhUrFtBeMP11eR
|
|
13
13
|
uk_bin_collection/uk_bin_collection/collect_data.py,sha256=dB7wWXsJX4fm5bIf84lexkvHIcO54CZ3JPxqmS-60YY,4654
|
14
14
|
uk_bin_collection/uk_bin_collection/common.py,sha256=oZQW8CYRGfbhNSyq7t7jEhrFl1tjb4H157xSx8QHWSY,10106
|
15
15
|
uk_bin_collection/uk_bin_collection/councils/AdurAndWorthingCouncils.py,sha256=ppbrmm-MzB1wOulK--CU_0j4P-djNf3ozMhHnmQFqLo,1511
|
16
|
+
uk_bin_collection/uk_bin_collection/councils/ArmaghBanbridgeCraigavonCouncil.py,sha256=o9NBbVCTdxKXnpYbP8-zxe1Gh8s57vwfV75Son_sAHE,2863
|
16
17
|
uk_bin_collection/uk_bin_collection/councils/ArunCouncil.py,sha256=yfhthv9nuogP19VOZ3TYQrq51qqjiCZcSel4sXhiKjs,4012
|
17
18
|
uk_bin_collection/uk_bin_collection/councils/AylesburyValeCouncil.py,sha256=LouqjspEMt1TkOGqWHs2zkxwOETIy3n7p64uKIlAgUg,2401
|
18
19
|
uk_bin_collection/uk_bin_collection/councils/BCPCouncil.py,sha256=W7QBx6Mgso8RYosuXsaYo3GGNAu-tiyBSmuYxr1JSOU,1707
|
19
|
-
uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py,sha256=
|
20
|
+
uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py,sha256=NccMd0830aiNRQ8SH6mc4r5Hlugfey5-PyUd24QLo4s,8478
|
20
21
|
uk_bin_collection/uk_bin_collection/councils/BarnsleyMBCouncil.py,sha256=MgF_7XyIcIoNzFR0OJsjBkLCZKgWxBrV6nTcutMxO1Q,4244
|
21
22
|
uk_bin_collection/uk_bin_collection/councils/BasildonCouncil.py,sha256=SBvAa0GZM3V7ygK8ARawbHAPH6R_303U30RH8WYPi5Q,3020
|
22
23
|
uk_bin_collection/uk_bin_collection/councils/BasingstokeCouncil.py,sha256=VPWGljnH4C3q8qs5ZmCtqjNjgWQvviALzjk00q3EZeQ,2632
|
@@ -24,7 +25,7 @@ uk_bin_collection/uk_bin_collection/councils/BathAndNorthEastSomersetCouncil.py,
|
|
24
25
|
uk_bin_collection/uk_bin_collection/councils/BedfordBoroughCouncil.py,sha256=CvGB7w9HMn7XyEtwfd9MWZE_HlZ75pDcaKMsQJz0xhk,1669
|
25
26
|
uk_bin_collection/uk_bin_collection/councils/BedfordshireCouncil.py,sha256=U1HOr9YLMAlFoZysfw5n04E0bVuCliO5Yj1FMiiwcHA,2549
|
26
27
|
uk_bin_collection/uk_bin_collection/councils/BelfastCityCouncil.py,sha256=mspYVHO8fgoVIwogT6V2Go1tbf3PDbEmr8kZMJug5Gs,4235
|
27
|
-
uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py,sha256=
|
28
|
+
uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py,sha256=nzet8yKKzpiAQXDr0tmqXuJ4R-EdJqtxSNGiUCh79bA,5591
|
28
29
|
uk_bin_collection/uk_bin_collection/councils/BirminghamCityCouncil.py,sha256=now2xgpfshYM33UWC18j6xa6BuBydO5Sl7OrDQOo6b0,4687
|
29
30
|
uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py,sha256=jHbCK8sL09vdmdP7Xnh8lIrU5AHTnJLEZfOLephPvWg,4090
|
30
31
|
uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py,sha256=r7wjrRPT2EinRMnYjGxmsCD6aMFhEOyRNd8_3R9PdQU,4117
|
@@ -94,6 +95,7 @@ uk_bin_collection/uk_bin_collection/councils/KirkleesCouncil.py,sha256=gtAtM6Fap
|
|
94
95
|
uk_bin_collection/uk_bin_collection/councils/KnowsleyMBCouncil.py,sha256=VdlWDESoHfr_X0r8-UMaLMUQhKZOa2BnpVPkX-1u3EQ,5605
|
95
96
|
uk_bin_collection/uk_bin_collection/councils/LancasterCityCouncil.py,sha256=FmHT6oyD4BwWuhxA80PHnGA7HPrLuyjP_54Cg8hT6k4,2537
|
96
97
|
uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py,sha256=iSZApZ9oSfSatQ6dAxmykSfti91jGuY6n2BwEkVMOiU,5144
|
98
|
+
uk_bin_collection/uk_bin_collection/councils/LichfieldDistrictCouncil.py,sha256=l3zgTWuKOW8fgb8PmXv0OTI6HaiGBPndefNQk8MM4oY,1810
|
97
99
|
uk_bin_collection/uk_bin_collection/councils/LisburnCastlereaghCityCouncil.py,sha256=vSOzdEwp9ZeUhed7E3eVv9ReD-2XgbSkpyAbVnfc-Gk,3309
|
98
100
|
uk_bin_collection/uk_bin_collection/councils/LiverpoolCityCouncil.py,sha256=n17OqZrCGrPrnxGUfHc-RGkb4oJ9Bx6uUWiLdzxfQlY,2587
|
99
101
|
uk_bin_collection/uk_bin_collection/councils/LondonBoroughEaling.py,sha256=QDx2Izr-6hUSFMi4UWqsgo3p6U8aRZ9d_Cu9cBSp2rY,1653
|
@@ -183,6 +185,7 @@ uk_bin_collection/uk_bin_collection/councils/UttlesfordDistrictCouncil.py,sha256
|
|
183
185
|
uk_bin_collection/uk_bin_collection/councils/ValeofGlamorganCouncil.py,sha256=Phgb_ECiUOOkqOx6OsfsTHMCW5VQfRmOC2zgYIQhuZA,5044
|
184
186
|
uk_bin_collection/uk_bin_collection/councils/ValeofWhiteHorseCouncil.py,sha256=5nZLbU5YVKNsJ2X_wuybrNLFAzjVAxkazu-bYP4IGXw,4292
|
185
187
|
uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py,sha256=-xqJOzHTrT4jOB3rHPXFYeqLaHyK9XmCPi92whaYBhw,4671
|
188
|
+
uk_bin_collection/uk_bin_collection/councils/WalsallCouncil.py,sha256=_anovUnXMr40lZLHyX3opIP73BwauCllKy-Z2SBrzPw,2076
|
186
189
|
uk_bin_collection/uk_bin_collection/councils/WalthamForest.py,sha256=P7MMw0EhpRmDbbnHb25tY5_yvYuZUFwJ1br4TOv24sY,4997
|
187
190
|
uk_bin_collection/uk_bin_collection/councils/WarwickDistrictCouncil.py,sha256=3WQrAxzYzKoV4LyOqNTp9xINVsNi1xW9t8etducGeag,1146
|
188
191
|
uk_bin_collection/uk_bin_collection/councils/WaverleyBoroughCouncil.py,sha256=tp9l7vdgSGRzNNG0pDfnNuFj4D2bpRJUJmAiTJ6bM0g,4662
|
@@ -191,12 +194,14 @@ uk_bin_collection/uk_bin_collection/councils/WelhatCouncil.py,sha256=ikUft37dYNJ
|
|
191
194
|
uk_bin_collection/uk_bin_collection/councils/WestBerkshireCouncil.py,sha256=r90AIUdPgo85VuuvN_NeCDUy3NEJXdO4Ntt93yKo6qI,5110
|
192
195
|
uk_bin_collection/uk_bin_collection/councils/WestLindseyDistrictCouncil.py,sha256=JFWUy4w0CKulGq16PfbRDKAdQEbokVEuabwlZYigdEU,4606
|
193
196
|
uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py,sha256=dq0jimtARvRkZiGbVFrXXZgY-BODtz3uYZ5UKn0bf64,4114
|
197
|
+
uk_bin_collection/uk_bin_collection/councils/WestMorlandAndFurness.py,sha256=jbqV3460rn9D0yTBGWjpSe1IvWWcdGur5pzgj-hJcQ4,2513
|
194
198
|
uk_bin_collection/uk_bin_collection/councils/WestNorthamptonshireCouncil.py,sha256=F1GeJUGND4DN_HuM6N0Elpeb0DAMm9_KeqG6qtIgZf4,1079
|
195
199
|
uk_bin_collection/uk_bin_collection/councils/WestSuffolkCouncil.py,sha256=HMFWxM7VMhBuC7iubNGbZYEoCVWi--gRHDJMVdPPFOM,2633
|
196
200
|
uk_bin_collection/uk_bin_collection/councils/WiganBoroughCouncil.py,sha256=3gqFA4-BVx_In6QOu3KUNqPN4Fkn9iMlZTeopMK9p6A,3746
|
197
201
|
uk_bin_collection/uk_bin_collection/councils/WiltshireCouncil.py,sha256=it2Oh5Kmq3lD30gAZgk2bzZPNCtJcFHyQO1NgOQtfvU,5653
|
198
202
|
uk_bin_collection/uk_bin_collection/councils/WinchesterCityCouncil.py,sha256=W2k00N5n9-1MzjMEqsNjldsQdOJPEPMjK7OGSinZm5Y,4335
|
199
203
|
uk_bin_collection/uk_bin_collection/councils/WindsorAndMaidenheadCouncil.py,sha256=7Qhznj95ktAQjpWm5C8pbD5UcvfXm7Mwb7_DQxwjGSM,1777
|
204
|
+
uk_bin_collection/uk_bin_collection/councils/WirralCouncil.py,sha256=X_e9zXEZAl_Mp6nPORHc9CTmf3QHdoMY3BCnKrXEr1I,2131
|
200
205
|
uk_bin_collection/uk_bin_collection/councils/WokingBoroughCouncil.py,sha256=37igH9g0xe4XIhRhcJ-ZJBU8MxTp5yzgpadWbdE33Yg,5205
|
201
206
|
uk_bin_collection/uk_bin_collection/councils/WokinghamBoroughCouncil.py,sha256=qQOg5HjvHMTRuy3SVX8vbXhSqYBFZmN-htk_Dtoae0M,4126
|
202
207
|
uk_bin_collection/uk_bin_collection/councils/WychavonDistrictCouncil.py,sha256=YuZdzEW0CZLwusm1VQcGRIKXAab_UDFLaCnN60itt_E,5776
|
@@ -205,8 +210,8 @@ uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py,sha256=I2kBYMlsD4bId
|
|
205
210
|
uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=4s9ODGPAwPqwXc8SrTX5Wlfmizs3_58iXUtHc4Ir86o,1162
|
206
211
|
uk_bin_collection/uk_bin_collection/create_new_council.py,sha256=m-IhmWmeWQlFsTZC4OxuFvtw5ZtB8EAJHxJTH4O59lQ,1536
|
207
212
|
uk_bin_collection/uk_bin_collection/get_bin_data.py,sha256=YvmHfZqanwrJ8ToGch34x-L-7yPe31nB_x77_Mgl_vo,4545
|
208
|
-
uk_bin_collection-0.
|
209
|
-
uk_bin_collection-0.
|
210
|
-
uk_bin_collection-0.
|
211
|
-
uk_bin_collection-0.
|
212
|
-
uk_bin_collection-0.
|
213
|
+
uk_bin_collection-0.98.1.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
|
214
|
+
uk_bin_collection-0.98.1.dist-info/METADATA,sha256=dmWxsxgW5nOAahaqJsFqK-bn6f5rbGSQB9uq1x0EsYk,16843
|
215
|
+
uk_bin_collection-0.98.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
216
|
+
uk_bin_collection-0.98.1.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
|
217
|
+
uk_bin_collection-0.98.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|