uk_bin_collection 0.152.3__py3-none-any.whl → 0.152.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- uk_bin_collection/tests/input.json +4 -4
- uk_bin_collection/uk_bin_collection/councils/SouthRibbleCouncil.py +100 -103
- {uk_bin_collection-0.152.3.dist-info → uk_bin_collection-0.152.5.dist-info}/METADATA +1 -1
- {uk_bin_collection-0.152.3.dist-info → uk_bin_collection-0.152.5.dist-info}/RECORD +7 -7
- {uk_bin_collection-0.152.3.dist-info → uk_bin_collection-0.152.5.dist-info}/LICENSE +0 -0
- {uk_bin_collection-0.152.3.dist-info → uk_bin_collection-0.152.5.dist-info}/WHEEL +0 -0
- {uk_bin_collection-0.152.3.dist-info → uk_bin_collection-0.152.5.dist-info}/entry_points.txt +0 -0
|
@@ -2180,10 +2180,10 @@
|
|
|
2180
2180
|
"LAD24CD": "E07000179"
|
|
2181
2181
|
},
|
|
2182
2182
|
"SouthRibbleCouncil": {
|
|
2183
|
-
"uprn": "
|
|
2184
|
-
"postcode": "
|
|
2185
|
-
"url": "https://
|
|
2186
|
-
"wiki_command_url_override": "https://
|
|
2183
|
+
"uprn": "10013243496",
|
|
2184
|
+
"postcode": "PR266QW",
|
|
2185
|
+
"url": "https://forms.chorleysouthribble.gov.uk/xfp/form/70",
|
|
2186
|
+
"wiki_command_url_override": "https://forms.chorleysouthribble.gov.uk/xfp/form/70",
|
|
2187
2187
|
"wiki_name": "South Ribble",
|
|
2188
2188
|
"wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN.",
|
|
2189
2189
|
"LAD24CD": "E07000126"
|
|
@@ -2,132 +2,129 @@ from typing import Dict, List, Any, Optional
|
|
|
2
2
|
from bs4 import BeautifulSoup
|
|
3
3
|
from dateutil.relativedelta import relativedelta
|
|
4
4
|
import requests
|
|
5
|
-
import logging
|
|
6
5
|
import re
|
|
7
6
|
from datetime import datetime
|
|
8
|
-
from uk_bin_collection.uk_bin_collection.common import
|
|
9
|
-
from dateutil.parser import parse
|
|
10
|
-
|
|
11
|
-
from uk_bin_collection.uk_bin_collection.common import check_uprn, check_postcode
|
|
7
|
+
from uk_bin_collection.uk_bin_collection.common import check_uprn, check_postcode, date_format
|
|
12
8
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def get_token(page) -> str:
|
|
16
|
-
"""
|
|
17
|
-
Get a __token to include in the form data
|
|
18
|
-
:param page: Page html
|
|
19
|
-
:return: Form __token
|
|
20
|
-
"""
|
|
21
|
-
soup = BeautifulSoup(page.text, features="html.parser")
|
|
22
|
-
soup.prettify()
|
|
23
|
-
token = soup.find("input", {"name": "__token"}).get("value")
|
|
24
|
-
return token
|
|
9
|
+
from dateutil.parser import parse
|
|
25
10
|
|
|
26
11
|
|
|
27
12
|
class CouncilClass(AbstractGetBinDataClass):
|
|
28
|
-
"""
|
|
29
|
-
Concrete classes have to implement all abstract operations of the
|
|
30
|
-
base class. They can also override some operations with a default
|
|
31
|
-
implementation.
|
|
32
|
-
"""
|
|
33
|
-
|
|
34
13
|
def get_data(self, url: str) -> str:
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
Keyword arguments:
|
|
38
|
-
url -- the url to get the data from
|
|
39
|
-
"""
|
|
40
|
-
# Set a user agent so we look like a browser ;-)
|
|
41
|
-
user_agent = (
|
|
42
|
-
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
43
|
-
"Chrome/108.0.0.0 Safari/537.36"
|
|
44
|
-
)
|
|
45
|
-
headers = {"User-Agent": user_agent}
|
|
46
|
-
requests.packages.urllib3.disable_warnings()
|
|
47
|
-
|
|
48
|
-
# Make the Request - change the URL - find out your property number
|
|
49
|
-
try:
|
|
50
|
-
session = requests.Session()
|
|
51
|
-
session.headers.update(headers)
|
|
52
|
-
full_page = session.get(url)
|
|
53
|
-
return full_page
|
|
54
|
-
except requests.exceptions.HTTPError as errh:
|
|
55
|
-
logging.error(f"Http Error: {errh}")
|
|
56
|
-
raise
|
|
57
|
-
except requests.exceptions.ConnectionError as errc:
|
|
58
|
-
logging.error(f"Error Connecting: {errc}")
|
|
59
|
-
raise
|
|
60
|
-
except requests.exceptions.Timeout as errt:
|
|
61
|
-
logging.error(f"Timeout Error: {errt}")
|
|
62
|
-
raise
|
|
63
|
-
except requests.exceptions.RequestException as err:
|
|
64
|
-
logging.error(f"Oops: Something Else {err}")
|
|
65
|
-
raise
|
|
14
|
+
# This method is not used in the current implementation
|
|
15
|
+
return ""
|
|
66
16
|
|
|
67
17
|
def parse_data(self, page: str, **kwargs: Any) -> Dict[str, List[Dict[str, str]]]:
|
|
68
|
-
uprn: Optional[str] = kwargs.get("uprn")
|
|
69
18
|
postcode: Optional[str] = kwargs.get("postcode")
|
|
19
|
+
uprn: Optional[str] = kwargs.get("uprn")
|
|
70
20
|
|
|
71
|
-
if uprn is None:
|
|
72
|
-
raise ValueError("
|
|
73
|
-
if postcode is None:
|
|
74
|
-
raise ValueError("Postcode is required and must be a non-empty string.")
|
|
21
|
+
if postcode is None or uprn is None:
|
|
22
|
+
raise ValueError("Both postcode and UPRN are required.")
|
|
75
23
|
|
|
76
|
-
check_uprn(uprn)
|
|
77
24
|
check_postcode(postcode)
|
|
25
|
+
check_uprn(uprn)
|
|
78
26
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
"
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
"next": "Next",
|
|
27
|
+
session = requests.Session()
|
|
28
|
+
headers = {
|
|
29
|
+
"User-Agent": (
|
|
30
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
31
|
+
"(KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36"
|
|
32
|
+
)
|
|
86
33
|
}
|
|
87
|
-
headers
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
34
|
+
session.headers.update(headers)
|
|
35
|
+
|
|
36
|
+
# Step 1: Load form and get token + field names
|
|
37
|
+
initial_url = "https://forms.chorleysouthribble.gov.uk/xfp/form/70"
|
|
38
|
+
get_resp = session.get(initial_url)
|
|
39
|
+
soup = BeautifulSoup(get_resp.text, "html.parser")
|
|
40
|
+
|
|
41
|
+
token = soup.find("input", {"name": "__token"})["value"]
|
|
42
|
+
page_id = soup.find("input", {"name": "page"})["value"]
|
|
43
|
+
postcode_field = soup.find("input", {"type": "text", "name": re.compile(".*_0_0")})["name"]
|
|
44
|
+
|
|
45
|
+
# Step 2: Submit postcode
|
|
46
|
+
post_resp = session.post(
|
|
47
|
+
initial_url,
|
|
48
|
+
data={
|
|
49
|
+
"__token": token,
|
|
50
|
+
"page": page_id,
|
|
51
|
+
"locale": "en_GB",
|
|
52
|
+
postcode_field: postcode,
|
|
53
|
+
"next": "Next",
|
|
54
|
+
},
|
|
94
55
|
)
|
|
95
56
|
|
|
96
|
-
soup = BeautifulSoup(
|
|
57
|
+
soup = BeautifulSoup(post_resp.text, "html.parser")
|
|
58
|
+
token = soup.find("input", {"name": "__token"})["value"]
|
|
59
|
+
address_field_el = soup.find("select", {"name": re.compile(".*_1_0")})
|
|
60
|
+
if not address_field_el:
|
|
61
|
+
raise ValueError("Failed to find address dropdown after postcode submission.")
|
|
62
|
+
|
|
63
|
+
address_field = address_field_el["name"]
|
|
64
|
+
|
|
65
|
+
# Step 3: Submit UPRN and retrieve bin data
|
|
66
|
+
final_resp = session.post(
|
|
67
|
+
initial_url,
|
|
68
|
+
data={
|
|
69
|
+
"__token": token,
|
|
70
|
+
"page": page_id,
|
|
71
|
+
"locale": "en_GB",
|
|
72
|
+
postcode_field: postcode,
|
|
73
|
+
address_field: uprn,
|
|
74
|
+
"next": "Next",
|
|
75
|
+
},
|
|
76
|
+
)
|
|
97
77
|
|
|
98
|
-
|
|
78
|
+
soup = BeautifulSoup(final_resp.text, "html.parser")
|
|
79
|
+
table = soup.find("table", class_="data-table")
|
|
80
|
+
if not table:
|
|
81
|
+
raise ValueError("Could not find bin collection table.")
|
|
99
82
|
|
|
100
|
-
|
|
83
|
+
rows = table.find("tbody").find_all("tr")
|
|
101
84
|
data: Dict[str, List[Dict[str, str]]] = {"bins": []}
|
|
102
85
|
|
|
103
|
-
#
|
|
86
|
+
# Extract bin type mapping from JavaScript
|
|
87
|
+
bin_type_map = {}
|
|
88
|
+
scripts = soup.find_all("script", type="text/javascript")
|
|
89
|
+
for script in scripts:
|
|
90
|
+
if script.string and "const bintype = {" in script.string:
|
|
91
|
+
match = re.search(r'const bintype = \{([^}]+)\}', script.string, re.DOTALL)
|
|
92
|
+
if match:
|
|
93
|
+
bintype_content = match.group(1)
|
|
94
|
+
for line in bintype_content.split('\n'):
|
|
95
|
+
line = line.strip()
|
|
96
|
+
if '"' in line and ':' in line:
|
|
97
|
+
parts = line.split(':', 1)
|
|
98
|
+
if len(parts) == 2:
|
|
99
|
+
key = parts[0].strip().strip('"').strip("'")
|
|
100
|
+
value = parts[1].strip().rstrip(',').strip().strip('"').strip("'")
|
|
101
|
+
bin_type_map[key] = value
|
|
102
|
+
break
|
|
103
|
+
|
|
104
104
|
for row in rows:
|
|
105
105
|
cells = row.find_all("td")
|
|
106
|
-
if cells:
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
106
|
+
if len(cells) >= 2:
|
|
107
|
+
bin_type_cell = cells[0]
|
|
108
|
+
bin_type = bin_type_cell.get_text(strip=True)
|
|
109
|
+
bin_type = bin_type_map.get(bin_type, bin_type)
|
|
110
|
+
|
|
111
|
+
date_text = cells[1].get_text(strip=True)
|
|
112
|
+
date_parts = date_text.split(", ")
|
|
113
|
+
date_str = date_parts[1] if len(date_parts) == 2 else date_text
|
|
114
|
+
|
|
115
|
+
try:
|
|
116
|
+
day, month, year = date_str.split('/')
|
|
117
|
+
year = int(year)
|
|
118
|
+
if year < 100:
|
|
119
|
+
year = 2000 + year
|
|
120
|
+
|
|
121
|
+
date_obj = datetime(year, int(month), int(day)).date()
|
|
122
|
+
|
|
123
|
+
data["bins"].append({
|
|
124
|
+
"type": bin_type,
|
|
125
|
+
"collectionDate": date_obj.strftime(date_format)
|
|
126
|
+
})
|
|
127
|
+
except Exception:
|
|
113
128
|
continue
|
|
114
129
|
|
|
115
|
-
collection_date_obj = parse(
|
|
116
|
-
re.sub(r"[()]", "", collection_date[0])
|
|
117
|
-
).date()
|
|
118
|
-
|
|
119
|
-
# since we only have the next collection day, if the parsed date is in the past,
|
|
120
|
-
# assume the day is instead next month
|
|
121
|
-
if collection_date_obj < datetime.now().date():
|
|
122
|
-
collection_date_obj += relativedelta(months=1)
|
|
123
|
-
|
|
124
|
-
# Make each Bin element in the JSON
|
|
125
|
-
dict_data = {
|
|
126
|
-
"type": bin_type,
|
|
127
|
-
"collectionDate": collection_date_obj.strftime(date_format),
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
# Add data to the main JSON Wrapper
|
|
131
|
-
data["bins"].append(dict_data)
|
|
132
|
-
|
|
133
130
|
return data
|
|
@@ -7,7 +7,7 @@ uk_bin_collection/tests/council_feature_input_parity.py,sha256=DO6Mk4ImYgM5ZCZ-c
|
|
|
7
7
|
uk_bin_collection/tests/features/environment.py,sha256=VQZjJdJI_kZn08M0j5cUgvKT4k3iTw8icJge1DGOkoA,127
|
|
8
8
|
uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=SJK-Vc737hrf03tssxxbeg_JIvAH-ddB8f6gU1LTbuQ,251
|
|
9
9
|
uk_bin_collection/tests/generate_map_test_results.py,sha256=CKnGK2ZgiSXomRGkomX90DitgMP-X7wkHhyKORDcL2E,1144
|
|
10
|
-
uk_bin_collection/tests/input.json,sha256=
|
|
10
|
+
uk_bin_collection/tests/input.json,sha256=A0di9py05xaAUGIAuhzmYPoZYqPZFP9mubA3UgHCwpg,132589
|
|
11
11
|
uk_bin_collection/tests/output.schema,sha256=ZwKQBwYyTDEM4G2hJwfLUVM-5v1vKRvRK9W9SS1sd18,1086
|
|
12
12
|
uk_bin_collection/tests/step_defs/step_helpers/file_handler.py,sha256=Ygzi4V0S1MIHqbdstUlIqtRIwnynvhu4UtpweJ6-5N8,1474
|
|
13
13
|
uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=VZ0a81sioJULD7syAYHjvK_-nT_Rd36tUyzPetSA0gk,3475
|
|
@@ -271,7 +271,7 @@ uk_bin_collection/uk_bin_collection/councils/SouthKestevenDistrictCouncil.py,sha
|
|
|
271
271
|
uk_bin_collection/uk_bin_collection/councils/SouthLanarkshireCouncil.py,sha256=fj-eZI0yrvQVCv8GvhcovZ3b9bV6Xv_ws3IunWjnv4U,3126
|
|
272
272
|
uk_bin_collection/uk_bin_collection/councils/SouthNorfolkCouncil.py,sha256=C2qIZjjbl9JnuukX9OH2RbfP0hSdp3uX76APGY33qKs,4622
|
|
273
273
|
uk_bin_collection/uk_bin_collection/councils/SouthOxfordshireCouncil.py,sha256=f9d2YDGv5hnN7Ul-u_I63h_BbpBU7CJFdgv-lOviRGc,4031
|
|
274
|
-
uk_bin_collection/uk_bin_collection/councils/SouthRibbleCouncil.py,sha256=
|
|
274
|
+
uk_bin_collection/uk_bin_collection/councils/SouthRibbleCouncil.py,sha256=juX7pg7dwqxzZUXHcGeFz8h7SNoUsZWUl8uiEOptE3k,5012
|
|
275
275
|
uk_bin_collection/uk_bin_collection/councils/SouthStaffordshireDistrictCouncil.py,sha256=ACQMHWyamnj1ag3gNF-8Jhp-DKUok1GhFdnzH4nCzwU,3201
|
|
276
276
|
uk_bin_collection/uk_bin_collection/councils/SouthTynesideCouncil.py,sha256=dxXGrJfg_fn2IPTBgq6Duwy0WY8GYLafMuisaCjOnbs,3426
|
|
277
277
|
uk_bin_collection/uk_bin_collection/councils/SouthamptonCityCouncil.py,sha256=exNoI-Vun_C5FowCYhZ_600MBUe_OPR7MdGZEMNLL0I,1542
|
|
@@ -346,8 +346,8 @@ uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py,sha256=I2kBYMlsD4bId
|
|
|
346
346
|
uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=QD4v4xpsEE0QheR_fGaNOIRMc2FatcUfKkkhAhseyVU,1159
|
|
347
347
|
uk_bin_collection/uk_bin_collection/create_new_council.py,sha256=m-IhmWmeWQlFsTZC4OxuFvtw5ZtB8EAJHxJTH4O59lQ,1536
|
|
348
348
|
uk_bin_collection/uk_bin_collection/get_bin_data.py,sha256=YvmHfZqanwrJ8ToGch34x-L-7yPe31nB_x77_Mgl_vo,4545
|
|
349
|
-
uk_bin_collection-0.152.
|
|
350
|
-
uk_bin_collection-0.152.
|
|
351
|
-
uk_bin_collection-0.152.
|
|
352
|
-
uk_bin_collection-0.152.
|
|
353
|
-
uk_bin_collection-0.152.
|
|
349
|
+
uk_bin_collection-0.152.5.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
|
|
350
|
+
uk_bin_collection-0.152.5.dist-info/METADATA,sha256=_fyAz52D1iFURq-XyITh526l91Pc6YQGO4rn7xGotQ8,26688
|
|
351
|
+
uk_bin_collection-0.152.5.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
|
352
|
+
uk_bin_collection-0.152.5.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
|
|
353
|
+
uk_bin_collection-0.152.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
{uk_bin_collection-0.152.3.dist-info → uk_bin_collection-0.152.5.dist-info}/entry_points.txt
RENAMED
|
File without changes
|