uk_bin_collection 0.74.1__py3-none-any.whl → 0.75.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- uk_bin_collection/tests/test_common_functions.py +1 -1
- uk_bin_collection/uk_bin_collection/common.py +4 -1
- uk_bin_collection/uk_bin_collection/councils/ArunCouncil.py +6 -5
- uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py +2 -3
- uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py +5 -5
- uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py +7 -5
- uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py +5 -4
- uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py +10 -10
- uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py +6 -6
- uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py +3 -2
- uk_bin_collection/uk_bin_collection/councils/CalderdaleCouncil.py +9 -10
- uk_bin_collection/uk_bin_collection/councils/ChelmsfordCityCouncil.py +10 -10
- uk_bin_collection/uk_bin_collection/councils/CheshireWestAndChesterCouncil.py +35 -11
- uk_bin_collection/uk_bin_collection/councils/ChorleyCouncil.py +2 -1
- uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/EastRidingCouncil.py +6 -7
- uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py +3 -2
- uk_bin_collection/uk_bin_collection/councils/ForestOfDeanDistrictCouncil.py +5 -5
- uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/GuildfordCouncil.py +10 -10
- uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py +5 -4
- uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/KingstonUponThamesCouncil.py +6 -4
- uk_bin_collection/uk_bin_collection/councils/KirkleesCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/KnowsleyMBCouncil.py +4 -3
- uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py +6 -7
- uk_bin_collection/uk_bin_collection/councils/LondonBoroughRedbridge.py +10 -10
- uk_bin_collection/uk_bin_collection/councils/MidAndEastAntrimBoroughCouncil.py +4 -3
- uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py +2 -1
- uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py +3 -3
- uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/NorthWestLeicestershire.py +4 -4
- uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py +2 -2
- uk_bin_collection/uk_bin_collection/councils/PortsmouthCityCouncil.py +4 -4
- uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py +4 -4
- uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/RenfrewshireCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/SevenoaksDistrictCouncil.py +4 -4
- uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/SunderlandCityCouncil.py +4 -4
- uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py +1 -1
- uk_bin_collection/uk_bin_collection/councils/WestBerkshireCouncil.py +1 -4
- uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py +1 -1
- {uk_bin_collection-0.74.1.dist-info → uk_bin_collection-0.75.0.dist-info}/METADATA +1 -1
- {uk_bin_collection-0.74.1.dist-info → uk_bin_collection-0.75.0.dist-info}/RECORD +51 -51
- {uk_bin_collection-0.74.1.dist-info → uk_bin_collection-0.75.0.dist-info}/LICENSE +0 -0
- {uk_bin_collection-0.74.1.dist-info → uk_bin_collection-0.75.0.dist-info}/WHEEL +0 -0
- {uk_bin_collection-0.74.1.dist-info → uk_bin_collection-0.75.0.dist-info}/entry_points.txt +0 -0
@@ -332,7 +332,7 @@ def test_contains_date_with_mixed_content():
|
|
332
332
|
|
333
333
|
|
334
334
|
def test_create_webdriver_local():
|
335
|
-
result = create_webdriver(None, headless=True, user_agent="FireFox")
|
335
|
+
result = create_webdriver(None, headless=True, user_agent="FireFox", session_name="test-session")
|
336
336
|
assert result.name in ["chrome","chrome-headless-shell"]
|
337
337
|
|
338
338
|
|
@@ -258,7 +258,7 @@ def contains_date(string, fuzzy=False) -> bool:
|
|
258
258
|
|
259
259
|
|
260
260
|
def create_webdriver(
|
261
|
-
web_driver: str = None, headless: bool = True, user_agent: str = None
|
261
|
+
web_driver: str = None, headless: bool = True, user_agent: str = None, session_name: str = None
|
262
262
|
) -> webdriver.Chrome:
|
263
263
|
"""
|
264
264
|
Create and return a Chrome WebDriver configured for optional headless operation.
|
@@ -266,6 +266,7 @@ def create_webdriver(
|
|
266
266
|
:param web_driver: URL to the Selenium server for remote web drivers. If None, a local driver is created.
|
267
267
|
:param headless: Whether to run the browser in headless mode.
|
268
268
|
:param user_agent: Optional custom user agent string.
|
269
|
+
:param session_name: Optional custom session name string.
|
269
270
|
:return: An instance of a Chrome WebDriver.
|
270
271
|
:raises WebDriverException: If the WebDriver cannot be created.
|
271
272
|
"""
|
@@ -279,6 +280,8 @@ def create_webdriver(
|
|
279
280
|
if user_agent:
|
280
281
|
options.add_argument(f"--user-agent={user_agent}")
|
281
282
|
options.add_experimental_option("excludeSwitches", ["enable-logging"])
|
283
|
+
if session_name and web_driver:
|
284
|
+
options.set_capability("se:name", session_name)
|
282
285
|
|
283
286
|
try:
|
284
287
|
if web_driver:
|
@@ -1,11 +1,10 @@
|
|
1
1
|
import time
|
2
2
|
|
3
|
+
from bs4 import BeautifulSoup
|
3
4
|
from selenium import webdriver
|
4
5
|
from selenium.webdriver.common.by import By
|
5
|
-
from selenium.webdriver.support.ui import WebDriverWait
|
6
6
|
from selenium.webdriver.support import expected_conditions as EC
|
7
|
-
from selenium.webdriver.support.ui import Select
|
8
|
-
from bs4 import BeautifulSoup
|
7
|
+
from selenium.webdriver.support.ui import Select, WebDriverWait
|
9
8
|
|
10
9
|
from uk_bin_collection.uk_bin_collection.common import *
|
11
10
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
@@ -28,7 +27,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
28
27
|
user_postcode = kwargs.get("postcode")
|
29
28
|
headless = kwargs.get("headless")
|
30
29
|
web_driver = kwargs.get("web_driver")
|
31
|
-
driver = create_webdriver(web_driver, headless)
|
30
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
32
31
|
page = "https://www1.arun.gov.uk/when-are-my-bins-collected/"
|
33
32
|
check_paon(user_paon)
|
34
33
|
check_postcode(user_postcode)
|
@@ -78,7 +77,9 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
78
77
|
collection_type = (
|
79
78
|
row.find("th", class_="govuk-table__header").text.strip().split(" ")
|
80
79
|
)[0]
|
81
|
-
collection_date = row.find(
|
80
|
+
collection_date = row.find(
|
81
|
+
"td", class_="govuk-table__cell"
|
82
|
+
).text.strip()
|
82
83
|
|
83
84
|
# Append the information to the data structure
|
84
85
|
data["bins"].append(
|
@@ -3,8 +3,7 @@ import time
|
|
3
3
|
from bs4 import BeautifulSoup
|
4
4
|
from selenium.webdriver.common.by import By
|
5
5
|
from selenium.webdriver.support import expected_conditions as EC
|
6
|
-
from selenium.webdriver.support.ui import Select
|
7
|
-
from selenium.webdriver.support.ui import WebDriverWait
|
6
|
+
from selenium.webdriver.support.ui import Select, WebDriverWait
|
8
7
|
|
9
8
|
from uk_bin_collection.uk_bin_collection.common import *
|
10
9
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
@@ -70,7 +69,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
70
69
|
check_paon(user_paon)
|
71
70
|
headless = kwargs.get("headless")
|
72
71
|
web_driver = kwargs.get("web_driver")
|
73
|
-
driver = create_webdriver(web_driver, headless)
|
72
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
74
73
|
page = "https://account.barnet.gov.uk/Forms/Home/Redirector/Index/?id=6a2ac067-3322-46e5-96e4-16c0c214454a&mod=OA&casetype=BAR&formname=BNTCOLDATE"
|
75
74
|
driver.get(page)
|
76
75
|
|
@@ -1,16 +1,16 @@
|
|
1
|
-
|
1
|
+
import time
|
2
2
|
from datetime import datetime
|
3
|
+
|
4
|
+
from bs4 import BeautifulSoup
|
3
5
|
from selenium.webdriver.common.by import By
|
6
|
+
from selenium.webdriver.common.keys import Keys
|
4
7
|
from selenium.webdriver.support import expected_conditions as EC
|
5
8
|
from selenium.webdriver.support.ui import Select
|
6
9
|
from selenium.webdriver.support.wait import WebDriverWait
|
7
|
-
from selenium.webdriver.common.keys import Keys
|
8
10
|
|
9
|
-
import time
|
10
11
|
from uk_bin_collection.uk_bin_collection.common import *
|
11
12
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
12
13
|
|
13
|
-
|
14
14
|
# import the wonderful Beautiful Soup and the URL grabber
|
15
15
|
|
16
16
|
|
@@ -35,7 +35,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
35
35
|
headless = kwargs.get("headless")
|
36
36
|
|
37
37
|
# Create Selenium webdriver
|
38
|
-
driver = create_webdriver(web_driver, headless)
|
38
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
39
39
|
driver.get(page)
|
40
40
|
|
41
41
|
# If you bang in the house number (or property name) and postcode in the box it should find your property
|
@@ -1,13 +1,15 @@
|
|
1
1
|
import json
|
2
|
+
import logging
|
3
|
+
import ssl
|
2
4
|
from collections import OrderedDict
|
3
5
|
from datetime import datetime
|
4
|
-
|
6
|
+
|
5
7
|
import requests
|
8
|
+
import urllib3
|
9
|
+
from bs4 import BeautifulSoup
|
10
|
+
|
6
11
|
from uk_bin_collection.uk_bin_collection.common import *
|
7
12
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
8
|
-
import ssl
|
9
|
-
import urllib3
|
10
|
-
import logging
|
11
13
|
|
12
14
|
|
13
15
|
class CustomHttpAdapter(requests.adapters.HTTPAdapter):
|
@@ -48,7 +50,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
48
50
|
f"https://mybins.blackburn.gov.uk/api/mybins/getbincollectiondays?uprn={uprn}&month={current_month}"
|
49
51
|
f"&year={current_year}"
|
50
52
|
)
|
51
|
-
driver = create_webdriver(web_driver, headless)
|
53
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
52
54
|
driver.get(url)
|
53
55
|
|
54
56
|
soup = BeautifulSoup(driver.page_source, "html.parser")
|
@@ -1,12 +1,13 @@
|
|
1
|
-
|
1
|
+
import time
|
2
2
|
from datetime import datetime
|
3
|
+
|
4
|
+
from bs4 import BeautifulSoup
|
3
5
|
from selenium.webdriver.common.by import By
|
6
|
+
from selenium.webdriver.common.keys import Keys
|
4
7
|
from selenium.webdriver.support import expected_conditions as EC
|
5
8
|
from selenium.webdriver.support.ui import Select
|
6
9
|
from selenium.webdriver.support.wait import WebDriverWait
|
7
|
-
from selenium.webdriver.common.keys import Keys
|
8
10
|
|
9
|
-
import time
|
10
11
|
from uk_bin_collection.uk_bin_collection.common import *
|
11
12
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
12
13
|
|
@@ -34,7 +35,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
34
35
|
# Get our initial session running
|
35
36
|
page = "https://carehomes.bolton.gov.uk/bins.aspx"
|
36
37
|
|
37
|
-
driver = create_webdriver(web_driver, headless)
|
38
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
38
39
|
driver.get(page)
|
39
40
|
|
40
41
|
# If you bang in the house number (or property name) and postcode in the box it should find your property
|
@@ -1,19 +1,19 @@
|
|
1
|
-
import re
|
2
|
-
import requests
|
3
|
-
from bs4 import BeautifulSoup
|
4
|
-
|
5
|
-
from uk_bin_collection.uk_bin_collection.common import *
|
6
|
-
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
7
|
-
|
8
1
|
# This script pulls (in one hit) the data from Bromley Council Bins Data
|
9
2
|
import datetime
|
3
|
+
import re
|
4
|
+
import time
|
10
5
|
from datetime import datetime
|
6
|
+
|
7
|
+
import requests
|
8
|
+
from bs4 import BeautifulSoup
|
11
9
|
from selenium.webdriver.common.by import By
|
10
|
+
from selenium.webdriver.common.keys import Keys
|
12
11
|
from selenium.webdriver.support import expected_conditions as EC
|
13
12
|
from selenium.webdriver.support.ui import Select
|
14
13
|
from selenium.webdriver.support.wait import WebDriverWait
|
15
|
-
|
16
|
-
import
|
14
|
+
|
15
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
16
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
17
17
|
|
18
18
|
|
19
19
|
# import the wonderful Beautiful Soup and the URL grabber
|
@@ -35,7 +35,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
35
35
|
postcode = kwargs.get("postcode")
|
36
36
|
web_driver = kwargs.get("web_driver")
|
37
37
|
headless = kwargs.get("headless")
|
38
|
-
driver = create_webdriver(web_driver, headless)
|
38
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
39
39
|
driver.get(kwargs.get("url"))
|
40
40
|
|
41
41
|
wait = WebDriverWait(driver, 60)
|
@@ -1,16 +1,16 @@
|
|
1
1
|
# This script pulls (in one hit) the data from Bromley Council Bins Data
|
2
2
|
import datetime
|
3
|
-
|
3
|
+
import time
|
4
4
|
from datetime import datetime
|
5
|
+
|
6
|
+
from bs4 import BeautifulSoup
|
7
|
+
from dateutil.relativedelta import relativedelta
|
5
8
|
from selenium.webdriver.common.by import By
|
9
|
+
from selenium.webdriver.common.keys import Keys
|
6
10
|
from selenium.webdriver.support import expected_conditions as EC
|
7
11
|
from selenium.webdriver.support.ui import Select
|
8
12
|
from selenium.webdriver.support.wait import WebDriverWait
|
9
|
-
from selenium.webdriver.common.keys import Keys
|
10
|
-
import time
|
11
13
|
|
12
|
-
from dateutil.relativedelta import relativedelta
|
13
|
-
from bs4 import BeautifulSoup
|
14
14
|
from uk_bin_collection.uk_bin_collection.common import *
|
15
15
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
16
16
|
|
@@ -35,7 +35,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
35
35
|
data = {"bins": []}
|
36
36
|
|
37
37
|
# Get our initial session running
|
38
|
-
driver = create_webdriver(web_driver, headless)
|
38
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
39
39
|
driver.get(kwargs.get("url"))
|
40
40
|
|
41
41
|
wait = WebDriverWait(driver, 30)
|
@@ -31,7 +31,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
31
31
|
check_postcode(user_postcode)
|
32
32
|
|
33
33
|
# Create Selenium webdriver
|
34
|
-
driver = create_webdriver(web_driver, headless)
|
34
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
35
35
|
driver.get(page)
|
36
36
|
|
37
37
|
# Populate postcode field
|
@@ -1,5 +1,6 @@
|
|
1
|
-
import pandas as pd
|
2
1
|
import time
|
2
|
+
|
3
|
+
import pandas as pd
|
3
4
|
from selenium.webdriver.common.by import By
|
4
5
|
from selenium.webdriver.common.keys import Keys
|
5
6
|
from selenium.webdriver.support.ui import Select
|
@@ -42,7 +43,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
42
43
|
headless = kwargs.get("headless")
|
43
44
|
|
44
45
|
# Create Selenium webdriver
|
45
|
-
driver = create_webdriver(web_driver, headless)
|
46
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
46
47
|
driver.get(page)
|
47
48
|
|
48
49
|
# Enter postcode in text box and wait
|
@@ -1,19 +1,18 @@
|
|
1
|
-
import requests
|
2
|
-
from bs4 import BeautifulSoup
|
3
|
-
|
4
|
-
from uk_bin_collection.uk_bin_collection.common import *
|
5
|
-
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
6
|
-
|
7
1
|
# This script pulls (in one hit) the data from Bromley Council Bins Data
|
8
2
|
import datetime
|
9
|
-
|
3
|
+
import time
|
10
4
|
from datetime import datetime
|
5
|
+
|
6
|
+
import requests
|
7
|
+
from bs4 import BeautifulSoup
|
11
8
|
from selenium.webdriver.common.by import By
|
9
|
+
from selenium.webdriver.common.keys import Keys
|
12
10
|
from selenium.webdriver.support import expected_conditions as EC
|
13
11
|
from selenium.webdriver.support.ui import Select
|
14
12
|
from selenium.webdriver.support.wait import WebDriverWait
|
15
|
-
|
16
|
-
import
|
13
|
+
|
14
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
15
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
17
16
|
|
18
17
|
|
19
18
|
# import the wonderful Beautiful Soup and the URL grabber
|
@@ -40,7 +39,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
40
39
|
data = {"bins": []}
|
41
40
|
|
42
41
|
# Get our initial session running
|
43
|
-
driver = create_webdriver(web_driver, headless)
|
42
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
44
43
|
driver.get(kwargs.get("url"))
|
45
44
|
|
46
45
|
wait = WebDriverWait(driver, 30)
|
@@ -1,19 +1,19 @@
|
|
1
|
-
import re
|
2
|
-
import requests
|
3
|
-
from bs4 import BeautifulSoup
|
4
|
-
|
5
|
-
from uk_bin_collection.uk_bin_collection.common import *
|
6
|
-
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
7
|
-
|
8
1
|
# This script pulls (in one hit) the data from Bromley Council Bins Data
|
9
2
|
import datetime
|
3
|
+
import re
|
4
|
+
import time
|
10
5
|
from datetime import datetime
|
6
|
+
|
7
|
+
import requests
|
8
|
+
from bs4 import BeautifulSoup
|
11
9
|
from selenium.webdriver.common.by import By
|
10
|
+
from selenium.webdriver.common.keys import Keys
|
12
11
|
from selenium.webdriver.support import expected_conditions as EC
|
13
12
|
from selenium.webdriver.support.ui import Select
|
14
13
|
from selenium.webdriver.support.wait import WebDriverWait
|
15
|
-
|
16
|
-
import
|
14
|
+
|
15
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
16
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
17
17
|
|
18
18
|
|
19
19
|
# import the wonderful Beautiful Soup and the URL grabber
|
@@ -35,7 +35,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
35
35
|
user_paon = kwargs.get("paon")
|
36
36
|
web_driver = kwargs.get("web_driver")
|
37
37
|
headless = kwargs.get("headless")
|
38
|
-
driver = create_webdriver(web_driver, headless)
|
38
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
39
39
|
url = kwargs.get("url")
|
40
40
|
|
41
41
|
driver.execute_script(f"window.location.href='{url}'")
|
@@ -1,15 +1,20 @@
|
|
1
|
-
import time
|
2
1
|
import logging
|
2
|
+
import time
|
3
|
+
|
3
4
|
from bs4 import BeautifulSoup
|
4
5
|
from selenium.webdriver.common.by import By
|
5
6
|
from selenium.webdriver.support import expected_conditions as EC
|
6
7
|
from selenium.webdriver.support.ui import Select
|
7
8
|
from selenium.webdriver.support.wait import WebDriverWait
|
9
|
+
|
8
10
|
from uk_bin_collection.uk_bin_collection.common import *
|
9
11
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
10
12
|
|
11
13
|
# Set up logging
|
12
|
-
logging.basicConfig(
|
14
|
+
logging.basicConfig(
|
15
|
+
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
|
16
|
+
)
|
17
|
+
|
13
18
|
|
14
19
|
class CouncilClass(AbstractGetBinDataClass):
|
15
20
|
def parse_data(self, page: str, **kwargs) -> dict:
|
@@ -26,11 +31,13 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
26
31
|
check_postcode(user_postcode)
|
27
32
|
|
28
33
|
# Create Selenium webdriver
|
29
|
-
driver = create_webdriver(web_driver, headless)
|
34
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
30
35
|
if headless:
|
31
36
|
driver.set_window_size(1920, 1080)
|
32
37
|
|
33
|
-
driver.get(
|
38
|
+
driver.get(
|
39
|
+
"https://www.cheshirewestandchester.gov.uk/residents/waste-and-recycling/your-bin-collection/collection-day"
|
40
|
+
)
|
34
41
|
wait = WebDriverWait(driver, 60)
|
35
42
|
|
36
43
|
def click_element(by, value):
|
@@ -45,26 +52,44 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
45
52
|
click_element(By.LINK_TEXT, "Find your collection day")
|
46
53
|
|
47
54
|
logging.info("Switching to iframe")
|
48
|
-
iframe_presence = wait.until(
|
55
|
+
iframe_presence = wait.until(
|
56
|
+
EC.presence_of_element_located((By.ID, "fillform-frame-1"))
|
57
|
+
)
|
49
58
|
driver.switch_to.frame(iframe_presence)
|
50
59
|
|
51
60
|
logging.info("Entering postcode")
|
52
|
-
input_element_postcode = wait.until(
|
61
|
+
input_element_postcode = wait.until(
|
62
|
+
EC.presence_of_element_located(
|
63
|
+
(By.XPATH, '//input[@id="postcode_search"]')
|
64
|
+
)
|
65
|
+
)
|
53
66
|
input_element_postcode.send_keys(user_postcode)
|
54
67
|
|
55
|
-
pcsearch_btn = wait.until(
|
68
|
+
pcsearch_btn = wait.until(
|
69
|
+
EC.element_to_be_clickable((By.XPATH, "//input[@id='postcode_search']"))
|
70
|
+
)
|
56
71
|
click_element(By.XPATH, "//input[@id='postcode_search']")
|
57
72
|
|
58
73
|
logging.info("Selecting address")
|
59
74
|
dropdown = wait.until(EC.element_to_be_clickable((By.ID, "Choose_Address")))
|
60
|
-
dropdown_options = wait.until(
|
75
|
+
dropdown_options = wait.until(
|
76
|
+
EC.presence_of_element_located((By.CLASS_NAME, "lookup-option"))
|
77
|
+
)
|
61
78
|
drop_down_values = Select(dropdown)
|
62
|
-
option_element = wait.until(
|
79
|
+
option_element = wait.until(
|
80
|
+
EC.presence_of_element_located(
|
81
|
+
(By.CSS_SELECTOR, f'option.lookup-option[value="{str(user_uprn)}"]')
|
82
|
+
)
|
83
|
+
)
|
63
84
|
driver.execute_script("arguments[0].scrollIntoView();", option_element)
|
64
85
|
drop_down_values.select_by_value(str(user_uprn))
|
65
86
|
|
66
87
|
logging.info("Waiting for bin schedule")
|
67
|
-
wait.until(
|
88
|
+
wait.until(
|
89
|
+
EC.presence_of_element_located(
|
90
|
+
(By.CLASS_NAME, "bin-schedule-content-bin-card")
|
91
|
+
)
|
92
|
+
)
|
68
93
|
|
69
94
|
logging.info("Extracting bin collection data")
|
70
95
|
soup = BeautifulSoup(driver.page_source, features="html.parser")
|
@@ -97,4 +122,3 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
97
122
|
finally:
|
98
123
|
if driver:
|
99
124
|
driver.quit()
|
100
|
-
|
@@ -1,5 +1,6 @@
|
|
1
1
|
import time
|
2
2
|
import urllib.parse
|
3
|
+
|
3
4
|
from bs4 import BeautifulSoup
|
4
5
|
from selenium.webdriver.common.by import By
|
5
6
|
from selenium.webdriver.support import expected_conditions as EC
|
@@ -42,7 +43,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
42
43
|
|
43
44
|
# Create Selenium webdriver
|
44
45
|
user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
45
|
-
driver = create_webdriver(web_driver, headless, user_agent)
|
46
|
+
driver = create_webdriver(web_driver, headless, user_agent, __name__)
|
46
47
|
driver.get("https://myaccount.chorley.gov.uk/wastecollections.aspx")
|
47
48
|
|
48
49
|
# Accept cookies banner
|
@@ -31,7 +31,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
31
31
|
check_postcode(user_postcode)
|
32
32
|
|
33
33
|
# Create Selenium webdriver
|
34
|
-
driver = create_webdriver(web_driver, headless)
|
34
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
35
35
|
driver.get(page)
|
36
36
|
|
37
37
|
# Populate postcode field
|
@@ -27,7 +27,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
27
27
|
check_postcode(user_postcode)
|
28
28
|
|
29
29
|
# Create Selenium webdriver
|
30
|
-
driver = create_webdriver(web_driver, headless)
|
30
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
31
31
|
driver.get(
|
32
32
|
"https://www.e-lindsey.gov.uk/article/6714/Your-Waste-Collection-Days"
|
33
33
|
)
|
@@ -1,17 +1,16 @@
|
|
1
|
+
import urllib.request
|
1
2
|
from datetime import datetime
|
2
|
-
from uk_bin_collection.uk_bin_collection.common import *
|
3
|
-
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
4
3
|
|
4
|
+
import pandas as pd
|
5
5
|
from bs4 import BeautifulSoup
|
6
|
-
from datetime import datetime
|
7
6
|
from selenium.webdriver.common.by import By
|
7
|
+
from selenium.webdriver.common.keys import Keys
|
8
8
|
from selenium.webdriver.support import expected_conditions as EC
|
9
9
|
from selenium.webdriver.support.ui import Select
|
10
10
|
from selenium.webdriver.support.wait import WebDriverWait
|
11
|
-
from selenium.webdriver.common.keys import Keys
|
12
11
|
|
13
|
-
|
14
|
-
import
|
12
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
13
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
15
14
|
|
16
15
|
|
17
16
|
class CouncilClass(AbstractGetBinDataClass):
|
@@ -32,7 +31,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
32
31
|
# Create Selenium webdriver
|
33
32
|
page = f"https://www.eastriding.gov.uk/environment/bins-rubbish-recycling/bins-and-collections/bin-collection-dates/"
|
34
33
|
|
35
|
-
driver = create_webdriver(web_driver, headless)
|
34
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
36
35
|
driver.get(page)
|
37
36
|
|
38
37
|
wait = WebDriverWait(driver, 60)
|
@@ -1,8 +1,9 @@
|
|
1
1
|
from time import sleep
|
2
|
+
|
2
3
|
from bs4 import BeautifulSoup
|
3
4
|
from selenium.webdriver.common.by import By
|
4
5
|
from selenium.webdriver.support import expected_conditions as EC
|
5
|
-
from selenium.webdriver.support.ui import
|
6
|
+
from selenium.webdriver.support.ui import Select, WebDriverWait
|
6
7
|
|
7
8
|
from uk_bin_collection.uk_bin_collection.common import *
|
8
9
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
@@ -27,7 +28,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
27
28
|
check_postcode(user_postcode)
|
28
29
|
|
29
30
|
# Create Selenium webdriver
|
30
|
-
driver = create_webdriver(web_driver, headless)
|
31
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
31
32
|
driver.get(
|
32
33
|
"https://my.eastsuffolk.gov.uk/service/Bin_collection_dates_finder"
|
33
34
|
)
|
@@ -1,16 +1,16 @@
|
|
1
|
-
|
1
|
+
import time
|
2
2
|
from datetime import datetime
|
3
|
+
|
4
|
+
from bs4 import BeautifulSoup
|
3
5
|
from selenium.webdriver.common.by import By
|
6
|
+
from selenium.webdriver.common.keys import Keys
|
4
7
|
from selenium.webdriver.support import expected_conditions as EC
|
5
8
|
from selenium.webdriver.support.ui import Select
|
6
9
|
from selenium.webdriver.support.wait import WebDriverWait
|
7
|
-
from selenium.webdriver.common.keys import Keys
|
8
|
-
import time
|
9
10
|
|
10
11
|
from uk_bin_collection.uk_bin_collection.common import *
|
11
12
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
12
13
|
|
13
|
-
|
14
14
|
# import the wonderful Beautiful Soup and the URL grabber
|
15
15
|
|
16
16
|
|
@@ -35,7 +35,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
35
35
|
headless = kwargs.get("headless")
|
36
36
|
|
37
37
|
# Create Selenium webdriver
|
38
|
-
driver = create_webdriver(web_driver, headless)
|
38
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
39
39
|
driver.get(page)
|
40
40
|
|
41
41
|
# If you bang in the house number (or property name) and postcode in the box it should find your property
|
@@ -27,7 +27,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
27
27
|
check_postcode(user_postcode)
|
28
28
|
|
29
29
|
# Create Selenium webdriver
|
30
|
-
driver = create_webdriver(web_driver, headless)
|
30
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
31
31
|
driver.get(
|
32
32
|
"https://www.gateshead.gov.uk/article/3150/Bin-collection-day-checker"
|
33
33
|
)
|
@@ -1,19 +1,19 @@
|
|
1
|
-
import re
|
2
|
-
import requests
|
3
|
-
from bs4 import BeautifulSoup
|
4
|
-
|
5
|
-
from uk_bin_collection.uk_bin_collection.common import *
|
6
|
-
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
7
|
-
|
8
1
|
# This script pulls (in one hit) the data from Bromley Council Bins Data
|
9
2
|
import datetime
|
3
|
+
import re
|
4
|
+
import time
|
10
5
|
from datetime import datetime
|
6
|
+
|
7
|
+
import requests
|
8
|
+
from bs4 import BeautifulSoup
|
11
9
|
from selenium.webdriver.common.by import By
|
10
|
+
from selenium.webdriver.common.keys import Keys
|
12
11
|
from selenium.webdriver.support import expected_conditions as EC
|
13
12
|
from selenium.webdriver.support.ui import Select
|
14
13
|
from selenium.webdriver.support.wait import WebDriverWait
|
15
|
-
|
16
|
-
import
|
14
|
+
|
15
|
+
from uk_bin_collection.uk_bin_collection.common import *
|
16
|
+
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
17
17
|
|
18
18
|
|
19
19
|
# import the wonderful Beautiful Soup and the URL grabber
|
@@ -36,7 +36,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
36
36
|
web_driver = kwargs.get("web_driver")
|
37
37
|
headless = kwargs.get("headless")
|
38
38
|
|
39
|
-
driver = create_webdriver(web_driver, headless)
|
39
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
40
40
|
driver.get(kwargs.get("url"))
|
41
41
|
|
42
42
|
wait = WebDriverWait(driver, 120)
|
@@ -1,15 +1,16 @@
|
|
1
|
-
|
1
|
+
import time
|
2
2
|
from datetime import datetime
|
3
|
+
|
4
|
+
from bs4 import BeautifulSoup
|
3
5
|
from selenium.webdriver.common.by import By
|
4
6
|
from selenium.webdriver.common.keys import Keys
|
5
7
|
from selenium.webdriver.support import expected_conditions as EC
|
6
8
|
from selenium.webdriver.support.ui import Select
|
7
9
|
from selenium.webdriver.support.wait import WebDriverWait
|
8
|
-
|
10
|
+
|
9
11
|
from uk_bin_collection.uk_bin_collection.common import *
|
10
12
|
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
11
13
|
|
12
|
-
|
13
14
|
# import the wonderful Beautiful Soup and the URL grabber
|
14
15
|
|
15
16
|
|
@@ -34,7 +35,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
34
35
|
f"https://webapp.halton.gov.uk/PublicWebForms/WasteServiceSearchv1.aspx"
|
35
36
|
)
|
36
37
|
|
37
|
-
driver = create_webdriver(web_driver, headless)
|
38
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
38
39
|
driver.get(page)
|
39
40
|
|
40
41
|
# If you bang in the house number (or property name) and postcode in the box it should find your property
|
@@ -66,7 +66,7 @@ class CouncilClass(AbstractGetBinDataClass):
|
|
66
66
|
headless = kwargs.get("headless")
|
67
67
|
|
68
68
|
# Create Selenium webdriver
|
69
|
-
driver = create_webdriver(web_driver, headless)
|
69
|
+
driver = create_webdriver(web_driver, headless, None, __name__)
|
70
70
|
driver.get(page)
|
71
71
|
|
72
72
|
# Hide Cookies
|