uk_bin_collection 0.77.0__py3-none-any.whl → 0.79.0__py3-none-any.whl

uk_bin_collection/tests/input.json

@@ -222,6 +222,15 @@
         "web_driver": "http://selenium:4444",
         "wiki_name": "Cheshire West and Chester Council"
     },
+    "ChichesterDistrictCouncil": {
+        "house_number": "7, Plaistow Road, Kirdford, Billingshurst, West Sussex",
+        "postcode": "RH14 0JT",
+        "skip_get_url": true,
+        "url": "https://www.chichester.gov.uk/checkyourbinday",
+        "web_driver": "http://selenium:4444",
+        "wiki_name": "ChichesterDistrictCouncil",
+        "wiki_note": "Needs the full address and postcode as appears on page https://www.chichester.gov.uk/checkyourbinday"
+    },
     "ChorleyCouncil": {
         "postcode": "PR6 7PG",
         "skip_get_url": true,
@@ -253,6 +262,12 @@
         "url": "https://service.croydon.gov.uk/wasteservices/w/webpage/bin-day-enter-address",
         "wiki_name": "Croydon Council"
     },
+    "DartfordBoroughCouncil": {
+        "url": "https://windmz.dartford.gov.uk/ufs/WS_CHECK_COLLECTIONS.eb?UPRN=010094157511",
+        "wiki_name": "DartfordBoroughCouncil",
+        "uprn": "010094157511",
+        "wiki_note": "Use https://uprn.uk/ to find your UPRN "
+    },
     "DerbyshireDalesDistrictCouncil": {
         "postcode": "DE4 3AS",
         "skip_get_url": true,
@@ -336,12 +351,23 @@
         "wiki_name": "Environment First",
         "wiki_note": "For properties with collections managed by Environment First, such as Lewes and Eastbourne.\nReplace the XXXXXXXXXXX with the UPRN of your property - you can use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find this."
     },
+    "EppingForestDistrictCouncil": {
+        "postcode": "IG9 6EP",
+        "url": "https://eppingforestdc.maps.arcgis.com/apps/instant/lookup/index.html?appid=bfca32b46e2a47cd9c0a84f2d8cdde17&find=IG9%206EP",
+        "wiki_name": "Epping Forest District Council"
+    },
     "ErewashBoroughCouncil": {
         "skip_get_url": true,
         "uprn": "10003582028",
         "url": "https://map.erewash.gov.uk/isharelive.web/myerewash.aspx",
         "wiki_name": "Erewash Borough Council"
     },
+    "FarehamBoroughCouncil": {
+        "postcode": "PO14 4NR",
+        "skip_get_url": true,
+        "url": "https://www.fareham.gov.uk/internetlookups/search_data.aspx?type=JSON&list=DomesticBinCollections&Road=&Postcode=PO14%204NR",
+        "wiki_name": "Fareham Borough Council"
+    },
     "FenlandDistrictCouncil": {
         "skip_get_url": true,
         "uprn": "200002981143",
@@ -421,6 +447,15 @@
         "wiki_name": "High Peak Council",
         "wiki_note": "Pass the name of the street with the house number parameter, wrapped in double quotes"
     },
+    "HounslowCouncil": {
+        "house_number": "17A LAMPTON PARK ROAD, HOUNSLOW",
+        "postcode": "TW3 4HS",
+        "skip_get_url": true,
+        "uprn": "10091596698",
+        "url": "https://www.hounslow.gov.uk/info/20272/recycling_and_waste_collection_day_finder",
+        "web_driver": "http://selenium:4444",
+        "wiki_name": "HounslowCouncil"
+    },
     "HullCityCouncil": {
         "skip_get_url": true,
         "uprn": "21033995",
@@ -555,7 +590,7 @@
         "postcode": "RH4 1SJ",
         "skip_get_url": true,
         "uprn": "200000171235",
-        "url": "https://molevalley.cloudmappin.com/mmv/",
+        "url": "https://myproperty.molevalley.gov.uk/molevalley/",
         "wiki_name": "Mole Valley District Council",
         "wiki_note": "UPRN can only be parsed with a valid postcode."
     },
@@ -828,6 +863,15 @@
         "url": "https://beta.southglos.gov.uk/waste-and-recycling-collection-date",
         "wiki_name": "South Gloucestershire Council"
     },
+    "SouthKestevenDistrictCouncil": {
+        "house_number": "2 Althorpe Close, Market Deeping, PE6 8BL",
+        "postcode": "PE68BL",
+        "skip_get_url": true,
+        "url": "https://pre.southkesteven.gov.uk/BinSearch.aspx",
+        "web_driver": "http://selenium:4444",
+        "wiki_name": "SouthKestevenDistrictCouncil",
+        "wiki_note": ""
+    },
     "SouthLanarkshireCouncil": {
         "url": "https://www.southlanarkshire.gov.uk/directory_record/579973/abbeyhill_crescent_lesmahagow",
         "wiki_command_url_override": "https://www.southlanarkshire.gov.uk/directory_record/XXXXX/XXXXX",
@@ -892,6 +936,13 @@
         "url": "https://www.stratford.gov.uk/waste-recycling/when-we-collect.cfm/part/calendar",
         "wiki_name": "Stratford Upon Avon Council"
     },
+    "StroudDistrictCouncil": {
+        "postcode": "GL10 3BH",
+        "uprn": "100120512183",
+        "url": "https://www.stroud.gov.uk/my-house?uprn=100120512183&postcode=GL10+3BH",
+        "wiki_name": "Stroud District Council",
+        "wiki_note": "Find your uprn and replace it in the url do the same for the postcode."
+    },
     "SunderlandCityCouncil": {
         "house_number": "13",
         "postcode": "SR4 6BJ",
@@ -926,6 +977,14 @@
         "url": "https://dac.telford.gov.uk/bindayfinder/",
         "wiki_name": "Telford and Wrekin Co-operative Council"
     },
+    "TendringDistrictCouncil": {
+        "postcode": "CO15 4EU",
+        "skip_get_url": true,
+        "uprn": "100090604247",
+        "url": "https://tendring-self.achieveservice.com/en/service/Rubbish_and_recycling_collection_days",
+        "web_driver": "http://selenium:4444",
+        "wiki_name": "Tendring District Council"
+    },
     "TestValleyBoroughCouncil": {
         "postcode": "SO51 9ZD",
         "skip_get_url": true,
@@ -974,6 +1033,16 @@
         "wiki_name": "Wakefield City Council",
         "wiki_note": "Follow the instructions [here](https://www.wakefield.gov.uk/where-i-live/) until you get the page that includes a \"Bin Collections\" section then copy the URL and replace the URL in the command."
     },
+    "WalthamForest": {
+        "house_number": "17 Chingford Road, Walthamstow",
+        "postcode": "E17 4PW",
+        "skip_get_url": true,
+        "uprn": "200001415697",
+        "url": "https://portal.walthamforest.gov.uk/AchieveForms/?mode=fill&consentMessage=yes&form_uri=sandbox-publish://AF-Process-d62ccdd2-3de9-48eb-a229-8e20cbdd6393/AF-Stage-8bf39bf9-5391-4c24-857f-0dc2025c67f4/definition.json&process=1&process_uri=sandbox-processes://AF-Process-d62ccdd2-3de9-48eb-a229-8e20cbdd6393&process_id=AF-Process-d62ccdd2-3de9-48eb-a229-8e20cbdd6393",
+        "web_driver": "http://selenium:4444",
+        "wiki_name": "Waltham Forest",
+        "wiki_note": "Use to find your UPRN https://uprn.uk/"
+    },
     "WarwickDistrictCouncil": {
         "url": "https://estates7.warwickdc.gov.uk/PropertyPortal/Property/Recycling/100070263793",
         "wiki_command_url_override": "https://estates7.warwickdc.gov.uk/PropertyPortal/Property/Recycling/XXXXXXXX",

uk_bin_collection/tests/step_defs/test_validate_council.py

@@ -1,6 +1,7 @@
 import logging
 import traceback
 from typing import Any, Generator, Callable
+import json
 
 import pytest
 from pytest_bdd import scenario, given, when, then, parsers
@@ -11,10 +12,26 @@ from uk_bin_collection.uk_bin_collection import collect_data
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
 
+
+def get_council_list():
+    json_file_path = "uk_bin_collection/tests/input.json"  # Specify the correct path to the JSON file
+    with open(json_file_path, "r") as file:
+        data = json.load(file)
+    logging.info(f"Council List: {list(data.keys())}")
+    return list(data.keys())
+
+
+@pytest.fixture(params=get_council_list())
+def council(request):
+    print(f"Running test for council: {request.param}")
+    return request.param
+
+
 @scenario("../features/validate_council_outputs.feature", "Validate Council Output")
-def test_scenario_outline() -> None:
+def test_scenario_outline(council) -> None:
     pass
 
+
 def handle_test_errors(func: Callable[..., Any]) -> Callable[..., Any]:
     @wraps(func)
     def wrapper(*args: Any, **kwargs: Any) -> Any:
@@ -24,30 +41,37 @@ def handle_test_errors(func: Callable[..., Any]) -> Callable[..., Any]:
             logging.error(f"Error in test '{func.__name__}': {e}")
             logging.error(traceback.format_exc())
             raise e
+
     return wrapper
 
-@pytest.fixture
-@handle_test_errors
-def context() -> Generator[Any, None, None]:
-    class Context:
-        metadata: dict[str, Any]
-        council: str
-        parse_result: Any
 
+class Context:
+    def __init__(self):
+        self.metadata: dict[str, Any] = {}
+        self.council: str = ""
+        self.parse_result: Any = None
+
+
+@pytest.fixture(scope="module")
+def context():
     return Context()
 
+
 @handle_test_errors
-@given(parsers.parse("the council: {council_name}"))
-def get_council_step(context: Any, council_name: str) -> None:
+@given(parsers.parse("the council"))
+def get_council_step(context, council) -> None:
     council_input_data = file_handler.load_json_file("input.json")
-    context.metadata = council_input_data[council_name]
+    context.metadata = council_input_data[council]
+    context.council = council
+
 
 @handle_test_errors
-@when(parsers.parse("we scrape the data from {council}"))
-def scrape_step(context: Any, council: str, headless_mode: str, local_browser: str, selenium_url: str) -> None:
-    context.council = council
+@when(parsers.parse("we scrape the data from the council"))
+def scrape_step(
+    context: Any, headless_mode: str, local_browser: str, selenium_url: str
+) -> None:
 
-    args = [council, context.metadata["url"]]
+    args = [context.council, context.metadata["url"]]
 
     if "uprn" in context.metadata:
         uprn = context.metadata["uprn"]
@@ -75,13 +99,17 @@ def scrape_step(context: Any, council: str, headless_mode: str, local_browser: s
     CollectData.set_args(args)
     context.parse_result = CollectData.run()
 
+
 @handle_test_errors
 @then("the result is valid json")
 def validate_json_step(context: Any) -> None:
     assert file_handler.validate_json(context.parse_result), "Invalid JSON output"
 
+
 @handle_test_errors
 @then("the output should validate against the schema")
 def validate_output_step(context: Any) -> None:
     council_schema = file_handler.load_json_file("output.schema")
-    assert file_handler.validate_json_schema(context.parse_result, council_schema), "Schema validation failed"
+    assert file_handler.validate_json_schema(
+        context.parse_result, council_schema
+    ), "Schema validation failed"

uk_bin_collection/tests/test_common_functions.py

@@ -332,8 +332,10 @@ def test_contains_date_with_mixed_content():
 
 
 def test_create_webdriver_local():
-    result = create_webdriver(None, headless=True, user_agent="FireFox", session_name="test-session")
-    assert result.name in  ["chrome","chrome-headless-shell"]
+    result = create_webdriver(
+        None, headless=True, user_agent="FireFox", session_name="test-session"
+    )
+    assert result.name in ["chrome", "chrome-headless-shell"]
 
 
 def test_create_webdriver_remote_failure():

uk_bin_collection/uk_bin_collection/common.py

@@ -258,7 +258,10 @@ def contains_date(string, fuzzy=False) -> bool:
 
 
 def create_webdriver(
-    web_driver: str = None, headless: bool = True, user_agent: str = None, session_name: str = None
+    web_driver: str = None,
+    headless: bool = True,
+    user_agent: str = None,
+    session_name: str = None,
 ) -> webdriver.Chrome:
     """
     Create and return a Chrome WebDriver configured for optional headless operation.

uk_bin_collection/uk_bin_collection/councils/ChichesterDistrictCouncil.py

@@ -0,0 +1,110 @@
+import time
+from datetime import datetime
+
+from selenium.webdriver.support.ui import Select
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.keys import Keys
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            # Make a BS4 object
+
+            page = "https://www.chichester.gov.uk/checkyourbinday"
+
+            user_postcode = kwargs.get("postcode")
+            user_uprn = kwargs.get("uprn")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            house_number = kwargs.get("paon")
+
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(page)
+
+            wait = WebDriverWait(driver, 60)
+
+            inputElement_postcodesearch = wait.until(
+                EC.visibility_of_element_located(
+                    (By.ID, "WASTECOLLECTIONCALENDARV5_CALENDAR_ADDRESSLOOKUPPOSTCODE")
+                )
+            )
+
+            inputElement_postcodesearch.send_keys(user_postcode)
+
+            inputElement_postcodesearch_btn = wait.until(
+                EC.visibility_of_element_located(
+                    (By.ID, "WASTECOLLECTIONCALENDARV5_CALENDAR_ADDRESSLOOKUPSEARCH")
+                )
+            )
+            inputElement_postcodesearch_btn.send_keys(Keys.ENTER)
+
+            inputElement_select_address = wait.until(
+                EC.element_to_be_clickable(
+                    (By.ID, "WASTECOLLECTIONCALENDARV5_CALENDAR_ADDRESSLOOKUPADDRESS")
+                )
+            )
+            dropdown_element = driver.find_element(
+                By.ID, "WASTECOLLECTIONCALENDARV5_CALENDAR_ADDRESSLOOKUPADDRESS"
+            )
+
+            # Now create a Select object based on the found element
+            dropdown = Select(dropdown_element)
+
+            # Select the option by visible text
+            dropdown.select_by_visible_text(house_number)
+
+            results = wait.until(
+                EC.element_to_be_clickable(
+                    (By.CLASS_NAME, "bin-collection-dates-container")
+                )
+            )
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            soup.prettify()
+
+            # Extract data from the table
+            bin_collection_data = []
+            rows = soup.find(
+                "table", class_="defaultgeneral bin-collection-dates"
+            ).find_all("tr")
+            for row in rows:
+                cells = row.find_all("td")
+                if cells:
+                    date_str = cells[0].text.strip()
+                    bin_type = cells[1].text.strip()
+                    # Convert date string to the required format DD/MM/YYYY
+                    date_obj = datetime.strptime(date_str, "%d %B %Y")
+                    date_formatted = date_obj.strftime(date_format)
+                    bin_collection_data.append(
+                        {"collectionDate": date_formatted, "type": bin_type}
+                    )
+
+            # Convert to JSON
+            json_data = {"bins": bin_collection_data}
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return json_data

uk_bin_collection/uk_bin_collection/councils/DartfordBoroughCouncil.py

@@ -0,0 +1,44 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        # Extract data
+        bin_data = {
+            "bins": []
+        }
+
+        # Find the table containing the bin collection data
+        table = soup.find('table', {'class': 'eb-EVDNdR1G-tableContent'})
+
+        if table:
+            rows = table.find_all('tr', class_='eb-EVDNdR1G-tableRow')
+            
+            for row in rows:
+                columns = row.find_all('td')
+                if len(columns) >= 4:
+                    collection_type = columns[1].get_text(strip=True)
+                    collection_date = columns[3].get_text(strip=True)
+                    
+                    # Validate collection_date format
+                    if re.match(r"\d{2}/\d{2}/\d{4}", collection_date):
+                        bin_entry = {
+                            "type": collection_type,
+                            "collectionDate": collection_date
+                        }
+                        bin_data["bins"].append(bin_entry)
+
+        return bin_data

uk_bin_collection/uk_bin_collection/councils/DoverDistrictCouncil.py

@@ -4,9 +4,10 @@ import re
 from uk_bin_collection.uk_bin_collection.common import *  # Consider specific imports
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
+
 class CouncilClass(AbstractGetBinDataClass):
     def parse_data(self, page: str, **kwargs) -> dict:
-        soup = BeautifulSoup(page.text, 'html.parser')
+        soup = BeautifulSoup(page.text, "html.parser")
 
         bins_data = {"bins": []}
         bin_collections = []
@@ -23,7 +24,9 @@ class CouncilClass(AbstractGetBinDataClass):
             if service_name and next_service:
                 bin_type = service_name.get_text().replace("Collection", "bin").strip()
                 date_span = next_service.find("span", {"class": "table-label"})
-                date_text = date_span.next_sibling.get_text().strip() if date_span else None
+                date_text = (
+                    date_span.next_sibling.get_text().strip() if date_span else None
+                )
 
                 if date_text and re.match(r"\d{2}/\d{2}/\d{4}", date_text):
                     try:
@@ -33,9 +36,11 @@ class CouncilClass(AbstractGetBinDataClass):
                         continue
 
         for bin_type, bin_date in sorted(bin_collections, key=lambda x: x[1]):
-            bins_data["bins"].append({
-                "type": bin_type.capitalize(),
-                "collectionDate": bin_date.strftime("%d/%m/%Y"),
-            })
+            bins_data["bins"].append(
+                {
+                    "type": bin_type.capitalize(),
+                    "collectionDate": bin_date.strftime("%d/%m/%Y"),
+                }
+            )
 
         return bins_data

uk_bin_collection/uk_bin_collection/councils/EppingForestDistrictCouncil.py

@@ -0,0 +1,51 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+from selenium import webdriver
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from datetime import datetime
+from uk_bin_collection.uk_bin_collection.common import date_format
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    def parse_data(self, page: str, **kwargs) -> dict:
+        postcode = kwargs.get("postcode", "")
+        web_driver = kwargs.get("web_driver")
+        headless = kwargs.get("headless")
+
+        options = webdriver.ChromeOptions()
+        if headless:
+            options.add_argument("--headless")
+        driver = create_webdriver(web_driver, headless)
+
+        try:
+            driver.get(
+                f"https://eppingforestdc.maps.arcgis.com/apps/instant/lookup/index.html?appid=bfca32b46e2a47cd9c0a84f2d8cdde17&find={postcode}"
+            )
+            wait = WebDriverWait(driver, 10)
+            WebDriverWait(driver, 10).until(
+                EC.visibility_of_element_located(
+                    (By.CSS_SELECTOR, ".esri-feature-content")
+                )
+            )
+            html_content = driver.page_source
+
+            soup = BeautifulSoup(html_content, "html.parser")
+            bin_info_divs = soup.select(".esri-feature-content p")
+            data = {"bins": []}
+            for div in bin_info_divs:
+                if "collection day is" in div.text:
+                    bin_type, date_str = div.text.split(" collection day is ")
+                    bin_dates = datetime.strptime(
+                        date_str.strip(), "%d/%m/%Y"
+                    ).strftime(date_format)
+                    data["bins"].append(
+                        {"type": bin_type.strip(), "collectionDate": bin_dates}
+                    )
+
+            return data
+        finally:
+            driver.quit()

uk_bin_collection/uk_bin_collection/councils/FarehamBoroughCouncil.py

@@ -0,0 +1,68 @@
+import json
+
+import requests
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_postcode = kwargs.get("postcode")
+        check_postcode(user_postcode)
+
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36",
+        }
+        params = {
+            "type": "JSON",
+            "list": "DomesticBinCollections",
+            "Road": "",
+            "Postcode": user_postcode,
+        }
+
+        response = requests.get(
+            "https://www.fareham.gov.uk/internetlookups/search_data.aspx",
+            params=params,
+            headers=headers,
+        )
+
+        bin_data = response.json()["data"]
+        data = {"bins": []}
+
+        if "rows" in bin_data:
+            collection_str = bin_data["rows"][0]["DomesticBinDay"]
+
+            results = re.findall(r"(\d\d?\/\d\d?\/\d{4}) \((\w*)\)", collection_str)
+
+            if results:
+                for result in results:
+                    collection_date = datetime.strptime(result[0], "%d/%m/%Y")
+                    dict_data = {
+                        "type": result[1],
+                        "collectionDate": collection_date.strftime(date_format),
+                    }
+                    data["bins"].append(dict_data)
+
+                    # Garden waste is also collected on recycling day
+                    if dict_data["type"] == "Recycling":
+                        garden_data = {
+                            "type": "Garden",
+                            "collectionDate": dict_data["collectionDate"],
+                        }
+                        data["bins"].append(garden_data)
+            else:
+                raise RuntimeError("Dates not parsed correctly.")
+        else:
+            raise ValueError("Postcode not found on website.")
+
+        data["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return data