uk_bin_collection 0.76.1__tar.gz → 0.78.0__tar.gz

{uk_bin_collection-0.76.1 → uk_bin_collection-0.78.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.76.1
+Version: 0.78.0
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.76.1 → uk_bin_collection-0.78.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.76.1"
+version = "0.78.0"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

uk_bin_collection-0.78.0/uk_bin_collection/tests/council_feature_input_parity.py

@@ -0,0 +1,128 @@
+import json
+import re
+import requests
+import sys
+from tabulate import tabulate
+
+def get_councils_from_files(repo, branch):
+    url = f"https://api.github.com/repos/{repo}/contents/uk_bin_collection/uk_bin_collection/councils?ref={branch}"
+    print(f"Fetching councils from files at URL: {url}")
+    response = requests.get(url)
+    
+    # Debugging lines to check the response content and type
+    print(f"Response Status Code (Files): {response.status_code}")
+    
+    if response.status_code == 200:
+        try:
+            data = response.json()
+            #print(f"Parsed JSON Data (Files): {data}")
+            # Ensure 'data' is a list before proceeding
+            if isinstance(data, list):
+                return [item['name'].replace('.py', '') for item in data if item['name'].endswith('.py')]
+            else:
+                print("Expected a list from the JSON response but got something else.")
+                raise ValueError("Expected a list from the JSON response but got something else.")
+        except json.JSONDecodeError as e:
+            print(f"JSON decoding error: {e}")
+            raise
+    else:
+        print(f"Failed to fetch councils from files: {response.content}")
+        return []
+
+def get_councils_from_json(repo, branch):
+    url = f"https://raw.githubusercontent.com/{repo}/{branch}/uk_bin_collection/tests/input.json"
+    print(f"Fetching councils from JSON at URL: {url}")
+    response = requests.get(url)
+    
+    # Debugging lines to check the response content and type
+    print(f"Response Status Code (JSON): {response.status_code}")
+    
+    if response.status_code == 200:
+        try:
+            data = json.loads(response.text)
+            #print(f"Parsed JSON Data: {data}")
+            return list(data.keys())
+        except json.JSONDecodeError as e:
+            print(f"JSON decoding error: {e}")
+            raise
+    else:
+        print(f"Failed to fetch councils from JSON: {response.content}")
+        return []
+
+def get_councils_from_features(repo, branch):
+    url = f"https://raw.githubusercontent.com/{repo}/{branch}/uk_bin_collection/tests/features/validate_council_outputs.feature"
+    print(f"Fetching councils from features at URL: {url}")
+    response = requests.get(url)
+    
+    # Debugging lines to check the response content and type
+    print(f"Response Status Code (Features): {response.status_code}")
+    
+    if response.status_code == 200:
+        content = response.text
+        #print(f"Fetched Features Content: {content[:500]}...")  # Print only the first 500 characters for brevity
+        return re.findall(r'Examples:\s+(\w+)', content)
+    else:
+        print(f"Failed to fetch councils from features: {response.content}")
+        return []
+
+def compare_councils(councils1, councils2, councils3):
+    set1 = set(councils1)
+    set2 = set(councils2)
+    set3 = set(councils3)
+    all_councils = set1 | set2 | set3
+    all_council_data = {}
+    discrepancies_found = False
+    for council in all_councils:
+        in_files = council in set1
+        in_json = council in set2
+        in_features = council in set3
+        discrepancies_count = [in_files, in_json, in_features].count(False)
+        all_council_data[council] = {
+            'in_files': in_files,
+            'in_json': in_json,
+            'in_features': in_features,
+            'discrepancies_count': discrepancies_count
+        }
+        if discrepancies_count > 0:
+            discrepancies_found = True
+    return all_council_data, discrepancies_found
+
+def main(repo="robbrad/UKBinCollectionData", branch="master"):
+    # Execute and print the comparison
+    print(f"Starting comparison for repo: {repo}, branch: {branch}")
+    file_councils = get_councils_from_files(repo, branch)
+    json_councils = get_councils_from_json(repo, branch)
+    feature_councils = get_councils_from_features(repo, branch)
+
+    #print(f"Councils from files: {file_councils}")
+    #print(f"Councils from JSON: {json_councils}")
+    #print(f"Councils from features: {feature_councils}")
+
+    all_councils_data, discrepancies_found = compare_councils(file_councils, json_councils, feature_councils)
+
+    # Create a list of lists for tabulate, sort by discrepancies count and then by name
+    table_data = []
+    headers = ["Council Name", "In Files", "In JSON", "In Features", "Discrepancies"]
+    for council, presence in sorted(all_councils_data.items(), key=lambda x: (x[1]['discrepancies_count'], x[0])):
+        row = [
+            council,
+            "✔" if presence['in_files'] else "✘",
+            "✔" if presence['in_json'] else "✘",
+            "✔" if presence['in_features'] else "✘",
+            presence['discrepancies_count']
+        ]
+        table_data.append(row)
+
+    # Print the table using tabulate
+    print(tabulate(table_data, headers=headers, tablefmt='grid'))
+
+    if discrepancies_found:
+        print("Discrepancies found! Failing the workflow.")
+        sys.exit(1)
+    else:
+        print("No discrepancies found. Workflow successful.")
+
+if __name__ == "__main__":
+    repo = sys.argv[1] if len(sys.argv) > 1 else "robbrad/UKBinCollectionData"
+    branch = sys.argv[2] if len(sys.argv) > 2 else "master"
+    main(repo, branch)

{uk_bin_collection-0.76.1 → uk_bin_collection-0.78.0}/uk_bin_collection/tests/features/validate_council_outputs.feature

@@ -76,6 +76,11 @@ Feature: Test each council output matches expected results
 		| council |
 		| BoltonCouncil |
 
+        @BracknellForestCouncil
+		Examples: BracknellForestCouncil
+		| council |
+		| BracknellForestCouncil |
+
         @BradfordMDC
 		Examples: BradfordMDC
 		| council |
@@ -236,6 +241,11 @@ Feature: Test each council output matches expected results
 		| council |
 		| ErewashBoroughCouncil |
 
+        @FarehamBoroughCouncil
+		Examples: FarehamBoroughCouncil
+		| council |
+		| FarehamBoroughCouncil |
+
         @FenlandDistrictCouncil
 		Examples: FenlandDistrictCouncil
 		| council |

{uk_bin_collection-0.76.1 → uk_bin_collection-0.78.0}/uk_bin_collection/tests/input.json

@@ -101,6 +101,13 @@
         "wiki_name": "Bolton Council",
         "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search). Previously required single field that was UPRN and full address, now requires UPRN and postcode as separate fields."
     },
+    "BracknellForestCouncil": {
+        "paon": "57",
+        "postcode": "GU47 9BS",
+        "skip_get_url": true,
+        "url": "https://selfservice.mybfc.bracknell-forest.gov.uk/w/webpage/waste-collection-days",
+        "wiki_name": "Bracknell Forest Council"
+    },
     "BradfordMDC": {
         "custom_component_show_url_field": false,
         "skip_get_url": true,
@@ -209,8 +216,8 @@
     "CheshireWestAndChesterCouncil": {
         "house_number": "Hill View House",
         "postcode": "CH3 9ER",
-        "uprn": "100012346655",
         "skip_get_url": true,
+        "uprn": "100012346655",
         "url": "https://www.cheshirewestandchester.gov.uk/residents/waste-and-recycling/your-bin-collection/collection-day",
         "web_driver": "http://selenium:4444",
         "wiki_name": "Cheshire West and Chester Council"
@@ -335,6 +342,12 @@
         "url": "https://map.erewash.gov.uk/isharelive.web/myerewash.aspx",
         "wiki_name": "Erewash Borough Council"
     },
+    "FarehamBoroughCouncil": {
+        "postcode": "PO14 4NR",        
+        "skip_get_url": true,
+        "url": "https://www.fareham.gov.uk/internetlookups/search_data.aspx?type=JSON&list=DomesticBinCollections&Road=&Postcode=PO14%204NR",
+        "wiki_name": "Fareham Borough Council"
+    },
     "FenlandDistrictCouncil": {
         "skip_get_url": true,
         "uprn": "200002981143",
@@ -1027,8 +1040,8 @@
     },
     "WestSuffolkCouncil": {
         "postcode": "IP28 6DR",
-        "uprn": "10009739960",
         "skip_get_url": true,
+        "uprn": "10009739960",
         "url": "https://maps.westsuffolk.gov.uk/MyWestSuffolk.aspx",
         "wiki_name": "West Suffolk Council"
     },

uk_bin_collection-0.78.0/uk_bin_collection/uk_bin_collection/councils/BracknellForestCouncil.py

@@ -0,0 +1,246 @@
+import time
+
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+def get_headers(base_url: str, method: str) -> dict[str, str]:
+    """
+    Gets request headers
+        :rtype: dict[str, str]
+        :param base_url: Base URL to use
+        :param method: Method to use
+        :return: Request headers
+    """
+    headers = {
+        "Accept-Encoding": "gzip, deflate, br",
+        "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
+        "Cache-Control": "max-age=0",
+        "Connection": "keep-alive",
+        "Host": "selfservice.mybfc.bracknell-forest.gov.uk",
+        "Origin": base_url,
+        "sec-ch-ua": '"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"',
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": "Windows",
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-User": "?1",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)"
+        " Chrome/109.0.0.0 Safari/537.36",
+    }
+    if method.lower() == "post":
+        headers["Accept"] = "application/json, text/javascript, */*; q=0.01"
+        headers["Content-Type"] = "application/x-www-form-urlencoded; charset=UTF-8"
+        headers["Sec-Fetch-Mode"] = "cors"
+        headers["Sec-Fetch-Mode"] = "same-origin"
+        headers["X-Requested-With"] = "XMLHttpRequest"
+    else:
+        headers["Accept"] = (
+            "text/html,application/xhtml+xml,application/xml;"
+            "q=0.9,image/avif,image/webp,image/apng,*/*;"
+            "q=0.8,application/signed-exchange;v=b3;q=0.9"
+        )
+        headers["Sec-Fetch-Mode"] = "navigate"
+        headers["Sec-Fetch-Mode"] = "none"
+    return headers
+
+
+def get_session_storage_global() -> object:
+    """
+    Gets session storage global object
+        :rtype: object
+        :return: Session storage global object
+    """
+    return {
+        "destination_stack": [
+            "w/webpage/waste-collection-days",
+        ],
+        "last_context_record_id": "86086077",
+    }
+
+
+def get_csrf_token(s: requests.session, base_url: str) -> str:
+    """
+    Gets a CSRF token
+        :rtype: str
+        :param s: requests.session() to use
+        :param base_url: Base URL to use
+        :return: CSRF token
+    """
+    csrf_token = ""
+    response = s.get(
+        base_url + "/w/webpage/waste-collection-days",
+        headers=get_headers(base_url, "GET"),
+    )
+    if response.status_code == 200:
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+        app_body = soup.find("div", {"class": "app-body"})
+        script = app_body.find("script", {"type": "text/javascript"}).string
+        p = re.compile("var CSRF = ('|\")(.*?)('|\");")
+        m = p.search(script)
+        csrf_token = m.groups()[1]
+    else:
+        raise ValueError(
+            "Code 1: Failed to get a CSRF token. Please ensure the council website is online first,"
+            " then open an issue on GitHub."
+        )
+    return csrf_token
+
+
+def get_address_id(
+    s: requests.session, base_url: str, csrf_token: str, postcode: str, paon: str
+) -> str:
+    """
+    Gets the address ID
+        :rtype: str
+        :param s: requests.session() to use
+        :param base_url: Base URL to use
+        :param csrf_token: CSRF token to use
+        :param postcode: Postcode to use
+        :param paon: House number/address to find
+        :return: address ID
+    """
+    address_id = "0"
+    # Get the addresses for the postcode
+    form_data = {
+        "code_action": "find_addresses",
+        "code_params": '{"search":"' + postcode + '"}',
+        "_session_storage": json.dumps(
+            {
+                "/w/webpage/waste-collection-days": {},
+                "_global": get_session_storage_global(),
+            }
+        ),
+        "action_cell_id": "PCL0003988FEFFB1",
+        "action_page_id": "PAG0000570FEFFB1",
+        "form_check_ajax": csrf_token,
+    }
+    response = s.post(
+        base_url
+        + "/w/webpage/waste-collection-days?webpage_subpage_id=PAG0000570FEFFB1"
+        "&webpage_token=390170046582b0e3d7ca68ef1d6b4829ccff0b1ae9c531047219c6f9b5295738"
+        "&widget_action=handle_event",
+        headers=get_headers(base_url, "POST"),
+        data=form_data,
+    )
+    if response.status_code == 200:
+        json_response = json.loads(response.text)
+        addresses = json_response["response"]["addresses"]["items"]
+        # Find the matching address id for the paon
+        for address in addresses:
+            # Check for full matches first
+            if address.get("Description") == paon:
+                address_id = address.get("Id")
+                break
+        # Check for matching start if no full match found
+        if address_id == "0":
+            for address in addresses:
+                if address.get("Description").split()[0] == paon.strip():
+                    address_id = address.get("Id")
+                    break
+        # Check match was found
+        if address_id == "0":
+            raise ValueError(
+                "Code 2: No matching address for house number/full address found."
+            )
+    else:
+        raise ValueError("Code 3: No addresses found for provided postcode.")
+    return address_id
+
+
+def get_collection_data(
+    s: requests.session, base_url: str, csrf_token: str, address_id: str
+) -> str:
+    """
+    Gets the collection data
+        :rtype: str
+        :param s: requests.session() to use
+        :param base_url: Base URL to use
+        :param csrf_token: CSRF token to use
+        :param address_id: Address id to use
+        :param retries: Retries count
+        :return: Collection data
+    """
+    collection_data = ""
+    if address_id != "0":
+        form_data = {
+            "code_action": "find_rounds",
+            "code_params": '{"addressId":"' + address_id + '"}',
+            "_session_storage": json.dumps(
+                {
+                    "/w/webpage/waste-collection-days": {},
+                    "_global": get_session_storage_global(),
+                }
+            ),
+            "action_cell_id": "PCL0003988FEFFB1",
+            "action_page_id": "PAG0000570FEFFB1",
+            "form_check_ajax": csrf_token,
+        }
+        response = s.post(
+            base_url
+            + "/w/webpage/waste-collection-days?webpage_subpage_id=PAG0000570FEFFB1"
+            "&webpage_token=390170046582b0e3d7ca68ef1d6b4829ccff0b1ae9c531047219c6f9b5295738"
+            "&widget_action=handle_event",
+            headers=get_headers(base_url, "POST"),
+            data=form_data,
+        )
+        if response.status_code == 200 and len(response.text) > 0:
+            json_response = json.loads(response.text)
+            collection_data = json_response["response"]["collections"]
+        else:
+            raise ValueError("Code 4: Failed to get bin data.")
+    return collection_data
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        requests.packages.urllib3.disable_warnings()
+        s = requests.session()
+        base_url = "https://selfservice.mybfc.bracknell-forest.gov.uk"
+        paon = kwargs.get("paon")
+        postcode = kwargs.get("postcode")
+        check_paon(paon)
+        check_postcode(postcode)
+
+        # Firstly, get a CSRF (cross-site request forgery) token
+        csrf_token = get_csrf_token(s, base_url)
+        # Next, get the address_id
+        address_id = get_address_id(s, base_url, csrf_token, postcode, paon)
+        # Finally, use the address_id to get the collection data
+        collection_data = get_collection_data(s, base_url, csrf_token, address_id)
+        if collection_data != "":
+            # Form a JSON wrapper
+            data = {"bins": []}
+
+            for c in collection_data:
+                collection_type = c["round"]
+                for c_date in c["upcomingCollections"]:
+                    collection_date = (
+                        re.search(r"Your (.*) is(.*)", c_date).group(2).strip()
+                    )
+                    dict_data = {
+                        "type": collection_type,
+                        "collectionDate": datetime.strptime(
+                            collection_date, "%A %d %B %Y"
+                        ).strftime(date_format),
+                    }
+                    data["bins"].append(dict_data)
+
+            if len(data["bins"]) == 0:
+                raise ValueError(
+                    "Code 5: No bin data found. Please ensure the council website is showing data first,"
+                    " then open an issue on GitHub."
+                )
+
+            data["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+            )
+            return data

uk_bin_collection-0.78.0/uk_bin_collection/uk_bin_collection/councils/FarehamBoroughCouncil.py

@@ -0,0 +1,69 @@
+import json
+
+import requests
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_postcode = kwargs.get("postcode")
+        check_postcode(user_postcode)
+
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36",
+        }
+        params = {
+            "type": "JSON",
+            "list": "DomesticBinCollections",
+            "Road": "",
+            "Postcode": user_postcode
+        }
+
+        response = requests.get(
+            "https://www.fareham.gov.uk/internetlookups/search_data.aspx",
+            params  = params,
+            headers = headers
+        )
+
+        bin_data = response.json()["data"]
+        data = {"bins": []}
+
+        if "rows" in bin_data:
+            collection_str = bin_data["rows"][0]["DomesticBinDay"]
+            
+            results = re.findall(r"(\d\d?\/\d\d?\/\d{4}) \((\w*)\)", collection_str)
+
+            if results:
+                    for result in results:
+                        collection_date = datetime.strptime(result[0], "%d/%m/%Y")
+                        dict_data = {
+                            "type": result[1],
+                            "collectionDate": collection_date.strftime(date_format),
+                        }
+                        data["bins"].append(dict_data)
+
+                        # Garden waste is also collected on recycling day
+                        if (dict_data["type"] == "Recycling"):
+                            garden_data = {
+                                "type": "Garden",
+                                "collectionDate": dict_data["collectionDate"],
+                            }
+                            data["bins"].append(garden_data)
+            else:
+                raise RuntimeError("Dates not parsed correctly.")
+        else:
+            raise ValueError("Postcode not found on website.")
+
+        data["bins"].sort(
+            key=lambda x: datetime.strptime(
+            x.get("collectionDate"), "%d/%m/%Y"
+            )
+        )
+
+        return data

uk_bin_collection-0.76.1/uk_bin_collection/tests/council_feature_input_parity.py

@@ -1,91 +0,0 @@
-import json
-import re
-import requests
-import sys
-from tabulate import tabulate
-
-def get_councils_from_files(branch):
-    url = f"https://api.github.com/repos/robbrad/UKBinCollectionData/contents/uk_bin_collection/uk_bin_collection/councils?ref={branch}"
-    response = requests.get(url)
-    data = response.json()
-    
-    # Debugging lines to check the response content and type
-    print(f"Response Status Code: {response.status_code}")
-    print(f"Response Content: {response.content}")
-    print(f"Parsed JSON Data: {data}")
-    print(f"Data Type: {type(data)}")
-    
-    # Ensure 'data' is a list before proceeding
-    if isinstance(data, list):
-        return [item['name'].replace('.py', '') for item in data if item['name'].endswith('.py')]
-    else:
-        raise ValueError("Expected a list from the JSON response but got something else.")
-
-
-def get_councils_from_json(branch):
-    url = f"https://raw.githubusercontent.com/robbrad/UKBinCollectionData/{branch}/uk_bin_collection/tests/input.json"
-    response = requests.get(url)
-    data = json.loads(response.text)
-    return list(data.keys())
-
-def get_councils_from_features(branch):
-    url = f"https://raw.githubusercontent.com/robbrad/UKBinCollectionData/{branch}/uk_bin_collection/tests/features/validate_council_outputs.feature"
-    response = requests.get(url)
-    content = response.text
-    return re.findall(r'Examples:\s+(\w+)', content)
-
-def compare_councils(councils1, councils2, councils3):
-    set1 = set(councils1)
-    set2 = set(councils2)
-    set3 = set(councils3)
-    all_councils = set1 | set2 | set3
-    all_council_data = {}
-    discrepancies_found = False
-    for council in all_councils:
-        in_files = council in set1
-        in_json = council in set2
-        in_features = council in set3
-        discrepancies_count = [in_files, in_json, in_features].count(False)
-        all_council_data[council] = {
-            'in_files': in_files,
-            'in_json': in_json,
-            'in_features': in_features,
-            'discrepancies_count': discrepancies_count
-        }
-        if discrepancies_count > 0:
-            discrepancies_found = True
-    return all_council_data, discrepancies_found
-
-def main(branch="master"):
-    # Execute and print the comparison
-    file_councils = get_councils_from_files(branch)
-    json_councils = get_councils_from_json(branch)
-    feature_councils = get_councils_from_features(branch)
-
-    all_councils_data, discrepancies_found = compare_councils(file_councils, json_councils, feature_councils)
-
-    # Create a list of lists for tabulate, sort by discrepancies count and then by name
-    table_data = []
-    headers = ["Council Name", "In Files", "In JSON", "In Features", "Discrepancies"]
-    for council, presence in sorted(all_councils_data.items(), key=lambda x: (x[1]['discrepancies_count'], x[0])):
-        row = [
-            council,
-            "✔" if presence['in_files'] else "✘",
-            "✔" if presence['in_json'] else "✘",
-            "✔" if presence['in_features'] else "✘",
-            presence['discrepancies_count']
-        ]
-        table_data.append(row)
-
-    # Print the table using tabulate
-    print(tabulate(table_data, headers=headers, tablefmt='grid'))
-
-    if discrepancies_found:
-        print("Discrepancies found! Failing the workflow.")
-        sys.exit(1)
-    else:
-        print("No discrepancies found. Workflow successful.")
-
-if __name__ == "__main__":
-    branch = sys.argv[1] if len(sys.argv) > 1 else "master"
-    main(branch)