uk_bin_collection 0.76.1__py3-none-any.whl → 0.77.0__py3-none-any.whl

uk_bin_collection/tests/council_feature_input_parity.py

@@ -4,35 +4,66 @@ import requests
 import sys
 from tabulate import tabulate
 
-def get_councils_from_files(branch):
-    url = f"https://api.github.com/repos/robbrad/UKBinCollectionData/contents/uk_bin_collection/uk_bin_collection/councils?ref={branch}"
+def get_councils_from_files(repo, branch):
+    url = f"https://api.github.com/repos/{repo}/contents/uk_bin_collection/uk_bin_collection/councils?ref={branch}"
+    print(f"Fetching councils from files at URL: {url}")
     response = requests.get(url)
-    data = response.json()
     
     # Debugging lines to check the response content and type
-    print(f"Response Status Code: {response.status_code}")
-    print(f"Response Content: {response.content}")
-    print(f"Parsed JSON Data: {data}")
-    print(f"Data Type: {type(data)}")
+    print(f"Response Status Code (Files): {response.status_code}")
     
-    # Ensure 'data' is a list before proceeding
-    if isinstance(data, list):
-        return [item['name'].replace('.py', '') for item in data if item['name'].endswith('.py')]
+    if response.status_code == 200:
+        try:
+            data = response.json()
+            #print(f"Parsed JSON Data (Files): {data}")
+            # Ensure 'data' is a list before proceeding
+            if isinstance(data, list):
+                return [item['name'].replace('.py', '') for item in data if item['name'].endswith('.py')]
+            else:
+                print("Expected a list from the JSON response but got something else.")
+                raise ValueError("Expected a list from the JSON response but got something else.")
+        except json.JSONDecodeError as e:
+            print(f"JSON decoding error: {e}")
+            raise
     else:
-        raise ValueError("Expected a list from the JSON response but got something else.")
+        print(f"Failed to fetch councils from files: {response.content}")
+        return []
 
-
-def get_councils_from_json(branch):
-    url = f"https://raw.githubusercontent.com/robbrad/UKBinCollectionData/{branch}/uk_bin_collection/tests/input.json"
+def get_councils_from_json(repo, branch):
+    url = f"https://raw.githubusercontent.com/{repo}/{branch}/uk_bin_collection/tests/input.json"
+    print(f"Fetching councils from JSON at URL: {url}")
     response = requests.get(url)
-    data = json.loads(response.text)
-    return list(data.keys())
+    
+    # Debugging lines to check the response content and type
+    print(f"Response Status Code (JSON): {response.status_code}")
+    
+    if response.status_code == 200:
+        try:
+            data = json.loads(response.text)
+            #print(f"Parsed JSON Data: {data}")
+            return list(data.keys())
+        except json.JSONDecodeError as e:
+            print(f"JSON decoding error: {e}")
+            raise
+    else:
+        print(f"Failed to fetch councils from JSON: {response.content}")
+        return []
 
-def get_councils_from_features(branch):
-    url = f"https://raw.githubusercontent.com/robbrad/UKBinCollectionData/{branch}/uk_bin_collection/tests/features/validate_council_outputs.feature"
+def get_councils_from_features(repo, branch):
+    url = f"https://raw.githubusercontent.com/{repo}/{branch}/uk_bin_collection/tests/features/validate_council_outputs.feature"
+    print(f"Fetching councils from features at URL: {url}")
     response = requests.get(url)
-    content = response.text
-    return re.findall(r'Examples:\s+(\w+)', content)
+    
+    # Debugging lines to check the response content and type
+    print(f"Response Status Code (Features): {response.status_code}")
+    
+    if response.status_code == 200:
+        content = response.text
+        #print(f"Fetched Features Content: {content[:500]}...")  # Print only the first 500 characters for brevity
+        return re.findall(r'Examples:\s+(\w+)', content)
+    else:
+        print(f"Failed to fetch councils from features: {response.content}")
+        return []
 
 def compare_councils(councils1, councils2, councils3):
     set1 = set(councils1)
@@ -56,11 +87,16 @@ def compare_councils(councils1, councils2, councils3):
             discrepancies_found = True
     return all_council_data, discrepancies_found
 
-def main(branch="master"):
+def main(repo="robbrad/UKBinCollectionData", branch="master"):
     # Execute and print the comparison
-    file_councils = get_councils_from_files(branch)
-    json_councils = get_councils_from_json(branch)
-    feature_councils = get_councils_from_features(branch)
+    print(f"Starting comparison for repo: {repo}, branch: {branch}")
+    file_councils = get_councils_from_files(repo, branch)
+    json_councils = get_councils_from_json(repo, branch)
+    feature_councils = get_councils_from_features(repo, branch)
+
+    #print(f"Councils from files: {file_councils}")
+    #print(f"Councils from JSON: {json_councils}")
+    #print(f"Councils from features: {feature_councils}")
 
     all_councils_data, discrepancies_found = compare_councils(file_councils, json_councils, feature_councils)
 
@@ -87,5 +123,6 @@ def main(branch="master"):
         print("No discrepancies found. Workflow successful.")
 
 if __name__ == "__main__":
-    branch = sys.argv[1] if len(sys.argv) > 1 else "master"
-    main(branch)
+    repo = sys.argv[1] if len(sys.argv) > 1 else "robbrad/UKBinCollectionData"
+    branch = sys.argv[2] if len(sys.argv) > 2 else "master"
+    main(repo, branch)

uk_bin_collection/tests/features/validate_council_outputs.feature

@@ -76,6 +76,11 @@ Feature: Test each council output matches expected results
 		| council |
 		| BoltonCouncil |
 
+        @BracknellForestCouncil
+		Examples: BracknellForestCouncil
+		| council |
+		| BracknellForestCouncil |
+
         @BradfordMDC
 		Examples: BradfordMDC
 		| council |

uk_bin_collection/tests/input.json

@@ -101,6 +101,13 @@
         "wiki_name": "Bolton Council",
         "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search). Previously required single field that was UPRN and full address, now requires UPRN and postcode as separate fields."
     },
+    "BracknellForestCouncil": {
+        "paon": "57",
+        "postcode": "GU47 9BS",
+        "skip_get_url": true,
+        "url": "https://selfservice.mybfc.bracknell-forest.gov.uk/w/webpage/waste-collection-days",
+        "wiki_name": "Bracknell Forest Council"
+    },
     "BradfordMDC": {
         "custom_component_show_url_field": false,
         "skip_get_url": true,
@@ -209,8 +216,8 @@
     "CheshireWestAndChesterCouncil": {
         "house_number": "Hill View House",
         "postcode": "CH3 9ER",
-        "uprn": "100012346655",
         "skip_get_url": true,
+        "uprn": "100012346655",
         "url": "https://www.cheshirewestandchester.gov.uk/residents/waste-and-recycling/your-bin-collection/collection-day",
         "web_driver": "http://selenium:4444",
         "wiki_name": "Cheshire West and Chester Council"
@@ -1027,8 +1034,8 @@
     },
     "WestSuffolkCouncil": {
         "postcode": "IP28 6DR",
-        "uprn": "10009739960",
         "skip_get_url": true,
+        "uprn": "10009739960",
         "url": "https://maps.westsuffolk.gov.uk/MyWestSuffolk.aspx",
         "wiki_name": "West Suffolk Council"
     },

uk_bin_collection/uk_bin_collection/councils/BracknellForestCouncil.py

@@ -0,0 +1,246 @@
+import time
+
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+def get_headers(base_url: str, method: str) -> dict[str, str]:
+    """
+    Gets request headers
+        :rtype: dict[str, str]
+        :param base_url: Base URL to use
+        :param method: Method to use
+        :return: Request headers
+    """
+    headers = {
+        "Accept-Encoding": "gzip, deflate, br",
+        "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
+        "Cache-Control": "max-age=0",
+        "Connection": "keep-alive",
+        "Host": "selfservice.mybfc.bracknell-forest.gov.uk",
+        "Origin": base_url,
+        "sec-ch-ua": '"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"',
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": "Windows",
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-User": "?1",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)"
+        " Chrome/109.0.0.0 Safari/537.36",
+    }
+    if method.lower() == "post":
+        headers["Accept"] = "application/json, text/javascript, */*; q=0.01"
+        headers["Content-Type"] = "application/x-www-form-urlencoded; charset=UTF-8"
+        headers["Sec-Fetch-Mode"] = "cors"
+        headers["Sec-Fetch-Mode"] = "same-origin"
+        headers["X-Requested-With"] = "XMLHttpRequest"
+    else:
+        headers["Accept"] = (
+            "text/html,application/xhtml+xml,application/xml;"
+            "q=0.9,image/avif,image/webp,image/apng,*/*;"
+            "q=0.8,application/signed-exchange;v=b3;q=0.9"
+        )
+        headers["Sec-Fetch-Mode"] = "navigate"
+        headers["Sec-Fetch-Mode"] = "none"
+    return headers
+
+
+def get_session_storage_global() -> object:
+    """
+    Gets session storage global object
+        :rtype: object
+        :return: Session storage global object
+    """
+    return {
+        "destination_stack": [
+            "w/webpage/waste-collection-days",
+        ],
+        "last_context_record_id": "86086077",
+    }
+
+
+def get_csrf_token(s: requests.session, base_url: str) -> str:
+    """
+    Gets a CSRF token
+        :rtype: str
+        :param s: requests.session() to use
+        :param base_url: Base URL to use
+        :return: CSRF token
+    """
+    csrf_token = ""
+    response = s.get(
+        base_url + "/w/webpage/waste-collection-days",
+        headers=get_headers(base_url, "GET"),
+    )
+    if response.status_code == 200:
+        soup = BeautifulSoup(response.text, features="html.parser")
+        soup.prettify()
+        app_body = soup.find("div", {"class": "app-body"})
+        script = app_body.find("script", {"type": "text/javascript"}).string
+        p = re.compile("var CSRF = ('|\")(.*?)('|\");")
+        m = p.search(script)
+        csrf_token = m.groups()[1]
+    else:
+        raise ValueError(
+            "Code 1: Failed to get a CSRF token. Please ensure the council website is online first,"
+            " then open an issue on GitHub."
+        )
+    return csrf_token
+
+
+def get_address_id(
+    s: requests.session, base_url: str, csrf_token: str, postcode: str, paon: str
+) -> str:
+    """
+    Gets the address ID
+        :rtype: str
+        :param s: requests.session() to use
+        :param base_url: Base URL to use
+        :param csrf_token: CSRF token to use
+        :param postcode: Postcode to use
+        :param paon: House number/address to find
+        :return: address ID
+    """
+    address_id = "0"
+    # Get the addresses for the postcode
+    form_data = {
+        "code_action": "find_addresses",
+        "code_params": '{"search":"' + postcode + '"}',
+        "_session_storage": json.dumps(
+            {
+                "/w/webpage/waste-collection-days": {},
+                "_global": get_session_storage_global(),
+            }
+        ),
+        "action_cell_id": "PCL0003988FEFFB1",
+        "action_page_id": "PAG0000570FEFFB1",
+        "form_check_ajax": csrf_token,
+    }
+    response = s.post(
+        base_url
+        + "/w/webpage/waste-collection-days?webpage_subpage_id=PAG0000570FEFFB1"
+        "&webpage_token=390170046582b0e3d7ca68ef1d6b4829ccff0b1ae9c531047219c6f9b5295738"
+        "&widget_action=handle_event",
+        headers=get_headers(base_url, "POST"),
+        data=form_data,
+    )
+    if response.status_code == 200:
+        json_response = json.loads(response.text)
+        addresses = json_response["response"]["addresses"]["items"]
+        # Find the matching address id for the paon
+        for address in addresses:
+            # Check for full matches first
+            if address.get("Description") == paon:
+                address_id = address.get("Id")
+                break
+        # Check for matching start if no full match found
+        if address_id == "0":
+            for address in addresses:
+                if address.get("Description").split()[0] == paon.strip():
+                    address_id = address.get("Id")
+                    break
+        # Check match was found
+        if address_id == "0":
+            raise ValueError(
+                "Code 2: No matching address for house number/full address found."
+            )
+    else:
+        raise ValueError("Code 3: No addresses found for provided postcode.")
+    return address_id
+
+
+def get_collection_data(
+    s: requests.session, base_url: str, csrf_token: str, address_id: str
+) -> str:
+    """
+    Gets the collection data
+        :rtype: str
+        :param s: requests.session() to use
+        :param base_url: Base URL to use
+        :param csrf_token: CSRF token to use
+        :param address_id: Address id to use
+        :param retries: Retries count
+        :return: Collection data
+    """
+    collection_data = ""
+    if address_id != "0":
+        form_data = {
+            "code_action": "find_rounds",
+            "code_params": '{"addressId":"' + address_id + '"}',
+            "_session_storage": json.dumps(
+                {
+                    "/w/webpage/waste-collection-days": {},
+                    "_global": get_session_storage_global(),
+                }
+            ),
+            "action_cell_id": "PCL0003988FEFFB1",
+            "action_page_id": "PAG0000570FEFFB1",
+            "form_check_ajax": csrf_token,
+        }
+        response = s.post(
+            base_url
+            + "/w/webpage/waste-collection-days?webpage_subpage_id=PAG0000570FEFFB1"
+            "&webpage_token=390170046582b0e3d7ca68ef1d6b4829ccff0b1ae9c531047219c6f9b5295738"
+            "&widget_action=handle_event",
+            headers=get_headers(base_url, "POST"),
+            data=form_data,
+        )
+        if response.status_code == 200 and len(response.text) > 0:
+            json_response = json.loads(response.text)
+            collection_data = json_response["response"]["collections"]
+        else:
+            raise ValueError("Code 4: Failed to get bin data.")
+    return collection_data
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        requests.packages.urllib3.disable_warnings()
+        s = requests.session()
+        base_url = "https://selfservice.mybfc.bracknell-forest.gov.uk"
+        paon = kwargs.get("paon")
+        postcode = kwargs.get("postcode")
+        check_paon(paon)
+        check_postcode(postcode)
+
+        # Firstly, get a CSRF (cross-site request forgery) token
+        csrf_token = get_csrf_token(s, base_url)
+        # Next, get the address_id
+        address_id = get_address_id(s, base_url, csrf_token, postcode, paon)
+        # Finally, use the address_id to get the collection data
+        collection_data = get_collection_data(s, base_url, csrf_token, address_id)
+        if collection_data != "":
+            # Form a JSON wrapper
+            data = {"bins": []}
+
+            for c in collection_data:
+                collection_type = c["round"]
+                for c_date in c["upcomingCollections"]:
+                    collection_date = (
+                        re.search(r"Your (.*) is(.*)", c_date).group(2).strip()
+                    )
+                    dict_data = {
+                        "type": collection_type,
+                        "collectionDate": datetime.strptime(
+                            collection_date, "%A %d %B %Y"
+                        ).strftime(date_format),
+                    }
+                    data["bins"].append(dict_data)
+
+            if len(data["bins"]) == 0:
+                raise ValueError(
+                    "Code 5: No bin data found. Please ensure the council website is showing data first,"
+                    " then open an issue on GitHub."
+                )
+
+            data["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+            )
+            return data

{uk_bin_collection-0.76.1.dist-info → uk_bin_collection-0.77.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.76.1
+Version: 0.77.0
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.76.1.dist-info → uk_bin_collection-0.77.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
 uk_bin_collection/README.rst,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-uk_bin_collection/tests/council_feature_input_parity.py,sha256=66MrXc-3SnkricUV7LyAs9GFux7wEH31xcs9xbdB5Qo,3515
+uk_bin_collection/tests/council_feature_input_parity.py,sha256=_BINFpWMsMg91Jf1MYydPykOi8gu06x8C-E2AdT0A3Q,5205
 uk_bin_collection/tests/features/environment.py,sha256=VQZjJdJI_kZn08M0j5cUgvKT4k3iTw8icJge1DGOkoA,127
-uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=XIYnckDPBwBd7jTX2CN7Nv4mhUQsiJYOnYzvXVGfpdk,16246
-uk_bin_collection/tests/input.json,sha256=lbJZCvGd0kvIvI-0dD7Q9clMnkk_zX2UWbp8j2Ji1ow,53474
+uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=JfARfiOGbj6ENoieIpFw0XKmkGgm4krWg8sbofgx9k4,16357
+uk_bin_collection/tests/input.json,sha256=PaincPdmWl4MqajMT9-79qGNOK2Zz2cgAY2_bTuy4gk,53745
 uk_bin_collection/tests/output.schema,sha256=ZwKQBwYyTDEM4G2hJwfLUVM-5v1vKRvRK9W9SS1sd18,1086
 uk_bin_collection/tests/step_defs/step_helpers/file_handler.py,sha256=Ygzi4V0S1MIHqbdstUlIqtRIwnynvhu4UtpweJ6-5N8,1474
 uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=Pg3Z7c3zrebx1w-yIZ9xTJ3E-okVndlNIAJL-F46HKU,2962
@@ -24,6 +24,7 @@ uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py,sha256=9MrbpfR17R6
 uk_bin_collection/uk_bin_collection/councils/BirminghamCityCouncil.py,sha256=wIJgp7gjUL2zVALZNEz0YapDpvAXmriOswE03EqH49o,4130
 uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py,sha256=jHbCK8sL09vdmdP7Xnh8lIrU5AHTnJLEZfOLephPvWg,4090
 uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py,sha256=MrSzfK3EukRVasgP1nxAqg3lTkZzlAkDLJ2Nu5dJJ-0,4023
+uk_bin_collection/uk_bin_collection/councils/BracknellForestCouncil.py,sha256=35P9f4j6LpteYMaUhpE-NdfzupTRg0wEQyojH1n-OUw,9015
 uk_bin_collection/uk_bin_collection/councils/BradfordMDC.py,sha256=9EDTdhbGb8BQ5PLEw9eiWZ3BrS0TtoFpYOHQU44wc2k,4308
 uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py,sha256=k6qt4cds-Ejd97Z-__pw2BYvGVbFdc9SUfF73PPrTNA,5823
 uk_bin_collection/uk_bin_collection/councils/BristolCityCouncil.py,sha256=VT7k0AzcWL_h6E7Yr84NXs-FJ7qbYed7giR4HClHYE0,5575
@@ -164,8 +165,8 @@ uk_bin_collection/uk_bin_collection/councils/WyreCouncil.py,sha256=zDDa7n4K_zm5P
 uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py,sha256=I2kBYMlsD4bIdsvmoSzBjJAvTTi6yPfJa8xjJx1ys2w,1490
 uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=4s9ODGPAwPqwXc8SrTX5Wlfmizs3_58iXUtHc4Ir86o,1162
 uk_bin_collection/uk_bin_collection/get_bin_data.py,sha256=9qppF2oPkhmOoK8-ZkRIU1M6vhBh-yUCWAZEEd07iLk,5414
-uk_bin_collection-0.76.1.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
-uk_bin_collection-0.76.1.dist-info/METADATA,sha256=hxUuy3JDHnEOpdpGmTvly1ME2eNlTxOBM5n9ln6EmEg,12594
-uk_bin_collection-0.76.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-uk_bin_collection-0.76.1.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
-uk_bin_collection-0.76.1.dist-info/RECORD,,
+uk_bin_collection-0.77.0.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
+uk_bin_collection-0.77.0.dist-info/METADATA,sha256=dWmzwfpIqThI-9oJf3jASJodK51-kB5TO1bgM_XJdY0,12594
+uk_bin_collection-0.77.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+uk_bin_collection-0.77.0.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
+uk_bin_collection-0.77.0.dist-info/RECORD,,