uk_bin_collection 0.134.3__tar.gz → 0.135.0__tar.gz

{uk_bin_collection-0.134.3 → uk_bin_collection-0.135.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.134.3
+Version: 0.135.0
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.134.3 → uk_bin_collection-0.135.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.134.3"
+version = "0.135.0"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

uk_bin_collection-0.135.0/uk_bin_collection/tests/check_selenium_url_in_input.json.py

@@ -0,0 +1,209 @@
+import json
+import requests
+import sys
+import base64
+from tabulate import tabulate
+
+
+def get_council_files(repo, branch):
+    """
+    Get a list of all .py council files in the 'councils' directory
+    from the GitHub repo (via API), plus a mapping from council name
+    to the file's GitHub 'download_url' or 'contents_url'.
+    """
+    url = f"https://api.github.com/repos/{repo}/contents/uk_bin_collection/uk_bin_collection/councils?ref={branch}"
+    print(f"Fetching council files from: {url}")
+    response = requests.get(url, headers={"Accept": "application/vnd.github.v3+json"})
+    if response.status_code == 200:
+        data = response.json()
+        # data should be a list of items in that folder
+        if isinstance(data, list):
+            councils = {}
+            for item in data:
+                name = item["name"]
+                if name.endswith(".py"):
+                    council_name = name.replace(".py", "")
+                    councils[council_name] = item["url"]  # 'url' gives API-based content URL
+            return councils
+        else:
+            raise ValueError("Expected a list from the GitHub response but got something else.")
+    else:
+        print(f"Failed to fetch councils from files: {response.content}")
+        return {}
+
+
+def get_council_file_content(api_url):
+    """
+    Given the API URL for a file in GitHub, fetch its content (decoded).
+    The 'download_url' is direct raw, but the 'url' is the API URL for the content.
+    We'll use the latter, decode base64, and return the text.
+    """
+    # Example: https://api.github.com/repos/robbrad/UKBinCollectionData/contents/...
+    response = requests.get(api_url, headers={"Accept": "application/vnd.github.v3+json"})
+    if response.status_code == 200:
+        file_json = response.json()
+        # file_json["content"] is base64-encoded
+        content = file_json.get("content", "")
+        decoded = base64.b64decode(content).decode("utf-8")
+        return decoded
+    else:
+        print(f"Failed to fetch file content: {response.content}")
+        return ""
+
+
+def get_input_json_data(repo, branch):
+    """
+    Fetch the entire input.json from GitHub and return it as a Python dict.
+    """
+    url = f"https://api.github.com/repos/{repo}/contents/uk_bin_collection/tests/input.json?ref={branch}"
+    print(f"Fetching input JSON from: {url}")
+    response = requests.get(url, headers={"Accept": "application/vnd.github.v3+json"})
+    if response.status_code == 200:
+        try:
+            file_json = response.json()
+            content = file_json.get("content", "")
+            decoded = base64.b64decode(content).decode("utf-8")
+            data = json.loads(decoded)
+            return data
+        except json.JSONDecodeError as e:
+            print(f"JSON decoding error: {e}")
+            raise
+    else:
+        print(f"Failed to fetch input JSON: {response.content}")
+        return {}
+
+
+def council_needs_update(council_name, json_data, council_file_content):
+    """
+    Check if the given council needs an update:
+      - We say 'needs update' if 'web_driver' is missing in the JSON,
+        BUT the script uses 'create_webdriver' in code.
+    """
+    # If the council isn't in the JSON at all, we can't do the check
+    # (or we assume no JSON data => no web_driver?).
+    council_data = json_data.get(council_name, {})
+    web_driver_missing = ("web_driver" not in council_data)
+    create_webdriver_present = ("create_webdriver" in council_file_content)
+
+    return web_driver_missing and create_webdriver_present
+
+
+def compare_councils(file_council_dict, json_data):
+    """
+    Compare councils in files vs councils in JSON, check for needs_update,
+    and gather everything for final tabulation.
+
+    Returns:
+      - all_councils_data: dict keyed by council name:
+            {
+              "in_files": bool,
+              "in_json": bool,
+              "discrepancies_count": int,
+              "needs_update": bool
+            }
+      - any_discrepancies_found: bool (if any differences in in_files vs in_json)
+      - any_updates_needed: bool (if any council needs update)
+    """
+    file_councils = set(file_council_dict.keys())
+    json_councils = set(json_data.keys())
+
+    all_councils = file_councils.union(json_councils)
+    all_council_data = {}
+
+    any_discrepancies_found = False
+    any_updates_needed = False
+
+    for council in all_councils:
+        in_files = council in file_councils
+        in_json = council in json_councils
+        # Count how many are False
+        discrepancies_count = [in_files, in_json].count(False)
+
+        # If the file is in the repo, fetch its content for checking
+        content = ""
+        if in_files:
+            file_api_url = file_council_dict[council]
+            content = get_council_file_content(file_api_url)
+
+        # Evaluate "needs_update" only if the file is in place
+        # (If there's no file, you might consider it "False" by default)
+        needs_update = False
+        if in_files:
+            needs_update = council_needs_update(council, json_data, content)
+
+        if discrepancies_count > 0:
+            any_discrepancies_found = True
+        if needs_update:
+            any_updates_needed = True
+
+        all_council_data[council] = {
+            "in_files": in_files,
+            "in_json": in_json,
+            "discrepancies_count": discrepancies_count,
+            "needs_update": needs_update,
+        }
+
+    return all_council_data, any_discrepancies_found, any_updates_needed
+
+
+def main(repo="robbrad/UKBinCollectionData", branch="master"):
+    print(f"Starting comparison for repo: {repo}, branch: {branch}")
+
+    # 1) Get council file data (dict: { council_name: content_api_url, ... })
+    file_council_dict = get_council_files(repo, branch)
+
+    # 2) Get the entire JSON data
+    json_data = get_input_json_data(repo, branch)
+
+    # 3) Compare
+    (
+        all_councils_data,
+        discrepancies_found,
+        updates_needed,
+    ) = compare_councils(file_council_dict, json_data)
+
+    # 4) Print results
+    table_data = []
+    headers = ["Council Name", "In Files", "In JSON", "Needs Update?", "Discrepancies"]
+    # Sort councils so that ones with the highest discrepancy or update appear first
+    # Then alphabetical if tie:
+    def sort_key(item):
+        # item is (council_name, data_dict)
+        return (
+            item[1]["needs_update"],         # sort by needs_update (False < True)
+            item[1]["discrepancies_count"],  # then by discrepancies
+            item[0],                         # then by name
+        )
+
+    # We'll sort descending for "needs_update", so invert the boolean or reverse later
+    sorted_councils = sorted(
+        all_councils_data.items(),
+        key=lambda x: (not x[1]["needs_update"], x[1]["discrepancies_count"], x[0])
+    )
+
+    for council, presence in sorted_councils:
+        row = [
+            council,
+            "✔" if presence["in_files"] else "✘",
+            "✔" if presence["in_json"] else "✘",
+            "Yes" if presence["needs_update"] else "No",
+            presence["discrepancies_count"],
+        ]
+        table_data.append(row)
+
+    print(tabulate(table_data, headers=headers, tablefmt="grid"))
+
+    # 5) Determine exit code:
+    #    If any discrepancies OR any council needs updates -> fail
+    if discrepancies_found or updates_needed:
+        print("Some discrepancies found or updates are needed. Failing workflow.")
+        sys.exit(1)
+    else:
+        print("No discrepancies found and no updates needed. Workflow successful.")
+
+
+if __name__ == "__main__":
+    # Optional CLI args: python script.py <repo> <branch>
+    repo_arg = sys.argv[1] if len(sys.argv) > 1 else "robbrad/UKBinCollectionData"
+    branch_arg = sys.argv[2] if len(sys.argv) > 2 else "master"
+    main(repo_arg, branch_arg)

{uk_bin_collection-0.134.3 → uk_bin_collection-0.135.0}/uk_bin_collection/tests/input.json

@@ -18,6 +18,12 @@
         "wiki_name": "Adur and Worthing Councils",
         "wiki_note": "Replace XXXXXXXX with your UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find it."
     },
+    "AmberValleyBoroughCouncil": {
+        "url": "https://ambervalley.gov.uk",
+        "uprn": "100030026621",
+        "wiki_name": "Amber Valley Borough Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "AntrimAndNewtonabbeyCouncil": {
         "url": "https://antrimandnewtownabbey.gov.uk/residents/bins-recycling/bins-schedule/?Id=643",
         "wiki_command_url_override": "https://antrimandnewtownabbey.gov.uk/residents/bins-recycling/bins-schedule/?Id=XXXX",
@@ -197,6 +203,12 @@
         "wiki_name": "Blaenau Gwent County Borough Council",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
+    "BolsoverCouncil": {
+        "url": "https://bolsover.gov.uk",
+        "uprn": "100030066827",
+        "wiki_name": "Bolsover Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "BoltonCouncil": {
         "postcode": "BL1 5PQ",
         "skip_get_url": true,
@@ -391,6 +403,12 @@
         "wiki_name": "Cheltenham Borough Council",
         "wiki_note": "Pass the UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search)."
     },
+    "CherwellDistrictCouncil": {
+        "uprn": "100121292407",
+        "url": "https://www.cherwell.gov.uk",
+        "wiki_name": "Cherwell District Council",
+        "wiki_note": "Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN."
+    },
     "CheshireEastCouncil": {
         "url": "https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/GetBartecJobList?uprn=100012791226&onelineaddress=3%20COBBLERS%20YARD,%20SK9%207DZ&_=1689413260149",
         "wiki_command_url_override": "https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/GetBartecJobList?uprn=XXXXXXXX&onelineaddress=XXXXXXXX&_=1689413260149",
@@ -440,11 +458,10 @@
         "wiki_note": "Pass the house name/number in the house number parameter, wrapped in double quotes."
     },
     "ConwyCountyBorough": {
-        "postcode": "LL30 2DF",
         "uprn": "100100429249",
-        "url": "https://www.conwy.gov.uk/Contensis-Forms/erf/collection-result-soap-xmas.asp?ilangid=1&uprn=100100429249",
+        "url": "https://www.conwy.gov.uk",
         "wiki_name": "Conwy County Borough Council",
-        "wiki_note": "Conwy County Borough Council uses a straight UPRN in the URL, e.g., `&uprn=XXXXXXXXXXXXX`."
+        "wiki_note": "Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN."
     },
     "CopelandBoroughCouncil": {
         "uprn": "100110734613",
@@ -535,7 +552,6 @@
         "skip_get_url": true,
         "uprn": "10070102161",
         "url": "https://www.derbyshiredales.gov.uk/",
-        "web_driver": "http://selenium:4444",
         "wiki_name": "Derbyshire Dales District Council",
         "wiki_note": "Pass the UPRN and postcode. To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)."
     },
@@ -709,6 +725,12 @@
         "wiki_name": "Epping Forest District Council",
         "wiki_note": "Replace the postcode in the URL with your own."
     },
+    "EpsomandEwellBoroughCouncil": {
+        "uprn": "100061349083",
+        "url": "https://www.epsom-ewell.gov.uk",
+        "wiki_name": "Epsom and Ewell Borough Council",
+        "wiki_note": "Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN."
+    },
     "ErewashBoroughCouncil": {
         "skip_get_url": true,
         "uprn": "10003582028",
@@ -1162,6 +1184,14 @@
         "wiki_name": "Mid Devon Council",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
+    "MiddlesbroughCouncil": {
+        "house_number": "12 Constantine Court Park Road North, Middlesbrough",
+        "skip_get_url": true,
+        "url": "https://www.midsussex.gov.uk/waste-recycling/bin-collection/",
+        "web_driver": "http://selenium:4444",
+        "wiki_name": "Middlesbrough Council",
+        "wiki_note": "Pass the entire address without postcode as it appears when you type it on the website. This parser requires a Selenium webdriver."
+    },
     "MidSuffolkDistrictCouncil": {
         "house_number": "Monday",
         "postcode": "Week 2",
@@ -1457,6 +1487,14 @@
         "wiki_name": "Reading Borough Council",
         "wiki_note": "Replace XXXXXXXX with your property's UPRN."
     },
+    "RedcarandClevelandCouncil": {
+        "house_number": "11",
+        "postcode": "TS10 2RE",
+        "skip_get_url": true,
+        "url": "https://www.redcar-cleveland.gov.uk",
+        "wiki_name": "Redcar and Cleveland Council",
+        "wiki_note": "Pass the house name/number and postcode in their respective parameters"
+    },
     "RedditchBoroughCouncil": {
         "uprn": "10094557691",
         "url": "https://redditchbc.gov.uk",
@@ -1531,6 +1569,13 @@
         "wiki_name": "Rugby Borough Council",
         "wiki_note": "Provide your UPRN and postcode. You can find your UPRN using [FindMyAddress](https://www.findmyaddress.co.uk/search)."
     },
+    "RunnymedeBoroughCouncil": {
+        "skip_get_url": true,
+        "uprn": "100061483636",
+        "url": "https://www.runnymede.gov.uk/",
+        "wiki_name": "Runnymede Borough Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "RushcliffeBoroughCouncil": {
         "postcode": "NG13 8TZ",
         "skip_get_url": true,

uk_bin_collection-0.135.0/uk_bin_collection/uk_bin_collection/councils/AmberValleyBoroughCouncil.py

@@ -0,0 +1,60 @@
+from datetime import datetime
+
+import requests
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        WASTE_TYPES_DATE_KEY = {
+            "REFUSE": "refuseNextDate",
+            "RECYCLING": "recyclingNextDate",
+            "GREEN": "greenNextDate",
+            "COMMUNAL REFUSE": "communalRefNextDate",
+            "COMMUNAL RECYCLING": "communalRycNextDate",
+        }
+
+        URI = f"https://info.ambervalley.gov.uk/WebServices/AVBCFeeds/WasteCollectionJSON.asmx/GetCollectionDetailsByUPRN?uprn={user_uprn}"
+
+        # Make the GET request
+        response = requests.get(URI)
+
+        # Parse the JSON response
+        bin_collection = response.json()
+
+        # print(bin_collection)
+
+        for bin, datge_key in WASTE_TYPES_DATE_KEY.items():
+            date_ = datetime.strptime(
+                bin_collection[datge_key], "%Y-%m-%dT%H:%M:%S"
+            ).strftime(date_format)
+            if date_ == "01/01/1":
+                continue
+            elif date_ == "01/01/1900":
+                continue
+
+            dict_data = {
+                "type": bin,
+                "collectionDate": date_,
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata