uk_bin_collection 0.134.0__py3-none-any.whl → 0.134.2__py3-none-any.whl

uk_bin_collection/tests/input.json

@@ -383,12 +383,12 @@
         "wiki_note": "Follow the instructions [here](https://www.chelmsford.gov.uk/myhome/) until you get the page listing your address, then copy the entire address text and use that in the house number field."
     },
     "CheltenhamBoroughCouncil": {
-        "house_number": "Monday",
-        "postcode": "Week 1",
+        "uprn": "100120372027",
+        "postcode": "GL51 3NA",
         "skip_get_url": true,
         "url": "https://www.cheltenham.gov.uk",
         "wiki_name": "Cheltenham Borough Council",
-        "wiki_note": "Use the House Number field to pass the DAY of the week for your collections. [Monday/Tuesday/Wednesday/Thursday/Friday]. Use the 'postcode' field to pass the WEEK (wrapped in quotes) for your collections. [Week 1/Week 2]."
+        "wiki_note": "Pass the UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search)."
     },
     "CheshireEastCouncil": {
         "url": "https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/GetBartecJobList?uprn=100012791226&onelineaddress=3%20COBBLERS%20YARD,%20SK9%207DZ&_=1689413260149",
@@ -609,6 +609,7 @@
         "house_number": "1",
         "postcode": "CM20 2FZ",
         "skip_get_url": true,
+        "web_driver": "http://selenium:4444",
         "url": "https://www.eastherts.gov.uk",
         "wiki_name": "East Herts Council",
         "wiki_note": "Pass the house number and postcode in their respective parameters."
@@ -2173,4 +2174,4 @@
         "wiki_name": "York Council",
         "wiki_note": "Provide your UPRN."
     }
-}
+}

uk_bin_collection/uk_bin_collection/councils/CheltenhamBoroughCouncil.py

@@ -1,18 +1,18 @@
-import re
-import time
+# import re
 
 import requests
-from bs4 import BeautifulSoup
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.support.ui import Select
-from selenium.webdriver.support.wait import WebDriverWait
 
-from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.common import (
+    check_postcode,
+    check_uprn,
+    datetime,
+    get_dates_every_x_days,
+    json,
+    timedelta,
+)
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
-# import the wonderful Beautiful Soup and the URL grabber
 class CouncilClass(AbstractGetBinDataClass):
     """
     Concrete classes have to implement all abstract operations of the
@@ -20,42 +20,193 @@ class CouncilClass(AbstractGetBinDataClass):
     implementation.
     """
 
-    def parse_data(self, page: str, **kwargs) -> dict:
+    def parse_data(self, page: str, **kwargs: str) -> dict[str, list[dict[str, str]]]:
+        if (postcode := kwargs.get("postcode")) is None:
+            raise KeyError("Missing: postcode")
+        if (uprn := kwargs.get("uprn")) is None:
+            raise KeyError("Missing: uprn")
+        check_postcode(postcode)
+        check_uprn(uprn)
+        bindata: dict[str, list[dict[str, str]]] = {"bins": []}
+        location_x: int = 0
+        location_y: int = 0
+        location_usrn: str = ""
 
-        collection_day = kwargs.get("paon")
-        collection_week = kwargs.get("postcode")
-        bindata = {"bins": []}
+        # Ensure any cookies set are maintained in a requests session
+        s = requests.session()
+
+        # Ask for a new SessionId from the server
+        session_id_url = "https://maps.cheltenham.gov.uk/map/Aurora.svc/"\
+            "RequestSession?userName=guest+CBC&password=&"\
+            "script=%5CAurora%5CCBC+Waste+Streets.AuroraScript%24"
+        session_id_response = s.get(session_id_url)
+        session_id_response.raise_for_status()
+        session_id = session_id_response.json().get("Session").get("SessionId")
+
+        # Ask what tasks we can do within the session
+        tasks_url = f"https://maps.cheltenham.gov.uk/map/Aurora.svc/"\
+            f"GetWorkflow?sessionId={session_id}&workflowId=wastestreet"
+        tasks_response = s.get(tasks_url)
+        tasks_response.raise_for_status()
+        # JSON response contained a BOM marker
+        tasks = json.loads(tasks_response.text[1:])
+        retrieve_results_task_id, initialise_map_task_id, drilldown_task_id = None, None, None
+        # Pull out the ID's of the tasks we will need
+        for task in tasks.get("Tasks"):
+            if task.get("$type") == "StatMap.Aurora.FetchResultSetTask, StatMapService":
+                retrieve_results_task_id = task.get("Id")
+            elif task.get("$type") == "StatMap.Aurora.SaveStateTask, StatMapService":
+                initialise_map_task_id = task.get("Id")
+            elif task.get("$type") == "StatMap.Aurora.DrillDownTask, StatMapService":
+                drilldown_task_id = task.get("Id")
+        if not all([retrieve_results_task_id, initialise_map_task_id, drilldown_task_id]):
+            raise ValueError("Not all task ID's found")
+
+        # Find the X / Y coordinates for the requested postcode
+        postcode_search_url = "https://maps.cheltenham.gov.uk/map/Aurora.svc/FindLocation?"\
+            f"sessionId={session_id}&address={postcode}&limit=1000"
+        postcode_search_response = s.get(postcode_search_url)
+        postcode_search_response.raise_for_status()
+        if len(locations_list := postcode_search_response.json().get("Locations")) == 0:
+            raise ValueError("Address locations empty")
+        for location in locations_list:
+            location_search_url = "https://maps.cheltenham.gov.uk/map/Aurora.svc/FindLocation?"\
+                f"sessionId={session_id}&locationId={location.get('Id')}"
+            location_search_response = s.get(location_search_url)
+            location_search_response.raise_for_status()
+            if not (location_list := location_search_response.json().get("Locations")):
+                raise KeyError("Locations wasn't present in results")
+            if not (location_detail := location_list[0].get("Details")):
+                raise KeyError("Details wasn't present in location")
+            location_uprn = [detail.get(
+                "Value") for detail in location_detail if detail.get("Name") == "UPRN"][0]
+            if str(location_uprn) == uprn:
+                location_usrn = str([detail.get(
+                    "Value") for detail in location_detail if detail.get("Name") == "USRN"][0])
+                location_x = location_list[0].get("X")
+                location_y = location_list[0].get("Y")
+                break
+
+        # Needed to initialise the server to allow follow on call
+        open_map_url = "https://maps.cheltenham.gov.uk/map/Aurora.svc/OpenScriptMap?"\
+            f"sessionId={session_id}"
+        if res := s.get(open_map_url):
+            res.raise_for_status()
+
+        # Needed to initialise the server to allow follow on call
+        save_state_map_url = "https://maps.cheltenham.gov.uk/map/Aurora.svc/ExecuteTaskJob?"\
+            f"sessionId={session_id}&taskId={initialise_map_task_id}&job="\
+            "%7BTask%3A+%7B+%24type%3A+%27StatMap.Aurora.SaveStateTask%2C"\
+            "+StatMapService%27+%7D%7D"
+        if res := s.get(save_state_map_url):
+            res.raise_for_status()
+
+        # Start search for address given by x / y coord
+        drilldown_map_url = "https://maps.cheltenham.gov.uk/map/Aurora.svc/ExecuteTaskJob?"\
+            f"sessionId={session_id}&taskId={drilldown_task_id}&job=%7B%22"\
+            f"QueryX%22%3A{location_x}%2C%22QueryY%22%3A{location_y}%2C%22"\
+            "Task%22%3A%7B%22Type%22%3A%22StatMap.Aurora.DrillDownTask%2C"\
+            "+StatMapService%22%7D%7D"
+        if res := s.get(drilldown_map_url):
+            res.raise_for_status()
+
+        # Get results from search for address given by x / y coord
+        address_details_url = "https://maps.cheltenham.gov.uk/map/Aurora.svc/ExecuteTaskJob?"\
+            f"sessionId={session_id}&taskId={retrieve_results_task_id}"\
+            f"&job=%7B%22QueryX%22%3A{location_x}%2C%22QueryY%22%3A"\
+            f"{location_y}%2C%22Task%22%3A%7B%22Type%22%3A%22"\
+            "StatMap.Aurora.FetchResultSetTask%2C+StatMapService"\
+            "%22%2C%22ResultSetName%22%3A%22inspection%22%7D%7D"
+        address_details_response = s.get(address_details_url)
+        address_details_response.raise_for_status()
+        # JSON response contained a BOM marker, skip first character
+        address_details = json.loads(address_details_response.text[1:])
+        if not (task_results := address_details.get("TaskResult")):
+            raise KeyError("TaskResult wasn't present in results")
+        if not (distance_export_set := task_results.get("DistanceOrderedSet")):
+            raise KeyError("DistanceOrderedSet wasn't present in TaskResult")
+        if not (result_set := distance_export_set.get("ResultSet")):
+            raise KeyError("ResultSet wasn't present in DistanceOrderedSet")
+        if not (result_tables := result_set.get("Tables")):
+            raise KeyError("Tables wasn't present in ResultSet")
+        result = result_tables[0]
+        column_names: dict[int, str] = {}
+        result_dict: dict[str, str | int] = {}
+        for column in result.get("ColumnDefinitions"):
+            column_names[column.get("ColumnIndex")] = column.get("ColumnName")
+        for r in result.get("Records"):
+            result_dict: dict[str, str | int] = {}
+            for idx, column_value in enumerate(r):
+                if not (column_name := column_names.get(idx)):
+                    raise IndexError("Column index out of range")
+                result_dict[column_name.upper()] = column_value
+            # Validate the street against the USRN. Some locations can return multiple results.
+            # Break on first match of USRN
+            # TODO: Need to select the correct option out of all available options
+            if location_usrn == str(result_dict.get("USRN")):
+                break
+
+        refuse_week, recycling_week, garden_week = 0, 0, 0
+        # After we've got the correct result, pull out the week number each bin type is taken on
+        if (refuse_week_raw := result_dict.get("New_Refuse_Week".upper())) is not None:
+            refuse_week = int(refuse_week_raw)
+        if (recycling_week_raw := result_dict.get("New_Recycling_Week".upper())) is not None:
+            recycling_week = int(recycling_week_raw)
+        if (garden_week_raw := result_dict.get("Garden_Bin_Week".upper())) is not None:
+            garden_week = int(garden_week_raw)
+
+        if not all([refuse_week, recycling_week, garden_week]):
+            raise KeyError("Not all week numbers found")
 
         days_of_week = [
-            "Monday",
-            "Tuesday",
-            "Wednesday",
-            "Thursday",
-            "Friday",
-            "Saturday",
-            "Sunday",
+            "MON",
+            "TUE",
+            "WED",
+            "THU",
+            "FRI",
+            "SAT",
+            "SUN",
         ]
 
-        week = ["Week 1", "Week 2"]
+        refuse_day_offset = days_of_week.index(
+            str(result_dict.get("New_Refuse_Day_internal".upper())).upper())
+        recycling_day_offset = days_of_week.index(
+            str(result_dict.get("New_Recycling_Day".upper())).upper())
+        garden_day_offset = days_of_week.index(
+            str(result_dict.get("New_Garden_Day".upper())).upper())
+        food_day_offset = days_of_week.index(
+            str(result_dict.get("New_Food_Day".upper())).upper())
 
-        offset_days = days_of_week.index(collection_day)
-        collection_week = week.index(collection_week)
+        # Initialise WEEK-1/WEEK-2 based on known details
+        week_1_epoch = datetime(2025, 1, 13)
 
-        if collection_week == 0:
-            refusestartDate = datetime(2024, 11, 25)
-            recyclingstartDate = datetime(2024, 11, 18)
-        else:
-            refusestartDate = datetime(2024, 11, 18)
-            recyclingstartDate = datetime(2024, 11, 25)
+        # Start of this week
+        this_week = datetime.now() - timedelta(days=datetime.now().weekday())
 
-        refuse_dates = get_dates_every_x_days(refusestartDate, 14, 28)
-        recycling_dates = get_dates_every_x_days(recyclingstartDate, 14, 28)
-        food_dates = get_dates_every_x_days(recyclingstartDate, 7, 56)
+        # If there's an even number of weeks between the week-1
+        # epoch and this week, then this week is of type week-1
+        if (((this_week - week_1_epoch).days // 7)) % 2 == 0:
+            week = {
+                1: this_week,
+                2: this_week + timedelta(days=7)
+            }
+        else:
+            week = {
+                1: this_week - timedelta(days=7),
+                2: this_week
+            }
 
-        for refuseDate in refuse_dates:
+        refuse_dates: list[str] = get_dates_every_x_days(
+            week[refuse_week], 14, 28)
+        recycling_dates: list[str] = get_dates_every_x_days(
+            week[recycling_week], 14, 28)
+        garden_dates: list[str] = get_dates_every_x_days(
+            week[garden_week], 14, 28)
 
+        for refuse_date in refuse_dates:
             collection_date = (
-                datetime.strptime(refuseDate, "%d/%m/%Y") + timedelta(days=offset_days)
+                datetime.strptime(refuse_date, "%d/%m/%Y") +
+                timedelta(days=refuse_day_offset)
             ).strftime("%d/%m/%Y")
 
             dict_data = {
@@ -64,11 +215,11 @@ class CouncilClass(AbstractGetBinDataClass):
             }
             bindata["bins"].append(dict_data)
 
-        for recyclingDate in recycling_dates:
+        for recycling_date in recycling_dates:
 
             collection_date = (
-                datetime.strptime(recyclingDate, "%d/%m/%Y")
-                + timedelta(days=offset_days)
+                datetime.strptime(recycling_date, "%d/%m/%Y") +
+                timedelta(days=recycling_day_offset)
             ).strftime("%d/%m/%Y")
 
             dict_data = {
@@ -77,26 +228,81 @@ class CouncilClass(AbstractGetBinDataClass):
             }
             bindata["bins"].append(dict_data)
 
-            dict_data = {
-                "type": "Garden Waste Bin",
-                "collectionDate": collection_date,
-            }
-            bindata["bins"].append(dict_data)
-
-        for food_date in food_dates:
+        for garden_date in garden_dates:
 
             collection_date = (
-                datetime.strptime(food_date, "%d/%m/%Y") + timedelta(days=offset_days)
+                datetime.strptime(garden_date, "%d/%m/%Y") +
+                timedelta(days=garden_day_offset)
             ).strftime("%d/%m/%Y")
 
             dict_data = {
-                "type": "Food Waste Bin",
+                "type": "Garden Waste Bin",
                 "collectionDate": collection_date,
             }
             bindata["bins"].append(dict_data)
 
+        if ((food_waste_week := str(result_dict.get("FOOD_WASTE_WEEK_EXTERNAL", "")).upper())
+                == "weekly".upper()):
+            food_dates: list[str] = get_dates_every_x_days(week[1], 7, 56)
+
+            for food_date in food_dates:
+
+                collection_date = (
+                    datetime.strptime(food_date, "%d/%m/%Y") +
+                    timedelta(days=food_day_offset)
+                ).strftime("%d/%m/%Y")
+
+                dict_data = {
+                    "type": "Food Waste Bin",
+                    "collectionDate": collection_date,
+                }
+                bindata["bins"].append(dict_data)
+        # Sometimes the food bin is collected on different days between
+        # week-1 and week-2
+        else:
+            first_week: str | int
+            second_week_detail: str
+            first_week, _, second_week_detail = food_waste_week.partition("&")
+            first_week = int(first_week.strip())
+
+            second_week_day, _, second_week_number = second_week_detail.partition(
+                "WEEK")
+            second_week_number = int(second_week_number.strip())
+            second_week_day: str = second_week_day.strip()[:3]
+
+            food_dates_first: list[str] = get_dates_every_x_days(
+                week[first_week], 14, 28)
+            food_dates_second: list[str] = get_dates_every_x_days(
+                week[second_week_number], 14, 28)
+            second_week_offset = days_of_week.index(second_week_day)
+
+            for food_date in food_dates_first:
+
+                collection_date = (
+                    datetime.strptime(food_date, "%d/%m/%Y") +
+                    timedelta(days=food_day_offset)
+                ).strftime("%d/%m/%Y")
+
+                dict_data = {
+                    "type": "Food Waste Bin",
+                    "collectionDate": collection_date,
+                }
+                bindata["bins"].append(dict_data)
+            for food_date in food_dates_second:
+
+                collection_date = (
+                    datetime.strptime(food_date, "%d/%m/%Y") +
+                    timedelta(days=second_week_offset)
+                ).strftime("%d/%m/%Y")
+
+                dict_data = {
+                    "type": "Food Waste Bin",
+                    "collectionDate": collection_date,
+                }
+                bindata["bins"].append(dict_data)
+
         bindata["bins"].sort(
-            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+            key=lambda x: datetime.strptime(
+                x.get("collectionDate", ""), "%d/%m/%Y")
         )
-
         return bindata

{uk_bin_collection-0.134.0.dist-info → uk_bin_collection-0.134.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.134.0
+Version: 0.134.2
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.134.0.dist-info → uk_bin_collection-0.134.2.dist-info}/RECORD

@@ -2,7 +2,7 @@ uk_bin_collection/README.rst,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 uk_bin_collection/tests/council_feature_input_parity.py,sha256=DO6Mk4ImYgM5ZCZ-cutwz5RoYYWZRLYx2tr6zIs_9Rc,3843
 uk_bin_collection/tests/features/environment.py,sha256=VQZjJdJI_kZn08M0j5cUgvKT4k3iTw8icJge1DGOkoA,127
 uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=SJK-Vc737hrf03tssxxbeg_JIvAH-ddB8f6gU1LTbuQ,251
-uk_bin_collection/tests/input.json,sha256=b5kS_5XsRt-cTRw-YdKTLciyPZHAk1mzHXPSqNgUbJ0,114203
+uk_bin_collection/tests/input.json,sha256=93b85jhY57G2qIx1RZh-DL_J3kia3UJZ_oo9d-Sq_OI,114121
 uk_bin_collection/tests/output.schema,sha256=ZwKQBwYyTDEM4G2hJwfLUVM-5v1vKRvRK9W9SS1sd18,1086
 uk_bin_collection/tests/step_defs/step_helpers/file_handler.py,sha256=Ygzi4V0S1MIHqbdstUlIqtRIwnynvhu4UtpweJ6-5N8,1474
 uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=VZ0a81sioJULD7syAYHjvK_-nT_Rd36tUyzPetSA0gk,3475
@@ -62,7 +62,7 @@ uk_bin_collection/uk_bin_collection/councils/CarmarthenshireCountyCouncil.py,sha
 uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py,sha256=JVPYUIlU2ISgbUSr5AOOXNK6IFQFtQmhZyYIMAOedD4,3858
 uk_bin_collection/uk_bin_collection/councils/CharnwoodBoroughCouncil.py,sha256=tXfzMetN6wxahuGGRp2mIyCCDSL4F2aG61HhUxw6COQ,2172
 uk_bin_collection/uk_bin_collection/councils/ChelmsfordCityCouncil.py,sha256=EB88D0MNJwuDZ2GX1ENc5maGYx17mnHTCtNl6s-v11E,5090
-uk_bin_collection/uk_bin_collection/councils/CheltenhamBoroughCouncil.py,sha256=z59g9h6QwdeCNzonM5AI-6NvpLKocD-DT8eSvqk8RwU,3198
+uk_bin_collection/uk_bin_collection/councils/CheltenhamBoroughCouncil.py,sha256=fo9n2AuD8a174h3YMa9awa3S4YO2XU1KaInQ4ULz6s4,13686
 uk_bin_collection/uk_bin_collection/councils/CheshireEastCouncil.py,sha256=K95UaS67nxGSFzLoyqMSZcLmfhwjDImVXznMZC_bFa4,1680
 uk_bin_collection/uk_bin_collection/councils/CheshireWestAndChesterCouncil.py,sha256=5mKZf22NgdyBY-SqV0c2q8b8IJobkoZrsfGEVUcxUyM,3544
 uk_bin_collection/uk_bin_collection/councils/ChesterfieldBoroughCouncil.py,sha256=mZiM8Ugm_OP0JkC5pLaQmi4i79mAp4SNNrcIdsREjHw,7198
@@ -307,8 +307,8 @@ uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py,sha256=I2kBYMlsD4bId
 uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=EQWRhZ2pEejlvm0fPyOTsOHKvUZmPnxEYO_OWRGKTjs,1158
 uk_bin_collection/uk_bin_collection/create_new_council.py,sha256=m-IhmWmeWQlFsTZC4OxuFvtw5ZtB8EAJHxJTH4O59lQ,1536
 uk_bin_collection/uk_bin_collection/get_bin_data.py,sha256=YvmHfZqanwrJ8ToGch34x-L-7yPe31nB_x77_Mgl_vo,4545
-uk_bin_collection-0.134.0.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
-uk_bin_collection-0.134.0.dist-info/METADATA,sha256=Svl74e_j58YAsZNRQkgc2gzL1rrJNozpkXcz-nwQWSo,19741
-uk_bin_collection-0.134.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-uk_bin_collection-0.134.0.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
-uk_bin_collection-0.134.0.dist-info/RECORD,,
+uk_bin_collection-0.134.2.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
+uk_bin_collection-0.134.2.dist-info/METADATA,sha256=AJh8NQGTC8kAz_sSl8NK8gd1ksOeXucIsEIyX11Eazg,19741
+uk_bin_collection-0.134.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+uk_bin_collection-0.134.2.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
+uk_bin_collection-0.134.2.dist-info/RECORD,,