uk_bin_collection 0.117.0__py3-none-any.whl → 0.119.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/CrawleyBoroughCouncil.py

@@ -1,4 +1,6 @@
-from bs4 import BeautifulSoup
+import time
+
+import requests
 from dateutil.relativedelta import relativedelta
 
 from uk_bin_collection.uk_bin_collection.common import *
@@ -19,43 +21,92 @@ class CouncilClass(AbstractGetBinDataClass):
         usrn = kwargs.get("paon")
         check_uprn(uprn)
         check_usrn(usrn)
+        bindata = {"bins": []}
+
+        SESSION_URL = "https://crawleybc-self.achieveservice.com/authapi/isauthenticated?uri=https%253A%252F%252Fcrawleybc-self.achieveservice.com%252Fen%252FAchieveForms%252F%253Fform_uri%253Dsandbox-publish%253A%252F%252FAF-Process-fb73f73e-e8f5-4441-9f83-8b5d04d889d6%252FAF-Stage-ec9ada91-d2d9-43bc-9730-597d15fc8108%252Fdefinition.json%2526redirectlink%253D%252Fen%2526cancelRedirectLink%253D%252Fen%2526noLoginPrompt%253D1%2526accept%253Dyes&hostname=crawleybc-self.achieveservice.com&withCredentials=true"
+
+        API_URL = "https://crawleybc-self.achieveservice.com/apibroker/"
+
+        currentdate = datetime.now().strftime("%d/%m/%Y")
+
+        data = {
+            "formValues": {
+                "Address": {
+                    "address": {
+                        "value": {
+                            "Address": {
+                                "usrn": {
+                                    "value": usrn,
+                                },
+                                "uprn": {
+                                    "value": uprn,
+                                },
+                            }
+                        },
+                    },
+                    "dayConverted": {
+                        "value": currentdate,
+                    },
+                    "getCollection": {
+                        "value": "true",
+                    },
+                    "getWorksheets": {
+                        "value": "false",
+                    },
+                },
+            },
+        }
+
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+            "User-Agent": "Mozilla/5.0",
+            "X-Requested-With": "XMLHttpRequest",
+            "Referer": "https://crawleybc-self.achieveservice.com/fillform/?iframe_id=fillform-frame-1&db_id=",
+        }
+        s = requests.session()
+        r = s.get(SESSION_URL)
+        r.raise_for_status()
+        session_data = r.json()
+        sid = session_data["auth-session"]
+        params = {
+            "api": "RunLookup",
+            "id": "5b4f0ec5f13f4",
+            "repeat_against": "",
+            "noRetry": "true",
+            "getOnlyTokens": "undefined",
+            "log_id": "",
+            "app_name": "AF-Renderer::Self",
+            # unix_timestamp
+            "_": str(int(time.time() * 1000)),
+            "sid": sid,
+        }
+
+        r = s.post(API_URL, json=data, headers=headers, params=params)
+        r.raise_for_status()
+
+        data = r.json()
+        rows_data = data["integration"]["transformed"]["rows_data"]["0"]
+        if not isinstance(rows_data, dict):
+            raise ValueError("Invalid data returned from API")
+
+        # Extract each service's relevant details for the bin schedule
+        for key, value in rows_data.items():
+            if key.endswith("DateNext"):
+                BinType = key.replace("DateNext", "Service")
+                for key2, value2 in rows_data.items():
+                    if key2 == BinType:
+                        BinType = value2
+                next_collection = datetime.strptime(value, "%A %d %B").replace(
+                    year=datetime.now().year
+                )
+                if datetime.now().month == 12 and next_collection.month == 1:
+                    next_collection = next_collection + relativedelta(years=1)
+
+                dict_data = {
+                    "type": BinType,
+                    "collectionDate": next_collection.strftime(date_format),
+                }
+                bindata["bins"].append(dict_data)
 
-        day = datetime.now().date().strftime("%d")
-        month = datetime.now().date().strftime("%m")
-        year = datetime.now().date().strftime("%Y")
-
-        api_url = (
-            f"https://my.crawley.gov.uk/appshost/firmstep/self/apps/custompage/waste?language=en&uprn={uprn}"
-            f"&usrn={usrn}&day={day}&month={month}&year={year}"
-        )
-        response = requests.get(api_url)
-
-        soup = BeautifulSoup(response.text, features="html.parser")
-        soup.prettify()
-
-        data = {"bins": []}
-
-        titles = [title.text for title in soup.select(".block-title")]
-        collection_tag = soup.body.find_all(
-            "div", {"class": "col-md-6 col-sm-6 col-xs-6"}, string="Next collection"
-        )
-        bin_index = 0
-        for tag in collection_tag:
-            for item in tag.next_elements:
-                if (
-                    str(item).startswith('<div class="date text-right text-grey">')
-                    and str(item) != ""
-                ):
-                    collection_date = datetime.strptime(item.text, "%A %d %B")
-                    next_collection = collection_date.replace(year=datetime.now().year)
-                    if datetime.now().month == 12 and next_collection.month == 1:
-                        next_collection = next_collection + relativedelta(years=1)
-
-                    dict_data = {
-                        "type": titles[bin_index].strip(),
-                        "collectionDate": next_collection.strftime(date_format),
-                    }
-                    data["bins"].append(dict_data)
-                    bin_index += 1
-                    break
-        return data
+        return bindata

uk_bin_collection/uk_bin_collection/councils/EdinburghCityCouncil.py

@@ -0,0 +1,98 @@
+import re
+import time
+
+import requests
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        collection_day = kwargs.get("paon")
+        collection_week = kwargs.get("postcode")
+        bindata = {"bins": []}
+
+        days_of_week = [
+            "Monday",
+            "Tuesday",
+            "Wednesday",
+            "Thursday",
+            "Friday",
+            "Saturday",
+            "Sunday",
+        ]
+
+        collection_weeks = ["Week 1", "Week 2"]
+        collection_week = collection_weeks.index(collection_week)
+
+        offset_days = days_of_week.index(collection_day)
+
+        if collection_week == 0:
+            recyclingstartDate = datetime(2024, 11, 4)
+            glassstartDate = datetime(2024, 11, 4)
+            refusestartDate = datetime(2024, 11, 11)
+        elif collection_week == 1:
+            recyclingstartDate = datetime(2024, 11, 11)
+            glassstartDate = datetime(2024, 11, 11)
+            refusestartDate = datetime(2024, 11, 4)
+
+        refuse_dates = get_dates_every_x_days(refusestartDate, 14, 28)
+        glass_dates = get_dates_every_x_days(glassstartDate, 14, 28)
+        recycling_dates = get_dates_every_x_days(recyclingstartDate, 14, 28)
+
+        for refuseDate in refuse_dates:
+
+            collection_date = (
+                datetime.strptime(refuseDate, "%d/%m/%Y") + timedelta(days=offset_days)
+            ).strftime("%d/%m/%Y")
+
+            dict_data = {
+                "type": "Grey Bin",
+                "collectionDate": collection_date,
+            }
+            bindata["bins"].append(dict_data)
+
+        for recyclingDate in recycling_dates:
+
+            collection_date = (
+                datetime.strptime(recyclingDate, "%d/%m/%Y")
+                + timedelta(days=offset_days)
+            ).strftime("%d/%m/%Y")
+
+            dict_data = {
+                "type": "Green Bin",
+                "collectionDate": collection_date,
+            }
+            bindata["bins"].append(dict_data)
+
+        for glassDate in glass_dates:
+
+            collection_date = (
+                datetime.strptime(glassDate, "%d/%m/%Y") + timedelta(days=offset_days)
+            ).strftime("%d/%m/%Y")
+
+            dict_data = {
+                "type": "Glass Box",
+                "collectionDate": collection_date,
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/ExeterCityCouncil.py

@@ -0,0 +1,52 @@
+import time
+
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        URI = f"https://exeter.gov.uk/repositories/hidden-pages/address-finder/?qsource=UPRN&qtype=bins&term={user_uprn}"
+
+        response = requests.get(URI)
+        response.raise_for_status()
+
+        data = response.json()
+
+        soup = BeautifulSoup(data[0]["Results"], "html.parser")
+        soup.prettify()
+
+        # Extract bin schedule
+        for section in soup.find_all("h2"):
+            bin_type = section.text.strip()
+            collection_date = section.find_next("h3").text.strip()
+
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": datetime.strptime(
+                    remove_ordinal_indicator_from_date_string(collection_date),
+                    "%A, %d %B %Y",
+                ).strftime(date_format),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+        )
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/MidSuffolkDistrictCouncil.py

@@ -23,6 +23,7 @@ class CouncilClass(AbstractGetBinDataClass):
     def parse_data(self, page: str, **kwargs) -> dict:
 
         collection_day = kwargs.get("paon")
+        garden_collection_week = kwargs.get("postcode")
         bindata = {"bins": []}
 
         days_of_week = [
@@ -35,10 +36,14 @@ class CouncilClass(AbstractGetBinDataClass):
             "Sunday",
         ]
 
+        garden_week = ["Week 1", "Week 2"]
+
         refusestartDate = datetime(2024, 11, 11)
         recyclingstartDate = datetime(2024, 11, 4)
 
         offset_days = days_of_week.index(collection_day)
+        if garden_collection_week:
+            garden_collection = garden_week.index(garden_collection_week)
 
         refuse_dates = get_dates_every_x_days(refusestartDate, 14, 28)
         recycling_dates = get_dates_every_x_days(recyclingstartDate, 14, 28)
@@ -125,6 +130,63 @@ class CouncilClass(AbstractGetBinDataClass):
             }
             bindata["bins"].append(dict_data)
 
+        if garden_collection_week:
+            if garden_collection == 0:
+                gardenstartDate = datetime(2024, 11, 11)
+            elif garden_collection == 1:
+                gardenstartDate = datetime(2024, 11, 4)
+
+            garden_dates = get_dates_every_x_days(gardenstartDate, 14, 28)
+
+            garden_bank_holidays = [
+                ("23/12/2024", 1),
+                ("24/12/2024", 1),
+                ("25/12/2024", 1),
+                ("26/12/2024", 1),
+                ("27/12/2024", 1),
+                ("30/12/2024", 1),
+                ("31/12/2024", 1),
+                ("01/01/2025", 1),
+                ("02/01/2025", 1),
+                ("03/01/2025", 1),
+            ]
+
+            for gardenDate in garden_dates:
+
+                collection_date = (
+                    datetime.strptime(gardenDate, "%d/%m/%Y")
+                    + timedelta(days=offset_days)
+                ).strftime("%d/%m/%Y")
+
+                garden_holiday = next(
+                    (
+                        value
+                        for date, value in garden_bank_holidays
+                        if date == collection_date
+                    ),
+                    0,
+                )
+
+                if garden_holiday > 0:
+                    continue
+
+                holiday_offset = next(
+                    (value for date, value in bank_holidays if date == collection_date),
+                    0,
+                )
+
+                if holiday_offset > 0:
+                    collection_date = (
+                        datetime.strptime(collection_date, "%d/%m/%Y")
+                        + timedelta(days=holiday_offset)
+                    ).strftime("%d/%m/%Y")
+
+                dict_data = {
+                    "type": "Garden Bin",
+                    "collectionDate": collection_date,
+                }
+                bindata["bins"].append(dict_data)
+
         bindata["bins"].sort(
             key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
         )

uk_bin_collection/uk_bin_collection/councils/RotherDistrictCouncil.py

@@ -0,0 +1,84 @@
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+from dateutil.relativedelta import relativedelta
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Get and check UPRN
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        uri = "https://www.rother.gov.uk/wp-admin/admin-ajax.php"
+        params = {
+            "action": "get_address_data",
+            "uprn": user_uprn,
+            "context": "full-page",
+        }
+
+        headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+        }
+
+        # Send a POST request with form data and headers
+        r = requests.post(uri, data=params, headers=headers, verify=False)
+
+        result = r.json()
+
+        if result["success"]:
+            # Parse the HTML with BeautifulSoup
+            soup = BeautifulSoup(result["data"], "html.parser")
+            soup.prettify()
+
+            # print(soup)
+
+            # Find the div elements with class "bindays-item"
+            bin_days = soup.find_all("div", class_="bindays-item")
+
+            # Loop through each bin item and extract type and date
+            for bin_day in bin_days:
+                # Extract bin type from the <h3> tag
+                bin_type = bin_day.find("h3").get_text(strip=True).replace(":", "")
+
+                # Extract date (or check if it's a subscription link for Garden Waste)
+                date_span = bin_day.find("span", class_="find-my-nearest-bindays-date")
+                if date_span:
+                    if date_span.find("a"):
+                        # If there is a link, this is the Garden bin signup link
+                        continue
+                    else:
+                        # Otherwise, get the date text directly
+                        date = date_span.get_text(strip=True)
+                else:
+                    date = None
+
+                date = datetime.strptime(
+                    remove_ordinal_indicator_from_date_string(date),
+                    "%A %d %B",
+                ).replace(year=datetime.now().year)
+                if datetime.now().month == 12 and date.month == 1:
+                    date = date + relativedelta(years=1)
+
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": date.strftime(date_format),
+                }
+                bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+        return bindata

uk_bin_collection/uk_bin_collection/councils/SouthHamsDistrictCouncil.py

@@ -0,0 +1,90 @@
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+from dateutil.relativedelta import relativedelta
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Get and check UPRN
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        uri = "https://waste.southhams.gov.uk/mycollections"
+
+        s = requests.session()
+        r = s.get(uri)
+        for cookie in r.cookies:
+            if cookie.name == "fcc_session_cookie":
+                fcc_session_token = cookie.value
+
+        uri = "https://waste.southhams.gov.uk/mycollections/getcollectiondetails"
+
+        params = {
+            "fcc_session_token": fcc_session_token,
+            "uprn": user_uprn,
+        }
+
+        headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+            "Referer": "https://waste.southhams.gov.uk/mycollections",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+        # Send a POST request with form data and headers
+        r = s.post(uri, data=params, headers=headers)
+
+        result = r.json()
+
+        for collection in result["binCollections"]["tile"]:
+
+            # Parse the HTML with BeautifulSoup
+            soup = BeautifulSoup(collection[0], "html.parser")
+            soup.prettify()
+
+            # Find all collectionDiv elements
+            collections = soup.find_all("div", class_="collectionDiv")
+
+            # Process each collectionDiv
+            for collection in collections:
+                # Extract the service name
+                service_name = collection.find("h3").text.strip()
+
+                # Extract collection frequency and day
+                details = collection.find("div", class_="detWrap").text.strip()
+
+                # Extract the next collection date
+                next_collection = details.split("Your next scheduled collection is ")[
+                    1
+                ].split(".")[0]
+
+                if next_collection.startswith("today"):
+                    next_collection = next_collection.split("today, ")[1]
+                elif next_collection.startswith("tomorrow"):
+                    next_collection = next_collection.split("tomorrow, ")[1]
+
+                dict_data = {
+                    "type": service_name,
+                    "collectionDate": datetime.strptime(
+                        next_collection, "%A, %d %B %Y"
+                    ).strftime(date_format),
+                }
+                bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/StevenageBoroughCouncil.py

@@ -0,0 +1,101 @@
+import time
+
+import requests
+from dateutil.relativedelta import relativedelta
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+        uprn = kwargs.get("uprn")
+        check_uprn(uprn)
+        bindata = {"bins": []}
+
+        SESSION_URL = "https://stevenage-self.achieveservice.com/authapi/isauthenticated?uri=https%253A%252F%252Fstevenage-self.achieveservice.com%252Fservice%252Fmy_bin_collection_schedule&hostname=stevenage-self.achieveservice.com&withCredentials=true"
+        TOKEN_URL = "https://stevenage-self.achieveservice.com/apibroker/runLookup?id=5e55337a540d4"
+        API_URL = "https://stevenage-self.achieveservice.com/apibroker/runLookup"
+
+        data = {
+            "formValues": {
+                "Section 1": {
+                    "token": {"value": ""},
+                    "LLPGUPRN": {
+                        "value": uprn,
+                    },
+                    "MinimumDateLookAhead": {
+                        "value": time.strftime("%Y-%m-%d"),
+                    },
+                    "MaximumDateLookAhead": {
+                        "value": str(int(time.strftime("%Y")) + 1)
+                        + time.strftime("-%m-%d"),
+                    },
+                },
+            },
+        }
+
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+            "User-Agent": "Mozilla/5.0",
+            "X-Requested-With": "XMLHttpRequest",
+            "Referer": "https://stevenage-self.achieveservice.com/fillform/?iframe_id=fillform-frame-1&db_id=",
+        }
+        s = requests.session()
+        r = s.get(SESSION_URL)
+        r.raise_for_status()
+        session_data = r.json()
+        sid = session_data["auth-session"]
+
+        t = s.get(TOKEN_URL)
+        t.raise_for_status()
+        token_data = t.json()
+        data["formValues"]["Section 1"]["token"]["value"] = token_data["integration"][
+            "transformed"
+        ]["rows_data"]["0"]["token"]
+
+        params = {
+            "id": "64ba8cee353e6",
+            "repeat_against": "",
+            "noRetry": "false",
+            "getOnlyTokens": "undefined",
+            "log_id": "",
+            "app_name": "AF-Renderer::Self",
+            # unix_timestamp
+            "_": str(int(time.time() * 1000)),
+            "sid": sid,
+        }
+
+        r = s.post(API_URL, json=data, headers=headers, params=params)
+        r.raise_for_status()
+
+        data = r.json()
+        rows_data = data["integration"]["transformed"]["rows_data"]
+        if not isinstance(rows_data, dict):
+            raise ValueError("Invalid data returned from API")
+
+        for key in rows_data:
+            value = rows_data[key]
+            bin_type = value["bintype"].strip()
+
+            try:
+                date = datetime.strptime(value["collectiondate"], "%A %d %B %Y").date()
+            except ValueError:
+                continue
+
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": date.strftime(date_format),
+            }
+            bindata["bins"].append(dict_data)
+
+        return bindata