uk_bin_collection 0.135.4__tar.gz → 0.136.0__tar.gz

{uk_bin_collection-0.135.4 → uk_bin_collection-0.136.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.135.4
+Version: 0.136.0
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.135.4 → uk_bin_collection-0.136.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.135.4"
+version = "0.136.0"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

{uk_bin_collection-0.135.4 → uk_bin_collection-0.136.0}/uk_bin_collection/tests/input.json

@@ -162,11 +162,10 @@
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
     "BexleyCouncil": {
-        "house_number": "1 Dorchester Avenue, Bexley",
+        "house_number": "1",
         "postcode": "DA5 3AH",
         "skip_get_url": true,
-        "uprn": "100020196143",
-        "url": "https://mybexley.bexley.gov.uk/service/When_is_my_collection_day",
+        "url": "https://waste.bexley.gov.uk/waste",
         "web_driver": "http://selenium:4444",
         "wiki_name": "Bexley Council",
         "wiki_note": "In order to use this parser, you will need to sign up to [Bexley's @Home app](https://www.bexley.gov.uk/services/rubbish-and-recycling/bexley-home-recycling-app/about-app). Complete the setup by entering your email and setting your address with postcode and address line. Once you can see the calendar, you should be good to run the parser. Just pass the email you used in quotes in the UPRN parameter."
@@ -345,6 +344,12 @@
         "wiki_name": "Calderdale Council",
         "wiki_note": "Pass the UPRN and postcode. To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)."
     },
+    "CambridgeCityCouncil": {
+        "uprn": "200004159750",
+        "url": "https://www.cambridge.gov.uk/",
+        "wiki_name": "Cambridge City Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "CannockChaseDistrictCouncil": {
         "postcode": "WS15 1JA",
         "skip_get_url": true,
@@ -1647,6 +1652,13 @@
         "wiki_name": "Somerset Council",
         "wiki_note": "Provide your UPRN and postcode. Find your UPRN using [FindMyAddress](https://www.findmyaddress.co.uk/search)."
     },
+    "SouthamptonCityCouncil": {
+        "skip_get_url": true,
+        "uprn": "100060731893",
+        "url": "https://www.southampton.gov.uk",
+        "wiki_name": "Southampton City Council",
+        "wiki_note": "Pass the UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search)."
+    },
     "SouthAyrshireCouncil": {
         "postcode": "KA19 7BN",
         "skip_get_url": true,
@@ -1740,6 +1752,13 @@
         "wiki_name": "Southwark Council",
         "wiki_note": "Replace `XXXXXXXX` with your UPRN. Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN."
     },
+    "SpelthorneBoroughCouncil": {
+        "house_number": "1",
+        "postcode": "TW18 2PR",
+        "skip_get_url": true,
+        "url": "https://www.spelthorne.gov.uk",
+        "wiki_name": "Spelthorne Borough Council"
+    },
     "StAlbansCityAndDistrictCouncil": {
         "skip_get_url": true,
         "uprn": "100081153583",

uk_bin_collection-0.136.0/uk_bin_collection/uk_bin_collection/councils/CambridgeCityCouncil.py

@@ -0,0 +1,61 @@
+import time
+
+import requests
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        URI = f"https://servicelayer3c.azure-api.net/wastecalendar/collection/search/{user_uprn}/?authority=CCC/?numberOfCollections=255"
+
+        # Make the GET request
+        response = requests.get(URI)
+
+        # Parse the JSON response
+        bin_collection = response.json()
+
+        # Loop through each collection in bin_collection
+        for collection in bin_collection["collections"]:
+            bin_types = collection["roundTypes"]
+            collection_date = collection["date"]
+
+            # Loop through the dates for each collection type
+            for bin_type in bin_types:
+                # print(f"Bin Type: {bin_type}")
+                # print(f"Collection Date: {collection_date}")
+
+                if bin_type == "ORGANIC":
+                    bin_type = "Green Bin"
+                if bin_type == "RECYCLE":
+                    bin_type = "Blue Bin"
+                if bin_type == "DOMESTIC":
+                    bin_type = "Black Bin"
+
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": datetime.strptime(
+                        collection_date,
+                        "%Y-%m-%dT%H:%M:%SZ",
+                    ).strftime("%d/%m/%Y"),
+                }
+                bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

{uk_bin_collection-0.135.4 → uk_bin_collection-0.136.0}/uk_bin_collection/uk_bin_collection/councils/CanterburyCityCouncil.py

@@ -22,12 +22,17 @@ class CouncilClass(AbstractGetBinDataClass):
 
         data = {"uprn": user_uprn, "usrn": "1"}
 
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
+            "Accept": "application/json",
+        }
+
         URI = (
             "https://zbr7r13ke2.execute-api.eu-west-2.amazonaws.com/Beta/get-bin-dates"
         )
 
         # Make the GET request
-        response = requests.post(URI, json=data)
+        response = requests.post(URI, json=data, headers=headers)
         response.raise_for_status()
 
         # Parse the JSON response

{uk_bin_collection-0.135.4 → uk_bin_collection-0.136.0}/uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py

@@ -1,4 +1,5 @@
 from bs4 import BeautifulSoup
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
@@ -25,7 +26,7 @@ class CouncilClass(AbstractGetBinDataClass):
         uprn = kwargs.get("uprn")
         check_uprn(uprn)
 
-        post_url = "https://apps.castlepoint.gov.uk/cpapps/index.cfm?fa=wastecalendar.displayDetails"
+        post_url = "https://apps.castlepoint.gov.uk/cpapps/index.cfm?fa=myStreet.displayDetails"
         post_header_str = (
             "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,"
             "image/apng,"
@@ -41,7 +42,7 @@ class CouncilClass(AbstractGetBinDataClass):
         )
 
         post_headers = parse_header(post_header_str)
-        form_data = {"roadID": uprn}
+        form_data = {"USRN": uprn}
         post_response = requests.post(
             post_url, headers=post_headers, data=form_data, verify=False
         )
@@ -53,36 +54,28 @@ class CouncilClass(AbstractGetBinDataClass):
         data = {"bins": []}
         collection_tuple = []
 
-        for i in range(1, 3):
-            calendar = soup.select(
-                f"#wasteCalendarContainer > div:nth-child(2) > div:nth-child({i}) > div"
-            )[0]
-            month = datetime.strptime(
-                calendar.find_next("h2").get_text(), "%B %Y"
-            ).strftime("%m")
-            year = datetime.strptime(
-                calendar.find_next("h2").get_text(), "%B %Y"
-            ).strftime("%Y")
-
-            pink_days = [
-                day.get_text().strip() for day in calendar.find_all("td", class_="pink")
-            ]
-            black_days = [
-                day.get_text().strip()
-                for day in calendar.find_all("td", class_="normal")
-            ]
-
-            for day in pink_days:
-                collection_date = datetime(
-                    year=int(year), month=int(month), day=int(day)
-                )
-                collection_tuple.append(("Pink collection", collection_date))
-
-            for day in black_days:
-                collection_date = datetime(
-                    year=int(year), month=int(month), day=int(day)
-                )
-                collection_tuple.append(("Normal collection", collection_date))
+        calendar = soup.find("table", class_="calendar")
+        month = datetime.strptime(
+            soup.find("div", class_="calMonthCurrent").get_text(), "[%b]"
+        ).strftime("%m")
+        year = datetime.strptime(
+            soup.find("h1").get_text(), "About my Street - %B %Y"
+        ).strftime("%Y")
+
+        pink_days = [
+            day.get_text().strip() for day in calendar.find_all("td", class_="pink")
+        ]
+        black_days = [
+            day.get_text().strip() for day in calendar.find_all("td", class_="normal")
+        ]
+
+        for day in pink_days:
+            collection_date = datetime(year=int(year), month=int(month), day=int(day))
+            collection_tuple.append(("Pink collection", collection_date))
+
+        for day in black_days:
+            collection_date = datetime(year=int(year), month=int(month), day=int(day))
+            collection_tuple.append(("Normal collection", collection_date))
 
         ordered_data = sorted(collection_tuple, key=lambda x: x[1])
 

{uk_bin_collection-0.135.4 → uk_bin_collection-0.136.0}/uk_bin_collection/uk_bin_collection/councils/HinckleyandBosworthBoroughCouncil.py

@@ -16,6 +16,7 @@ class CouncilClass(AbstractGetBinDataClass):
     def parse_data(self, page: str, **kwargs) -> dict:
 
         user_uprn = kwargs.get("uprn")
+        user_uprn = str(user_uprn).zfill(12)
         check_uprn(user_uprn)
         bindata = {"bins": []}
 

uk_bin_collection-0.136.0/uk_bin_collection/uk_bin_collection/councils/SouthamptonCityCouncil.py

@@ -0,0 +1,52 @@
+import time
+
+import requests
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        REGEX = r"(Glass|Recycling|General Waste|Garden Waste).*?([0-9]{1,2}\/[0-9]{1,2}\/[0-9]{4})"
+
+        s = requests.Session()
+        r = s.get(
+            f"https://www.southampton.gov.uk/whereilive/waste-calendar?UPRN={user_uprn}"
+        )
+        r.raise_for_status()
+
+        # Limit search scope to avoid duplicates
+        calendar_view_only = re.search(
+            r"#calendar1.*?listView", r.text, flags=re.DOTALL
+        )[0]
+
+        results = re.findall(REGEX, calendar_view_only)
+
+        for item in results:
+
+            dict_data = {
+                "type": item[0],
+                "collectionDate": datetime.strptime(item[1], "%m/%d/%Y").strftime(
+                    "%d/%m/%Y"
+                ),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+
+        return bindata

uk_bin_collection-0.136.0/uk_bin_collection/uk_bin_collection/councils/SpelthorneBoroughCouncil.py

@@ -0,0 +1,161 @@
+import time
+
+import requests
+from dateutil.relativedelta import relativedelta
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_postcode = kwargs.get("postcode")
+        check_postcode(user_postcode)
+        user_paon = kwargs.get("paon")
+        check_paon(user_paon)
+        bindata = {"bins": []}
+
+        SESSION_URL = "https://spelthorne-self.achieveservice.com/authapi/isauthenticated?uri=https%253A%252F%252Fspelthorne-self.achieveservice.com%252Fen%252FAchieveForms%252F%253Fmode%253Dfill%2526consentMessage%253Dyes%2526form_uri%253Dsandbox-publish%253A%252F%252FAF-Process-8d0df9d6-6bd0-487e-bbfe-38815dcc780d%252FAF-Stage-bce7fc80-bcd7-45f1-bf55-a76d38dbebba%252Fdefinition.json%2526process%253D1%2526process_uri%253Dsandbox-processes%253A%252F%252FAF-Process-8d0df9d6-6bd0-487e-bbfe-38815dcc780d%2526process_id%253DAF-Process-8d0df9d6-6bd0-487e-bbfe-38815dcc780d%2526noLoginPrompt%253D1&hostname=spelthorne-self.achieveservice.com&withCredentials=true"
+
+        API_URL = "https://spelthorne-self.achieveservice.com/apibroker/runLookup"
+
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+            "User-Agent": "Mozilla/5.0",
+            "X-Requested-With": "XMLHttpRequest",
+            "Referer": "https://spelthorne-self.achieveservice.com/fillform/?iframe_id=fillform-frame-1&db_id=",
+        }
+        s = requests.session()
+        r = s.get(SESSION_URL)
+        r.raise_for_status()
+        session_data = r.json()
+        sid = session_data["auth-session"]
+        data = {
+            "formValues": {
+                "Property details": {
+                    "postalcode": {"value": user_postcode},
+                }
+            }
+        }
+        params = {
+            "id": "59b4477d6d84e",
+            "repeat_against": "",
+            "noRetry": "false",
+            "getOnlyTokens": "undefined",
+            "log_id": "",
+            "app_name": "AF-Renderer::Self",
+            # unix_timestamp
+            "_": str(int(time.time() * 1000)),
+            "sid": sid,
+        }
+
+        r = s.post(API_URL, json=data, headers=headers, params=params)
+        r.raise_for_status()
+
+        data = r.json()
+
+        rows_data = data["integration"]["transformed"]["rows_data"]
+
+        for key, value in rows_data.items():
+
+            address = value.get("display")
+
+            if address.startswith(user_paon):
+                user_uprn = value.get("value")
+                break
+
+        params = {
+            "id": "5f97e6e09fedd",
+            "repeat_against": "",
+            "noRetry": "true",
+            "getOnlyTokens": "undefined",
+            "log_id": "",
+            "app_name": "AF-Renderer::Self",
+            # unix_timestamp
+            "_": str(int(time.time() * 1000)),
+            "sid": sid,
+        }
+
+        r = s.post(API_URL, headers=headers, params=params)
+        r.raise_for_status()
+
+        data = r.json()
+
+        rows_data = data["integration"]["transformed"]["rows_data"]["0"]
+
+        token = rows_data.get("tokenString")
+
+        start_date = datetime.today() + relativedelta(weeks=2)
+        last2weeks = datetime.today() - relativedelta(weeks=2)
+
+        # Format as YYYY-MM-DD
+        formatted_date = start_date.strftime("%Y-%m-%d")
+        last2weeks = last2weeks.strftime("%Y-%m-%d")
+
+        data = {
+            "formValues": {
+                "Property details": {
+                    "uprn1": {"value": user_uprn},
+                    "endDate": {"value": formatted_date},
+                    "last2Weeks": {"value": last2weeks},
+                    "token": {"value": token},
+                }
+            }
+        }
+
+        params = {
+            "id": "66042a164c9a5",
+            "repeat_against": "",
+            "noRetry": "true",
+            "getOnlyTokens": "undefined",
+            "log_id": "",
+            "app_name": "AF-Renderer::Self",
+            # unix_timestamp
+            "_": str(int(time.time() * 1000)),
+            "sid": sid,
+        }
+
+        r = s.post(API_URL, json=data, headers=headers, params=params)
+        r.raise_for_status()
+
+        data = r.json()
+
+        rows_data = data["integration"]["transformed"]["rows_data"]["0"]
+
+        if not isinstance(rows_data, dict):
+            raise ValueError("Invalid data returned from API")
+
+        use_new = any(k.endswith("New") and v for k, v in rows_data.items())
+        next_date_key = "NextDateNew" if use_new else "NextDateOld"
+
+        for key, value in rows_data.items():
+            if not (key.endswith("NextCollection") or key.endswith(next_date_key)):
+                continue
+
+            bin_type = key.split("NextCollection")[0]
+            if bin_type == "Gw":
+                bin_type = ["Garden Waste"]
+            if bin_type == "Rec":
+                bin_type = ["Recycling Bin", "Food Waste Bin"]
+            if bin_type == "Ref":
+                bin_type = ["Rubbish Bin", "Food Waste Bin"]
+
+            try:
+                date = datetime.strptime(value, "%Y-%m-%d").strftime("%d/%m/%Y")
+            except ValueError:
+                continue
+
+            for bin in bin_type:
+                dict_data = {"type": bin, "collectionDate": date}
+                bindata["bins"].append(dict_data)
+
+        return bindata

{uk_bin_collection-0.135.4 → uk_bin_collection-0.136.0}/uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py

@@ -1,7 +1,9 @@
+from datetime import datetime
+
 from bs4 import BeautifulSoup
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
-from datetime import datetime
 
 
 class CouncilClass(AbstractGetBinDataClass):