uk_bin_collection 0.95.0__tar.gz → 0.96.0__tar.gz

{uk_bin_collection-0.95.0 → uk_bin_collection-0.96.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.95.0
+Version: 0.96.0
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.95.0 → uk_bin_collection-0.96.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.95.0"
+version = "0.96.0"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

{uk_bin_collection-0.95.0 → uk_bin_collection-0.96.0}/uk_bin_collection/tests/input.json

@@ -399,6 +399,13 @@
         "url": "https://www.eastleigh.gov.uk/waste-bins-and-recycling/collection-dates/your-waste-bin-and-recycling-collections?uprn=",
         "wiki_name": "Eastleigh Borough Council"
     },
+    "ElmbridgeBoroughCouncil": {
+        "url": "https://www.elmbridge.gov.uk",
+        "wiki_command_url_override": "https://www.elmbridge.gov.uk",
+        "uprn": "10013119164",
+        "wiki_name": "Elmbridge Borough Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "EnfieldCouncil": {
         "house_number": "111",
         "postcode": "N13 5AJ",
@@ -509,6 +516,13 @@
         "wiki_name": "Harrogate Borough Council",
         "wiki_note": "Pass the UPRN which can be found at https://secure.harrogate.gov.uk/inmyarea URL doesn't need to be passed."
     },
+    "HighlandCouncil": {
+        "url": "https://www.highland.gov.uk",
+        "wiki_command_url_override": "https://www.highland.gov.uk",
+        "uprn": "130072429",
+        "wiki_name": "Highland Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "HighPeakCouncil": {
         "house_number": "9 Ellison Street, Glossop",
         "postcode": "SK13 8BX",
@@ -961,6 +975,13 @@
         "url": "https://www.scambs.gov.uk/recycling-and-bins/find-your-household-bin-collection-day/",
         "wiki_name": "South Cambridgeshire Council"
     },
+    "SouthDerbyshireDistrictCouncil": {
+        "url": "https://maps.southderbyshire.gov.uk/iShareLIVE.web//getdata.aspx?RequestType=LocalInfo&ms=mapsources/MyHouse&format=JSONP&group=Recycling%20Bins%20and%20Waste|Next%20Bin%20Collections&uid=",
+        "wiki_command_url_override": "https://maps.southderbyshire.gov.uk/iShareLIVE.web//getdata.aspx?RequestType=LocalInfo&ms=mapsources/MyHouse&format=JSONP&group=Recycling%20Bins%20and%20Waste|Next%20Bin%20Collections&uid=XXXXXXXX",
+        "uprn": "10000820668",
+        "wiki_name": "South Derbyshire District Council",
+        "wiki_note": "Replace XXXXXXXX with UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "SouthGloucestershireCouncil": {
         "skip_get_url": true,
         "uprn": "566419",
@@ -1001,6 +1022,13 @@
         "url": "https://www.southtyneside.gov.uk/article/33352/Bin-collection-dates",
         "wiki_name": "South Tyneside Council"
     },
+    "SouthwarkCouncil": {
+        "url": "https://www.southwark.gov.uk/bins/lookup/",
+        "wiki_command_url_override": "https://www.southwark.gov.uk/bins/lookup/XXXXXXXX",
+        "uprn": "200003469271",
+        "wiki_name": "Southwark Council",
+        "wiki_note": "Replace XXXXXXXX with UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "StAlbansCityAndDistrictCouncil": {
         "skip_get_url": true,
         "uprn": "100081153583",

uk_bin_collection-0.96.0/uk_bin_collection/uk_bin_collection/councils/ElmbridgeBoroughCouncil.py

@@ -0,0 +1,91 @@
+import time
+
+import requests
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        BASE_URL = "https://elmbridge-self.achieveservice.com"
+        INTIAL_URL = f"{BASE_URL}/service/Your_bin_collection_days"
+        AUTH_URL = f"{BASE_URL}/authapi/isauthenticated"
+        AUTH_TEST = f"{BASE_URL}/apibroker/domain/elmbridge-self.achieveservice.com"
+        API_URL = f"{BASE_URL}/apibroker/runLookup"
+
+        self._session = requests.Session()
+        r = self._session.get(INTIAL_URL)
+        r.raise_for_status()
+        params: dict[str, str | int] = {
+            "uri": r.url,
+            "hostname": "elmbridge-self.achieveservice.com",
+            "withCredentials": "true",
+        }
+        r = self._session.get(AUTH_URL, params=params)
+        r.raise_for_status()
+        data = r.json()
+        session_key = data["auth-session"]
+
+        params = {
+            "sid": session_key,
+            "_": int(datetime.now().timestamp() * 1000),
+        }
+        r = self._session.get(AUTH_TEST, params=params)
+        r.raise_for_status()
+
+        params: dict[str, int | str] = {
+            "id": "663b557cdaece",
+            "repeat_against": "",
+            "noRetry": "false",
+            "getOnlyTokens": "undefined",
+            "log_id": "",
+            "app_name": "AF-Renderer::Self",
+            "_": int(datetime.now().timestamp() * 1000),
+            "sid": session_key,
+        }
+        payload = {
+            "formValues": {
+                "Section 1": {
+                    "UPRN": {"value": user_uprn},
+                }
+            }
+        }
+        r = self._session.post(API_URL, params=params, json=payload)
+        r.raise_for_status()
+
+        data = r.json()
+        collections = list(r.json()["integration"]["transformed"]["rows_data"].values())
+
+        for collection in collections:
+            date = collection["Date"]
+            for service in [
+                collection["Service1"],
+                collection["Service2"],
+                collection["Service3"],
+            ]:
+                if not service:
+                    continue
+                service = service.removesuffix(" Collection Service")
+
+                dict_data = {
+                    "type": service,
+                    "collectionDate": datetime.strptime(
+                        date, "%d/%m/%Y %H:%M:%S"
+                    ).strftime("%d/%m/%Y"),
+                }
+                bindata["bins"].append(dict_data)
+
+        return bindata

uk_bin_collection-0.96.0/uk_bin_collection/uk_bin_collection/councils/HighlandCouncil.py

@@ -0,0 +1,88 @@
+import time
+
+import requests
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        SESSION_URL = "https://highland-self.achieveservice.com/authapi/isauthenticated?uri=https%3A%2F%2Fhighland-self.achieveservice.com%2Fen%2Fservice%2FCheck_your_household_bin_collection_days&hostname=highland-self.achieveservice.com&withCredentials=true"
+
+        API_URL = "https://highland-self.achieveservice.com/apibroker/runLookup"
+
+        data = {
+            "formValues": {"Your address": {"propertyuprn": {"value": user_uprn}}},
+        }
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+            "User-Agent": "Mozilla/5.0",
+            "X-Requested-With": "XMLHttpRequest",
+            "Referer": "https://highland-self.achieveservice.com/fillform/?iframe_id=fillform-frame-1&db_id=",
+        }
+        s = requests.session()
+        r = s.get(SESSION_URL)
+        r.raise_for_status()
+        session_data = r.json()
+        sid = session_data["auth-session"]
+        params = {
+            "id": "660d44a698632",
+            "repeat_against": "",
+            "noRetry": "false",
+            "getOnlyTokens": "undefined",
+            "log_id": "",
+            "app_name": "AF-Renderer::Self",
+            # unix_timestamp
+            "_": str(int(time.time() * 1000)),
+            "sid": sid,
+        }
+
+        r = s.post(API_URL, json=data, headers=headers, params=params)
+        r.raise_for_status()
+
+        data = r.json()
+        rows_data = data["integration"]["transformed"]["rows_data"]["0"]
+        if not isinstance(rows_data, dict):
+            raise ValueError("Invalid data returned from API")
+
+        use_new = any(k.endswith("New") and v for k, v in rows_data.items())
+        next_date_key = "NextDateNew" if use_new else "NextDateOld"
+
+        for key, value in rows_data.items():
+            if not (key.endswith("NextDate") or key.endswith(next_date_key)):
+                continue
+
+            bin_type = key.split("NextDate")[0]
+            if bin_type == "refuse":
+                bin_type = "Non-recyclable waste"
+            if bin_type == "fibres":
+                bin_type = "Paper, card and cardboard recycling"
+            if bin_type == "containers":
+                bin_type = "Plastics, metals and cartons recycling"
+            if bin_type == "garden":
+                bin_type = "Garden waste"
+            if bin_type == "food":
+                bin_type = "Food waste"
+
+            try:
+                date = datetime.strptime(value, "%Y-%m-%d").strftime("%d/%m/%Y")
+            except ValueError:
+                continue
+            dict_data = {"type": bin_type, "collectionDate": date}
+            bindata["bins"].append(dict_data)
+
+        return bindata

uk_bin_collection-0.96.0/uk_bin_collection/uk_bin_collection/councils/SouthDerbyshireDistrictCouncil.py

@@ -0,0 +1,60 @@
+import requests
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        data = {"bins": []}
+
+        baseurl = "https://maps.southderbyshire.gov.uk/iShareLIVE.web//getdata.aspx?RequestType=LocalInfo&ms=mapsources/MyHouse&format=JSONP&group=Recycling%20Bins%20and%20Waste|Next%20Bin%20Collections&uid="
+        url = baseurl + user_uprn
+
+        # Make the web request
+        response = requests.get(url).text
+
+        # Remove the JSONP wrapper using a regular expression
+        jsonp_pattern = r"\{.*\}"
+        json_match = re.search(jsonp_pattern, response)
+
+        if json_match:
+            # Extract the JSON part
+            json_data = json_match.group(0)
+
+            # Parse the JSON
+            parsed_data = json.loads(json_data)
+
+            # Extract the embedded HTML string
+            html_content = parsed_data["Results"]["Next_Bin_Collections"]["_"]
+
+            # Parse the HTML to extract dates and bin types using regex
+            matches = re.findall(
+                r"<span.*?>(\d{2} \w+ \d{4})</span>.*?<span.*?>(.*?)</span>",
+                html_content,
+                re.S,
+            )
+
+            # Output the parsed bin collection details
+            for match in matches:
+                dict_data = {
+                    "type": match[1],
+                    "collectionDate": datetime.strptime(match[0], "%d %B %Y").strftime(
+                        "%d/%m/%Y"
+                    ),
+                }
+                data["bins"].append(dict_data)
+        else:
+            print("No valid JSON found in the response.")
+
+        return data

uk_bin_collection-0.96.0/uk_bin_collection/uk_bin_collection/councils/SouthwarkCouncil.py

@@ -0,0 +1,136 @@
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        data = {"bins": []}
+
+        baseurl = "https://www.southwark.gov.uk/bins/lookup/"
+        url = baseurl + user_uprn
+
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"
+        }
+
+        # Make the web request
+        response = requests.get(url, headers=headers).text
+
+        soup = BeautifulSoup(response, "html.parser")
+
+        # Extract recycling collection information
+        recycling_section = soup.find(
+            "div", {"aria-labelledby": "recyclingCollectionTitle"}
+        )
+        if recycling_section:
+            recycling_title = recycling_section.find(
+                "p", {"id": "recyclingCollectionTitle"}
+            ).text
+            recycling_next_collection = (
+                recycling_section.find(text=lambda text: "Next collection" in text)
+                .strip()
+                .split(": ")[1]
+            )
+
+            dict_data = {
+                "type": recycling_title,
+                "collectionDate": datetime.strptime(
+                    recycling_next_collection, "%a, %d %B %Y"
+                ).strftime("%d/%m/%Y"),
+            }
+            data["bins"].append(dict_data)
+
+        # Extract refuse collection information
+        refuse_section = soup.find("div", {"aria-labelledby": "refuseCollectionTitle"})
+        if refuse_section:
+            refuse_title = refuse_section.find(
+                "p", {"id": "refuseCollectionTitle"}
+            ).text
+            refuse_next_collection = (
+                refuse_section.find(text=lambda text: "Next collection" in text)
+                .strip()
+                .split(": ")[1]
+            )
+
+            dict_data = {
+                "type": refuse_title,
+                "collectionDate": datetime.strptime(
+                    refuse_next_collection, "%a, %d %B %Y"
+                ).strftime("%d/%m/%Y"),
+            }
+            data["bins"].append(dict_data)
+
+        # Extract food waste collection information
+        food_section = soup.find("div", {"aria-labelledby": "organicsCollectionTitle"})
+        if food_section:
+            food_title = food_section.find("p", {"id": "organicsCollectionTitle"}).text
+            food_next_collection = (
+                food_section.find(text=lambda text: "Next collection" in text)
+                .strip()
+                .split(": ")[1]
+            )
+
+            dict_data = {
+                "type": food_title,
+                "collectionDate": datetime.strptime(
+                    food_next_collection, "%a, %d %B %Y"
+                ).strftime("%d/%m/%Y"),
+            }
+            data["bins"].append(dict_data)
+
+        comrec_section = soup.find(
+            "div", {"aria-labelledby": "recyclingCommunalCollectionTitle"}
+        )
+        if comrec_section:
+            comrec_title = comrec_section.find(
+                "p", {"id": "recyclingCommunalCollectionTitle"}
+            ).text
+            comrec_next_collection = (
+                comrec_section.find(text=lambda text: "Next collection" in text)
+                .strip()
+                .split(": ")[1]
+            )
+
+            dict_data = {
+                "type": comrec_title,
+                "collectionDate": datetime.strptime(
+                    comrec_next_collection, "%a, %d %B %Y"
+                ).strftime("%d/%m/%Y"),
+            }
+            data["bins"].append(dict_data)
+
+        comref_section = soup.find(
+            "div", {"aria-labelledby": "refuseCommunalCollectionTitle"}
+        )
+        if comref_section:
+            comref_title = comref_section.find(
+                "p", {"id": "refuseCommunalCollectionTitle"}
+            ).text
+            comref_next_collection = (
+                comref_section.find(text=lambda text: "Next collection" in text)
+                .strip()
+                .split(": ")[1]
+            )
+
+            dict_data = {
+                "type": comref_title,
+                "collectionDate": datetime.strptime(
+                    comref_next_collection, "%a, %d %B %Y"
+                ).strftime("%d/%m/%Y"),
+            }
+            data["bins"].append(dict_data)
+
+        return data