uk_bin_collection 0.74.0__py3-none-any.whl

uk_bin_collection/uk_bin_collection/councils/WindsorAndMaidenheadCouncil.py

@@ -0,0 +1,134 @@
+from datetime import datetime
+from urllib.parse import urlparse
+
+import requests
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        root_url = "https://my.rbwm.gov.uk"
+        href_url = ""
+        api_url = "https://my.rbwm.gov.uk/block_refresh/block/47/node/136968?"
+        user_postcode = kwargs.get("postcode")
+        user_paon = kwargs.get("paon")
+
+        data = {"bins": []}
+
+        requests.packages.urllib3.disable_warnings()
+        s = requests.session()
+        # Form start
+        headers = {
+            "authority": "my.rbwm.gov.uk",
+            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+            "accept-language": "en-GB,en;q=0.8",
+            "cache-control": "max-age=0",
+            "referer": "https://my.rbwm.gov.uk/special/your-collection-dates?uprn=100080371082&subdate=2022-08-19&addr=11%20Douglas%20Lane%20Wraysbury%20Staines%20TW19%205NF",
+            "sec-fetch-dest": "document",
+            "sec-fetch-mode": "navigate",
+            "sec-fetch-site": "same-origin",
+            "sec-fetch-user": "?1",
+            "sec-gpc": "1",
+            "upgrade-insecure-requests": "1",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36",
+        }
+        s.get(
+            "https://my.rbwm.gov.uk/special/find-your-collection-dates", headers=headers
+        )
+
+        # Select address
+        headers = {
+            "authority": "my.rbwm.gov.uk",
+            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+            "accept-language": "en-GB,en;q=0.8",
+            "cache-control": "max-age=0",
+            "origin": "https://my.rbwm.gov.uk",
+            "referer": "https://my.rbwm.gov.uk/special/find-your-collection-dates",
+            "sec-fetch-dest": "document",
+            "sec-fetch-mode": "navigate",
+            "sec-fetch-site": "same-origin",
+            "sec-fetch-user": "?1",
+            "sec-gpc": "1",
+            "upgrade-insecure-requests": "1",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36",
+        }
+        request_data = {
+            "atTxtStreet": user_postcode,
+            "nodeid": "x",
+            "formname": "x",
+            "pg": "20",
+            "start": "1",
+            "selectaddress": "Select this address",
+            "selectheading": "The following addresses match the address you entered - choose your address",
+            "arg": "",
+        }
+        response = s.post(
+            "https://my.rbwm.gov.uk/special/address-selector-collection-dates",
+            headers=headers,
+            data=request_data,
+        )
+
+        soup = BeautifulSoup(response.content, features="html.parser")
+        soup.prettify()
+
+        table = soup.find("table")
+        if table:
+            table_rows = table.find_all("tr")
+            for tr in table_rows:
+                td = tr.find_all("td")
+                # row = [i.text for i in td]
+                for item in td:
+                    if user_paon in item.text and user_postcode in item.text:
+                        href_url = td[1].find("a").get("href")
+                        continue
+
+            # Getting to bin data
+            headers = {
+                "authority": "my.rbwm.gov.uk",
+                "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+                "accept-language": "en-GB,en;q=0.8",
+                "referer": "https://my.rbwm.gov.uk/special/address-selector-collection-dates",
+                "sec-fetch-dest": "document",
+                "sec-fetch-mode": "navigate",
+                "sec-fetch-site": "same-origin",
+                "sec-fetch-user": "?1",
+                "sec-gpc": "1",
+                "upgrade-insecure-requests": "1",
+                "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36",
+            }
+            params = {}
+            parsed_params = urlparse(href_url).query.split("&")
+            for item in parsed_params:
+                values = item.split("=")
+                params.update({values[0]: values[1]})
+
+            s.get(root_url + href_url, params=params, headers=headers)
+            response = s.get(
+                api_url + href_url.split("?")[1], params=params, headers=headers
+            )
+
+            soup = BeautifulSoup(response.content, features="html.parser")
+            soup.prettify()
+
+            table_rows = soup.find_all("tr")
+            for tr in table_rows:
+                td = tr.find_all("td")
+                row = [i.text for i in td]
+
+                if len(row) > 0:
+                    dict_data = {
+                        "type": row[0],
+                        "collectionDate": row[1],
+                    }
+                    data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/WokingBoroughCouncil.py

@@ -0,0 +1,114 @@
+from bs4 import BeautifulSoup
+import urllib
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        requests.packages.urllib3.disable_warnings()
+        root_url = "https://asjwsw-wrpwokingmunicipal-live.whitespacews.com/"
+        # Get the house number and postcode from the commandline
+        user_paon = kwargs.get("paon")
+        user_postcode = kwargs.get("postcode")
+        check_postcode(user_postcode)
+
+        # Start a new session for the form, and get the chosen URL from the commandline
+        session = requests.Session()
+        req = session.get(root_url)
+
+        # Parse the requested URL to get a link to the "View My Collections" portal with a unique service ID
+        start = BeautifulSoup(req.text, features="html.parser")
+        start.prettify()
+        base_link = start.select(
+            "#menu-content > div > div:nth-child(1) > p.govuk-body.govuk-\!-margin-bottom-0.colorblue.lineheight15 > a"
+        )[0].attrs.get("href")
+
+        # We need to reorder the query parts from the unique URL, so split them up to make it easier
+        query_parts = urllib.parse.urlparse(base_link).query.split("&")
+        parts = base_link.split("?")
+        addr_link = (
+            parts[0] + "/mop.php?" + query_parts[1] + "&" + query_parts[0] + "&seq=2"
+        )
+
+        # Bring in some headers to emulate a browser, and put the UPRN and postcode into the form data.
+        # This is sent in a POST request, emulating browser behaviour.
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "en-GB,en;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Origin": "https://asjwsw-wrpwokingmunicipal-live.whitespacews.com",
+            "Pragma": "no-cache",
+            "Referer": "https://asjwsw-wrpwokingmunicipal-live.whitespacews.com/",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "same-origin",
+            "Sec-Fetch-User": "?1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 OPR/98.0.0.0",
+            "sec-ch-ua": '"Chromium";v="112", "Not_A Brand";v="24", "Opera GX";v="98"',
+            "sec-ch-ua-mobile": "?0",
+            "sec-ch-ua-platform": '"Windows"',
+        }
+        data = {
+            "address_name_number": user_paon,
+            "address_street": "",
+            "street_town": "",
+            "address_postcode": user_postcode,
+        }
+        addr_page = session.post(addr_link, headers=headers, data=data)
+        addr = BeautifulSoup(addr_page.text, features="html.parser")
+        addr.prettify()
+
+        # This page should only have one address, but regardless, select the first one and make a request to load the
+        # calendar page.
+        cal_link = root_url + addr.select("#property_list > ul > li > a")[0].attrs.get(
+            "href"
+        )
+        cal_page = session.get(cal_link)
+
+        # Parse the calendar page
+        soup = BeautifulSoup(cal_page.text, features="html.parser")
+        soup.prettify()
+        data = {"bins": []}
+
+        # For whatever reason, each row contains all the information for that row, and each one after it. This code
+        # essentially gets all items from each row, but ignores the whitespace that you get when splitting using \n.
+        # This produces a big list of dates then bin types, so we split them up into a list of lists - each pair is
+        # a date and the bin type.
+        items = [
+            i
+            for i in soup.find(
+                "u1",
+                {
+                    "class": "displayinlineblock justifycontentleft alignitemscenter margin0 padding0"
+                },
+            ).text.split("\n")
+            if i != ""
+        ]
+        pairs = [items[i : i + 2] for i in range(0, len(items), 2)]
+
+        # Loop through the paired bin dates and types
+        for pair in pairs:
+            # This isn't necessary, but better safe than sorry
+            collection_date = datetime.strptime(pair[0], date_format).strftime(
+                date_format
+            )
+            # Change the formatting of the purple bins to replace the hyphens with slashes
+            if pair[1] == "Batteries-small electricals-textiles":
+                bin_type = pair[1].replace("-", "/").strip()
+            else:
+                bin_type = pair[1]
+
+            # Add the data into the dictionary
+            data["bins"].append({"type": bin_type, "collectionDate": collection_date})
+
+        return data

uk_bin_collection/uk_bin_collection/councils/WyreCouncil.py

@@ -0,0 +1,89 @@
+import requests
+import json
+import urllib.parse
+from dateutil.relativedelta import relativedelta
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        user_postcode = kwargs.get("postcode")
+        check_postcode(user_postcode)
+
+        data = {"bins": []}
+        collections = []
+
+        headers = {
+            "authority": "www.wyre.gov.uk",
+            "accept": "application/json, text/javascript, */*; q=0.01",
+            "accept-language": "en-GB,en;q=0.9",
+            "cache-control": "no-cache",
+            # 'content-length': '0',
+            # 'cookie': 'PHPSESSID=ApMqEd65JEQwNgj2AHeeekU9YA5%2C8Tc8YW-nWYSmWkfYq3mS1nvE1WLzMfeWgyoj',
+            "origin": "https://www.wyre.gov.uk",
+            "pragma": "no-cache",
+            "referer": "https://www.wyre.gov.uk/bincollections",
+            "sec-fetch-dest": "empty",
+            "sec-fetch-mode": "cors",
+            "sec-fetch-site": "same-origin",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.186 Safari/537.36",
+            "x-requested-with": "XMLHttpRequest",
+        }
+        postcode_api = f"https://www.wyre.gov.uk/singlepoint_ajax/{urllib.parse.quote(user_postcode)}"
+        addr_res = requests.post(postcode_api, headers=headers)
+        json_data = addr_res.json()
+
+        for v in json_data.values():
+            v_uprn = v.get("uprn")
+            if v_uprn == user_uprn:
+                addr_line = v.get("label")
+                break
+
+        api_url = f"https://www.wyre.gov.uk/bincollections?uprn={user_uprn}&address={urllib.parse.quote(addr_line)}&submit="
+        res = requests.get(api_url, headers=headers)
+
+        soup = BeautifulSoup(res.text, features="html.parser")
+        soup.prettify()
+
+        bins = soup.find_all("div", {"class": "boxed"})
+
+        for item in bins:
+            collection_title = " ".join(
+                item.find("h3", {"class": "bin-collection-tasks__heading"}).text.split(
+                    " "
+                )[2:4]
+            )
+            collection_date = datetime.strptime(
+                remove_ordinal_indicator_from_date_string(
+                    item.find("div", {"class": "bin-collection-tasks__content"})
+                    .text.strip()
+                    .replace("\n", " ")
+                ),
+                "%A %d %B",
+            )
+            next_collection = collection_date.replace(year=datetime.now().year)
+            if datetime.now().month == 12 and next_collection.month == 1:
+                next_collection = next_collection + relativedelta(years=1)
+            collections.append((collection_title, next_collection))
+
+        ordered_data = sorted(collections, key=lambda x: x[1])
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0].capitalize(),
+                "collectionDate": item[1].strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py

@@ -0,0 +1,45 @@
+import json
+from datetime import datetime
+
+import requests
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        api_url = (
+            "https://waste-api.york.gov.uk/api/Collections/GetBinCollectionDataForUprn/"
+        )
+        uprn = kwargs.get("uprn")
+        check_uprn(uprn)
+
+        requests.packages.urllib3.disable_warnings()
+        response = requests.get(f"{api_url}{uprn}")
+        json_response = json.loads(response.content)["services"]
+        data = {"bins": []}
+        collection_tuple = []
+
+        for item in json_response:
+            collection_date = datetime.strptime(
+                item.get("nextCollection"), "%Y-%m-%dT%H:%M:%S"
+            ).strftime(date_format)
+            collection_tuple.append((item.get("service"), collection_date))
+
+        ordered_data = sorted(collection_tuple, key=lambda x: x[1])
+
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0],
+                "collectionDate": item[1],
+            }
+            data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py

@@ -0,0 +1,33 @@
+from bs4 import BeautifulSoup
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BS4 object
+        soup = BeautifulSoup(page.text, features="html.parser")
+        soup.prettify()
+
+        data = {"bins": []}
+
+        for bins in soup.select('div[class*="service-item"]'):
+            bin_type = bins.div.h3.text.strip()
+            bin_collection = bins.select("div > p")[1]
+            if bin_collection:
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": datetime.strptime(
+                        bin_collection.get_text(strip=True), "%A, %d %B %Y"
+                    ),
+                }
+                data["bins"].append(dict_data)
+
+        return data

uk_bin_collection/uk_bin_collection/get_bin_data.py

@@ -0,0 +1,165 @@
+"""Module that contains an abstract class that can be imported to
+handle the data recieved from the provided council class.
+
+Keyword arguments: None
+"""
+
+import json
+import logging
+from abc import ABC, abstractmethod
+from logging.config import dictConfig
+import os
+
+import requests
+
+from uk_bin_collection.uk_bin_collection.common import update_input_json
+
+_LOGGER = logging.getLogger(__name__)
+
+LOGGING_CONFIG = dict(
+    version=1,
+    formatters={"f": {"format": "%(asctime)s %(name)-12s %(levelname)-8s %(message)s"}},
+    handlers={
+        "h": {"class": "logging.StreamHandler", "formatter": "f", "level": logging.INFO}
+    },
+    root={"handlers": ["h"], "level": logging.INFO},
+)
+
+
+def setup_logging(logging_config, logger_name):
+    try:
+        logging.config.dictConfig(logging_config)
+        logger = logging.getLogger(logger_name)
+        return logger
+    except Exception as exp:
+        raise exp
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+
+
+class AbstractGetBinDataClass(ABC):
+    """An abstract class that can be imported to handle the data recieved from the provided
+    council class.
+
+    Keyword arguments: None
+    """
+
+    def template_method(self, address_url: str, **kwargs) -> None:  # pragma: no cover
+        """The main template method that is constructed
+
+        Keyword arguments:
+        address_url -- the url to get the data from
+        """
+        this_url = address_url
+        this_postcode = kwargs.get("postcode", None)
+        this_paon = kwargs.get("paon", None)
+        this_uprn = kwargs.get("uprn", None)
+        this_usrn = kwargs.get("usrn", None)
+        this_web_driver = kwargs.get("web_driver", None)
+        this_headless = kwargs.get("headless", None)
+        skip_get_url = kwargs.get("skip_get_url", None)
+        dev_mode = kwargs.get("dev_mode", False)
+        council_module_str = kwargs.get("council_module_str", None)
+        if (
+            not skip_get_url or skip_get_url is False
+        ):  # we will not use the generic way to get data - needs a get data in the council class itself
+            page = self.get_data(address_url)
+            bin_data_dict = self.parse_data(
+                page,
+                postcode=this_postcode,
+                paon=this_paon,
+                uprn=this_uprn,
+                usrn=this_usrn,
+                web_driver=this_web_driver,
+                headless=this_headless,
+                url=this_url,
+            )
+            json_output = self.output_json(bin_data_dict)
+        else:
+            bin_data_dict = self.parse_data(
+                "",
+                postcode=this_postcode,
+                paon=this_paon,
+                uprn=this_uprn,
+                usrn=this_usrn,
+                web_driver=this_web_driver,
+                headless=this_headless,
+                url=this_url,
+            )
+            json_output = self.output_json(bin_data_dict)
+
+        # if dev mode create/update council's entry in the input.json
+        if dev_mode is not None and dev_mode is True:
+            cwd = os.getcwd()
+            input_file_path = os.path.join(
+                cwd, "uk_bin_collection", "tests", "input.json"
+            )
+            update_input_json(
+                council_module_str,
+                this_url,
+                input_file_path,
+                postcode=this_postcode,
+                paon=this_paon,
+                uprn=this_uprn,
+                usrn=this_usrn,
+                web_driver=this_web_driver,
+                skip_get_url=skip_get_url,
+            )
+
+        return json_output
+
+    @classmethod
+    def get_data(cls, url) -> str:
+        """This method makes the request to the council
+
+        Keyword arguments:
+        url -- the url to get the data from
+        """
+        # Set a user agent so we look like a browser ;-)
+        user_agent = (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/108.0.0.0 Safari/537.36"
+        )
+        headers = {"User-Agent": user_agent}
+        requests.packages.urllib3.disable_warnings()
+
+        # Make the Request - change the URL - find out your property number
+        try:
+            full_page = requests.get(url, headers, verify=False)
+            return full_page
+        except requests.exceptions.HTTPError as errh:
+            _LOGGER.error(f"Http Error: {errh}")
+            raise
+        except requests.exceptions.ConnectionError as errc:
+            _LOGGER.error(f"Error Connecting: {errc}")
+            raise
+        except requests.exceptions.Timeout as errt:
+            _LOGGER.error(f"Timeout Error: {errt}")
+            raise
+        except requests.exceptions.RequestException as err:
+            _LOGGER.error(f"Oops: Something Else {err}")
+            raise
+
+    @abstractmethod
+    def parse_data(self, page: str, **kwargs) -> dict:
+        """Abstract method that takes a page as a string
+
+        Keyword arguments:
+        page -- a string from the requested page
+        """
+
+    @classmethod
+    def output_json(cls, bin_data_dict: dict) -> str:
+        """Method to output the json as a pretty printed string
+
+        Keyword arguments:
+        bin_data_dict -- a dict parsed data
+        """
+        # Form a JSON wrapper
+        # Make the JSON
+
+        json_data = json.dumps(bin_data_dict, sort_keys=False, indent=4)
+
+        # Output the data
+        return json_data

uk_bin_collection-0.74.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Robert Bradley
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.