PyPI - uk_bin_collection - Versions diffs - 0.148.5__tar.gz → 0.148.6__tar.gz - Mend

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.148.5
+Version: 0.148.6
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.148.5"
+version = "0.148.6"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

@@ -2341,6 +2341,7 @@
     "ThanetDistrictCouncil": {
         "uprn": "100061111858",
         "url": "https://www.thanet.gov.uk",
+        "web_driver": "http://selenium:4444",
         "wiki_name": "Thanet",
         "wiki_note": "Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN.",
         "LAD24CD": "E07000114"

@@ -0,0 +1,78 @@
+import json
+import time
+from datetime import datetime
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+        url = f"https://www.thanet.gov.uk/wp-content/mu-plugins/collection-day/incl/mu-collection-day-calls.php?pAddress={user_uprn}"
+        web_driver = kwargs.get("web_driver")
+        headless = kwargs.get("headless")
+        # Create the Selenium WebDriver
+        driver = create_webdriver(web_driver, headless, None, __name__)
+        try:
+            print(f"Navigating to URL: {url}")
+            driver.get(url)
+            # Wait for Cloudflare to complete its check
+            WebDriverWait(driver, 30).until(
+                lambda d: d.execute_script("return document.readyState") == "complete"
+            )
+            print("Page loaded successfully.")
+            # Parse the page source with BeautifulSoup
+            soup = BeautifulSoup(driver.page_source, "html.parser")
+            # Extract the JSON data from the page
+            print("Extracting bin collection data...")
+            body_content = soup.find("body").text
+            if not body_content:
+                raise ValueError("Expected JSON data not found in the <body> tag.")
+            bin_collection = json.loads(body_content)
+            # Process the bin collection data
+            for collection in bin_collection:
+                bin_type = collection["type"]
+                collection_date = collection["nextDate"].split(" ")[0]
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": collection_date,
+                }
+                bindata["bins"].append(dict_data)
+            # Sort the bins by collection date
+            bindata["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+            )
+            print(bindata)
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            raise
+        finally:
+            print("Cleaning up WebDriver...")
+            driver.quit()
+        return bindata

@@ -1,51 +0,0 @@
-import time
-import requests
-from uk_bin_collection.uk_bin_collection.common import *
-from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
-# import the wonderful Beautiful Soup and the URL grabber
-class CouncilClass(AbstractGetBinDataClass):
-    """
-    Concrete classes have to implement all abstract operations of the
-    base class. They can also override some operations with a default
-    implementation.
-    """
-    def parse_data(self, page: str, **kwargs) -> dict:
-        user_uprn = kwargs.get("uprn")
-        check_uprn(user_uprn)
-        bindata = {"bins": []}
-        URI = f"https://www.thanet.gov.uk/wp-content/mu-plugins/collection-day/incl/mu-collection-day-calls.php?pAddress={user_uprn}"
-        headers = {
-            "x-requested-with": "XMLHttpRequest",
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-        }
-        # Make the GET request
-        response = requests.get(URI, headers=headers)
-        # Parse the JSON response
-        bin_collection = response.json()
-        # Loop through each collection in bin_collection
-        for collection in bin_collection:
-            bin_type = collection["type"]
-            collection_date = collection["nextDate"].split(" ")[0]
-            dict_data = {
-                "type": bin_type,
-                "collectionDate": collection_date,
-            }
-            bindata["bins"].append(dict_data)
-        bindata["bins"].sort(
-            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
-        )
-        return bindata

uk_bin_collection 0.148.5__tar.gz → 0.148.6__tar.gz

uk_bin_collection 0.148.5tar.gz → 0.148.6tar.gz