uk_bin_collection 0.148.5__py3-none-any.whl → 0.149.0__py3-none-any.whl

uk_bin_collection/tests/input.json

@@ -1772,6 +1772,13 @@
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN.",
         "LAD24CD": "E07000178"
     },
+    "PembrokeshireCountyCouncil": {
+        "url": "https://nearest.pembrokeshire.gov.uk/property/100100278790",
+        "wiki_command_url_override": "https://nearest.pembrokeshire.gov.uk/property/XXXXXXXXXX",
+        "wiki_name": "Pembrokeshire",
+        "wiki_note": "Replace XXXXXXXX with your UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find it.",
+        "LAD24CD": "W06000009"
+    },
     "PeterboroughCityCouncil": {
         "house_number": "7 Arundel Road, Peterborough, PE4 6JJ",
         "postcode": "PE4 6JJ",
@@ -2341,6 +2348,7 @@
     "ThanetDistrictCouncil": {
         "uprn": "100061111858",
         "url": "https://www.thanet.gov.uk",
+        "web_driver": "http://selenium:4444",
         "wiki_name": "Thanet",
         "wiki_note": "Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN.",
         "LAD24CD": "E07000114"

uk_bin_collection/uk_bin_collection/councils/PembrokeshireCountyCouncil.py

@@ -0,0 +1,96 @@
+from datetime import datetime
+
+import bs4.element
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Make a BeautifulSoup object
+        soup = BeautifulSoup(page.text, features="html.parser")
+
+        data = {"bins": []}
+
+        # Locate the section containing bin collection data
+        bin_collection_divs = soup.find_all(
+            "div", class_="col-6 col-md-4 text-center mb-3"
+        )
+
+        if not bin_collection_divs:
+            raise ValueError("No bin collection data found in the provided HTML.")
+
+        for bin_div in bin_collection_divs:
+            # Get the image tag first to check if this is a bin collection div
+            img_tag = bin_div.find("img")
+            if (
+                not img_tag
+                or not img_tag.get("src")
+                or "pembrokeshire.gov.uk/images" not in img_tag["src"]
+            ):
+                continue
+
+            # Extract bin type - first try the image title
+            bin_type = None
+            if img_tag.get("title"):
+                bin_type = img_tag["title"].strip()
+
+            # If no title, get all text nodes and join them
+            if not bin_type:
+                # Get all text nodes that are not within a <strong> tag (to exclude the date)
+                text_nodes = [
+                    text.strip()
+                    for text in bin_div.find_all(text=True, recursive=True)
+                    if text.strip()
+                    and not isinstance(text.parent, bs4.element.Tag)
+                    or text.parent.name != "strong"
+                ]
+                if text_nodes:
+                    bin_type = " ".join(text_nodes).strip()
+
+            if not bin_type:
+                continue  # Skip if we couldn't find a bin type
+
+            # Extract collection date
+            bin_date_tag = bin_div.find("strong")
+            if not bin_date_tag:
+                continue  # Skip if no date found
+
+            bin_date = bin_date_tag.text.strip()
+
+            try:
+                # Parse the date into a datetime object
+                collection_date = datetime.strptime(bin_date, "%d/%m/%Y")
+                # Format date back to DD/MM/YYYY format as required by schema
+                formatted_date = collection_date.strftime("%d/%m/%Y")
+            except ValueError:
+                continue  # Skip if date parsing fails
+
+            # Append the bin data to the list
+            dict_data = {
+                "type": bin_type,
+                "collectionDate": formatted_date,
+            }
+            data["bins"].append(dict_data)
+
+        if not data["bins"]:
+            raise ValueError(
+                "No valid bin collection data could be parsed from the HTML."
+            )
+
+        # Sort the bins by collection date
+        data["bins"].sort(
+            key=lambda x: datetime.strptime(x["collectionDate"], "%d/%m/%Y")
+        )
+
+        print(data)
+
+        return data

uk_bin_collection/uk_bin_collection/councils/ThanetDistrictCouncil.py

@@ -1,12 +1,16 @@
+import json
 import time
+from datetime import datetime
 
-import requests
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
 
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
-# import the wonderful Beautiful Soup and the URL grabber
 class CouncilClass(AbstractGetBinDataClass):
     """
     Concrete classes have to implement all abstract operations of the
@@ -15,37 +19,60 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page: str, **kwargs) -> dict:
-
         user_uprn = kwargs.get("uprn")
         check_uprn(user_uprn)
         bindata = {"bins": []}
 
-        URI = f"https://www.thanet.gov.uk/wp-content/mu-plugins/collection-day/incl/mu-collection-day-calls.php?pAddress={user_uprn}"
+        url = f"https://www.thanet.gov.uk/wp-content/mu-plugins/collection-day/incl/mu-collection-day-calls.php?pAddress={user_uprn}"
+        web_driver = kwargs.get("web_driver")
+        headless = kwargs.get("headless")
+
+        # Create the Selenium WebDriver
+        driver = create_webdriver(web_driver, headless, None, __name__)
+
+        try:
+            print(f"Navigating to URL: {url}")
+            driver.get(url)
+
+            # Wait for Cloudflare to complete its check
+            WebDriverWait(driver, 30).until(
+                lambda d: d.execute_script("return document.readyState") == "complete"
+            )
+            print("Page loaded successfully.")
+
+            # Parse the page source with BeautifulSoup
+            soup = BeautifulSoup(driver.page_source, "html.parser")
 
-        headers = {
-            "x-requested-with": "XMLHttpRequest",
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-        }
+            # Extract the JSON data from the page
+            print("Extracting bin collection data...")
+            body_content = soup.find("body").text
+            if not body_content:
+                raise ValueError("Expected JSON data not found in the <body> tag.")
 
-        # Make the GET request
-        response = requests.get(URI, headers=headers)
+            bin_collection = json.loads(body_content)
 
-        # Parse the JSON response
-        bin_collection = response.json()
+            # Process the bin collection data
+            for collection in bin_collection:
+                bin_type = collection["type"]
+                collection_date = collection["nextDate"].split(" ")[0]
 
-        # Loop through each collection in bin_collection
-        for collection in bin_collection:
-            bin_type = collection["type"]
-            collection_date = collection["nextDate"].split(" ")[0]
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": collection_date,
+                }
+                bindata["bins"].append(dict_data)
 
-            dict_data = {
-                "type": bin_type,
-                "collectionDate": collection_date,
-            }
-            bindata["bins"].append(dict_data)
+            # Sort the bins by collection date
+            bindata["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+            )
+            print(bindata)
 
-        bindata["bins"].sort(
-            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
-        )
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            raise
+        finally:
+            print("Cleaning up WebDriver...")
+            driver.quit()
 
         return bindata

{uk_bin_collection-0.148.5.dist-info → uk_bin_collection-0.149.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: uk_bin_collection
-Version: 0.148.5
+Version: 0.149.0
 Summary: Python Lib to collect UK Bin Data
 Author: Robert Bradley
 Author-email: robbrad182@gmail.com

{uk_bin_collection-0.148.5.dist-info → uk_bin_collection-0.149.0.dist-info}/RECORD

@@ -7,7 +7,7 @@ uk_bin_collection/tests/council_feature_input_parity.py,sha256=DO6Mk4ImYgM5ZCZ-c
 uk_bin_collection/tests/features/environment.py,sha256=VQZjJdJI_kZn08M0j5cUgvKT4k3iTw8icJge1DGOkoA,127
 uk_bin_collection/tests/features/validate_council_outputs.feature,sha256=SJK-Vc737hrf03tssxxbeg_JIvAH-ddB8f6gU1LTbuQ,251
 uk_bin_collection/tests/generate_map_test_results.py,sha256=CKnGK2ZgiSXomRGkomX90DitgMP-X7wkHhyKORDcL2E,1144
-uk_bin_collection/tests/input.json,sha256=Zjpi_QnZFlIojCGBJfBkQ8Tn9O2zbi48w_yKr0wbSXU,131194
+uk_bin_collection/tests/input.json,sha256=Uur26vWavpRAc9xJ5an1GUFQzNdct6NS24ZhWQxHad4,131672
 uk_bin_collection/tests/output.schema,sha256=ZwKQBwYyTDEM4G2hJwfLUVM-5v1vKRvRK9W9SS1sd18,1086
 uk_bin_collection/tests/step_defs/step_helpers/file_handler.py,sha256=Ygzi4V0S1MIHqbdstUlIqtRIwnynvhu4UtpweJ6-5N8,1474
 uk_bin_collection/tests/step_defs/test_validate_council.py,sha256=VZ0a81sioJULD7syAYHjvK_-nT_Rd36tUyzPetSA0gk,3475
@@ -224,6 +224,7 @@ uk_bin_collection/uk_bin_collection/councils/NuneatonBedworthBoroughCouncil.py,s
 uk_bin_collection/uk_bin_collection/councils/OadbyAndWigstonBoroughCouncil.py,sha256=Kgy5HA0xZ9hR4_cAydPfOfskhGUB4j93AQF2-9Fj-Cg,2179
 uk_bin_collection/uk_bin_collection/councils/OldhamCouncil.py,sha256=9dlesCxNoVXlmQaqZj7QFh00smnJbm1Gnjkr_Uvzurs,1771
 uk_bin_collection/uk_bin_collection/councils/OxfordCityCouncil.py,sha256=d_bY0cXRDH4kSoWGGCTNN61MNErapSOf2WSTYDJr2r8,2318
+uk_bin_collection/uk_bin_collection/councils/PembrokeshireCountyCouncil.py,sha256=GRAxjf_DuO5uZ660kEbZt_yCjP3n2maMxsYTPCTz3GQ,3324
 uk_bin_collection/uk_bin_collection/councils/PerthAndKinrossCouncil.py,sha256=Kos5GzN2co3Ij3tSHOXB9S71Yt78RROCfVRtnh7M1VU,3657
 uk_bin_collection/uk_bin_collection/councils/PeterboroughCityCouncil.py,sha256=lOrDD4jfJ-_C5UwCGqRcQ1G-U1F5X6rf255ypzYEBcg,6300
 uk_bin_collection/uk_bin_collection/councils/PlymouthCouncil.py,sha256=FJqpJ0GJhpjYeyZ9ioZPkKGl-zrqMD3y5iKa07e_i30,3202
@@ -291,7 +292,7 @@ uk_bin_collection/uk_bin_collection/councils/TeignbridgeCouncil.py,sha256=-NowMN
 uk_bin_collection/uk_bin_collection/councils/TelfordAndWrekinCouncil.py,sha256=p1ZS5R4EGxbEWlRBrkGXgKwE_lkyBT-R60yKFFhVObc,1844
 uk_bin_collection/uk_bin_collection/councils/TendringDistrictCouncil.py,sha256=1_CkpWPTfRUEP5YJ9R4_dJRLtb-O9i83hfWJc1shw_c,4283
 uk_bin_collection/uk_bin_collection/councils/TestValleyBoroughCouncil.py,sha256=Dtfkyrwt795W7gqFJxVGRR8t3R5WMNQZwTWJckLpZWE,8480
-uk_bin_collection/uk_bin_collection/councils/ThanetDistrictCouncil.py,sha256=-opmZG9GzjB_NvmWpN6nFZ7rlkSoaRrQICU5E8T0DEQ,1659
+uk_bin_collection/uk_bin_collection/councils/ThanetDistrictCouncil.py,sha256=Cxrf0tUryDL-wFclPH5yovVt8i7Sc7g-ZFrU9_wg6KY,2717
 uk_bin_collection/uk_bin_collection/councils/ThreeRiversDistrictCouncil.py,sha256=RHt3e9oeKzwxjjY-M8aC0nk-ZXhHIoyC81JzxkPVxsE,5531
 uk_bin_collection/uk_bin_collection/councils/ThurrockCouncil.py,sha256=vAZMm6mcsdEcOkP15xwxWy9gdXpmLYQFH7qRifurNoY,2935
 uk_bin_collection/uk_bin_collection/councils/TonbridgeAndMallingBC.py,sha256=UlgnHDoi8ecav2H5-HqKNDpqW1J3RN-c___5c08_Q7I,4859
@@ -337,8 +338,8 @@ uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py,sha256=I2kBYMlsD4bId
 uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py,sha256=QD4v4xpsEE0QheR_fGaNOIRMc2FatcUfKkkhAhseyVU,1159
 uk_bin_collection/uk_bin_collection/create_new_council.py,sha256=m-IhmWmeWQlFsTZC4OxuFvtw5ZtB8EAJHxJTH4O59lQ,1536
 uk_bin_collection/uk_bin_collection/get_bin_data.py,sha256=YvmHfZqanwrJ8ToGch34x-L-7yPe31nB_x77_Mgl_vo,4545
-uk_bin_collection-0.148.5.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
-uk_bin_collection-0.148.5.dist-info/METADATA,sha256=QFI4MOygjxoWUqezFNR-eFxcJM105HMe1volyN_IQjk,20914
-uk_bin_collection-0.148.5.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-uk_bin_collection-0.148.5.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
-uk_bin_collection-0.148.5.dist-info/RECORD,,
+uk_bin_collection-0.149.0.dist-info/LICENSE,sha256=vABBUOzcrgfaTKpzeo-si9YVEun6juDkndqA8RKdKGs,1071
+uk_bin_collection-0.149.0.dist-info/METADATA,sha256=88bzKT9T-AZ1OLv3p83LPJY3qbHpXS4-jk_NluPkBvA,20914
+uk_bin_collection-0.149.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+uk_bin_collection-0.149.0.dist-info/entry_points.txt,sha256=36WCSGMWSc916S3Hi1ZkazzDKHaJ6CD-4fCEFm5MIao,90
+uk_bin_collection-0.149.0.dist-info/RECORD,,